ixgbe_main.c 219 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058
  1. /*******************************************************************************
  2. Intel 10 Gigabit PCI Express Linux driver
  3. Copyright(c) 1999 - 2011 Intel Corporation.
  4. This program is free software; you can redistribute it and/or modify it
  5. under the terms and conditions of the GNU General Public License,
  6. version 2, as published by the Free Software Foundation.
  7. This program is distributed in the hope it will be useful, but WITHOUT
  8. ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  9. FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  10. more details.
  11. You should have received a copy of the GNU General Public License along with
  12. this program; if not, write to the Free Software Foundation, Inc.,
  13. 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  14. The full GNU General Public License is included in this distribution in
  15. the file called "COPYING".
  16. Contact Information:
  17. e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  18. Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  19. *******************************************************************************/
  20. #include <linux/types.h>
  21. #include <linux/module.h>
  22. #include <linux/pci.h>
  23. #include <linux/netdevice.h>
  24. #include <linux/vmalloc.h>
  25. #include <linux/string.h>
  26. #include <linux/in.h>
  27. #include <linux/interrupt.h>
  28. #include <linux/ip.h>
  29. #include <linux/tcp.h>
  30. #include <linux/sctp.h>
  31. #include <linux/pkt_sched.h>
  32. #include <linux/ipv6.h>
  33. #include <linux/slab.h>
  34. #include <net/checksum.h>
  35. #include <net/ip6_checksum.h>
  36. #include <linux/ethtool.h>
  37. #include <linux/if.h>
  38. #include <linux/if_vlan.h>
  39. #include <linux/prefetch.h>
  40. #include <scsi/fc/fc_fcoe.h>
  41. #include "ixgbe.h"
  42. #include "ixgbe_common.h"
  43. #include "ixgbe_dcb_82599.h"
  44. #include "ixgbe_sriov.h"
  45. char ixgbe_driver_name[] = "ixgbe";
  46. static const char ixgbe_driver_string[] =
  47. "Intel(R) 10 Gigabit PCI Express Network Driver";
  48. #define MAJ 3
  49. #define MIN 6
  50. #define BUILD 7
  51. #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  52. __stringify(BUILD) "-k"
  53. const char ixgbe_driver_version[] = DRV_VERSION;
  54. static const char ixgbe_copyright[] =
  55. "Copyright (c) 1999-2011 Intel Corporation.";
  56. static const struct ixgbe_info *ixgbe_info_tbl[] = {
  57. [board_82598] = &ixgbe_82598_info,
  58. [board_82599] = &ixgbe_82599_info,
  59. [board_X540] = &ixgbe_X540_info,
  60. };
  61. /* ixgbe_pci_tbl - PCI Device ID Table
  62. *
  63. * Wildcard entries (PCI_ANY_ID) should come last
  64. * Last entry must be all 0s
  65. *
  66. * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  67. * Class, Class Mask, private data (not used) }
  68. */
  69. static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
  70. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
  71. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
  72. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
  73. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
  74. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
  75. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
  76. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
  77. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
  78. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
  79. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
  80. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
  81. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
  82. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
  83. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
  84. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
  85. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
  86. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
  87. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
  88. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
  89. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
  90. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
  91. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
  92. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
  93. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
  94. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
  95. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
  96. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
  97. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
  98. /* required last entry */
  99. {0, }
  100. };
  101. MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
  102. #ifdef CONFIG_IXGBE_DCA
  103. static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
  104. void *p);
  105. static struct notifier_block dca_notifier = {
  106. .notifier_call = ixgbe_notify_dca,
  107. .next = NULL,
  108. .priority = 0
  109. };
  110. #endif
  111. #ifdef CONFIG_PCI_IOV
  112. static unsigned int max_vfs;
  113. module_param(max_vfs, uint, 0);
  114. MODULE_PARM_DESC(max_vfs,
  115. "Maximum number of virtual functions to allocate per physical function");
  116. #endif /* CONFIG_PCI_IOV */
  117. MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  118. MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
  119. MODULE_LICENSE("GPL");
  120. MODULE_VERSION(DRV_VERSION);
  121. #define DEFAULT_DEBUG_LEVEL_SHIFT 3
  122. static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
  123. {
  124. if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
  125. !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
  126. schedule_work(&adapter->service_task);
  127. }
  128. static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
  129. {
  130. BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
  131. /* flush memory to make sure state is correct before next watchdog */
  132. smp_mb__before_clear_bit();
  133. clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
  134. }
  135. struct ixgbe_reg_info {
  136. u32 ofs;
  137. char *name;
  138. };
  139. static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
  140. /* General Registers */
  141. {IXGBE_CTRL, "CTRL"},
  142. {IXGBE_STATUS, "STATUS"},
  143. {IXGBE_CTRL_EXT, "CTRL_EXT"},
  144. /* Interrupt Registers */
  145. {IXGBE_EICR, "EICR"},
  146. /* RX Registers */
  147. {IXGBE_SRRCTL(0), "SRRCTL"},
  148. {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
  149. {IXGBE_RDLEN(0), "RDLEN"},
  150. {IXGBE_RDH(0), "RDH"},
  151. {IXGBE_RDT(0), "RDT"},
  152. {IXGBE_RXDCTL(0), "RXDCTL"},
  153. {IXGBE_RDBAL(0), "RDBAL"},
  154. {IXGBE_RDBAH(0), "RDBAH"},
  155. /* TX Registers */
  156. {IXGBE_TDBAL(0), "TDBAL"},
  157. {IXGBE_TDBAH(0), "TDBAH"},
  158. {IXGBE_TDLEN(0), "TDLEN"},
  159. {IXGBE_TDH(0), "TDH"},
  160. {IXGBE_TDT(0), "TDT"},
  161. {IXGBE_TXDCTL(0), "TXDCTL"},
  162. /* List Terminator */
  163. {}
  164. };
  165. /*
  166. * ixgbe_regdump - register printout routine
  167. */
  168. static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
  169. {
  170. int i = 0, j = 0;
  171. char rname[16];
  172. u32 regs[64];
  173. switch (reginfo->ofs) {
  174. case IXGBE_SRRCTL(0):
  175. for (i = 0; i < 64; i++)
  176. regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
  177. break;
  178. case IXGBE_DCA_RXCTRL(0):
  179. for (i = 0; i < 64; i++)
  180. regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
  181. break;
  182. case IXGBE_RDLEN(0):
  183. for (i = 0; i < 64; i++)
  184. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
  185. break;
  186. case IXGBE_RDH(0):
  187. for (i = 0; i < 64; i++)
  188. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
  189. break;
  190. case IXGBE_RDT(0):
  191. for (i = 0; i < 64; i++)
  192. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
  193. break;
  194. case IXGBE_RXDCTL(0):
  195. for (i = 0; i < 64; i++)
  196. regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
  197. break;
  198. case IXGBE_RDBAL(0):
  199. for (i = 0; i < 64; i++)
  200. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
  201. break;
  202. case IXGBE_RDBAH(0):
  203. for (i = 0; i < 64; i++)
  204. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
  205. break;
  206. case IXGBE_TDBAL(0):
  207. for (i = 0; i < 64; i++)
  208. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
  209. break;
  210. case IXGBE_TDBAH(0):
  211. for (i = 0; i < 64; i++)
  212. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
  213. break;
  214. case IXGBE_TDLEN(0):
  215. for (i = 0; i < 64; i++)
  216. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
  217. break;
  218. case IXGBE_TDH(0):
  219. for (i = 0; i < 64; i++)
  220. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
  221. break;
  222. case IXGBE_TDT(0):
  223. for (i = 0; i < 64; i++)
  224. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
  225. break;
  226. case IXGBE_TXDCTL(0):
  227. for (i = 0; i < 64; i++)
  228. regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
  229. break;
  230. default:
  231. pr_info("%-15s %08x\n", reginfo->name,
  232. IXGBE_READ_REG(hw, reginfo->ofs));
  233. return;
  234. }
  235. for (i = 0; i < 8; i++) {
  236. snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
  237. pr_err("%-15s", rname);
  238. for (j = 0; j < 8; j++)
  239. pr_cont(" %08x", regs[i*8+j]);
  240. pr_cont("\n");
  241. }
  242. }
  243. /*
  244. * ixgbe_dump - Print registers, tx-rings and rx-rings
  245. */
  246. static void ixgbe_dump(struct ixgbe_adapter *adapter)
  247. {
  248. struct net_device *netdev = adapter->netdev;
  249. struct ixgbe_hw *hw = &adapter->hw;
  250. struct ixgbe_reg_info *reginfo;
  251. int n = 0;
  252. struct ixgbe_ring *tx_ring;
  253. struct ixgbe_tx_buffer *tx_buffer_info;
  254. union ixgbe_adv_tx_desc *tx_desc;
  255. struct my_u0 { u64 a; u64 b; } *u0;
  256. struct ixgbe_ring *rx_ring;
  257. union ixgbe_adv_rx_desc *rx_desc;
  258. struct ixgbe_rx_buffer *rx_buffer_info;
  259. u32 staterr;
  260. int i = 0;
  261. if (!netif_msg_hw(adapter))
  262. return;
  263. /* Print netdevice Info */
  264. if (netdev) {
  265. dev_info(&adapter->pdev->dev, "Net device Info\n");
  266. pr_info("Device Name state "
  267. "trans_start last_rx\n");
  268. pr_info("%-15s %016lX %016lX %016lX\n",
  269. netdev->name,
  270. netdev->state,
  271. netdev->trans_start,
  272. netdev->last_rx);
  273. }
  274. /* Print Registers */
  275. dev_info(&adapter->pdev->dev, "Register Dump\n");
  276. pr_info(" Register Name Value\n");
  277. for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
  278. reginfo->name; reginfo++) {
  279. ixgbe_regdump(hw, reginfo);
  280. }
  281. /* Print TX Ring Summary */
  282. if (!netdev || !netif_running(netdev))
  283. goto exit;
  284. dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
  285. pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
  286. for (n = 0; n < adapter->num_tx_queues; n++) {
  287. tx_ring = adapter->tx_ring[n];
  288. tx_buffer_info =
  289. &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
  290. pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
  291. n, tx_ring->next_to_use, tx_ring->next_to_clean,
  292. (u64)tx_buffer_info->dma,
  293. tx_buffer_info->length,
  294. tx_buffer_info->next_to_watch,
  295. (u64)tx_buffer_info->time_stamp);
  296. }
  297. /* Print TX Rings */
  298. if (!netif_msg_tx_done(adapter))
  299. goto rx_ring_summary;
  300. dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
  301. /* Transmit Descriptor Formats
  302. *
  303. * Advanced Transmit Descriptor
  304. * +--------------------------------------------------------------+
  305. * 0 | Buffer Address [63:0] |
  306. * +--------------------------------------------------------------+
  307. * 8 | PAYLEN | PORTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
  308. * +--------------------------------------------------------------+
  309. * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
  310. */
  311. for (n = 0; n < adapter->num_tx_queues; n++) {
  312. tx_ring = adapter->tx_ring[n];
  313. pr_info("------------------------------------\n");
  314. pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
  315. pr_info("------------------------------------\n");
  316. pr_info("T [desc] [address 63:0 ] "
  317. "[PlPOIdStDDt Ln] [bi->dma ] "
  318. "leng ntw timestamp bi->skb\n");
  319. for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
  320. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
  321. tx_buffer_info = &tx_ring->tx_buffer_info[i];
  322. u0 = (struct my_u0 *)tx_desc;
  323. pr_info("T [0x%03X] %016llX %016llX %016llX"
  324. " %04X %p %016llX %p", i,
  325. le64_to_cpu(u0->a),
  326. le64_to_cpu(u0->b),
  327. (u64)tx_buffer_info->dma,
  328. tx_buffer_info->length,
  329. tx_buffer_info->next_to_watch,
  330. (u64)tx_buffer_info->time_stamp,
  331. tx_buffer_info->skb);
  332. if (i == tx_ring->next_to_use &&
  333. i == tx_ring->next_to_clean)
  334. pr_cont(" NTC/U\n");
  335. else if (i == tx_ring->next_to_use)
  336. pr_cont(" NTU\n");
  337. else if (i == tx_ring->next_to_clean)
  338. pr_cont(" NTC\n");
  339. else
  340. pr_cont("\n");
  341. if (netif_msg_pktdata(adapter) &&
  342. tx_buffer_info->dma != 0)
  343. print_hex_dump(KERN_INFO, "",
  344. DUMP_PREFIX_ADDRESS, 16, 1,
  345. phys_to_virt(tx_buffer_info->dma),
  346. tx_buffer_info->length, true);
  347. }
  348. }
  349. /* Print RX Rings Summary */
  350. rx_ring_summary:
  351. dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
  352. pr_info("Queue [NTU] [NTC]\n");
  353. for (n = 0; n < adapter->num_rx_queues; n++) {
  354. rx_ring = adapter->rx_ring[n];
  355. pr_info("%5d %5X %5X\n",
  356. n, rx_ring->next_to_use, rx_ring->next_to_clean);
  357. }
  358. /* Print RX Rings */
  359. if (!netif_msg_rx_status(adapter))
  360. goto exit;
  361. dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
  362. /* Advanced Receive Descriptor (Read) Format
  363. * 63 1 0
  364. * +-----------------------------------------------------+
  365. * 0 | Packet Buffer Address [63:1] |A0/NSE|
  366. * +----------------------------------------------+------+
  367. * 8 | Header Buffer Address [63:1] | DD |
  368. * +-----------------------------------------------------+
  369. *
  370. *
  371. * Advanced Receive Descriptor (Write-Back) Format
  372. *
  373. * 63 48 47 32 31 30 21 20 16 15 4 3 0
  374. * +------------------------------------------------------+
  375. * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
  376. * | Checksum Ident | | | | Type | Type |
  377. * +------------------------------------------------------+
  378. * 8 | VLAN Tag | Length | Extended Error | Extended Status |
  379. * +------------------------------------------------------+
  380. * 63 48 47 32 31 20 19 0
  381. */
  382. for (n = 0; n < adapter->num_rx_queues; n++) {
  383. rx_ring = adapter->rx_ring[n];
  384. pr_info("------------------------------------\n");
  385. pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
  386. pr_info("------------------------------------\n");
  387. pr_info("R [desc] [ PktBuf A0] "
  388. "[ HeadBuf DD] [bi->dma ] [bi->skb] "
  389. "<-- Adv Rx Read format\n");
  390. pr_info("RWB[desc] [PcsmIpSHl PtRs] "
  391. "[vl er S cks ln] ---------------- [bi->skb] "
  392. "<-- Adv Rx Write-Back format\n");
  393. for (i = 0; i < rx_ring->count; i++) {
  394. rx_buffer_info = &rx_ring->rx_buffer_info[i];
  395. rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
  396. u0 = (struct my_u0 *)rx_desc;
  397. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  398. if (staterr & IXGBE_RXD_STAT_DD) {
  399. /* Descriptor Done */
  400. pr_info("RWB[0x%03X] %016llX "
  401. "%016llX ---------------- %p", i,
  402. le64_to_cpu(u0->a),
  403. le64_to_cpu(u0->b),
  404. rx_buffer_info->skb);
  405. } else {
  406. pr_info("R [0x%03X] %016llX "
  407. "%016llX %016llX %p", i,
  408. le64_to_cpu(u0->a),
  409. le64_to_cpu(u0->b),
  410. (u64)rx_buffer_info->dma,
  411. rx_buffer_info->skb);
  412. if (netif_msg_pktdata(adapter)) {
  413. print_hex_dump(KERN_INFO, "",
  414. DUMP_PREFIX_ADDRESS, 16, 1,
  415. phys_to_virt(rx_buffer_info->dma),
  416. rx_ring->rx_buf_len, true);
  417. if (rx_ring->rx_buf_len
  418. < IXGBE_RXBUFFER_2K)
  419. print_hex_dump(KERN_INFO, "",
  420. DUMP_PREFIX_ADDRESS, 16, 1,
  421. phys_to_virt(
  422. rx_buffer_info->page_dma +
  423. rx_buffer_info->page_offset
  424. ),
  425. PAGE_SIZE/2, true);
  426. }
  427. }
  428. if (i == rx_ring->next_to_use)
  429. pr_cont(" NTU\n");
  430. else if (i == rx_ring->next_to_clean)
  431. pr_cont(" NTC\n");
  432. else
  433. pr_cont("\n");
  434. }
  435. }
  436. exit:
  437. return;
  438. }
  439. static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
  440. {
  441. u32 ctrl_ext;
  442. /* Let firmware take over control of h/w */
  443. ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
  444. IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
  445. ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
  446. }
  447. static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
  448. {
  449. u32 ctrl_ext;
  450. /* Let firmware know the driver has taken over */
  451. ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
  452. IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
  453. ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
  454. }
  455. /*
  456. * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
  457. * @adapter: pointer to adapter struct
  458. * @direction: 0 for Rx, 1 for Tx, -1 for other causes
  459. * @queue: queue to map the corresponding interrupt to
  460. * @msix_vector: the vector to map to the corresponding queue
  461. *
  462. */
  463. static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
  464. u8 queue, u8 msix_vector)
  465. {
  466. u32 ivar, index;
  467. struct ixgbe_hw *hw = &adapter->hw;
  468. switch (hw->mac.type) {
  469. case ixgbe_mac_82598EB:
  470. msix_vector |= IXGBE_IVAR_ALLOC_VAL;
  471. if (direction == -1)
  472. direction = 0;
  473. index = (((direction * 64) + queue) >> 2) & 0x1F;
  474. ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
  475. ivar &= ~(0xFF << (8 * (queue & 0x3)));
  476. ivar |= (msix_vector << (8 * (queue & 0x3)));
  477. IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
  478. break;
  479. case ixgbe_mac_82599EB:
  480. case ixgbe_mac_X540:
  481. if (direction == -1) {
  482. /* other causes */
  483. msix_vector |= IXGBE_IVAR_ALLOC_VAL;
  484. index = ((queue & 1) * 8);
  485. ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
  486. ivar &= ~(0xFF << index);
  487. ivar |= (msix_vector << index);
  488. IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
  489. break;
  490. } else {
  491. /* tx or rx causes */
  492. msix_vector |= IXGBE_IVAR_ALLOC_VAL;
  493. index = ((16 * (queue & 1)) + (8 * direction));
  494. ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
  495. ivar &= ~(0xFF << index);
  496. ivar |= (msix_vector << index);
  497. IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
  498. break;
  499. }
  500. default:
  501. break;
  502. }
  503. }
  504. static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
  505. u64 qmask)
  506. {
  507. u32 mask;
  508. switch (adapter->hw.mac.type) {
  509. case ixgbe_mac_82598EB:
  510. mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
  511. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
  512. break;
  513. case ixgbe_mac_82599EB:
  514. case ixgbe_mac_X540:
  515. mask = (qmask & 0xFFFFFFFF);
  516. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
  517. mask = (qmask >> 32);
  518. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
  519. break;
  520. default:
  521. break;
  522. }
  523. }
  524. static inline void ixgbe_unmap_tx_resource(struct ixgbe_ring *ring,
  525. struct ixgbe_tx_buffer *tx_buffer)
  526. {
  527. if (tx_buffer->dma) {
  528. if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_MAPPED_AS_PAGE)
  529. dma_unmap_page(ring->dev,
  530. tx_buffer->dma,
  531. tx_buffer->length,
  532. DMA_TO_DEVICE);
  533. else
  534. dma_unmap_single(ring->dev,
  535. tx_buffer->dma,
  536. tx_buffer->length,
  537. DMA_TO_DEVICE);
  538. }
  539. tx_buffer->dma = 0;
  540. }
  541. void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
  542. struct ixgbe_tx_buffer *tx_buffer_info)
  543. {
  544. ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
  545. if (tx_buffer_info->skb)
  546. dev_kfree_skb_any(tx_buffer_info->skb);
  547. tx_buffer_info->skb = NULL;
  548. /* tx_buffer_info must be completely set up in the transmit path */
  549. }
  550. static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
  551. {
  552. struct ixgbe_hw *hw = &adapter->hw;
  553. struct ixgbe_hw_stats *hwstats = &adapter->stats;
  554. u32 data = 0;
  555. u32 xoff[8] = {0};
  556. int i;
  557. if ((hw->fc.current_mode == ixgbe_fc_full) ||
  558. (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
  559. switch (hw->mac.type) {
  560. case ixgbe_mac_82598EB:
  561. data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
  562. break;
  563. default:
  564. data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
  565. }
  566. hwstats->lxoffrxc += data;
  567. /* refill credits (no tx hang) if we received xoff */
  568. if (!data)
  569. return;
  570. for (i = 0; i < adapter->num_tx_queues; i++)
  571. clear_bit(__IXGBE_HANG_CHECK_ARMED,
  572. &adapter->tx_ring[i]->state);
  573. return;
  574. } else if (!(adapter->dcb_cfg.pfc_mode_enable))
  575. return;
  576. /* update stats for each tc, only valid with PFC enabled */
  577. for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
  578. switch (hw->mac.type) {
  579. case ixgbe_mac_82598EB:
  580. xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
  581. break;
  582. default:
  583. xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
  584. }
  585. hwstats->pxoffrxc[i] += xoff[i];
  586. }
  587. /* disarm tx queues that have received xoff frames */
  588. for (i = 0; i < adapter->num_tx_queues; i++) {
  589. struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
  590. u8 tc = tx_ring->dcb_tc;
  591. if (xoff[tc])
  592. clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
  593. }
  594. }
  595. static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
  596. {
  597. return ring->tx_stats.completed;
  598. }
  599. static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
  600. {
  601. struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
  602. struct ixgbe_hw *hw = &adapter->hw;
  603. u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
  604. u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
  605. if (head != tail)
  606. return (head < tail) ?
  607. tail - head : (tail + ring->count - head);
  608. return 0;
  609. }
  610. static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
  611. {
  612. u32 tx_done = ixgbe_get_tx_completed(tx_ring);
  613. u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
  614. u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
  615. bool ret = false;
  616. clear_check_for_tx_hang(tx_ring);
  617. /*
  618. * Check for a hung queue, but be thorough. This verifies
  619. * that a transmit has been completed since the previous
  620. * check AND there is at least one packet pending. The
  621. * ARMED bit is set to indicate a potential hang. The
  622. * bit is cleared if a pause frame is received to remove
  623. * false hang detection due to PFC or 802.3x frames. By
  624. * requiring this to fail twice we avoid races with
  625. * pfc clearing the ARMED bit and conditions where we
  626. * run the check_tx_hang logic with a transmit completion
  627. * pending but without time to complete it yet.
  628. */
  629. if ((tx_done_old == tx_done) && tx_pending) {
  630. /* make sure it is true for two checks in a row */
  631. ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
  632. &tx_ring->state);
  633. } else {
  634. /* update completed stats and continue */
  635. tx_ring->tx_stats.tx_done_old = tx_done;
  636. /* reset the countdown */
  637. clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
  638. }
  639. return ret;
  640. }
  641. /**
  642. * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
  643. * @adapter: driver private struct
  644. **/
  645. static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
  646. {
  647. /* Do the reset outside of interrupt context */
  648. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  649. adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
  650. ixgbe_service_event_schedule(adapter);
  651. }
  652. }
  653. /**
  654. * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
  655. * @q_vector: structure containing interrupt and ring information
  656. * @tx_ring: tx ring to clean
  657. **/
  658. static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
  659. struct ixgbe_ring *tx_ring)
  660. {
  661. struct ixgbe_adapter *adapter = q_vector->adapter;
  662. struct ixgbe_tx_buffer *tx_buffer;
  663. union ixgbe_adv_tx_desc *tx_desc;
  664. unsigned int total_bytes = 0, total_packets = 0;
  665. unsigned int budget = q_vector->tx.work_limit;
  666. u16 i = tx_ring->next_to_clean;
  667. tx_buffer = &tx_ring->tx_buffer_info[i];
  668. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
  669. for (; budget; budget--) {
  670. union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
  671. /* if next_to_watch is not set then there is no work pending */
  672. if (!eop_desc)
  673. break;
  674. /* if DD is not set pending work has not been completed */
  675. if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
  676. break;
  677. /* count the packet as being completed */
  678. tx_ring->tx_stats.completed++;
  679. /* clear next_to_watch to prevent false hangs */
  680. tx_buffer->next_to_watch = NULL;
  681. /* prevent any other reads prior to eop_desc being verified */
  682. rmb();
  683. do {
  684. ixgbe_unmap_tx_resource(tx_ring, tx_buffer);
  685. tx_desc->wb.status = 0;
  686. if (likely(tx_desc == eop_desc)) {
  687. eop_desc = NULL;
  688. dev_kfree_skb_any(tx_buffer->skb);
  689. tx_buffer->skb = NULL;
  690. total_bytes += tx_buffer->bytecount;
  691. total_packets += tx_buffer->gso_segs;
  692. }
  693. tx_buffer++;
  694. tx_desc++;
  695. i++;
  696. if (unlikely(i == tx_ring->count)) {
  697. i = 0;
  698. tx_buffer = tx_ring->tx_buffer_info;
  699. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
  700. }
  701. } while (eop_desc);
  702. }
  703. tx_ring->next_to_clean = i;
  704. u64_stats_update_begin(&tx_ring->syncp);
  705. tx_ring->stats.bytes += total_bytes;
  706. tx_ring->stats.packets += total_packets;
  707. u64_stats_update_end(&tx_ring->syncp);
  708. q_vector->tx.total_bytes += total_bytes;
  709. q_vector->tx.total_packets += total_packets;
  710. if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
  711. /* schedule immediate reset if we believe we hung */
  712. struct ixgbe_hw *hw = &adapter->hw;
  713. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
  714. e_err(drv, "Detected Tx Unit Hang\n"
  715. " Tx Queue <%d>\n"
  716. " TDH, TDT <%x>, <%x>\n"
  717. " next_to_use <%x>\n"
  718. " next_to_clean <%x>\n"
  719. "tx_buffer_info[next_to_clean]\n"
  720. " time_stamp <%lx>\n"
  721. " jiffies <%lx>\n",
  722. tx_ring->queue_index,
  723. IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
  724. IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
  725. tx_ring->next_to_use, i,
  726. tx_ring->tx_buffer_info[i].time_stamp, jiffies);
  727. netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
  728. e_info(probe,
  729. "tx hang %d detected on queue %d, resetting adapter\n",
  730. adapter->tx_timeout_count + 1, tx_ring->queue_index);
  731. /* schedule immediate reset if we believe we hung */
  732. ixgbe_tx_timeout_reset(adapter);
  733. /* the adapter is about to reset, no point in enabling stuff */
  734. return true;
  735. }
  736. #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
  737. if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
  738. (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
  739. /* Make sure that anybody stopping the queue after this
  740. * sees the new next_to_clean.
  741. */
  742. smp_mb();
  743. if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
  744. !test_bit(__IXGBE_DOWN, &adapter->state)) {
  745. netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
  746. ++tx_ring->tx_stats.restart_queue;
  747. }
  748. }
  749. return !!budget;
  750. }
  751. #ifdef CONFIG_IXGBE_DCA
  752. static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
  753. struct ixgbe_ring *rx_ring,
  754. int cpu)
  755. {
  756. struct ixgbe_hw *hw = &adapter->hw;
  757. u32 rxctrl;
  758. u8 reg_idx = rx_ring->reg_idx;
  759. rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
  760. switch (hw->mac.type) {
  761. case ixgbe_mac_82598EB:
  762. rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
  763. rxctrl |= dca3_get_tag(rx_ring->dev, cpu);
  764. break;
  765. case ixgbe_mac_82599EB:
  766. case ixgbe_mac_X540:
  767. rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
  768. rxctrl |= (dca3_get_tag(rx_ring->dev, cpu) <<
  769. IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
  770. break;
  771. default:
  772. break;
  773. }
  774. rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
  775. rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
  776. rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
  777. IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
  778. }
  779. static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
  780. struct ixgbe_ring *tx_ring,
  781. int cpu)
  782. {
  783. struct ixgbe_hw *hw = &adapter->hw;
  784. u32 txctrl;
  785. u8 reg_idx = tx_ring->reg_idx;
  786. switch (hw->mac.type) {
  787. case ixgbe_mac_82598EB:
  788. txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
  789. txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
  790. txctrl |= dca3_get_tag(tx_ring->dev, cpu);
  791. txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
  792. IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
  793. break;
  794. case ixgbe_mac_82599EB:
  795. case ixgbe_mac_X540:
  796. txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
  797. txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
  798. txctrl |= (dca3_get_tag(tx_ring->dev, cpu) <<
  799. IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
  800. txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
  801. IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
  802. break;
  803. default:
  804. break;
  805. }
  806. }
  807. static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
  808. {
  809. struct ixgbe_adapter *adapter = q_vector->adapter;
  810. struct ixgbe_ring *ring;
  811. int cpu = get_cpu();
  812. if (q_vector->cpu == cpu)
  813. goto out_no_update;
  814. for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
  815. ixgbe_update_tx_dca(adapter, ring, cpu);
  816. for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
  817. ixgbe_update_rx_dca(adapter, ring, cpu);
  818. q_vector->cpu = cpu;
  819. out_no_update:
  820. put_cpu();
  821. }
  822. static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
  823. {
  824. int num_q_vectors;
  825. int i;
  826. if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
  827. return;
  828. /* always use CB2 mode, difference is masked in the CB driver */
  829. IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
  830. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  831. num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  832. else
  833. num_q_vectors = 1;
  834. for (i = 0; i < num_q_vectors; i++) {
  835. adapter->q_vector[i]->cpu = -1;
  836. ixgbe_update_dca(adapter->q_vector[i]);
  837. }
  838. }
  839. static int __ixgbe_notify_dca(struct device *dev, void *data)
  840. {
  841. struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
  842. unsigned long event = *(unsigned long *)data;
  843. if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
  844. return 0;
  845. switch (event) {
  846. case DCA_PROVIDER_ADD:
  847. /* if we're already enabled, don't do it again */
  848. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
  849. break;
  850. if (dca_add_requester(dev) == 0) {
  851. adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
  852. ixgbe_setup_dca(adapter);
  853. break;
  854. }
  855. /* Fall Through since DCA is disabled. */
  856. case DCA_PROVIDER_REMOVE:
  857. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
  858. dca_remove_requester(dev);
  859. adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
  860. IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
  861. }
  862. break;
  863. }
  864. return 0;
  865. }
  866. #endif /* CONFIG_IXGBE_DCA */
  867. static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
  868. struct sk_buff *skb)
  869. {
  870. skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
  871. }
  872. /**
  873. * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
  874. * @adapter: address of board private structure
  875. * @rx_desc: advanced rx descriptor
  876. *
  877. * Returns : true if it is FCoE pkt
  878. */
  879. static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
  880. union ixgbe_adv_rx_desc *rx_desc)
  881. {
  882. __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
  883. return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
  884. ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
  885. (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
  886. IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
  887. }
  888. /**
  889. * ixgbe_receive_skb - Send a completed packet up the stack
  890. * @adapter: board private structure
  891. * @skb: packet to send up
  892. * @status: hardware indication of status of receive
  893. * @rx_ring: rx descriptor ring (for a specific queue) to setup
  894. * @rx_desc: rx descriptor
  895. **/
  896. static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
  897. struct sk_buff *skb, u8 status,
  898. struct ixgbe_ring *ring,
  899. union ixgbe_adv_rx_desc *rx_desc)
  900. {
  901. struct ixgbe_adapter *adapter = q_vector->adapter;
  902. struct napi_struct *napi = &q_vector->napi;
  903. bool is_vlan = (status & IXGBE_RXD_STAT_VP);
  904. u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
  905. if (is_vlan && (tag & VLAN_VID_MASK))
  906. __vlan_hwaccel_put_tag(skb, tag);
  907. if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
  908. napi_gro_receive(napi, skb);
  909. else
  910. netif_rx(skb);
  911. }
  912. /**
  913. * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
  914. * @adapter: address of board private structure
  915. * @status_err: hardware indication of status of receive
  916. * @skb: skb currently being received and modified
  917. * @status_err: status error value of last descriptor in packet
  918. **/
  919. static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
  920. union ixgbe_adv_rx_desc *rx_desc,
  921. struct sk_buff *skb,
  922. u32 status_err)
  923. {
  924. skb->ip_summed = CHECKSUM_NONE;
  925. /* Rx csum disabled */
  926. if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
  927. return;
  928. /* if IP and error */
  929. if ((status_err & IXGBE_RXD_STAT_IPCS) &&
  930. (status_err & IXGBE_RXDADV_ERR_IPE)) {
  931. adapter->hw_csum_rx_error++;
  932. return;
  933. }
  934. if (!(status_err & IXGBE_RXD_STAT_L4CS))
  935. return;
  936. if (status_err & IXGBE_RXDADV_ERR_TCPE) {
  937. u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
  938. /*
  939. * 82599 errata, UDP frames with a 0 checksum can be marked as
  940. * checksum errors.
  941. */
  942. if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
  943. (adapter->hw.mac.type == ixgbe_mac_82599EB))
  944. return;
  945. adapter->hw_csum_rx_error++;
  946. return;
  947. }
  948. /* It must be a TCP or UDP packet with a valid checksum */
  949. skb->ip_summed = CHECKSUM_UNNECESSARY;
  950. }
  951. static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
  952. {
  953. /*
  954. * Force memory writes to complete before letting h/w
  955. * know there are new descriptors to fetch. (Only
  956. * applicable for weak-ordered memory model archs,
  957. * such as IA-64).
  958. */
  959. wmb();
  960. writel(val, rx_ring->tail);
  961. }
  962. /**
  963. * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
  964. * @rx_ring: ring to place buffers on
  965. * @cleaned_count: number of buffers to replace
  966. **/
  967. void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
  968. {
  969. union ixgbe_adv_rx_desc *rx_desc;
  970. struct ixgbe_rx_buffer *bi;
  971. struct sk_buff *skb;
  972. u16 i = rx_ring->next_to_use;
  973. /* do nothing if no valid netdev defined */
  974. if (!rx_ring->netdev)
  975. return;
  976. while (cleaned_count--) {
  977. rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
  978. bi = &rx_ring->rx_buffer_info[i];
  979. skb = bi->skb;
  980. if (!skb) {
  981. skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
  982. rx_ring->rx_buf_len);
  983. if (!skb) {
  984. rx_ring->rx_stats.alloc_rx_buff_failed++;
  985. goto no_buffers;
  986. }
  987. /* initialize queue mapping */
  988. skb_record_rx_queue(skb, rx_ring->queue_index);
  989. bi->skb = skb;
  990. }
  991. if (!bi->dma) {
  992. bi->dma = dma_map_single(rx_ring->dev,
  993. skb->data,
  994. rx_ring->rx_buf_len,
  995. DMA_FROM_DEVICE);
  996. if (dma_mapping_error(rx_ring->dev, bi->dma)) {
  997. rx_ring->rx_stats.alloc_rx_buff_failed++;
  998. bi->dma = 0;
  999. goto no_buffers;
  1000. }
  1001. }
  1002. if (ring_is_ps_enabled(rx_ring)) {
  1003. if (!bi->page) {
  1004. bi->page = alloc_page(GFP_ATOMIC | __GFP_COLD);
  1005. if (!bi->page) {
  1006. rx_ring->rx_stats.alloc_rx_page_failed++;
  1007. goto no_buffers;
  1008. }
  1009. }
  1010. if (!bi->page_dma) {
  1011. /* use a half page if we're re-using */
  1012. bi->page_offset ^= PAGE_SIZE / 2;
  1013. bi->page_dma = dma_map_page(rx_ring->dev,
  1014. bi->page,
  1015. bi->page_offset,
  1016. PAGE_SIZE / 2,
  1017. DMA_FROM_DEVICE);
  1018. if (dma_mapping_error(rx_ring->dev,
  1019. bi->page_dma)) {
  1020. rx_ring->rx_stats.alloc_rx_page_failed++;
  1021. bi->page_dma = 0;
  1022. goto no_buffers;
  1023. }
  1024. }
  1025. /* Refresh the desc even if buffer_addrs didn't change
  1026. * because each write-back erases this info. */
  1027. rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
  1028. rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
  1029. } else {
  1030. rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
  1031. rx_desc->read.hdr_addr = 0;
  1032. }
  1033. i++;
  1034. if (i == rx_ring->count)
  1035. i = 0;
  1036. }
  1037. no_buffers:
  1038. if (rx_ring->next_to_use != i) {
  1039. rx_ring->next_to_use = i;
  1040. ixgbe_release_rx_desc(rx_ring, i);
  1041. }
  1042. }
  1043. static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
  1044. {
  1045. /* HW will not DMA in data larger than the given buffer, even if it
  1046. * parses the (NFS, of course) header to be larger. In that case, it
  1047. * fills the header buffer and spills the rest into the page.
  1048. */
  1049. u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
  1050. u16 hlen = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
  1051. IXGBE_RXDADV_HDRBUFLEN_SHIFT;
  1052. if (hlen > IXGBE_RX_HDR_SIZE)
  1053. hlen = IXGBE_RX_HDR_SIZE;
  1054. return hlen;
  1055. }
  1056. /**
  1057. * ixgbe_transform_rsc_queue - change rsc queue into a full packet
  1058. * @skb: pointer to the last skb in the rsc queue
  1059. *
  1060. * This function changes a queue full of hw rsc buffers into a completed
  1061. * packet. It uses the ->prev pointers to find the first packet and then
  1062. * turns it into the frag list owner.
  1063. **/
  1064. static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
  1065. {
  1066. unsigned int frag_list_size = 0;
  1067. unsigned int skb_cnt = 1;
  1068. while (skb->prev) {
  1069. struct sk_buff *prev = skb->prev;
  1070. frag_list_size += skb->len;
  1071. skb->prev = NULL;
  1072. skb = prev;
  1073. skb_cnt++;
  1074. }
  1075. skb_shinfo(skb)->frag_list = skb->next;
  1076. skb->next = NULL;
  1077. skb->len += frag_list_size;
  1078. skb->data_len += frag_list_size;
  1079. skb->truesize += frag_list_size;
  1080. IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;
  1081. return skb;
  1082. }
  1083. static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
  1084. {
  1085. return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
  1086. IXGBE_RXDADV_RSCCNT_MASK);
  1087. }
  1088. static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
  1089. struct ixgbe_ring *rx_ring,
  1090. int budget)
  1091. {
  1092. struct ixgbe_adapter *adapter = q_vector->adapter;
  1093. union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
  1094. struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
  1095. struct sk_buff *skb;
  1096. unsigned int total_rx_bytes = 0, total_rx_packets = 0;
  1097. const int current_node = numa_node_id();
  1098. #ifdef IXGBE_FCOE
  1099. int ddp_bytes = 0;
  1100. #endif /* IXGBE_FCOE */
  1101. u32 staterr;
  1102. u16 i;
  1103. u16 cleaned_count = 0;
  1104. bool pkt_is_rsc = false;
  1105. i = rx_ring->next_to_clean;
  1106. rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
  1107. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  1108. while (staterr & IXGBE_RXD_STAT_DD) {
  1109. u32 upper_len = 0;
  1110. rmb(); /* read descriptor and rx_buffer_info after status DD */
  1111. rx_buffer_info = &rx_ring->rx_buffer_info[i];
  1112. skb = rx_buffer_info->skb;
  1113. rx_buffer_info->skb = NULL;
  1114. prefetch(skb->data);
  1115. if (ring_is_rsc_enabled(rx_ring))
  1116. pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
  1117. /* linear means we are building an skb from multiple pages */
  1118. if (!skb_is_nonlinear(skb)) {
  1119. u16 hlen;
  1120. if (pkt_is_rsc &&
  1121. !(staterr & IXGBE_RXD_STAT_EOP) &&
  1122. !skb->prev) {
  1123. /*
  1124. * When HWRSC is enabled, delay unmapping
  1125. * of the first packet. It carries the
  1126. * header information, HW may still
  1127. * access the header after the writeback.
  1128. * Only unmap it when EOP is reached
  1129. */
  1130. IXGBE_RSC_CB(skb)->delay_unmap = true;
  1131. IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
  1132. } else {
  1133. dma_unmap_single(rx_ring->dev,
  1134. rx_buffer_info->dma,
  1135. rx_ring->rx_buf_len,
  1136. DMA_FROM_DEVICE);
  1137. }
  1138. rx_buffer_info->dma = 0;
  1139. if (ring_is_ps_enabled(rx_ring)) {
  1140. hlen = ixgbe_get_hlen(rx_desc);
  1141. upper_len = le16_to_cpu(rx_desc->wb.upper.length);
  1142. } else {
  1143. hlen = le16_to_cpu(rx_desc->wb.upper.length);
  1144. }
  1145. skb_put(skb, hlen);
  1146. } else {
  1147. /* assume packet split since header is unmapped */
  1148. upper_len = le16_to_cpu(rx_desc->wb.upper.length);
  1149. }
  1150. if (upper_len) {
  1151. dma_unmap_page(rx_ring->dev,
  1152. rx_buffer_info->page_dma,
  1153. PAGE_SIZE / 2,
  1154. DMA_FROM_DEVICE);
  1155. rx_buffer_info->page_dma = 0;
  1156. skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
  1157. rx_buffer_info->page,
  1158. rx_buffer_info->page_offset,
  1159. upper_len);
  1160. if ((page_count(rx_buffer_info->page) == 1) &&
  1161. (page_to_nid(rx_buffer_info->page) == current_node))
  1162. get_page(rx_buffer_info->page);
  1163. else
  1164. rx_buffer_info->page = NULL;
  1165. skb->len += upper_len;
  1166. skb->data_len += upper_len;
  1167. skb->truesize += PAGE_SIZE / 2;
  1168. }
  1169. i++;
  1170. if (i == rx_ring->count)
  1171. i = 0;
  1172. next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
  1173. prefetch(next_rxd);
  1174. cleaned_count++;
  1175. if (pkt_is_rsc) {
  1176. u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
  1177. IXGBE_RXDADV_NEXTP_SHIFT;
  1178. next_buffer = &rx_ring->rx_buffer_info[nextp];
  1179. } else {
  1180. next_buffer = &rx_ring->rx_buffer_info[i];
  1181. }
  1182. if (!(staterr & IXGBE_RXD_STAT_EOP)) {
  1183. if (ring_is_ps_enabled(rx_ring)) {
  1184. rx_buffer_info->skb = next_buffer->skb;
  1185. rx_buffer_info->dma = next_buffer->dma;
  1186. next_buffer->skb = skb;
  1187. next_buffer->dma = 0;
  1188. } else {
  1189. skb->next = next_buffer->skb;
  1190. skb->next->prev = skb;
  1191. }
  1192. rx_ring->rx_stats.non_eop_descs++;
  1193. goto next_desc;
  1194. }
  1195. if (skb->prev) {
  1196. skb = ixgbe_transform_rsc_queue(skb);
  1197. /* if we got here without RSC the packet is invalid */
  1198. if (!pkt_is_rsc) {
  1199. __pskb_trim(skb, 0);
  1200. rx_buffer_info->skb = skb;
  1201. goto next_desc;
  1202. }
  1203. }
  1204. if (ring_is_rsc_enabled(rx_ring)) {
  1205. if (IXGBE_RSC_CB(skb)->delay_unmap) {
  1206. dma_unmap_single(rx_ring->dev,
  1207. IXGBE_RSC_CB(skb)->dma,
  1208. rx_ring->rx_buf_len,
  1209. DMA_FROM_DEVICE);
  1210. IXGBE_RSC_CB(skb)->dma = 0;
  1211. IXGBE_RSC_CB(skb)->delay_unmap = false;
  1212. }
  1213. }
  1214. if (pkt_is_rsc) {
  1215. if (ring_is_ps_enabled(rx_ring))
  1216. rx_ring->rx_stats.rsc_count +=
  1217. skb_shinfo(skb)->nr_frags;
  1218. else
  1219. rx_ring->rx_stats.rsc_count +=
  1220. IXGBE_RSC_CB(skb)->skb_cnt;
  1221. rx_ring->rx_stats.rsc_flush++;
  1222. }
  1223. /* ERR_MASK will only have valid bits if EOP set */
  1224. if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
  1225. dev_kfree_skb_any(skb);
  1226. goto next_desc;
  1227. }
  1228. ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
  1229. if (adapter->netdev->features & NETIF_F_RXHASH)
  1230. ixgbe_rx_hash(rx_desc, skb);
  1231. /* probably a little skewed due to removing CRC */
  1232. total_rx_bytes += skb->len;
  1233. total_rx_packets++;
  1234. skb->protocol = eth_type_trans(skb, rx_ring->netdev);
  1235. #ifdef IXGBE_FCOE
  1236. /* if ddp, not passing to ULD unless for FCP_RSP or error */
  1237. if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
  1238. ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
  1239. staterr);
  1240. if (!ddp_bytes) {
  1241. dev_kfree_skb_any(skb);
  1242. goto next_desc;
  1243. }
  1244. }
  1245. #endif /* IXGBE_FCOE */
  1246. ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
  1247. budget--;
  1248. next_desc:
  1249. rx_desc->wb.upper.status_error = 0;
  1250. if (!budget)
  1251. break;
  1252. /* return some buffers to hardware, one at a time is too slow */
  1253. if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
  1254. ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
  1255. cleaned_count = 0;
  1256. }
  1257. /* use prefetched values */
  1258. rx_desc = next_rxd;
  1259. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  1260. }
  1261. rx_ring->next_to_clean = i;
  1262. cleaned_count = ixgbe_desc_unused(rx_ring);
  1263. if (cleaned_count)
  1264. ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
  1265. #ifdef IXGBE_FCOE
  1266. /* include DDPed FCoE data */
  1267. if (ddp_bytes > 0) {
  1268. unsigned int mss;
  1269. mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
  1270. sizeof(struct fc_frame_header) -
  1271. sizeof(struct fcoe_crc_eof);
  1272. if (mss > 512)
  1273. mss &= ~511;
  1274. total_rx_bytes += ddp_bytes;
  1275. total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
  1276. }
  1277. #endif /* IXGBE_FCOE */
  1278. u64_stats_update_begin(&rx_ring->syncp);
  1279. rx_ring->stats.packets += total_rx_packets;
  1280. rx_ring->stats.bytes += total_rx_bytes;
  1281. u64_stats_update_end(&rx_ring->syncp);
  1282. q_vector->rx.total_packets += total_rx_packets;
  1283. q_vector->rx.total_bytes += total_rx_bytes;
  1284. return !!budget;
  1285. }
  1286. /**
  1287. * ixgbe_configure_msix - Configure MSI-X hardware
  1288. * @adapter: board private structure
  1289. *
  1290. * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
  1291. * interrupts.
  1292. **/
  1293. static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
  1294. {
  1295. struct ixgbe_q_vector *q_vector;
  1296. int q_vectors, v_idx;
  1297. u32 mask;
  1298. q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  1299. /* Populate MSIX to EITR Select */
  1300. if (adapter->num_vfs > 32) {
  1301. u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
  1302. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
  1303. }
  1304. /*
  1305. * Populate the IVAR table and set the ITR values to the
  1306. * corresponding register.
  1307. */
  1308. for (v_idx = 0; v_idx < q_vectors; v_idx++) {
  1309. struct ixgbe_ring *ring;
  1310. q_vector = adapter->q_vector[v_idx];
  1311. for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
  1312. ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
  1313. for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
  1314. ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
  1315. if (q_vector->tx.ring && !q_vector->rx.ring) {
  1316. /* tx only vector */
  1317. if (adapter->tx_itr_setting == 1)
  1318. q_vector->itr = IXGBE_10K_ITR;
  1319. else
  1320. q_vector->itr = adapter->tx_itr_setting;
  1321. } else {
  1322. /* rx or rx/tx vector */
  1323. if (adapter->rx_itr_setting == 1)
  1324. q_vector->itr = IXGBE_20K_ITR;
  1325. else
  1326. q_vector->itr = adapter->rx_itr_setting;
  1327. }
  1328. ixgbe_write_eitr(q_vector);
  1329. }
  1330. switch (adapter->hw.mac.type) {
  1331. case ixgbe_mac_82598EB:
  1332. ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
  1333. v_idx);
  1334. break;
  1335. case ixgbe_mac_82599EB:
  1336. case ixgbe_mac_X540:
  1337. ixgbe_set_ivar(adapter, -1, 1, v_idx);
  1338. break;
  1339. default:
  1340. break;
  1341. }
  1342. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
  1343. /* set up to autoclear timer, and the vectors */
  1344. mask = IXGBE_EIMS_ENABLE_MASK;
  1345. mask &= ~(IXGBE_EIMS_OTHER |
  1346. IXGBE_EIMS_MAILBOX |
  1347. IXGBE_EIMS_LSC);
  1348. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
  1349. }
  1350. enum latency_range {
  1351. lowest_latency = 0,
  1352. low_latency = 1,
  1353. bulk_latency = 2,
  1354. latency_invalid = 255
  1355. };
  1356. /**
  1357. * ixgbe_update_itr - update the dynamic ITR value based on statistics
  1358. * @q_vector: structure containing interrupt and ring information
  1359. * @ring_container: structure containing ring performance data
  1360. *
  1361. * Stores a new ITR value based on packets and byte
  1362. * counts during the last interrupt. The advantage of per interrupt
  1363. * computation is faster updates and more accurate ITR for the current
  1364. * traffic pattern. Constants in this function were computed
  1365. * based on theoretical maximum wire speed and thresholds were set based
  1366. * on testing data as well as attempting to minimize response time
  1367. * while increasing bulk throughput.
  1368. * this functionality is controlled by the InterruptThrottleRate module
  1369. * parameter (see ixgbe_param.c)
  1370. **/
  1371. static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
  1372. struct ixgbe_ring_container *ring_container)
  1373. {
  1374. u64 bytes_perint;
  1375. struct ixgbe_adapter *adapter = q_vector->adapter;
  1376. int bytes = ring_container->total_bytes;
  1377. int packets = ring_container->total_packets;
  1378. u32 timepassed_us;
  1379. u8 itr_setting = ring_container->itr;
  1380. if (packets == 0)
  1381. return;
  1382. /* simple throttlerate management
  1383. * 0-20MB/s lowest (100000 ints/s)
  1384. * 20-100MB/s low (20000 ints/s)
  1385. * 100-1249MB/s bulk (8000 ints/s)
  1386. */
  1387. /* what was last interrupt timeslice? */
  1388. timepassed_us = q_vector->itr >> 2;
  1389. bytes_perint = bytes / timepassed_us; /* bytes/usec */
  1390. switch (itr_setting) {
  1391. case lowest_latency:
  1392. if (bytes_perint > adapter->eitr_low)
  1393. itr_setting = low_latency;
  1394. break;
  1395. case low_latency:
  1396. if (bytes_perint > adapter->eitr_high)
  1397. itr_setting = bulk_latency;
  1398. else if (bytes_perint <= adapter->eitr_low)
  1399. itr_setting = lowest_latency;
  1400. break;
  1401. case bulk_latency:
  1402. if (bytes_perint <= adapter->eitr_high)
  1403. itr_setting = low_latency;
  1404. break;
  1405. }
  1406. /* clear work counters since we have the values we need */
  1407. ring_container->total_bytes = 0;
  1408. ring_container->total_packets = 0;
  1409. /* write updated itr to ring container */
  1410. ring_container->itr = itr_setting;
  1411. }
  1412. /**
  1413. * ixgbe_write_eitr - write EITR register in hardware specific way
  1414. * @q_vector: structure containing interrupt and ring information
  1415. *
  1416. * This function is made to be called by ethtool and by the driver
  1417. * when it needs to update EITR registers at runtime. Hardware
  1418. * specific quirks/differences are taken care of here.
  1419. */
  1420. void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
  1421. {
  1422. struct ixgbe_adapter *adapter = q_vector->adapter;
  1423. struct ixgbe_hw *hw = &adapter->hw;
  1424. int v_idx = q_vector->v_idx;
  1425. u32 itr_reg = q_vector->itr;
  1426. switch (adapter->hw.mac.type) {
  1427. case ixgbe_mac_82598EB:
  1428. /* must write high and low 16 bits to reset counter */
  1429. itr_reg |= (itr_reg << 16);
  1430. break;
  1431. case ixgbe_mac_82599EB:
  1432. case ixgbe_mac_X540:
  1433. /*
  1434. * set the WDIS bit to not clear the timer bits and cause an
  1435. * immediate assertion of the interrupt
  1436. */
  1437. itr_reg |= IXGBE_EITR_CNT_WDIS;
  1438. break;
  1439. default:
  1440. break;
  1441. }
  1442. IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
  1443. }
  1444. static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
  1445. {
  1446. u32 new_itr = q_vector->itr;
  1447. u8 current_itr;
  1448. ixgbe_update_itr(q_vector, &q_vector->tx);
  1449. ixgbe_update_itr(q_vector, &q_vector->rx);
  1450. current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
  1451. switch (current_itr) {
  1452. /* counts and packets in update_itr are dependent on these numbers */
  1453. case lowest_latency:
  1454. new_itr = IXGBE_100K_ITR;
  1455. break;
  1456. case low_latency:
  1457. new_itr = IXGBE_20K_ITR;
  1458. break;
  1459. case bulk_latency:
  1460. new_itr = IXGBE_8K_ITR;
  1461. break;
  1462. default:
  1463. break;
  1464. }
  1465. if (new_itr != q_vector->itr) {
  1466. /* do an exponential smoothing */
  1467. new_itr = (10 * new_itr * q_vector->itr) /
  1468. ((9 * new_itr) + q_vector->itr);
  1469. /* save the algorithm value here */
  1470. q_vector->itr = new_itr & IXGBE_MAX_EITR;
  1471. ixgbe_write_eitr(q_vector);
  1472. }
  1473. }
  1474. /**
  1475. * ixgbe_check_overtemp_subtask - check for over tempurature
  1476. * @adapter: pointer to adapter
  1477. **/
  1478. static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
  1479. {
  1480. struct ixgbe_hw *hw = &adapter->hw;
  1481. u32 eicr = adapter->interrupt_event;
  1482. if (test_bit(__IXGBE_DOWN, &adapter->state))
  1483. return;
  1484. if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
  1485. !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
  1486. return;
  1487. adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
  1488. switch (hw->device_id) {
  1489. case IXGBE_DEV_ID_82599_T3_LOM:
  1490. /*
  1491. * Since the warning interrupt is for both ports
  1492. * we don't have to check if:
  1493. * - This interrupt wasn't for our port.
  1494. * - We may have missed the interrupt so always have to
  1495. * check if we got a LSC
  1496. */
  1497. if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
  1498. !(eicr & IXGBE_EICR_LSC))
  1499. return;
  1500. if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
  1501. u32 autoneg;
  1502. bool link_up = false;
  1503. hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
  1504. if (link_up)
  1505. return;
  1506. }
  1507. /* Check if this is not due to overtemp */
  1508. if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
  1509. return;
  1510. break;
  1511. default:
  1512. if (!(eicr & IXGBE_EICR_GPI_SDP0))
  1513. return;
  1514. break;
  1515. }
  1516. e_crit(drv,
  1517. "Network adapter has been stopped because it has over heated. "
  1518. "Restart the computer. If the problem persists, "
  1519. "power off the system and replace the adapter\n");
  1520. adapter->interrupt_event = 0;
  1521. }
  1522. static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
  1523. {
  1524. struct ixgbe_hw *hw = &adapter->hw;
  1525. if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
  1526. (eicr & IXGBE_EICR_GPI_SDP1)) {
  1527. e_crit(probe, "Fan has stopped, replace the adapter\n");
  1528. /* write to clear the interrupt */
  1529. IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
  1530. }
  1531. }
  1532. static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
  1533. {
  1534. if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
  1535. return;
  1536. switch (adapter->hw.mac.type) {
  1537. case ixgbe_mac_82599EB:
  1538. /*
  1539. * Need to check link state so complete overtemp check
  1540. * on service task
  1541. */
  1542. if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
  1543. (!test_bit(__IXGBE_DOWN, &adapter->state))) {
  1544. adapter->interrupt_event = eicr;
  1545. adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
  1546. ixgbe_service_event_schedule(adapter);
  1547. return;
  1548. }
  1549. return;
  1550. case ixgbe_mac_X540:
  1551. if (!(eicr & IXGBE_EICR_TS))
  1552. return;
  1553. break;
  1554. default:
  1555. return;
  1556. }
  1557. e_crit(drv,
  1558. "Network adapter has been stopped because it has over heated. "
  1559. "Restart the computer. If the problem persists, "
  1560. "power off the system and replace the adapter\n");
  1561. }
  1562. static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
  1563. {
  1564. struct ixgbe_hw *hw = &adapter->hw;
  1565. if (eicr & IXGBE_EICR_GPI_SDP2) {
  1566. /* Clear the interrupt */
  1567. IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
  1568. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  1569. adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
  1570. ixgbe_service_event_schedule(adapter);
  1571. }
  1572. }
  1573. if (eicr & IXGBE_EICR_GPI_SDP1) {
  1574. /* Clear the interrupt */
  1575. IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
  1576. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  1577. adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
  1578. ixgbe_service_event_schedule(adapter);
  1579. }
  1580. }
  1581. }
  1582. static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
  1583. {
  1584. struct ixgbe_hw *hw = &adapter->hw;
  1585. adapter->lsc_int++;
  1586. adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
  1587. adapter->link_check_timeout = jiffies;
  1588. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  1589. IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
  1590. IXGBE_WRITE_FLUSH(hw);
  1591. ixgbe_service_event_schedule(adapter);
  1592. }
  1593. }
  1594. static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
  1595. u64 qmask)
  1596. {
  1597. u32 mask;
  1598. struct ixgbe_hw *hw = &adapter->hw;
  1599. switch (hw->mac.type) {
  1600. case ixgbe_mac_82598EB:
  1601. mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
  1602. IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
  1603. break;
  1604. case ixgbe_mac_82599EB:
  1605. case ixgbe_mac_X540:
  1606. mask = (qmask & 0xFFFFFFFF);
  1607. if (mask)
  1608. IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
  1609. mask = (qmask >> 32);
  1610. if (mask)
  1611. IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
  1612. break;
  1613. default:
  1614. break;
  1615. }
  1616. /* skip the flush */
  1617. }
  1618. static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
  1619. u64 qmask)
  1620. {
  1621. u32 mask;
  1622. struct ixgbe_hw *hw = &adapter->hw;
  1623. switch (hw->mac.type) {
  1624. case ixgbe_mac_82598EB:
  1625. mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
  1626. IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
  1627. break;
  1628. case ixgbe_mac_82599EB:
  1629. case ixgbe_mac_X540:
  1630. mask = (qmask & 0xFFFFFFFF);
  1631. if (mask)
  1632. IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
  1633. mask = (qmask >> 32);
  1634. if (mask)
  1635. IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
  1636. break;
  1637. default:
  1638. break;
  1639. }
  1640. /* skip the flush */
  1641. }
  1642. /**
  1643. * ixgbe_irq_enable - Enable default interrupt generation settings
  1644. * @adapter: board private structure
  1645. **/
  1646. static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
  1647. bool flush)
  1648. {
  1649. u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
  1650. /* don't reenable LSC while waiting for link */
  1651. if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
  1652. mask &= ~IXGBE_EIMS_LSC;
  1653. if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
  1654. switch (adapter->hw.mac.type) {
  1655. case ixgbe_mac_82599EB:
  1656. mask |= IXGBE_EIMS_GPI_SDP0;
  1657. break;
  1658. case ixgbe_mac_X540:
  1659. mask |= IXGBE_EIMS_TS;
  1660. break;
  1661. default:
  1662. break;
  1663. }
  1664. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
  1665. mask |= IXGBE_EIMS_GPI_SDP1;
  1666. switch (adapter->hw.mac.type) {
  1667. case ixgbe_mac_82599EB:
  1668. mask |= IXGBE_EIMS_GPI_SDP1;
  1669. mask |= IXGBE_EIMS_GPI_SDP2;
  1670. case ixgbe_mac_X540:
  1671. mask |= IXGBE_EIMS_ECC;
  1672. mask |= IXGBE_EIMS_MAILBOX;
  1673. break;
  1674. default:
  1675. break;
  1676. }
  1677. if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
  1678. !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
  1679. mask |= IXGBE_EIMS_FLOW_DIR;
  1680. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
  1681. if (queues)
  1682. ixgbe_irq_enable_queues(adapter, ~0);
  1683. if (flush)
  1684. IXGBE_WRITE_FLUSH(&adapter->hw);
  1685. }
  1686. static irqreturn_t ixgbe_msix_other(int irq, void *data)
  1687. {
  1688. struct ixgbe_adapter *adapter = data;
  1689. struct ixgbe_hw *hw = &adapter->hw;
  1690. u32 eicr;
  1691. /*
  1692. * Workaround for Silicon errata. Use clear-by-write instead
  1693. * of clear-by-read. Reading with EICS will return the
  1694. * interrupt causes without clearing, which later be done
  1695. * with the write to EICR.
  1696. */
  1697. eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
  1698. IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
  1699. if (eicr & IXGBE_EICR_LSC)
  1700. ixgbe_check_lsc(adapter);
  1701. if (eicr & IXGBE_EICR_MAILBOX)
  1702. ixgbe_msg_task(adapter);
  1703. switch (hw->mac.type) {
  1704. case ixgbe_mac_82599EB:
  1705. case ixgbe_mac_X540:
  1706. if (eicr & IXGBE_EICR_ECC)
  1707. e_info(link, "Received unrecoverable ECC Err, please "
  1708. "reboot\n");
  1709. /* Handle Flow Director Full threshold interrupt */
  1710. if (eicr & IXGBE_EICR_FLOW_DIR) {
  1711. int reinit_count = 0;
  1712. int i;
  1713. for (i = 0; i < adapter->num_tx_queues; i++) {
  1714. struct ixgbe_ring *ring = adapter->tx_ring[i];
  1715. if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
  1716. &ring->state))
  1717. reinit_count++;
  1718. }
  1719. if (reinit_count) {
  1720. /* no more flow director interrupts until after init */
  1721. IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
  1722. adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
  1723. ixgbe_service_event_schedule(adapter);
  1724. }
  1725. }
  1726. ixgbe_check_sfp_event(adapter, eicr);
  1727. ixgbe_check_overtemp_event(adapter, eicr);
  1728. break;
  1729. default:
  1730. break;
  1731. }
  1732. ixgbe_check_fan_failure(adapter, eicr);
  1733. /* re-enable the original interrupt state, no lsc, no queues */
  1734. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  1735. ixgbe_irq_enable(adapter, false, false);
  1736. return IRQ_HANDLED;
  1737. }
  1738. static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
  1739. {
  1740. struct ixgbe_q_vector *q_vector = data;
  1741. /* EIAM disabled interrupts (on this vector) for us */
  1742. if (q_vector->rx.ring || q_vector->tx.ring)
  1743. napi_schedule(&q_vector->napi);
  1744. return IRQ_HANDLED;
  1745. }
  1746. static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
  1747. int r_idx)
  1748. {
  1749. struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
  1750. struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
  1751. rx_ring->q_vector = q_vector;
  1752. rx_ring->next = q_vector->rx.ring;
  1753. q_vector->rx.ring = rx_ring;
  1754. q_vector->rx.count++;
  1755. }
  1756. static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
  1757. int t_idx)
  1758. {
  1759. struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
  1760. struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
  1761. tx_ring->q_vector = q_vector;
  1762. tx_ring->next = q_vector->tx.ring;
  1763. q_vector->tx.ring = tx_ring;
  1764. q_vector->tx.count++;
  1765. q_vector->tx.work_limit = a->tx_work_limit;
  1766. }
  1767. /**
  1768. * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
  1769. * @adapter: board private structure to initialize
  1770. *
  1771. * This function maps descriptor rings to the queue-specific vectors
  1772. * we were allotted through the MSI-X enabling code. Ideally, we'd have
  1773. * one vector per ring/queue, but on a constrained vector budget, we
  1774. * group the rings as "efficiently" as possible. You would add new
  1775. * mapping configurations in here.
  1776. **/
  1777. static void ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
  1778. {
  1779. int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  1780. int rxr_remaining = adapter->num_rx_queues, rxr_idx = 0;
  1781. int txr_remaining = adapter->num_tx_queues, txr_idx = 0;
  1782. int v_start = 0;
  1783. /* only one q_vector if MSI-X is disabled. */
  1784. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
  1785. q_vectors = 1;
  1786. /*
  1787. * If we don't have enough vectors for a 1-to-1 mapping, we'll have to
  1788. * group them so there are multiple queues per vector.
  1789. *
  1790. * Re-adjusting *qpv takes care of the remainder.
  1791. */
  1792. for (; v_start < q_vectors && rxr_remaining; v_start++) {
  1793. int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_start);
  1794. for (; rqpv; rqpv--, rxr_idx++, rxr_remaining--)
  1795. map_vector_to_rxq(adapter, v_start, rxr_idx);
  1796. }
  1797. /*
  1798. * If there are not enough q_vectors for each ring to have it's own
  1799. * vector then we must pair up Rx/Tx on a each vector
  1800. */
  1801. if ((v_start + txr_remaining) > q_vectors)
  1802. v_start = 0;
  1803. for (; v_start < q_vectors && txr_remaining; v_start++) {
  1804. int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_start);
  1805. for (; tqpv; tqpv--, txr_idx++, txr_remaining--)
  1806. map_vector_to_txq(adapter, v_start, txr_idx);
  1807. }
  1808. }
  1809. /**
  1810. * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
  1811. * @adapter: board private structure
  1812. *
  1813. * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
  1814. * interrupts from the kernel.
  1815. **/
  1816. static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
  1817. {
  1818. struct net_device *netdev = adapter->netdev;
  1819. int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  1820. int vector, err;
  1821. int ri = 0, ti = 0;
  1822. for (vector = 0; vector < q_vectors; vector++) {
  1823. struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
  1824. struct msix_entry *entry = &adapter->msix_entries[vector];
  1825. if (q_vector->tx.ring && q_vector->rx.ring) {
  1826. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  1827. "%s-%s-%d", netdev->name, "TxRx", ri++);
  1828. ti++;
  1829. } else if (q_vector->rx.ring) {
  1830. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  1831. "%s-%s-%d", netdev->name, "rx", ri++);
  1832. } else if (q_vector->tx.ring) {
  1833. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  1834. "%s-%s-%d", netdev->name, "tx", ti++);
  1835. } else {
  1836. /* skip this unused q_vector */
  1837. continue;
  1838. }
  1839. err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
  1840. q_vector->name, q_vector);
  1841. if (err) {
  1842. e_err(probe, "request_irq failed for MSIX interrupt "
  1843. "Error: %d\n", err);
  1844. goto free_queue_irqs;
  1845. }
  1846. /* If Flow Director is enabled, set interrupt affinity */
  1847. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  1848. /* assign the mask for this irq */
  1849. irq_set_affinity_hint(entry->vector,
  1850. q_vector->affinity_mask);
  1851. }
  1852. }
  1853. err = request_irq(adapter->msix_entries[vector].vector,
  1854. ixgbe_msix_other, 0, netdev->name, adapter);
  1855. if (err) {
  1856. e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
  1857. goto free_queue_irqs;
  1858. }
  1859. return 0;
  1860. free_queue_irqs:
  1861. while (vector) {
  1862. vector--;
  1863. irq_set_affinity_hint(adapter->msix_entries[vector].vector,
  1864. NULL);
  1865. free_irq(adapter->msix_entries[vector].vector,
  1866. adapter->q_vector[vector]);
  1867. }
  1868. adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
  1869. pci_disable_msix(adapter->pdev);
  1870. kfree(adapter->msix_entries);
  1871. adapter->msix_entries = NULL;
  1872. return err;
  1873. }
  1874. /**
  1875. * ixgbe_intr - legacy mode Interrupt Handler
  1876. * @irq: interrupt number
  1877. * @data: pointer to a network interface device structure
  1878. **/
  1879. static irqreturn_t ixgbe_intr(int irq, void *data)
  1880. {
  1881. struct ixgbe_adapter *adapter = data;
  1882. struct ixgbe_hw *hw = &adapter->hw;
  1883. struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
  1884. u32 eicr;
  1885. /*
  1886. * Workaround for silicon errata on 82598. Mask the interrupts
  1887. * before the read of EICR.
  1888. */
  1889. IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
  1890. /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
  1891. * therefore no explicit interrupt disable is necessary */
  1892. eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
  1893. if (!eicr) {
  1894. /*
  1895. * shared interrupt alert!
  1896. * make sure interrupts are enabled because the read will
  1897. * have disabled interrupts due to EIAM
  1898. * finish the workaround of silicon errata on 82598. Unmask
  1899. * the interrupt that we masked before the EICR read.
  1900. */
  1901. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  1902. ixgbe_irq_enable(adapter, true, true);
  1903. return IRQ_NONE; /* Not our interrupt */
  1904. }
  1905. if (eicr & IXGBE_EICR_LSC)
  1906. ixgbe_check_lsc(adapter);
  1907. switch (hw->mac.type) {
  1908. case ixgbe_mac_82599EB:
  1909. ixgbe_check_sfp_event(adapter, eicr);
  1910. /* Fall through */
  1911. case ixgbe_mac_X540:
  1912. if (eicr & IXGBE_EICR_ECC)
  1913. e_info(link, "Received unrecoverable ECC err, please "
  1914. "reboot\n");
  1915. ixgbe_check_overtemp_event(adapter, eicr);
  1916. break;
  1917. default:
  1918. break;
  1919. }
  1920. ixgbe_check_fan_failure(adapter, eicr);
  1921. if (napi_schedule_prep(&(q_vector->napi))) {
  1922. /* would disable interrupts here but EIAM disabled it */
  1923. __napi_schedule(&(q_vector->napi));
  1924. }
  1925. /*
  1926. * re-enable link(maybe) and non-queue interrupts, no flush.
  1927. * ixgbe_poll will re-enable the queue interrupts
  1928. */
  1929. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  1930. ixgbe_irq_enable(adapter, false, false);
  1931. return IRQ_HANDLED;
  1932. }
  1933. static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
  1934. {
  1935. int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  1936. int i;
  1937. /* legacy and MSI only use one vector */
  1938. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
  1939. q_vectors = 1;
  1940. for (i = 0; i < adapter->num_rx_queues; i++) {
  1941. adapter->rx_ring[i]->q_vector = NULL;
  1942. adapter->rx_ring[i]->next = NULL;
  1943. }
  1944. for (i = 0; i < adapter->num_tx_queues; i++) {
  1945. adapter->tx_ring[i]->q_vector = NULL;
  1946. adapter->tx_ring[i]->next = NULL;
  1947. }
  1948. for (i = 0; i < q_vectors; i++) {
  1949. struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
  1950. memset(&q_vector->rx, 0, sizeof(struct ixgbe_ring_container));
  1951. memset(&q_vector->tx, 0, sizeof(struct ixgbe_ring_container));
  1952. }
  1953. }
  1954. /**
  1955. * ixgbe_request_irq - initialize interrupts
  1956. * @adapter: board private structure
  1957. *
  1958. * Attempts to configure interrupts using the best available
  1959. * capabilities of the hardware and kernel.
  1960. **/
  1961. static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
  1962. {
  1963. struct net_device *netdev = adapter->netdev;
  1964. int err;
  1965. /* map all of the rings to the q_vectors */
  1966. ixgbe_map_rings_to_vectors(adapter);
  1967. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  1968. err = ixgbe_request_msix_irqs(adapter);
  1969. else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
  1970. err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
  1971. netdev->name, adapter);
  1972. else
  1973. err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
  1974. netdev->name, adapter);
  1975. if (err) {
  1976. e_err(probe, "request_irq failed, Error %d\n", err);
  1977. /* place q_vectors and rings back into a known good state */
  1978. ixgbe_reset_q_vectors(adapter);
  1979. }
  1980. return err;
  1981. }
  1982. static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
  1983. {
  1984. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  1985. int i, q_vectors;
  1986. q_vectors = adapter->num_msix_vectors;
  1987. i = q_vectors - 1;
  1988. free_irq(adapter->msix_entries[i].vector, adapter);
  1989. i--;
  1990. for (; i >= 0; i--) {
  1991. /* free only the irqs that were actually requested */
  1992. if (!adapter->q_vector[i]->rx.ring &&
  1993. !adapter->q_vector[i]->tx.ring)
  1994. continue;
  1995. /* clear the affinity_mask in the IRQ descriptor */
  1996. irq_set_affinity_hint(adapter->msix_entries[i].vector,
  1997. NULL);
  1998. free_irq(adapter->msix_entries[i].vector,
  1999. adapter->q_vector[i]);
  2000. }
  2001. } else {
  2002. free_irq(adapter->pdev->irq, adapter);
  2003. }
  2004. /* clear q_vector state information */
  2005. ixgbe_reset_q_vectors(adapter);
  2006. }
  2007. /**
  2008. * ixgbe_irq_disable - Mask off interrupt generation on the NIC
  2009. * @adapter: board private structure
  2010. **/
  2011. static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
  2012. {
  2013. switch (adapter->hw.mac.type) {
  2014. case ixgbe_mac_82598EB:
  2015. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
  2016. break;
  2017. case ixgbe_mac_82599EB:
  2018. case ixgbe_mac_X540:
  2019. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
  2020. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
  2021. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
  2022. break;
  2023. default:
  2024. break;
  2025. }
  2026. IXGBE_WRITE_FLUSH(&adapter->hw);
  2027. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  2028. int i;
  2029. for (i = 0; i < adapter->num_msix_vectors; i++)
  2030. synchronize_irq(adapter->msix_entries[i].vector);
  2031. } else {
  2032. synchronize_irq(adapter->pdev->irq);
  2033. }
  2034. }
  2035. /**
  2036. * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
  2037. *
  2038. **/
  2039. static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
  2040. {
  2041. struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
  2042. /* rx/tx vector */
  2043. if (adapter->rx_itr_setting == 1)
  2044. q_vector->itr = IXGBE_20K_ITR;
  2045. else
  2046. q_vector->itr = adapter->rx_itr_setting;
  2047. ixgbe_write_eitr(q_vector);
  2048. ixgbe_set_ivar(adapter, 0, 0, 0);
  2049. ixgbe_set_ivar(adapter, 1, 0, 0);
  2050. e_info(hw, "Legacy interrupt IVAR setup done\n");
  2051. }
  2052. /**
  2053. * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
  2054. * @adapter: board private structure
  2055. * @ring: structure containing ring specific data
  2056. *
  2057. * Configure the Tx descriptor ring after a reset.
  2058. **/
  2059. void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
  2060. struct ixgbe_ring *ring)
  2061. {
  2062. struct ixgbe_hw *hw = &adapter->hw;
  2063. u64 tdba = ring->dma;
  2064. int wait_loop = 10;
  2065. u32 txdctl = IXGBE_TXDCTL_ENABLE;
  2066. u8 reg_idx = ring->reg_idx;
  2067. /* disable queue to avoid issues while updating state */
  2068. IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
  2069. IXGBE_WRITE_FLUSH(hw);
  2070. IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
  2071. (tdba & DMA_BIT_MASK(32)));
  2072. IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
  2073. IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
  2074. ring->count * sizeof(union ixgbe_adv_tx_desc));
  2075. IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
  2076. IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
  2077. ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
  2078. /*
  2079. * set WTHRESH to encourage burst writeback, it should not be set
  2080. * higher than 1 when ITR is 0 as it could cause false TX hangs
  2081. *
  2082. * In order to avoid issues WTHRESH + PTHRESH should always be equal
  2083. * to or less than the number of on chip descriptors, which is
  2084. * currently 40.
  2085. */
  2086. if (!adapter->tx_itr_setting || !adapter->rx_itr_setting)
  2087. txdctl |= (1 << 16); /* WTHRESH = 1 */
  2088. else
  2089. txdctl |= (8 << 16); /* WTHRESH = 8 */
  2090. /* PTHRESH=32 is needed to avoid a Tx hang with DFP enabled. */
  2091. txdctl |= (1 << 8) | /* HTHRESH = 1 */
  2092. 32; /* PTHRESH = 32 */
  2093. /* reinitialize flowdirector state */
  2094. if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
  2095. adapter->atr_sample_rate) {
  2096. ring->atr_sample_rate = adapter->atr_sample_rate;
  2097. ring->atr_count = 0;
  2098. set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
  2099. } else {
  2100. ring->atr_sample_rate = 0;
  2101. }
  2102. clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
  2103. /* enable queue */
  2104. IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
  2105. /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
  2106. if (hw->mac.type == ixgbe_mac_82598EB &&
  2107. !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
  2108. return;
  2109. /* poll to verify queue is enabled */
  2110. do {
  2111. usleep_range(1000, 2000);
  2112. txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
  2113. } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
  2114. if (!wait_loop)
  2115. e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
  2116. }
  2117. static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
  2118. {
  2119. struct ixgbe_hw *hw = &adapter->hw;
  2120. u32 rttdcs;
  2121. u32 reg;
  2122. u8 tcs = netdev_get_num_tc(adapter->netdev);
  2123. if (hw->mac.type == ixgbe_mac_82598EB)
  2124. return;
  2125. /* disable the arbiter while setting MTQC */
  2126. rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
  2127. rttdcs |= IXGBE_RTTDCS_ARBDIS;
  2128. IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
  2129. /* set transmit pool layout */
  2130. switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  2131. case (IXGBE_FLAG_SRIOV_ENABLED):
  2132. IXGBE_WRITE_REG(hw, IXGBE_MTQC,
  2133. (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
  2134. break;
  2135. default:
  2136. if (!tcs)
  2137. reg = IXGBE_MTQC_64Q_1PB;
  2138. else if (tcs <= 4)
  2139. reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
  2140. else
  2141. reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
  2142. IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
  2143. /* Enable Security TX Buffer IFG for multiple pb */
  2144. if (tcs) {
  2145. reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
  2146. reg |= IXGBE_SECTX_DCB;
  2147. IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
  2148. }
  2149. break;
  2150. }
  2151. /* re-enable the arbiter */
  2152. rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
  2153. IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
  2154. }
  2155. /**
  2156. * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
  2157. * @adapter: board private structure
  2158. *
  2159. * Configure the Tx unit of the MAC after a reset.
  2160. **/
  2161. static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
  2162. {
  2163. struct ixgbe_hw *hw = &adapter->hw;
  2164. u32 dmatxctl;
  2165. u32 i;
  2166. ixgbe_setup_mtqc(adapter);
  2167. if (hw->mac.type != ixgbe_mac_82598EB) {
  2168. /* DMATXCTL.EN must be before Tx queues are enabled */
  2169. dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
  2170. dmatxctl |= IXGBE_DMATXCTL_TE;
  2171. IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
  2172. }
  2173. /* Setup the HW Tx Head and Tail descriptor pointers */
  2174. for (i = 0; i < adapter->num_tx_queues; i++)
  2175. ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
  2176. }
  2177. #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
  2178. static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
  2179. struct ixgbe_ring *rx_ring)
  2180. {
  2181. u32 srrctl;
  2182. u8 reg_idx = rx_ring->reg_idx;
  2183. switch (adapter->hw.mac.type) {
  2184. case ixgbe_mac_82598EB: {
  2185. struct ixgbe_ring_feature *feature = adapter->ring_feature;
  2186. const int mask = feature[RING_F_RSS].mask;
  2187. reg_idx = reg_idx & mask;
  2188. }
  2189. break;
  2190. case ixgbe_mac_82599EB:
  2191. case ixgbe_mac_X540:
  2192. default:
  2193. break;
  2194. }
  2195. srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
  2196. srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
  2197. srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
  2198. if (adapter->num_vfs)
  2199. srrctl |= IXGBE_SRRCTL_DROP_EN;
  2200. srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
  2201. IXGBE_SRRCTL_BSIZEHDR_MASK;
  2202. if (ring_is_ps_enabled(rx_ring)) {
  2203. #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
  2204. srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
  2205. #else
  2206. srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
  2207. #endif
  2208. srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
  2209. } else {
  2210. srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
  2211. IXGBE_SRRCTL_BSIZEPKT_SHIFT;
  2212. srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
  2213. }
  2214. IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
  2215. }
  2216. static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
  2217. {
  2218. struct ixgbe_hw *hw = &adapter->hw;
  2219. static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
  2220. 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
  2221. 0x6A3E67EA, 0x14364D17, 0x3BED200D};
  2222. u32 mrqc = 0, reta = 0;
  2223. u32 rxcsum;
  2224. int i, j;
  2225. u8 tcs = netdev_get_num_tc(adapter->netdev);
  2226. int maxq = adapter->ring_feature[RING_F_RSS].indices;
  2227. if (tcs)
  2228. maxq = min(maxq, adapter->num_tx_queues / tcs);
  2229. /* Fill out hash function seeds */
  2230. for (i = 0; i < 10; i++)
  2231. IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
  2232. /* Fill out redirection table */
  2233. for (i = 0, j = 0; i < 128; i++, j++) {
  2234. if (j == maxq)
  2235. j = 0;
  2236. /* reta = 4-byte sliding window of
  2237. * 0x00..(indices-1)(indices-1)00..etc. */
  2238. reta = (reta << 8) | (j * 0x11);
  2239. if ((i & 3) == 3)
  2240. IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
  2241. }
  2242. /* Disable indicating checksum in descriptor, enables RSS hash */
  2243. rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
  2244. rxcsum |= IXGBE_RXCSUM_PCSD;
  2245. IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
  2246. if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
  2247. (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
  2248. mrqc = IXGBE_MRQC_RSSEN;
  2249. } else {
  2250. int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
  2251. | IXGBE_FLAG_SRIOV_ENABLED);
  2252. switch (mask) {
  2253. case (IXGBE_FLAG_RSS_ENABLED):
  2254. if (!tcs)
  2255. mrqc = IXGBE_MRQC_RSSEN;
  2256. else if (tcs <= 4)
  2257. mrqc = IXGBE_MRQC_RTRSS4TCEN;
  2258. else
  2259. mrqc = IXGBE_MRQC_RTRSS8TCEN;
  2260. break;
  2261. case (IXGBE_FLAG_SRIOV_ENABLED):
  2262. mrqc = IXGBE_MRQC_VMDQEN;
  2263. break;
  2264. default:
  2265. break;
  2266. }
  2267. }
  2268. /* Perform hash on these packet types */
  2269. mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
  2270. | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
  2271. | IXGBE_MRQC_RSS_FIELD_IPV6
  2272. | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
  2273. IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
  2274. }
  2275. /**
  2276. * ixgbe_configure_rscctl - enable RSC for the indicated ring
  2277. * @adapter: address of board private structure
  2278. * @index: index of ring to set
  2279. **/
  2280. static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
  2281. struct ixgbe_ring *ring)
  2282. {
  2283. struct ixgbe_hw *hw = &adapter->hw;
  2284. u32 rscctrl;
  2285. int rx_buf_len;
  2286. u8 reg_idx = ring->reg_idx;
  2287. if (!ring_is_rsc_enabled(ring))
  2288. return;
  2289. rx_buf_len = ring->rx_buf_len;
  2290. rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
  2291. rscctrl |= IXGBE_RSCCTL_RSCEN;
  2292. /*
  2293. * we must limit the number of descriptors so that the
  2294. * total size of max desc * buf_len is not greater
  2295. * than 65535
  2296. */
  2297. if (ring_is_ps_enabled(ring)) {
  2298. #if (MAX_SKB_FRAGS > 16)
  2299. rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
  2300. #elif (MAX_SKB_FRAGS > 8)
  2301. rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
  2302. #elif (MAX_SKB_FRAGS > 4)
  2303. rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
  2304. #else
  2305. rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
  2306. #endif
  2307. } else {
  2308. if (rx_buf_len < IXGBE_RXBUFFER_4K)
  2309. rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
  2310. else if (rx_buf_len < IXGBE_RXBUFFER_8K)
  2311. rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
  2312. else
  2313. rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
  2314. }
  2315. IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
  2316. }
  2317. /**
  2318. * ixgbe_set_uta - Set unicast filter table address
  2319. * @adapter: board private structure
  2320. *
  2321. * The unicast table address is a register array of 32-bit registers.
  2322. * The table is meant to be used in a way similar to how the MTA is used
  2323. * however due to certain limitations in the hardware it is necessary to
  2324. * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
  2325. * enable bit to allow vlan tag stripping when promiscuous mode is enabled
  2326. **/
  2327. static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
  2328. {
  2329. struct ixgbe_hw *hw = &adapter->hw;
  2330. int i;
  2331. /* The UTA table only exists on 82599 hardware and newer */
  2332. if (hw->mac.type < ixgbe_mac_82599EB)
  2333. return;
  2334. /* we only need to do this if VMDq is enabled */
  2335. if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
  2336. return;
  2337. for (i = 0; i < 128; i++)
  2338. IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
  2339. }
  2340. #define IXGBE_MAX_RX_DESC_POLL 10
  2341. static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
  2342. struct ixgbe_ring *ring)
  2343. {
  2344. struct ixgbe_hw *hw = &adapter->hw;
  2345. int wait_loop = IXGBE_MAX_RX_DESC_POLL;
  2346. u32 rxdctl;
  2347. u8 reg_idx = ring->reg_idx;
  2348. /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
  2349. if (hw->mac.type == ixgbe_mac_82598EB &&
  2350. !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
  2351. return;
  2352. do {
  2353. usleep_range(1000, 2000);
  2354. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2355. } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
  2356. if (!wait_loop) {
  2357. e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
  2358. "the polling period\n", reg_idx);
  2359. }
  2360. }
  2361. void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
  2362. struct ixgbe_ring *ring)
  2363. {
  2364. struct ixgbe_hw *hw = &adapter->hw;
  2365. int wait_loop = IXGBE_MAX_RX_DESC_POLL;
  2366. u32 rxdctl;
  2367. u8 reg_idx = ring->reg_idx;
  2368. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2369. rxdctl &= ~IXGBE_RXDCTL_ENABLE;
  2370. /* write value back with RXDCTL.ENABLE bit cleared */
  2371. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
  2372. if (hw->mac.type == ixgbe_mac_82598EB &&
  2373. !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
  2374. return;
  2375. /* the hardware may take up to 100us to really disable the rx queue */
  2376. do {
  2377. udelay(10);
  2378. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2379. } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
  2380. if (!wait_loop) {
  2381. e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
  2382. "the polling period\n", reg_idx);
  2383. }
  2384. }
  2385. void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
  2386. struct ixgbe_ring *ring)
  2387. {
  2388. struct ixgbe_hw *hw = &adapter->hw;
  2389. u64 rdba = ring->dma;
  2390. u32 rxdctl;
  2391. u8 reg_idx = ring->reg_idx;
  2392. /* disable queue to avoid issues while updating state */
  2393. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2394. ixgbe_disable_rx_queue(adapter, ring);
  2395. IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
  2396. IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
  2397. IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
  2398. ring->count * sizeof(union ixgbe_adv_rx_desc));
  2399. IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
  2400. IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
  2401. ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
  2402. ixgbe_configure_srrctl(adapter, ring);
  2403. ixgbe_configure_rscctl(adapter, ring);
  2404. /* If operating in IOV mode set RLPML for X540 */
  2405. if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
  2406. hw->mac.type == ixgbe_mac_X540) {
  2407. rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
  2408. rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
  2409. ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
  2410. }
  2411. if (hw->mac.type == ixgbe_mac_82598EB) {
  2412. /*
  2413. * enable cache line friendly hardware writes:
  2414. * PTHRESH=32 descriptors (half the internal cache),
  2415. * this also removes ugly rx_no_buffer_count increment
  2416. * HTHRESH=4 descriptors (to minimize latency on fetch)
  2417. * WTHRESH=8 burst writeback up to two cache lines
  2418. */
  2419. rxdctl &= ~0x3FFFFF;
  2420. rxdctl |= 0x080420;
  2421. }
  2422. /* enable receive descriptor ring */
  2423. rxdctl |= IXGBE_RXDCTL_ENABLE;
  2424. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
  2425. ixgbe_rx_desc_queue_enable(adapter, ring);
  2426. ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
  2427. }
  2428. static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
  2429. {
  2430. struct ixgbe_hw *hw = &adapter->hw;
  2431. int p;
  2432. /* PSRTYPE must be initialized in non 82598 adapters */
  2433. u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
  2434. IXGBE_PSRTYPE_UDPHDR |
  2435. IXGBE_PSRTYPE_IPV4HDR |
  2436. IXGBE_PSRTYPE_L2HDR |
  2437. IXGBE_PSRTYPE_IPV6HDR;
  2438. if (hw->mac.type == ixgbe_mac_82598EB)
  2439. return;
  2440. if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
  2441. psrtype |= (adapter->num_rx_queues_per_pool << 29);
  2442. for (p = 0; p < adapter->num_rx_pools; p++)
  2443. IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
  2444. psrtype);
  2445. }
  2446. static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
  2447. {
  2448. struct ixgbe_hw *hw = &adapter->hw;
  2449. u32 gcr_ext;
  2450. u32 vt_reg_bits;
  2451. u32 reg_offset, vf_shift;
  2452. u32 vmdctl;
  2453. int i;
  2454. if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
  2455. return;
  2456. vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
  2457. vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
  2458. vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
  2459. IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
  2460. vf_shift = adapter->num_vfs % 32;
  2461. reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
  2462. /* Enable only the PF's pool for Tx/Rx */
  2463. IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
  2464. IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
  2465. IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
  2466. IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
  2467. IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
  2468. /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
  2469. hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
  2470. /*
  2471. * Set up VF register offsets for selected VT Mode,
  2472. * i.e. 32 or 64 VFs for SR-IOV
  2473. */
  2474. gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
  2475. gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
  2476. gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
  2477. IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
  2478. /* enable Tx loopback for VF/PF communication */
  2479. IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
  2480. /* Enable MAC Anti-Spoofing */
  2481. hw->mac.ops.set_mac_anti_spoofing(hw,
  2482. (adapter->num_vfs != 0),
  2483. adapter->num_vfs);
  2484. /* For VFs that have spoof checking turned off */
  2485. for (i = 0; i < adapter->num_vfs; i++) {
  2486. if (!adapter->vfinfo[i].spoofchk_enabled)
  2487. ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
  2488. }
  2489. }
  2490. static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
  2491. {
  2492. struct ixgbe_hw *hw = &adapter->hw;
  2493. struct net_device *netdev = adapter->netdev;
  2494. int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
  2495. int rx_buf_len;
  2496. struct ixgbe_ring *rx_ring;
  2497. int i;
  2498. u32 mhadd, hlreg0;
  2499. /* Decide whether to use packet split mode or not */
  2500. /* On by default */
  2501. adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
  2502. /* Do not use packet split if we're in SR-IOV Mode */
  2503. if (adapter->num_vfs)
  2504. adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
  2505. /* Disable packet split due to 82599 erratum #45 */
  2506. if (hw->mac.type == ixgbe_mac_82599EB)
  2507. adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
  2508. #ifdef IXGBE_FCOE
  2509. /* adjust max frame to be able to do baby jumbo for FCoE */
  2510. if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
  2511. (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
  2512. max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
  2513. #endif /* IXGBE_FCOE */
  2514. mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
  2515. if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
  2516. mhadd &= ~IXGBE_MHADD_MFS_MASK;
  2517. mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
  2518. IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
  2519. }
  2520. /* MHADD will allow an extra 4 bytes past for vlan tagged frames */
  2521. max_frame += VLAN_HLEN;
  2522. /* Set the RX buffer length according to the mode */
  2523. if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
  2524. rx_buf_len = IXGBE_RX_HDR_SIZE;
  2525. } else {
  2526. if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
  2527. (netdev->mtu <= ETH_DATA_LEN))
  2528. rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
  2529. /*
  2530. * Make best use of allocation by using all but 1K of a
  2531. * power of 2 allocation that will be used for skb->head.
  2532. */
  2533. else if (max_frame <= IXGBE_RXBUFFER_3K)
  2534. rx_buf_len = IXGBE_RXBUFFER_3K;
  2535. else if (max_frame <= IXGBE_RXBUFFER_7K)
  2536. rx_buf_len = IXGBE_RXBUFFER_7K;
  2537. else if (max_frame <= IXGBE_RXBUFFER_15K)
  2538. rx_buf_len = IXGBE_RXBUFFER_15K;
  2539. else
  2540. rx_buf_len = IXGBE_MAX_RXBUFFER;
  2541. }
  2542. hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
  2543. /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
  2544. hlreg0 |= IXGBE_HLREG0_JUMBOEN;
  2545. IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
  2546. /*
  2547. * Setup the HW Rx Head and Tail Descriptor Pointers and
  2548. * the Base and Length of the Rx Descriptor Ring
  2549. */
  2550. for (i = 0; i < adapter->num_rx_queues; i++) {
  2551. rx_ring = adapter->rx_ring[i];
  2552. rx_ring->rx_buf_len = rx_buf_len;
  2553. if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
  2554. set_ring_ps_enabled(rx_ring);
  2555. else
  2556. clear_ring_ps_enabled(rx_ring);
  2557. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
  2558. set_ring_rsc_enabled(rx_ring);
  2559. else
  2560. clear_ring_rsc_enabled(rx_ring);
  2561. #ifdef IXGBE_FCOE
  2562. if (netdev->features & NETIF_F_FCOE_MTU) {
  2563. struct ixgbe_ring_feature *f;
  2564. f = &adapter->ring_feature[RING_F_FCOE];
  2565. if ((i >= f->mask) && (i < f->mask + f->indices)) {
  2566. clear_ring_ps_enabled(rx_ring);
  2567. if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE)
  2568. rx_ring->rx_buf_len =
  2569. IXGBE_FCOE_JUMBO_FRAME_SIZE;
  2570. } else if (!ring_is_rsc_enabled(rx_ring) &&
  2571. !ring_is_ps_enabled(rx_ring)) {
  2572. rx_ring->rx_buf_len =
  2573. IXGBE_FCOE_JUMBO_FRAME_SIZE;
  2574. }
  2575. }
  2576. #endif /* IXGBE_FCOE */
  2577. }
  2578. }
  2579. static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
  2580. {
  2581. struct ixgbe_hw *hw = &adapter->hw;
  2582. u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
  2583. switch (hw->mac.type) {
  2584. case ixgbe_mac_82598EB:
  2585. /*
  2586. * For VMDq support of different descriptor types or
  2587. * buffer sizes through the use of multiple SRRCTL
  2588. * registers, RDRXCTL.MVMEN must be set to 1
  2589. *
  2590. * also, the manual doesn't mention it clearly but DCA hints
  2591. * will only use queue 0's tags unless this bit is set. Side
  2592. * effects of setting this bit are only that SRRCTL must be
  2593. * fully programmed [0..15]
  2594. */
  2595. rdrxctl |= IXGBE_RDRXCTL_MVMEN;
  2596. break;
  2597. case ixgbe_mac_82599EB:
  2598. case ixgbe_mac_X540:
  2599. /* Disable RSC for ACK packets */
  2600. IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
  2601. (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
  2602. rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
  2603. /* hardware requires some bits to be set by default */
  2604. rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
  2605. rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
  2606. break;
  2607. default:
  2608. /* We should do nothing since we don't know this hardware */
  2609. return;
  2610. }
  2611. IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
  2612. }
  2613. /**
  2614. * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
  2615. * @adapter: board private structure
  2616. *
  2617. * Configure the Rx unit of the MAC after a reset.
  2618. **/
  2619. static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
  2620. {
  2621. struct ixgbe_hw *hw = &adapter->hw;
  2622. int i;
  2623. u32 rxctrl;
  2624. /* disable receives while setting up the descriptors */
  2625. rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
  2626. IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
  2627. ixgbe_setup_psrtype(adapter);
  2628. ixgbe_setup_rdrxctl(adapter);
  2629. /* Program registers for the distribution of queues */
  2630. ixgbe_setup_mrqc(adapter);
  2631. ixgbe_set_uta(adapter);
  2632. /* set_rx_buffer_len must be called before ring initialization */
  2633. ixgbe_set_rx_buffer_len(adapter);
  2634. /*
  2635. * Setup the HW Rx Head and Tail Descriptor Pointers and
  2636. * the Base and Length of the Rx Descriptor Ring
  2637. */
  2638. for (i = 0; i < adapter->num_rx_queues; i++)
  2639. ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
  2640. /* disable drop enable for 82598 parts */
  2641. if (hw->mac.type == ixgbe_mac_82598EB)
  2642. rxctrl |= IXGBE_RXCTRL_DMBYPS;
  2643. /* enable all receives */
  2644. rxctrl |= IXGBE_RXCTRL_RXEN;
  2645. hw->mac.ops.enable_rx_dma(hw, rxctrl);
  2646. }
  2647. static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
  2648. {
  2649. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  2650. struct ixgbe_hw *hw = &adapter->hw;
  2651. int pool_ndx = adapter->num_vfs;
  2652. /* add VID to filter table */
  2653. hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, true);
  2654. set_bit(vid, adapter->active_vlans);
  2655. return 0;
  2656. }
  2657. static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
  2658. {
  2659. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  2660. struct ixgbe_hw *hw = &adapter->hw;
  2661. int pool_ndx = adapter->num_vfs;
  2662. /* remove VID from filter table */
  2663. hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, false);
  2664. clear_bit(vid, adapter->active_vlans);
  2665. return 0;
  2666. }
  2667. /**
  2668. * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering
  2669. * @adapter: driver data
  2670. */
  2671. static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
  2672. {
  2673. struct ixgbe_hw *hw = &adapter->hw;
  2674. u32 vlnctrl;
  2675. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2676. vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
  2677. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2678. }
  2679. /**
  2680. * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering
  2681. * @adapter: driver data
  2682. */
  2683. static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
  2684. {
  2685. struct ixgbe_hw *hw = &adapter->hw;
  2686. u32 vlnctrl;
  2687. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2688. vlnctrl |= IXGBE_VLNCTRL_VFE;
  2689. vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
  2690. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2691. }
  2692. /**
  2693. * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
  2694. * @adapter: driver data
  2695. */
  2696. static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
  2697. {
  2698. struct ixgbe_hw *hw = &adapter->hw;
  2699. u32 vlnctrl;
  2700. int i, j;
  2701. switch (hw->mac.type) {
  2702. case ixgbe_mac_82598EB:
  2703. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2704. vlnctrl &= ~IXGBE_VLNCTRL_VME;
  2705. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2706. break;
  2707. case ixgbe_mac_82599EB:
  2708. case ixgbe_mac_X540:
  2709. for (i = 0; i < adapter->num_rx_queues; i++) {
  2710. j = adapter->rx_ring[i]->reg_idx;
  2711. vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
  2712. vlnctrl &= ~IXGBE_RXDCTL_VME;
  2713. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
  2714. }
  2715. break;
  2716. default:
  2717. break;
  2718. }
  2719. }
  2720. /**
  2721. * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
  2722. * @adapter: driver data
  2723. */
  2724. static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
  2725. {
  2726. struct ixgbe_hw *hw = &adapter->hw;
  2727. u32 vlnctrl;
  2728. int i, j;
  2729. switch (hw->mac.type) {
  2730. case ixgbe_mac_82598EB:
  2731. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2732. vlnctrl |= IXGBE_VLNCTRL_VME;
  2733. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2734. break;
  2735. case ixgbe_mac_82599EB:
  2736. case ixgbe_mac_X540:
  2737. for (i = 0; i < adapter->num_rx_queues; i++) {
  2738. j = adapter->rx_ring[i]->reg_idx;
  2739. vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
  2740. vlnctrl |= IXGBE_RXDCTL_VME;
  2741. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
  2742. }
  2743. break;
  2744. default:
  2745. break;
  2746. }
  2747. }
  2748. static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
  2749. {
  2750. u16 vid;
  2751. ixgbe_vlan_rx_add_vid(adapter->netdev, 0);
  2752. for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
  2753. ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
  2754. }
  2755. /**
  2756. * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
  2757. * @netdev: network interface device structure
  2758. *
  2759. * Writes unicast address list to the RAR table.
  2760. * Returns: -ENOMEM on failure/insufficient address space
  2761. * 0 on no addresses written
  2762. * X on writing X addresses to the RAR table
  2763. **/
  2764. static int ixgbe_write_uc_addr_list(struct net_device *netdev)
  2765. {
  2766. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  2767. struct ixgbe_hw *hw = &adapter->hw;
  2768. unsigned int vfn = adapter->num_vfs;
  2769. unsigned int rar_entries = IXGBE_MAX_PF_MACVLANS;
  2770. int count = 0;
  2771. /* return ENOMEM indicating insufficient memory for addresses */
  2772. if (netdev_uc_count(netdev) > rar_entries)
  2773. return -ENOMEM;
  2774. if (!netdev_uc_empty(netdev) && rar_entries) {
  2775. struct netdev_hw_addr *ha;
  2776. /* return error if we do not support writing to RAR table */
  2777. if (!hw->mac.ops.set_rar)
  2778. return -ENOMEM;
  2779. netdev_for_each_uc_addr(ha, netdev) {
  2780. if (!rar_entries)
  2781. break;
  2782. hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
  2783. vfn, IXGBE_RAH_AV);
  2784. count++;
  2785. }
  2786. }
  2787. /* write the addresses in reverse order to avoid write combining */
  2788. for (; rar_entries > 0 ; rar_entries--)
  2789. hw->mac.ops.clear_rar(hw, rar_entries);
  2790. return count;
  2791. }
  2792. /**
  2793. * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
  2794. * @netdev: network interface device structure
  2795. *
  2796. * The set_rx_method entry point is called whenever the unicast/multicast
  2797. * address list or the network interface flags are updated. This routine is
  2798. * responsible for configuring the hardware for proper unicast, multicast and
  2799. * promiscuous mode.
  2800. **/
  2801. void ixgbe_set_rx_mode(struct net_device *netdev)
  2802. {
  2803. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  2804. struct ixgbe_hw *hw = &adapter->hw;
  2805. u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
  2806. int count;
  2807. /* Check for Promiscuous and All Multicast modes */
  2808. fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
  2809. /* set all bits that we expect to always be set */
  2810. fctrl |= IXGBE_FCTRL_BAM;
  2811. fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
  2812. fctrl |= IXGBE_FCTRL_PMCF;
  2813. /* clear the bits we are changing the status of */
  2814. fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
  2815. if (netdev->flags & IFF_PROMISC) {
  2816. hw->addr_ctrl.user_set_promisc = true;
  2817. fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
  2818. vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
  2819. /* don't hardware filter vlans in promisc mode */
  2820. ixgbe_vlan_filter_disable(adapter);
  2821. } else {
  2822. if (netdev->flags & IFF_ALLMULTI) {
  2823. fctrl |= IXGBE_FCTRL_MPE;
  2824. vmolr |= IXGBE_VMOLR_MPE;
  2825. } else {
  2826. /*
  2827. * Write addresses to the MTA, if the attempt fails
  2828. * then we should just turn on promiscuous mode so
  2829. * that we can at least receive multicast traffic
  2830. */
  2831. hw->mac.ops.update_mc_addr_list(hw, netdev);
  2832. vmolr |= IXGBE_VMOLR_ROMPE;
  2833. }
  2834. ixgbe_vlan_filter_enable(adapter);
  2835. hw->addr_ctrl.user_set_promisc = false;
  2836. /*
  2837. * Write addresses to available RAR registers, if there is not
  2838. * sufficient space to store all the addresses then enable
  2839. * unicast promiscuous mode
  2840. */
  2841. count = ixgbe_write_uc_addr_list(netdev);
  2842. if (count < 0) {
  2843. fctrl |= IXGBE_FCTRL_UPE;
  2844. vmolr |= IXGBE_VMOLR_ROPE;
  2845. }
  2846. }
  2847. if (adapter->num_vfs) {
  2848. ixgbe_restore_vf_multicasts(adapter);
  2849. vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) &
  2850. ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
  2851. IXGBE_VMOLR_ROPE);
  2852. IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr);
  2853. }
  2854. IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
  2855. if (netdev->features & NETIF_F_HW_VLAN_RX)
  2856. ixgbe_vlan_strip_enable(adapter);
  2857. else
  2858. ixgbe_vlan_strip_disable(adapter);
  2859. }
  2860. static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
  2861. {
  2862. int q_idx;
  2863. struct ixgbe_q_vector *q_vector;
  2864. int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  2865. /* legacy and MSI only use one vector */
  2866. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
  2867. q_vectors = 1;
  2868. for (q_idx = 0; q_idx < q_vectors; q_idx++) {
  2869. q_vector = adapter->q_vector[q_idx];
  2870. napi_enable(&q_vector->napi);
  2871. }
  2872. }
  2873. static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
  2874. {
  2875. int q_idx;
  2876. struct ixgbe_q_vector *q_vector;
  2877. int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  2878. /* legacy and MSI only use one vector */
  2879. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
  2880. q_vectors = 1;
  2881. for (q_idx = 0; q_idx < q_vectors; q_idx++) {
  2882. q_vector = adapter->q_vector[q_idx];
  2883. napi_disable(&q_vector->napi);
  2884. }
  2885. }
  2886. #ifdef CONFIG_IXGBE_DCB
  2887. /*
  2888. * ixgbe_configure_dcb - Configure DCB hardware
  2889. * @adapter: ixgbe adapter struct
  2890. *
  2891. * This is called by the driver on open to configure the DCB hardware.
  2892. * This is also called by the gennetlink interface when reconfiguring
  2893. * the DCB state.
  2894. */
  2895. static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
  2896. {
  2897. struct ixgbe_hw *hw = &adapter->hw;
  2898. int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
  2899. if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
  2900. if (hw->mac.type == ixgbe_mac_82598EB)
  2901. netif_set_gso_max_size(adapter->netdev, 65536);
  2902. return;
  2903. }
  2904. if (hw->mac.type == ixgbe_mac_82598EB)
  2905. netif_set_gso_max_size(adapter->netdev, 32768);
  2906. /* Enable VLAN tag insert/strip */
  2907. adapter->netdev->features |= NETIF_F_HW_VLAN_RX;
  2908. hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true);
  2909. #ifdef IXGBE_FCOE
  2910. if (adapter->netdev->features & NETIF_F_FCOE_MTU)
  2911. max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
  2912. #endif
  2913. /* reconfigure the hardware */
  2914. if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
  2915. ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
  2916. DCB_TX_CONFIG);
  2917. ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
  2918. DCB_RX_CONFIG);
  2919. ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
  2920. } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
  2921. ixgbe_dcb_hw_ets(&adapter->hw,
  2922. adapter->ixgbe_ieee_ets,
  2923. max_frame);
  2924. ixgbe_dcb_hw_pfc_config(&adapter->hw,
  2925. adapter->ixgbe_ieee_pfc->pfc_en,
  2926. adapter->ixgbe_ieee_ets->prio_tc);
  2927. }
  2928. /* Enable RSS Hash per TC */
  2929. if (hw->mac.type != ixgbe_mac_82598EB) {
  2930. int i;
  2931. u32 reg = 0;
  2932. for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
  2933. u8 msb = 0;
  2934. u8 cnt = adapter->netdev->tc_to_txq[i].count;
  2935. while (cnt >>= 1)
  2936. msb++;
  2937. reg |= msb << IXGBE_RQTC_SHIFT_TC(i);
  2938. }
  2939. IXGBE_WRITE_REG(hw, IXGBE_RQTC, reg);
  2940. }
  2941. }
  2942. #endif
  2943. /* Additional bittime to account for IXGBE framing */
  2944. #define IXGBE_ETH_FRAMING 20
  2945. /*
  2946. * ixgbe_hpbthresh - calculate high water mark for flow control
  2947. *
  2948. * @adapter: board private structure to calculate for
  2949. * @pb - packet buffer to calculate
  2950. */
  2951. static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
  2952. {
  2953. struct ixgbe_hw *hw = &adapter->hw;
  2954. struct net_device *dev = adapter->netdev;
  2955. int link, tc, kb, marker;
  2956. u32 dv_id, rx_pba;
  2957. /* Calculate max LAN frame size */
  2958. tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
  2959. #ifdef IXGBE_FCOE
  2960. /* FCoE traffic class uses FCOE jumbo frames */
  2961. if (dev->features & NETIF_F_FCOE_MTU) {
  2962. int fcoe_pb = 0;
  2963. #ifdef CONFIG_IXGBE_DCB
  2964. fcoe_pb = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
  2965. #endif
  2966. if (fcoe_pb == pb && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE)
  2967. tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
  2968. }
  2969. #endif
  2970. /* Calculate delay value for device */
  2971. switch (hw->mac.type) {
  2972. case ixgbe_mac_X540:
  2973. dv_id = IXGBE_DV_X540(link, tc);
  2974. break;
  2975. default:
  2976. dv_id = IXGBE_DV(link, tc);
  2977. break;
  2978. }
  2979. /* Loopback switch introduces additional latency */
  2980. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  2981. dv_id += IXGBE_B2BT(tc);
  2982. /* Delay value is calculated in bit times convert to KB */
  2983. kb = IXGBE_BT2KB(dv_id);
  2984. rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
  2985. marker = rx_pba - kb;
  2986. /* It is possible that the packet buffer is not large enough
  2987. * to provide required headroom. In this case throw an error
  2988. * to user and a do the best we can.
  2989. */
  2990. if (marker < 0) {
  2991. e_warn(drv, "Packet Buffer(%i) can not provide enough"
  2992. "headroom to support flow control."
  2993. "Decrease MTU or number of traffic classes\n", pb);
  2994. marker = tc + 1;
  2995. }
  2996. return marker;
  2997. }
  2998. /*
  2999. * ixgbe_lpbthresh - calculate low water mark for for flow control
  3000. *
  3001. * @adapter: board private structure to calculate for
  3002. * @pb - packet buffer to calculate
  3003. */
  3004. static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter)
  3005. {
  3006. struct ixgbe_hw *hw = &adapter->hw;
  3007. struct net_device *dev = adapter->netdev;
  3008. int tc;
  3009. u32 dv_id;
  3010. /* Calculate max LAN frame size */
  3011. tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
  3012. /* Calculate delay value for device */
  3013. switch (hw->mac.type) {
  3014. case ixgbe_mac_X540:
  3015. dv_id = IXGBE_LOW_DV_X540(tc);
  3016. break;
  3017. default:
  3018. dv_id = IXGBE_LOW_DV(tc);
  3019. break;
  3020. }
  3021. /* Delay value is calculated in bit times convert to KB */
  3022. return IXGBE_BT2KB(dv_id);
  3023. }
  3024. /*
  3025. * ixgbe_pbthresh_setup - calculate and setup high low water marks
  3026. */
  3027. static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
  3028. {
  3029. struct ixgbe_hw *hw = &adapter->hw;
  3030. int num_tc = netdev_get_num_tc(adapter->netdev);
  3031. int i;
  3032. if (!num_tc)
  3033. num_tc = 1;
  3034. hw->fc.low_water = ixgbe_lpbthresh(adapter);
  3035. for (i = 0; i < num_tc; i++) {
  3036. hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);
  3037. /* Low water marks must not be larger than high water marks */
  3038. if (hw->fc.low_water > hw->fc.high_water[i])
  3039. hw->fc.low_water = 0;
  3040. }
  3041. }
  3042. static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
  3043. {
  3044. struct ixgbe_hw *hw = &adapter->hw;
  3045. int hdrm;
  3046. u8 tc = netdev_get_num_tc(adapter->netdev);
  3047. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
  3048. adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
  3049. hdrm = 32 << adapter->fdir_pballoc;
  3050. else
  3051. hdrm = 0;
  3052. hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
  3053. ixgbe_pbthresh_setup(adapter);
  3054. }
  3055. static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
  3056. {
  3057. struct ixgbe_hw *hw = &adapter->hw;
  3058. struct hlist_node *node, *node2;
  3059. struct ixgbe_fdir_filter *filter;
  3060. spin_lock(&adapter->fdir_perfect_lock);
  3061. if (!hlist_empty(&adapter->fdir_filter_list))
  3062. ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);
  3063. hlist_for_each_entry_safe(filter, node, node2,
  3064. &adapter->fdir_filter_list, fdir_node) {
  3065. ixgbe_fdir_write_perfect_filter_82599(hw,
  3066. &filter->filter,
  3067. filter->sw_idx,
  3068. (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
  3069. IXGBE_FDIR_DROP_QUEUE :
  3070. adapter->rx_ring[filter->action]->reg_idx);
  3071. }
  3072. spin_unlock(&adapter->fdir_perfect_lock);
  3073. }
  3074. static void ixgbe_configure(struct ixgbe_adapter *adapter)
  3075. {
  3076. ixgbe_configure_pb(adapter);
  3077. #ifdef CONFIG_IXGBE_DCB
  3078. ixgbe_configure_dcb(adapter);
  3079. #endif
  3080. ixgbe_set_rx_mode(adapter->netdev);
  3081. ixgbe_restore_vlan(adapter);
  3082. #ifdef IXGBE_FCOE
  3083. if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
  3084. ixgbe_configure_fcoe(adapter);
  3085. #endif /* IXGBE_FCOE */
  3086. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  3087. ixgbe_init_fdir_signature_82599(&adapter->hw,
  3088. adapter->fdir_pballoc);
  3089. } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
  3090. ixgbe_init_fdir_perfect_82599(&adapter->hw,
  3091. adapter->fdir_pballoc);
  3092. ixgbe_fdir_filter_restore(adapter);
  3093. }
  3094. ixgbe_configure_virtualization(adapter);
  3095. ixgbe_configure_tx(adapter);
  3096. ixgbe_configure_rx(adapter);
  3097. }
  3098. static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
  3099. {
  3100. switch (hw->phy.type) {
  3101. case ixgbe_phy_sfp_avago:
  3102. case ixgbe_phy_sfp_ftl:
  3103. case ixgbe_phy_sfp_intel:
  3104. case ixgbe_phy_sfp_unknown:
  3105. case ixgbe_phy_sfp_passive_tyco:
  3106. case ixgbe_phy_sfp_passive_unknown:
  3107. case ixgbe_phy_sfp_active_unknown:
  3108. case ixgbe_phy_sfp_ftl_active:
  3109. return true;
  3110. case ixgbe_phy_nl:
  3111. if (hw->mac.type == ixgbe_mac_82598EB)
  3112. return true;
  3113. default:
  3114. return false;
  3115. }
  3116. }
  3117. /**
  3118. * ixgbe_sfp_link_config - set up SFP+ link
  3119. * @adapter: pointer to private adapter struct
  3120. **/
  3121. static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
  3122. {
  3123. /*
  3124. * We are assuming the worst case scenario here, and that
  3125. * is that an SFP was inserted/removed after the reset
  3126. * but before SFP detection was enabled. As such the best
  3127. * solution is to just start searching as soon as we start
  3128. */
  3129. if (adapter->hw.mac.type == ixgbe_mac_82598EB)
  3130. adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
  3131. adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
  3132. }
  3133. /**
  3134. * ixgbe_non_sfp_link_config - set up non-SFP+ link
  3135. * @hw: pointer to private hardware struct
  3136. *
  3137. * Returns 0 on success, negative on failure
  3138. **/
  3139. static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
  3140. {
  3141. u32 autoneg;
  3142. bool negotiation, link_up = false;
  3143. u32 ret = IXGBE_ERR_LINK_SETUP;
  3144. if (hw->mac.ops.check_link)
  3145. ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
  3146. if (ret)
  3147. goto link_cfg_out;
  3148. autoneg = hw->phy.autoneg_advertised;
  3149. if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
  3150. ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
  3151. &negotiation);
  3152. if (ret)
  3153. goto link_cfg_out;
  3154. if (hw->mac.ops.setup_link)
  3155. ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
  3156. link_cfg_out:
  3157. return ret;
  3158. }
  3159. static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
  3160. {
  3161. struct ixgbe_hw *hw = &adapter->hw;
  3162. u32 gpie = 0;
  3163. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  3164. gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
  3165. IXGBE_GPIE_OCD;
  3166. gpie |= IXGBE_GPIE_EIAME;
  3167. /*
  3168. * use EIAM to auto-mask when MSI-X interrupt is asserted
  3169. * this saves a register write for every interrupt
  3170. */
  3171. switch (hw->mac.type) {
  3172. case ixgbe_mac_82598EB:
  3173. IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
  3174. break;
  3175. case ixgbe_mac_82599EB:
  3176. case ixgbe_mac_X540:
  3177. default:
  3178. IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
  3179. IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
  3180. break;
  3181. }
  3182. } else {
  3183. /* legacy interrupts, use EIAM to auto-mask when reading EICR,
  3184. * specifically only auto mask tx and rx interrupts */
  3185. IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
  3186. }
  3187. /* XXX: to interrupt immediately for EICS writes, enable this */
  3188. /* gpie |= IXGBE_GPIE_EIMEN; */
  3189. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  3190. gpie &= ~IXGBE_GPIE_VTMODE_MASK;
  3191. gpie |= IXGBE_GPIE_VTMODE_64;
  3192. }
  3193. /* Enable Thermal over heat sensor interrupt */
  3194. if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
  3195. switch (adapter->hw.mac.type) {
  3196. case ixgbe_mac_82599EB:
  3197. gpie |= IXGBE_SDP0_GPIEN;
  3198. break;
  3199. case ixgbe_mac_X540:
  3200. gpie |= IXGBE_EIMS_TS;
  3201. break;
  3202. default:
  3203. break;
  3204. }
  3205. }
  3206. /* Enable fan failure interrupt */
  3207. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
  3208. gpie |= IXGBE_SDP1_GPIEN;
  3209. if (hw->mac.type == ixgbe_mac_82599EB) {
  3210. gpie |= IXGBE_SDP1_GPIEN;
  3211. gpie |= IXGBE_SDP2_GPIEN;
  3212. }
  3213. IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
  3214. }
  3215. static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
  3216. {
  3217. struct ixgbe_hw *hw = &adapter->hw;
  3218. int err;
  3219. u32 ctrl_ext;
  3220. ixgbe_get_hw_control(adapter);
  3221. ixgbe_setup_gpie(adapter);
  3222. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  3223. ixgbe_configure_msix(adapter);
  3224. else
  3225. ixgbe_configure_msi_and_legacy(adapter);
  3226. /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
  3227. if (hw->mac.ops.enable_tx_laser &&
  3228. ((hw->phy.multispeed_fiber) ||
  3229. ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
  3230. (hw->mac.type == ixgbe_mac_82599EB))))
  3231. hw->mac.ops.enable_tx_laser(hw);
  3232. clear_bit(__IXGBE_DOWN, &adapter->state);
  3233. ixgbe_napi_enable_all(adapter);
  3234. if (ixgbe_is_sfp(hw)) {
  3235. ixgbe_sfp_link_config(adapter);
  3236. } else {
  3237. err = ixgbe_non_sfp_link_config(hw);
  3238. if (err)
  3239. e_err(probe, "link_config FAILED %d\n", err);
  3240. }
  3241. /* clear any pending interrupts, may auto mask */
  3242. IXGBE_READ_REG(hw, IXGBE_EICR);
  3243. ixgbe_irq_enable(adapter, true, true);
  3244. /*
  3245. * If this adapter has a fan, check to see if we had a failure
  3246. * before we enabled the interrupt.
  3247. */
  3248. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
  3249. u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
  3250. if (esdp & IXGBE_ESDP_SDP1)
  3251. e_crit(drv, "Fan has stopped, replace the adapter\n");
  3252. }
  3253. /* enable transmits */
  3254. netif_tx_start_all_queues(adapter->netdev);
  3255. /* bring the link up in the watchdog, this could race with our first
  3256. * link up interrupt but shouldn't be a problem */
  3257. adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
  3258. adapter->link_check_timeout = jiffies;
  3259. mod_timer(&adapter->service_timer, jiffies);
  3260. /* Set PF Reset Done bit so PF/VF Mail Ops can work */
  3261. ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
  3262. ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
  3263. IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
  3264. }
  3265. void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
  3266. {
  3267. WARN_ON(in_interrupt());
  3268. /* put off any impending NetWatchDogTimeout */
  3269. adapter->netdev->trans_start = jiffies;
  3270. while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
  3271. usleep_range(1000, 2000);
  3272. ixgbe_down(adapter);
  3273. /*
  3274. * If SR-IOV enabled then wait a bit before bringing the adapter
  3275. * back up to give the VFs time to respond to the reset. The
  3276. * two second wait is based upon the watchdog timer cycle in
  3277. * the VF driver.
  3278. */
  3279. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  3280. msleep(2000);
  3281. ixgbe_up(adapter);
  3282. clear_bit(__IXGBE_RESETTING, &adapter->state);
  3283. }
  3284. void ixgbe_up(struct ixgbe_adapter *adapter)
  3285. {
  3286. /* hardware has been reset, we need to reload some things */
  3287. ixgbe_configure(adapter);
  3288. ixgbe_up_complete(adapter);
  3289. }
  3290. void ixgbe_reset(struct ixgbe_adapter *adapter)
  3291. {
  3292. struct ixgbe_hw *hw = &adapter->hw;
  3293. int err;
  3294. /* lock SFP init bit to prevent race conditions with the watchdog */
  3295. while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
  3296. usleep_range(1000, 2000);
  3297. /* clear all SFP and link config related flags while holding SFP_INIT */
  3298. adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
  3299. IXGBE_FLAG2_SFP_NEEDS_RESET);
  3300. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
  3301. err = hw->mac.ops.init_hw(hw);
  3302. switch (err) {
  3303. case 0:
  3304. case IXGBE_ERR_SFP_NOT_PRESENT:
  3305. case IXGBE_ERR_SFP_NOT_SUPPORTED:
  3306. break;
  3307. case IXGBE_ERR_MASTER_REQUESTS_PENDING:
  3308. e_dev_err("master disable timed out\n");
  3309. break;
  3310. case IXGBE_ERR_EEPROM_VERSION:
  3311. /* We are running on a pre-production device, log a warning */
  3312. e_dev_warn("This device is a pre-production adapter/LOM. "
  3313. "Please be aware there may be issues associated with "
  3314. "your hardware. If you are experiencing problems "
  3315. "please contact your Intel or hardware "
  3316. "representative who provided you with this "
  3317. "hardware.\n");
  3318. break;
  3319. default:
  3320. e_dev_err("Hardware Error: %d\n", err);
  3321. }
  3322. clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
  3323. /* reprogram the RAR[0] in case user changed it. */
  3324. hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
  3325. IXGBE_RAH_AV);
  3326. }
  3327. /**
  3328. * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
  3329. * @rx_ring: ring to free buffers from
  3330. **/
  3331. static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
  3332. {
  3333. struct device *dev = rx_ring->dev;
  3334. unsigned long size;
  3335. u16 i;
  3336. /* ring already cleared, nothing to do */
  3337. if (!rx_ring->rx_buffer_info)
  3338. return;
  3339. /* Free all the Rx ring sk_buffs */
  3340. for (i = 0; i < rx_ring->count; i++) {
  3341. struct ixgbe_rx_buffer *rx_buffer_info;
  3342. rx_buffer_info = &rx_ring->rx_buffer_info[i];
  3343. if (rx_buffer_info->dma) {
  3344. dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
  3345. rx_ring->rx_buf_len,
  3346. DMA_FROM_DEVICE);
  3347. rx_buffer_info->dma = 0;
  3348. }
  3349. if (rx_buffer_info->skb) {
  3350. struct sk_buff *skb = rx_buffer_info->skb;
  3351. rx_buffer_info->skb = NULL;
  3352. do {
  3353. struct sk_buff *this = skb;
  3354. if (IXGBE_RSC_CB(this)->delay_unmap) {
  3355. dma_unmap_single(dev,
  3356. IXGBE_RSC_CB(this)->dma,
  3357. rx_ring->rx_buf_len,
  3358. DMA_FROM_DEVICE);
  3359. IXGBE_RSC_CB(this)->dma = 0;
  3360. IXGBE_RSC_CB(skb)->delay_unmap = false;
  3361. }
  3362. skb = skb->prev;
  3363. dev_kfree_skb(this);
  3364. } while (skb);
  3365. }
  3366. if (!rx_buffer_info->page)
  3367. continue;
  3368. if (rx_buffer_info->page_dma) {
  3369. dma_unmap_page(dev, rx_buffer_info->page_dma,
  3370. PAGE_SIZE / 2, DMA_FROM_DEVICE);
  3371. rx_buffer_info->page_dma = 0;
  3372. }
  3373. put_page(rx_buffer_info->page);
  3374. rx_buffer_info->page = NULL;
  3375. rx_buffer_info->page_offset = 0;
  3376. }
  3377. size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
  3378. memset(rx_ring->rx_buffer_info, 0, size);
  3379. /* Zero out the descriptor ring */
  3380. memset(rx_ring->desc, 0, rx_ring->size);
  3381. rx_ring->next_to_clean = 0;
  3382. rx_ring->next_to_use = 0;
  3383. }
  3384. /**
  3385. * ixgbe_clean_tx_ring - Free Tx Buffers
  3386. * @tx_ring: ring to be cleaned
  3387. **/
  3388. static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
  3389. {
  3390. struct ixgbe_tx_buffer *tx_buffer_info;
  3391. unsigned long size;
  3392. u16 i;
  3393. /* ring already cleared, nothing to do */
  3394. if (!tx_ring->tx_buffer_info)
  3395. return;
  3396. /* Free all the Tx ring sk_buffs */
  3397. for (i = 0; i < tx_ring->count; i++) {
  3398. tx_buffer_info = &tx_ring->tx_buffer_info[i];
  3399. ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
  3400. }
  3401. size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
  3402. memset(tx_ring->tx_buffer_info, 0, size);
  3403. /* Zero out the descriptor ring */
  3404. memset(tx_ring->desc, 0, tx_ring->size);
  3405. tx_ring->next_to_use = 0;
  3406. tx_ring->next_to_clean = 0;
  3407. }
  3408. /**
  3409. * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
  3410. * @adapter: board private structure
  3411. **/
  3412. static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
  3413. {
  3414. int i;
  3415. for (i = 0; i < adapter->num_rx_queues; i++)
  3416. ixgbe_clean_rx_ring(adapter->rx_ring[i]);
  3417. }
  3418. /**
  3419. * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
  3420. * @adapter: board private structure
  3421. **/
  3422. static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
  3423. {
  3424. int i;
  3425. for (i = 0; i < adapter->num_tx_queues; i++)
  3426. ixgbe_clean_tx_ring(adapter->tx_ring[i]);
  3427. }
  3428. static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
  3429. {
  3430. struct hlist_node *node, *node2;
  3431. struct ixgbe_fdir_filter *filter;
  3432. spin_lock(&adapter->fdir_perfect_lock);
  3433. hlist_for_each_entry_safe(filter, node, node2,
  3434. &adapter->fdir_filter_list, fdir_node) {
  3435. hlist_del(&filter->fdir_node);
  3436. kfree(filter);
  3437. }
  3438. adapter->fdir_filter_count = 0;
  3439. spin_unlock(&adapter->fdir_perfect_lock);
  3440. }
  3441. void ixgbe_down(struct ixgbe_adapter *adapter)
  3442. {
  3443. struct net_device *netdev = adapter->netdev;
  3444. struct ixgbe_hw *hw = &adapter->hw;
  3445. u32 rxctrl;
  3446. int i;
  3447. /* signal that we are down to the interrupt handler */
  3448. set_bit(__IXGBE_DOWN, &adapter->state);
  3449. /* disable receives */
  3450. rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
  3451. IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
  3452. /* disable all enabled rx queues */
  3453. for (i = 0; i < adapter->num_rx_queues; i++)
  3454. /* this call also flushes the previous write */
  3455. ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
  3456. usleep_range(10000, 20000);
  3457. netif_tx_stop_all_queues(netdev);
  3458. /* call carrier off first to avoid false dev_watchdog timeouts */
  3459. netif_carrier_off(netdev);
  3460. netif_tx_disable(netdev);
  3461. ixgbe_irq_disable(adapter);
  3462. ixgbe_napi_disable_all(adapter);
  3463. adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
  3464. IXGBE_FLAG2_RESET_REQUESTED);
  3465. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
  3466. del_timer_sync(&adapter->service_timer);
  3467. if (adapter->num_vfs) {
  3468. /* Clear EITR Select mapping */
  3469. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
  3470. /* Mark all the VFs as inactive */
  3471. for (i = 0 ; i < adapter->num_vfs; i++)
  3472. adapter->vfinfo[i].clear_to_send = false;
  3473. /* ping all the active vfs to let them know we are going down */
  3474. ixgbe_ping_all_vfs(adapter);
  3475. /* Disable all VFTE/VFRE TX/RX */
  3476. ixgbe_disable_tx_rx(adapter);
  3477. }
  3478. /* disable transmits in the hardware now that interrupts are off */
  3479. for (i = 0; i < adapter->num_tx_queues; i++) {
  3480. u8 reg_idx = adapter->tx_ring[i]->reg_idx;
  3481. IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
  3482. }
  3483. /* Disable the Tx DMA engine on 82599 and X540 */
  3484. switch (hw->mac.type) {
  3485. case ixgbe_mac_82599EB:
  3486. case ixgbe_mac_X540:
  3487. IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
  3488. (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
  3489. ~IXGBE_DMATXCTL_TE));
  3490. break;
  3491. default:
  3492. break;
  3493. }
  3494. if (!pci_channel_offline(adapter->pdev))
  3495. ixgbe_reset(adapter);
  3496. /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
  3497. if (hw->mac.ops.disable_tx_laser &&
  3498. ((hw->phy.multispeed_fiber) ||
  3499. ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
  3500. (hw->mac.type == ixgbe_mac_82599EB))))
  3501. hw->mac.ops.disable_tx_laser(hw);
  3502. ixgbe_clean_all_tx_rings(adapter);
  3503. ixgbe_clean_all_rx_rings(adapter);
  3504. #ifdef CONFIG_IXGBE_DCA
  3505. /* since we reset the hardware DCA settings were cleared */
  3506. ixgbe_setup_dca(adapter);
  3507. #endif
  3508. }
  3509. /**
  3510. * ixgbe_poll - NAPI Rx polling callback
  3511. * @napi: structure for representing this polling device
  3512. * @budget: how many packets driver is allowed to clean
  3513. *
  3514. * This function is used for legacy and MSI, NAPI mode
  3515. **/
  3516. static int ixgbe_poll(struct napi_struct *napi, int budget)
  3517. {
  3518. struct ixgbe_q_vector *q_vector =
  3519. container_of(napi, struct ixgbe_q_vector, napi);
  3520. struct ixgbe_adapter *adapter = q_vector->adapter;
  3521. struct ixgbe_ring *ring;
  3522. int per_ring_budget;
  3523. bool clean_complete = true;
  3524. #ifdef CONFIG_IXGBE_DCA
  3525. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
  3526. ixgbe_update_dca(q_vector);
  3527. #endif
  3528. for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
  3529. clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring);
  3530. /* attempt to distribute budget to each queue fairly, but don't allow
  3531. * the budget to go below 1 because we'll exit polling */
  3532. if (q_vector->rx.count > 1)
  3533. per_ring_budget = max(budget/q_vector->rx.count, 1);
  3534. else
  3535. per_ring_budget = budget;
  3536. for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
  3537. clean_complete &= ixgbe_clean_rx_irq(q_vector, ring,
  3538. per_ring_budget);
  3539. /* If all work not completed, return budget and keep polling */
  3540. if (!clean_complete)
  3541. return budget;
  3542. /* all work done, exit the polling mode */
  3543. napi_complete(napi);
  3544. if (adapter->rx_itr_setting & 1)
  3545. ixgbe_set_itr(q_vector);
  3546. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  3547. ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));
  3548. return 0;
  3549. }
  3550. /**
  3551. * ixgbe_tx_timeout - Respond to a Tx Hang
  3552. * @netdev: network interface device structure
  3553. **/
  3554. static void ixgbe_tx_timeout(struct net_device *netdev)
  3555. {
  3556. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  3557. /* Do the reset outside of interrupt context */
  3558. ixgbe_tx_timeout_reset(adapter);
  3559. }
  3560. /**
  3561. * ixgbe_set_rss_queues: Allocate queues for RSS
  3562. * @adapter: board private structure to initialize
  3563. *
  3564. * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
  3565. * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
  3566. *
  3567. **/
  3568. static inline bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter)
  3569. {
  3570. bool ret = false;
  3571. struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_RSS];
  3572. if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
  3573. f->mask = 0xF;
  3574. adapter->num_rx_queues = f->indices;
  3575. adapter->num_tx_queues = f->indices;
  3576. ret = true;
  3577. } else {
  3578. ret = false;
  3579. }
  3580. return ret;
  3581. }
  3582. /**
  3583. * ixgbe_set_fdir_queues: Allocate queues for Flow Director
  3584. * @adapter: board private structure to initialize
  3585. *
  3586. * Flow Director is an advanced Rx filter, attempting to get Rx flows back
  3587. * to the original CPU that initiated the Tx session. This runs in addition
  3588. * to RSS, so if a packet doesn't match an FDIR filter, we can still spread the
  3589. * Rx load across CPUs using RSS.
  3590. *
  3591. **/
  3592. static inline bool ixgbe_set_fdir_queues(struct ixgbe_adapter *adapter)
  3593. {
  3594. bool ret = false;
  3595. struct ixgbe_ring_feature *f_fdir = &adapter->ring_feature[RING_F_FDIR];
  3596. f_fdir->indices = min((int)num_online_cpus(), f_fdir->indices);
  3597. f_fdir->mask = 0;
  3598. /* Flow Director must have RSS enabled */
  3599. if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
  3600. (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
  3601. adapter->num_tx_queues = f_fdir->indices;
  3602. adapter->num_rx_queues = f_fdir->indices;
  3603. ret = true;
  3604. } else {
  3605. adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
  3606. }
  3607. return ret;
  3608. }
  3609. #ifdef IXGBE_FCOE
  3610. /**
  3611. * ixgbe_set_fcoe_queues: Allocate queues for Fiber Channel over Ethernet (FCoE)
  3612. * @adapter: board private structure to initialize
  3613. *
  3614. * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges.
  3615. * The ring feature mask is not used as a mask for FCoE, as it can take any 8
  3616. * rx queues out of the max number of rx queues, instead, it is used as the
  3617. * index of the first rx queue used by FCoE.
  3618. *
  3619. **/
  3620. static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter)
  3621. {
  3622. struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
  3623. if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
  3624. return false;
  3625. f->indices = min((int)num_online_cpus(), f->indices);
  3626. adapter->num_rx_queues = 1;
  3627. adapter->num_tx_queues = 1;
  3628. if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
  3629. e_info(probe, "FCoE enabled with RSS\n");
  3630. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
  3631. ixgbe_set_fdir_queues(adapter);
  3632. else
  3633. ixgbe_set_rss_queues(adapter);
  3634. }
  3635. /* adding FCoE rx rings to the end */
  3636. f->mask = adapter->num_rx_queues;
  3637. adapter->num_rx_queues += f->indices;
  3638. adapter->num_tx_queues += f->indices;
  3639. return true;
  3640. }
  3641. #endif /* IXGBE_FCOE */
  3642. /* Artificial max queue cap per traffic class in DCB mode */
  3643. #define DCB_QUEUE_CAP 8
  3644. #ifdef CONFIG_IXGBE_DCB
  3645. static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter)
  3646. {
  3647. int per_tc_q, q, i, offset = 0;
  3648. struct net_device *dev = adapter->netdev;
  3649. int tcs = netdev_get_num_tc(dev);
  3650. if (!tcs)
  3651. return false;
  3652. /* Map queue offset and counts onto allocated tx queues */
  3653. per_tc_q = min(dev->num_tx_queues / tcs, (unsigned int)DCB_QUEUE_CAP);
  3654. q = min((int)num_online_cpus(), per_tc_q);
  3655. for (i = 0; i < tcs; i++) {
  3656. netdev_set_tc_queue(dev, i, q, offset);
  3657. offset += q;
  3658. }
  3659. adapter->num_tx_queues = q * tcs;
  3660. adapter->num_rx_queues = q * tcs;
  3661. #ifdef IXGBE_FCOE
  3662. /* FCoE enabled queues require special configuration indexed
  3663. * by feature specific indices and mask. Here we map FCoE
  3664. * indices onto the DCB queue pairs allowing FCoE to own
  3665. * configuration later.
  3666. */
  3667. if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) {
  3668. int tc;
  3669. struct ixgbe_ring_feature *f =
  3670. &adapter->ring_feature[RING_F_FCOE];
  3671. tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
  3672. f->indices = dev->tc_to_txq[tc].count;
  3673. f->mask = dev->tc_to_txq[tc].offset;
  3674. }
  3675. #endif
  3676. return true;
  3677. }
  3678. #endif
  3679. /**
  3680. * ixgbe_set_sriov_queues: Allocate queues for IOV use
  3681. * @adapter: board private structure to initialize
  3682. *
  3683. * IOV doesn't actually use anything, so just NAK the
  3684. * request for now and let the other queue routines
  3685. * figure out what to do.
  3686. */
  3687. static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
  3688. {
  3689. return false;
  3690. }
  3691. /*
  3692. * ixgbe_set_num_queues: Allocate queues for device, feature dependent
  3693. * @adapter: board private structure to initialize
  3694. *
  3695. * This is the top level queue allocation routine. The order here is very
  3696. * important, starting with the "most" number of features turned on at once,
  3697. * and ending with the smallest set of features. This way large combinations
  3698. * can be allocated if they're turned on, and smaller combinations are the
  3699. * fallthrough conditions.
  3700. *
  3701. **/
  3702. static int ixgbe_set_num_queues(struct ixgbe_adapter *adapter)
  3703. {
  3704. /* Start with base case */
  3705. adapter->num_rx_queues = 1;
  3706. adapter->num_tx_queues = 1;
  3707. adapter->num_rx_pools = adapter->num_rx_queues;
  3708. adapter->num_rx_queues_per_pool = 1;
  3709. if (ixgbe_set_sriov_queues(adapter))
  3710. goto done;
  3711. #ifdef CONFIG_IXGBE_DCB
  3712. if (ixgbe_set_dcb_queues(adapter))
  3713. goto done;
  3714. #endif
  3715. #ifdef IXGBE_FCOE
  3716. if (ixgbe_set_fcoe_queues(adapter))
  3717. goto done;
  3718. #endif /* IXGBE_FCOE */
  3719. if (ixgbe_set_fdir_queues(adapter))
  3720. goto done;
  3721. if (ixgbe_set_rss_queues(adapter))
  3722. goto done;
  3723. /* fallback to base case */
  3724. adapter->num_rx_queues = 1;
  3725. adapter->num_tx_queues = 1;
  3726. done:
  3727. /* Notify the stack of the (possibly) reduced queue counts. */
  3728. netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
  3729. return netif_set_real_num_rx_queues(adapter->netdev,
  3730. adapter->num_rx_queues);
  3731. }
  3732. static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter,
  3733. int vectors)
  3734. {
  3735. int err, vector_threshold;
  3736. /* We'll want at least 3 (vector_threshold):
  3737. * 1) TxQ[0] Cleanup
  3738. * 2) RxQ[0] Cleanup
  3739. * 3) Other (Link Status Change, etc.)
  3740. * 4) TCP Timer (optional)
  3741. */
  3742. vector_threshold = MIN_MSIX_COUNT;
  3743. /* The more we get, the more we will assign to Tx/Rx Cleanup
  3744. * for the separate queues...where Rx Cleanup >= Tx Cleanup.
  3745. * Right now, we simply care about how many we'll get; we'll
  3746. * set them up later while requesting irq's.
  3747. */
  3748. while (vectors >= vector_threshold) {
  3749. err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
  3750. vectors);
  3751. if (!err) /* Success in acquiring all requested vectors. */
  3752. break;
  3753. else if (err < 0)
  3754. vectors = 0; /* Nasty failure, quit now */
  3755. else /* err == number of vectors we should try again with */
  3756. vectors = err;
  3757. }
  3758. if (vectors < vector_threshold) {
  3759. /* Can't allocate enough MSI-X interrupts? Oh well.
  3760. * This just means we'll go with either a single MSI
  3761. * vector or fall back to legacy interrupts.
  3762. */
  3763. netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
  3764. "Unable to allocate MSI-X interrupts\n");
  3765. adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
  3766. kfree(adapter->msix_entries);
  3767. adapter->msix_entries = NULL;
  3768. } else {
  3769. adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */
  3770. /*
  3771. * Adjust for only the vectors we'll use, which is minimum
  3772. * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
  3773. * vectors we were allocated.
  3774. */
  3775. adapter->num_msix_vectors = min(vectors,
  3776. adapter->max_msix_q_vectors + NON_Q_VECTORS);
  3777. }
  3778. }
  3779. /**
  3780. * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS
  3781. * @adapter: board private structure to initialize
  3782. *
  3783. * Cache the descriptor ring offsets for RSS to the assigned rings.
  3784. *
  3785. **/
  3786. static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter)
  3787. {
  3788. int i;
  3789. if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
  3790. return false;
  3791. for (i = 0; i < adapter->num_rx_queues; i++)
  3792. adapter->rx_ring[i]->reg_idx = i;
  3793. for (i = 0; i < adapter->num_tx_queues; i++)
  3794. adapter->tx_ring[i]->reg_idx = i;
  3795. return true;
  3796. }
  3797. #ifdef CONFIG_IXGBE_DCB
  3798. /* ixgbe_get_first_reg_idx - Return first register index associated with ring */
  3799. static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
  3800. unsigned int *tx, unsigned int *rx)
  3801. {
  3802. struct net_device *dev = adapter->netdev;
  3803. struct ixgbe_hw *hw = &adapter->hw;
  3804. u8 num_tcs = netdev_get_num_tc(dev);
  3805. *tx = 0;
  3806. *rx = 0;
  3807. switch (hw->mac.type) {
  3808. case ixgbe_mac_82598EB:
  3809. *tx = tc << 2;
  3810. *rx = tc << 3;
  3811. break;
  3812. case ixgbe_mac_82599EB:
  3813. case ixgbe_mac_X540:
  3814. if (num_tcs > 4) {
  3815. if (tc < 3) {
  3816. *tx = tc << 5;
  3817. *rx = tc << 4;
  3818. } else if (tc < 5) {
  3819. *tx = ((tc + 2) << 4);
  3820. *rx = tc << 4;
  3821. } else if (tc < num_tcs) {
  3822. *tx = ((tc + 8) << 3);
  3823. *rx = tc << 4;
  3824. }
  3825. } else {
  3826. *rx = tc << 5;
  3827. switch (tc) {
  3828. case 0:
  3829. *tx = 0;
  3830. break;
  3831. case 1:
  3832. *tx = 64;
  3833. break;
  3834. case 2:
  3835. *tx = 96;
  3836. break;
  3837. case 3:
  3838. *tx = 112;
  3839. break;
  3840. default:
  3841. break;
  3842. }
  3843. }
  3844. break;
  3845. default:
  3846. break;
  3847. }
  3848. }
  3849. /**
  3850. * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB
  3851. * @adapter: board private structure to initialize
  3852. *
  3853. * Cache the descriptor ring offsets for DCB to the assigned rings.
  3854. *
  3855. **/
  3856. static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
  3857. {
  3858. struct net_device *dev = adapter->netdev;
  3859. int i, j, k;
  3860. u8 num_tcs = netdev_get_num_tc(dev);
  3861. if (!num_tcs)
  3862. return false;
  3863. for (i = 0, k = 0; i < num_tcs; i++) {
  3864. unsigned int tx_s, rx_s;
  3865. u16 count = dev->tc_to_txq[i].count;
  3866. ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s);
  3867. for (j = 0; j < count; j++, k++) {
  3868. adapter->tx_ring[k]->reg_idx = tx_s + j;
  3869. adapter->rx_ring[k]->reg_idx = rx_s + j;
  3870. adapter->tx_ring[k]->dcb_tc = i;
  3871. adapter->rx_ring[k]->dcb_tc = i;
  3872. }
  3873. }
  3874. return true;
  3875. }
  3876. #endif
  3877. /**
  3878. * ixgbe_cache_ring_fdir - Descriptor ring to register mapping for Flow Director
  3879. * @adapter: board private structure to initialize
  3880. *
  3881. * Cache the descriptor ring offsets for Flow Director to the assigned rings.
  3882. *
  3883. **/
  3884. static inline bool ixgbe_cache_ring_fdir(struct ixgbe_adapter *adapter)
  3885. {
  3886. int i;
  3887. bool ret = false;
  3888. if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
  3889. (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
  3890. for (i = 0; i < adapter->num_rx_queues; i++)
  3891. adapter->rx_ring[i]->reg_idx = i;
  3892. for (i = 0; i < adapter->num_tx_queues; i++)
  3893. adapter->tx_ring[i]->reg_idx = i;
  3894. ret = true;
  3895. }
  3896. return ret;
  3897. }
  3898. #ifdef IXGBE_FCOE
  3899. /**
  3900. * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE
  3901. * @adapter: board private structure to initialize
  3902. *
  3903. * Cache the descriptor ring offsets for FCoE mode to the assigned rings.
  3904. *
  3905. */
  3906. static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter)
  3907. {
  3908. struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
  3909. int i;
  3910. u8 fcoe_rx_i = 0, fcoe_tx_i = 0;
  3911. if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
  3912. return false;
  3913. if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
  3914. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
  3915. ixgbe_cache_ring_fdir(adapter);
  3916. else
  3917. ixgbe_cache_ring_rss(adapter);
  3918. fcoe_rx_i = f->mask;
  3919. fcoe_tx_i = f->mask;
  3920. }
  3921. for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) {
  3922. adapter->rx_ring[f->mask + i]->reg_idx = fcoe_rx_i;
  3923. adapter->tx_ring[f->mask + i]->reg_idx = fcoe_tx_i;
  3924. }
  3925. return true;
  3926. }
  3927. #endif /* IXGBE_FCOE */
  3928. /**
  3929. * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov
  3930. * @adapter: board private structure to initialize
  3931. *
  3932. * SR-IOV doesn't use any descriptor rings but changes the default if
  3933. * no other mapping is used.
  3934. *
  3935. */
  3936. static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
  3937. {
  3938. adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2;
  3939. adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2;
  3940. if (adapter->num_vfs)
  3941. return true;
  3942. else
  3943. return false;
  3944. }
  3945. /**
  3946. * ixgbe_cache_ring_register - Descriptor ring to register mapping
  3947. * @adapter: board private structure to initialize
  3948. *
  3949. * Once we know the feature-set enabled for the device, we'll cache
  3950. * the register offset the descriptor ring is assigned to.
  3951. *
  3952. * Note, the order the various feature calls is important. It must start with
  3953. * the "most" features enabled at the same time, then trickle down to the
  3954. * least amount of features turned on at once.
  3955. **/
  3956. static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter)
  3957. {
  3958. /* start with default case */
  3959. adapter->rx_ring[0]->reg_idx = 0;
  3960. adapter->tx_ring[0]->reg_idx = 0;
  3961. if (ixgbe_cache_ring_sriov(adapter))
  3962. return;
  3963. #ifdef CONFIG_IXGBE_DCB
  3964. if (ixgbe_cache_ring_dcb(adapter))
  3965. return;
  3966. #endif
  3967. #ifdef IXGBE_FCOE
  3968. if (ixgbe_cache_ring_fcoe(adapter))
  3969. return;
  3970. #endif /* IXGBE_FCOE */
  3971. if (ixgbe_cache_ring_fdir(adapter))
  3972. return;
  3973. if (ixgbe_cache_ring_rss(adapter))
  3974. return;
  3975. }
  3976. /**
  3977. * ixgbe_alloc_queues - Allocate memory for all rings
  3978. * @adapter: board private structure to initialize
  3979. *
  3980. * We allocate one ring per queue at run-time since we don't know the
  3981. * number of queues at compile-time. The polling_netdev array is
  3982. * intended for Multiqueue, but should work fine with a single queue.
  3983. **/
  3984. static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter)
  3985. {
  3986. int rx = 0, tx = 0, nid = adapter->node;
  3987. if (nid < 0 || !node_online(nid))
  3988. nid = first_online_node;
  3989. for (; tx < adapter->num_tx_queues; tx++) {
  3990. struct ixgbe_ring *ring;
  3991. ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
  3992. if (!ring)
  3993. ring = kzalloc(sizeof(*ring), GFP_KERNEL);
  3994. if (!ring)
  3995. goto err_allocation;
  3996. ring->count = adapter->tx_ring_count;
  3997. ring->queue_index = tx;
  3998. ring->numa_node = nid;
  3999. ring->dev = &adapter->pdev->dev;
  4000. ring->netdev = adapter->netdev;
  4001. adapter->tx_ring[tx] = ring;
  4002. }
  4003. for (; rx < adapter->num_rx_queues; rx++) {
  4004. struct ixgbe_ring *ring;
  4005. ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
  4006. if (!ring)
  4007. ring = kzalloc(sizeof(*ring), GFP_KERNEL);
  4008. if (!ring)
  4009. goto err_allocation;
  4010. ring->count = adapter->rx_ring_count;
  4011. ring->queue_index = rx;
  4012. ring->numa_node = nid;
  4013. ring->dev = &adapter->pdev->dev;
  4014. ring->netdev = adapter->netdev;
  4015. adapter->rx_ring[rx] = ring;
  4016. }
  4017. ixgbe_cache_ring_register(adapter);
  4018. return 0;
  4019. err_allocation:
  4020. while (tx)
  4021. kfree(adapter->tx_ring[--tx]);
  4022. while (rx)
  4023. kfree(adapter->rx_ring[--rx]);
  4024. return -ENOMEM;
  4025. }
  4026. /**
  4027. * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported
  4028. * @adapter: board private structure to initialize
  4029. *
  4030. * Attempt to configure the interrupts using the best available
  4031. * capabilities of the hardware and the kernel.
  4032. **/
  4033. static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
  4034. {
  4035. struct ixgbe_hw *hw = &adapter->hw;
  4036. int err = 0;
  4037. int vector, v_budget;
  4038. /*
  4039. * It's easy to be greedy for MSI-X vectors, but it really
  4040. * doesn't do us much good if we have a lot more vectors
  4041. * than CPU's. So let's be conservative and only ask for
  4042. * (roughly) the same number of vectors as there are CPU's.
  4043. */
  4044. v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
  4045. (int)num_online_cpus()) + NON_Q_VECTORS;
  4046. /*
  4047. * At the same time, hardware can only support a maximum of
  4048. * hw.mac->max_msix_vectors vectors. With features
  4049. * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
  4050. * descriptor queues supported by our device. Thus, we cap it off in
  4051. * those rare cases where the cpu count also exceeds our vector limit.
  4052. */
  4053. v_budget = min(v_budget, (int)hw->mac.max_msix_vectors);
  4054. /* A failure in MSI-X entry allocation isn't fatal, but it does
  4055. * mean we disable MSI-X capabilities of the adapter. */
  4056. adapter->msix_entries = kcalloc(v_budget,
  4057. sizeof(struct msix_entry), GFP_KERNEL);
  4058. if (adapter->msix_entries) {
  4059. for (vector = 0; vector < v_budget; vector++)
  4060. adapter->msix_entries[vector].entry = vector;
  4061. ixgbe_acquire_msix_vectors(adapter, v_budget);
  4062. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  4063. goto out;
  4064. }
  4065. adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
  4066. adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED;
  4067. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  4068. e_err(probe,
  4069. "ATR is not supported while multiple "
  4070. "queues are disabled. Disabling Flow Director\n");
  4071. }
  4072. adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
  4073. adapter->atr_sample_rate = 0;
  4074. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  4075. ixgbe_disable_sriov(adapter);
  4076. err = ixgbe_set_num_queues(adapter);
  4077. if (err)
  4078. return err;
  4079. err = pci_enable_msi(adapter->pdev);
  4080. if (!err) {
  4081. adapter->flags |= IXGBE_FLAG_MSI_ENABLED;
  4082. } else {
  4083. netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
  4084. "Unable to allocate MSI interrupt, "
  4085. "falling back to legacy. Error: %d\n", err);
  4086. /* reset err */
  4087. err = 0;
  4088. }
  4089. out:
  4090. return err;
  4091. }
  4092. /**
  4093. * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors
  4094. * @adapter: board private structure to initialize
  4095. *
  4096. * We allocate one q_vector per queue interrupt. If allocation fails we
  4097. * return -ENOMEM.
  4098. **/
  4099. static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter)
  4100. {
  4101. int v_idx, num_q_vectors;
  4102. struct ixgbe_q_vector *q_vector;
  4103. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  4104. num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  4105. else
  4106. num_q_vectors = 1;
  4107. for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
  4108. q_vector = kzalloc_node(sizeof(struct ixgbe_q_vector),
  4109. GFP_KERNEL, adapter->node);
  4110. if (!q_vector)
  4111. q_vector = kzalloc(sizeof(struct ixgbe_q_vector),
  4112. GFP_KERNEL);
  4113. if (!q_vector)
  4114. goto err_out;
  4115. q_vector->adapter = adapter;
  4116. q_vector->v_idx = v_idx;
  4117. /* Allocate the affinity_hint cpumask, configure the mask */
  4118. if (!alloc_cpumask_var(&q_vector->affinity_mask, GFP_KERNEL))
  4119. goto err_out;
  4120. cpumask_set_cpu(v_idx, q_vector->affinity_mask);
  4121. netif_napi_add(adapter->netdev, &q_vector->napi,
  4122. ixgbe_poll, 64);
  4123. adapter->q_vector[v_idx] = q_vector;
  4124. }
  4125. return 0;
  4126. err_out:
  4127. while (v_idx) {
  4128. v_idx--;
  4129. q_vector = adapter->q_vector[v_idx];
  4130. netif_napi_del(&q_vector->napi);
  4131. free_cpumask_var(q_vector->affinity_mask);
  4132. kfree(q_vector);
  4133. adapter->q_vector[v_idx] = NULL;
  4134. }
  4135. return -ENOMEM;
  4136. }
  4137. /**
  4138. * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors
  4139. * @adapter: board private structure to initialize
  4140. *
  4141. * This function frees the memory allocated to the q_vectors. In addition if
  4142. * NAPI is enabled it will delete any references to the NAPI struct prior
  4143. * to freeing the q_vector.
  4144. **/
  4145. static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter)
  4146. {
  4147. int v_idx, num_q_vectors;
  4148. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  4149. num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  4150. else
  4151. num_q_vectors = 1;
  4152. for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
  4153. struct ixgbe_q_vector *q_vector = adapter->q_vector[v_idx];
  4154. adapter->q_vector[v_idx] = NULL;
  4155. netif_napi_del(&q_vector->napi);
  4156. free_cpumask_var(q_vector->affinity_mask);
  4157. kfree(q_vector);
  4158. }
  4159. }
  4160. static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter)
  4161. {
  4162. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  4163. adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
  4164. pci_disable_msix(adapter->pdev);
  4165. kfree(adapter->msix_entries);
  4166. adapter->msix_entries = NULL;
  4167. } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
  4168. adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED;
  4169. pci_disable_msi(adapter->pdev);
  4170. }
  4171. }
  4172. /**
  4173. * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme
  4174. * @adapter: board private structure to initialize
  4175. *
  4176. * We determine which interrupt scheme to use based on...
  4177. * - Kernel support (MSI, MSI-X)
  4178. * - which can be user-defined (via MODULE_PARAM)
  4179. * - Hardware queue count (num_*_queues)
  4180. * - defined by miscellaneous hardware support/features (RSS, etc.)
  4181. **/
  4182. int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter)
  4183. {
  4184. int err;
  4185. /* Number of supported queues */
  4186. err = ixgbe_set_num_queues(adapter);
  4187. if (err)
  4188. return err;
  4189. err = ixgbe_set_interrupt_capability(adapter);
  4190. if (err) {
  4191. e_dev_err("Unable to setup interrupt capabilities\n");
  4192. goto err_set_interrupt;
  4193. }
  4194. err = ixgbe_alloc_q_vectors(adapter);
  4195. if (err) {
  4196. e_dev_err("Unable to allocate memory for queue vectors\n");
  4197. goto err_alloc_q_vectors;
  4198. }
  4199. err = ixgbe_alloc_queues(adapter);
  4200. if (err) {
  4201. e_dev_err("Unable to allocate memory for queues\n");
  4202. goto err_alloc_queues;
  4203. }
  4204. e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n",
  4205. (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
  4206. adapter->num_rx_queues, adapter->num_tx_queues);
  4207. set_bit(__IXGBE_DOWN, &adapter->state);
  4208. return 0;
  4209. err_alloc_queues:
  4210. ixgbe_free_q_vectors(adapter);
  4211. err_alloc_q_vectors:
  4212. ixgbe_reset_interrupt_capability(adapter);
  4213. err_set_interrupt:
  4214. return err;
  4215. }
  4216. /**
  4217. * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings
  4218. * @adapter: board private structure to clear interrupt scheme on
  4219. *
  4220. * We go through and clear interrupt specific resources and reset the structure
  4221. * to pre-load conditions
  4222. **/
  4223. void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter)
  4224. {
  4225. int i;
  4226. for (i = 0; i < adapter->num_tx_queues; i++) {
  4227. kfree(adapter->tx_ring[i]);
  4228. adapter->tx_ring[i] = NULL;
  4229. }
  4230. for (i = 0; i < adapter->num_rx_queues; i++) {
  4231. struct ixgbe_ring *ring = adapter->rx_ring[i];
  4232. /* ixgbe_get_stats64() might access this ring, we must wait
  4233. * a grace period before freeing it.
  4234. */
  4235. kfree_rcu(ring, rcu);
  4236. adapter->rx_ring[i] = NULL;
  4237. }
  4238. adapter->num_tx_queues = 0;
  4239. adapter->num_rx_queues = 0;
  4240. ixgbe_free_q_vectors(adapter);
  4241. ixgbe_reset_interrupt_capability(adapter);
  4242. }
  4243. /**
  4244. * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
  4245. * @adapter: board private structure to initialize
  4246. *
  4247. * ixgbe_sw_init initializes the Adapter private data structure.
  4248. * Fields are initialized based on PCI device information and
  4249. * OS network device settings (MTU size).
  4250. **/
  4251. static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
  4252. {
  4253. struct ixgbe_hw *hw = &adapter->hw;
  4254. struct pci_dev *pdev = adapter->pdev;
  4255. unsigned int rss;
  4256. #ifdef CONFIG_IXGBE_DCB
  4257. int j;
  4258. struct tc_configuration *tc;
  4259. #endif
  4260. /* PCI config space info */
  4261. hw->vendor_id = pdev->vendor;
  4262. hw->device_id = pdev->device;
  4263. hw->revision_id = pdev->revision;
  4264. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  4265. hw->subsystem_device_id = pdev->subsystem_device;
  4266. /* Set capability flags */
  4267. rss = min(IXGBE_MAX_RSS_INDICES, (int)num_online_cpus());
  4268. adapter->ring_feature[RING_F_RSS].indices = rss;
  4269. adapter->flags |= IXGBE_FLAG_RSS_ENABLED;
  4270. switch (hw->mac.type) {
  4271. case ixgbe_mac_82598EB:
  4272. if (hw->device_id == IXGBE_DEV_ID_82598AT)
  4273. adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
  4274. adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598;
  4275. break;
  4276. case ixgbe_mac_X540:
  4277. adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
  4278. case ixgbe_mac_82599EB:
  4279. adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599;
  4280. adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
  4281. adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
  4282. if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
  4283. adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
  4284. /* Flow Director hash filters enabled */
  4285. adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
  4286. adapter->atr_sample_rate = 20;
  4287. adapter->ring_feature[RING_F_FDIR].indices =
  4288. IXGBE_MAX_FDIR_INDICES;
  4289. adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
  4290. #ifdef IXGBE_FCOE
  4291. adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
  4292. adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
  4293. adapter->ring_feature[RING_F_FCOE].indices = 0;
  4294. #ifdef CONFIG_IXGBE_DCB
  4295. /* Default traffic class to use for FCoE */
  4296. adapter->fcoe.up = IXGBE_FCOE_DEFTC;
  4297. #endif
  4298. #endif /* IXGBE_FCOE */
  4299. break;
  4300. default:
  4301. break;
  4302. }
  4303. /* n-tuple support exists, always init our spinlock */
  4304. spin_lock_init(&adapter->fdir_perfect_lock);
  4305. #ifdef CONFIG_IXGBE_DCB
  4306. switch (hw->mac.type) {
  4307. case ixgbe_mac_X540:
  4308. adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
  4309. adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
  4310. break;
  4311. default:
  4312. adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
  4313. adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
  4314. break;
  4315. }
  4316. /* Configure DCB traffic classes */
  4317. for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
  4318. tc = &adapter->dcb_cfg.tc_config[j];
  4319. tc->path[DCB_TX_CONFIG].bwg_id = 0;
  4320. tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
  4321. tc->path[DCB_RX_CONFIG].bwg_id = 0;
  4322. tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
  4323. tc->dcb_pfc = pfc_disabled;
  4324. }
  4325. /* Initialize default user to priority mapping, UPx->TC0 */
  4326. tc = &adapter->dcb_cfg.tc_config[0];
  4327. tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
  4328. tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
  4329. adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
  4330. adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
  4331. adapter->dcb_cfg.pfc_mode_enable = false;
  4332. adapter->dcb_set_bitmap = 0x00;
  4333. adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
  4334. ixgbe_copy_dcb_cfg(&adapter->dcb_cfg, &adapter->temp_dcb_cfg,
  4335. MAX_TRAFFIC_CLASS);
  4336. #endif
  4337. /* default flow control settings */
  4338. hw->fc.requested_mode = ixgbe_fc_full;
  4339. hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
  4340. #ifdef CONFIG_DCB
  4341. adapter->last_lfc_mode = hw->fc.current_mode;
  4342. #endif
  4343. ixgbe_pbthresh_setup(adapter);
  4344. hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
  4345. hw->fc.send_xon = true;
  4346. hw->fc.disable_fc_autoneg = false;
  4347. /* enable itr by default in dynamic mode */
  4348. adapter->rx_itr_setting = 1;
  4349. adapter->tx_itr_setting = 1;
  4350. /* set defaults for eitr in MegaBytes */
  4351. adapter->eitr_low = 10;
  4352. adapter->eitr_high = 20;
  4353. /* set default ring sizes */
  4354. adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
  4355. adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
  4356. /* set default work limits */
  4357. adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
  4358. /* initialize eeprom parameters */
  4359. if (ixgbe_init_eeprom_params_generic(hw)) {
  4360. e_dev_err("EEPROM initialization failed\n");
  4361. return -EIO;
  4362. }
  4363. /* enable rx csum by default */
  4364. adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
  4365. /* get assigned NUMA node */
  4366. adapter->node = dev_to_node(&pdev->dev);
  4367. set_bit(__IXGBE_DOWN, &adapter->state);
  4368. return 0;
  4369. }
  4370. /**
  4371. * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
  4372. * @tx_ring: tx descriptor ring (for a specific queue) to setup
  4373. *
  4374. * Return 0 on success, negative on failure
  4375. **/
  4376. int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
  4377. {
  4378. struct device *dev = tx_ring->dev;
  4379. int size;
  4380. size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
  4381. tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
  4382. if (!tx_ring->tx_buffer_info)
  4383. tx_ring->tx_buffer_info = vzalloc(size);
  4384. if (!tx_ring->tx_buffer_info)
  4385. goto err;
  4386. /* round up to nearest 4K */
  4387. tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
  4388. tx_ring->size = ALIGN(tx_ring->size, 4096);
  4389. tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
  4390. &tx_ring->dma, GFP_KERNEL);
  4391. if (!tx_ring->desc)
  4392. goto err;
  4393. tx_ring->next_to_use = 0;
  4394. tx_ring->next_to_clean = 0;
  4395. return 0;
  4396. err:
  4397. vfree(tx_ring->tx_buffer_info);
  4398. tx_ring->tx_buffer_info = NULL;
  4399. dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
  4400. return -ENOMEM;
  4401. }
  4402. /**
  4403. * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
  4404. * @adapter: board private structure
  4405. *
  4406. * If this function returns with an error, then it's possible one or
  4407. * more of the rings is populated (while the rest are not). It is the
  4408. * callers duty to clean those orphaned rings.
  4409. *
  4410. * Return 0 on success, negative on failure
  4411. **/
  4412. static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
  4413. {
  4414. int i, err = 0;
  4415. for (i = 0; i < adapter->num_tx_queues; i++) {
  4416. err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
  4417. if (!err)
  4418. continue;
  4419. e_err(probe, "Allocation for Tx Queue %u failed\n", i);
  4420. break;
  4421. }
  4422. return err;
  4423. }
  4424. /**
  4425. * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
  4426. * @rx_ring: rx descriptor ring (for a specific queue) to setup
  4427. *
  4428. * Returns 0 on success, negative on failure
  4429. **/
  4430. int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
  4431. {
  4432. struct device *dev = rx_ring->dev;
  4433. int size;
  4434. size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
  4435. rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
  4436. if (!rx_ring->rx_buffer_info)
  4437. rx_ring->rx_buffer_info = vzalloc(size);
  4438. if (!rx_ring->rx_buffer_info)
  4439. goto err;
  4440. /* Round up to nearest 4K */
  4441. rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
  4442. rx_ring->size = ALIGN(rx_ring->size, 4096);
  4443. rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
  4444. &rx_ring->dma, GFP_KERNEL);
  4445. if (!rx_ring->desc)
  4446. goto err;
  4447. rx_ring->next_to_clean = 0;
  4448. rx_ring->next_to_use = 0;
  4449. return 0;
  4450. err:
  4451. vfree(rx_ring->rx_buffer_info);
  4452. rx_ring->rx_buffer_info = NULL;
  4453. dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
  4454. return -ENOMEM;
  4455. }
  4456. /**
  4457. * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
  4458. * @adapter: board private structure
  4459. *
  4460. * If this function returns with an error, then it's possible one or
  4461. * more of the rings is populated (while the rest are not). It is the
  4462. * callers duty to clean those orphaned rings.
  4463. *
  4464. * Return 0 on success, negative on failure
  4465. **/
  4466. static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
  4467. {
  4468. int i, err = 0;
  4469. for (i = 0; i < adapter->num_rx_queues; i++) {
  4470. err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
  4471. if (!err)
  4472. continue;
  4473. e_err(probe, "Allocation for Rx Queue %u failed\n", i);
  4474. break;
  4475. }
  4476. return err;
  4477. }
  4478. /**
  4479. * ixgbe_free_tx_resources - Free Tx Resources per Queue
  4480. * @tx_ring: Tx descriptor ring for a specific queue
  4481. *
  4482. * Free all transmit software resources
  4483. **/
  4484. void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
  4485. {
  4486. ixgbe_clean_tx_ring(tx_ring);
  4487. vfree(tx_ring->tx_buffer_info);
  4488. tx_ring->tx_buffer_info = NULL;
  4489. /* if not set, then don't free */
  4490. if (!tx_ring->desc)
  4491. return;
  4492. dma_free_coherent(tx_ring->dev, tx_ring->size,
  4493. tx_ring->desc, tx_ring->dma);
  4494. tx_ring->desc = NULL;
  4495. }
  4496. /**
  4497. * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
  4498. * @adapter: board private structure
  4499. *
  4500. * Free all transmit software resources
  4501. **/
  4502. static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
  4503. {
  4504. int i;
  4505. for (i = 0; i < adapter->num_tx_queues; i++)
  4506. if (adapter->tx_ring[i]->desc)
  4507. ixgbe_free_tx_resources(adapter->tx_ring[i]);
  4508. }
  4509. /**
  4510. * ixgbe_free_rx_resources - Free Rx Resources
  4511. * @rx_ring: ring to clean the resources from
  4512. *
  4513. * Free all receive software resources
  4514. **/
  4515. void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
  4516. {
  4517. ixgbe_clean_rx_ring(rx_ring);
  4518. vfree(rx_ring->rx_buffer_info);
  4519. rx_ring->rx_buffer_info = NULL;
  4520. /* if not set, then don't free */
  4521. if (!rx_ring->desc)
  4522. return;
  4523. dma_free_coherent(rx_ring->dev, rx_ring->size,
  4524. rx_ring->desc, rx_ring->dma);
  4525. rx_ring->desc = NULL;
  4526. }
  4527. /**
  4528. * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
  4529. * @adapter: board private structure
  4530. *
  4531. * Free all receive software resources
  4532. **/
  4533. static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
  4534. {
  4535. int i;
  4536. for (i = 0; i < adapter->num_rx_queues; i++)
  4537. if (adapter->rx_ring[i]->desc)
  4538. ixgbe_free_rx_resources(adapter->rx_ring[i]);
  4539. }
  4540. /**
  4541. * ixgbe_change_mtu - Change the Maximum Transfer Unit
  4542. * @netdev: network interface device structure
  4543. * @new_mtu: new value for maximum frame size
  4544. *
  4545. * Returns 0 on success, negative on failure
  4546. **/
  4547. static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
  4548. {
  4549. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  4550. struct ixgbe_hw *hw = &adapter->hw;
  4551. int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
  4552. /* MTU < 68 is an error and causes problems on some kernels */
  4553. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED &&
  4554. hw->mac.type != ixgbe_mac_X540) {
  4555. if ((new_mtu < 68) || (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
  4556. return -EINVAL;
  4557. } else {
  4558. if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
  4559. return -EINVAL;
  4560. }
  4561. e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
  4562. /* must set new MTU before calling down or up */
  4563. netdev->mtu = new_mtu;
  4564. if (netif_running(netdev))
  4565. ixgbe_reinit_locked(adapter);
  4566. return 0;
  4567. }
  4568. /**
  4569. * ixgbe_open - Called when a network interface is made active
  4570. * @netdev: network interface device structure
  4571. *
  4572. * Returns 0 on success, negative value on failure
  4573. *
  4574. * The open entry point is called when a network interface is made
  4575. * active by the system (IFF_UP). At this point all resources needed
  4576. * for transmit and receive operations are allocated, the interrupt
  4577. * handler is registered with the OS, the watchdog timer is started,
  4578. * and the stack is notified that the interface is ready.
  4579. **/
  4580. static int ixgbe_open(struct net_device *netdev)
  4581. {
  4582. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  4583. int err;
  4584. /* disallow open during test */
  4585. if (test_bit(__IXGBE_TESTING, &adapter->state))
  4586. return -EBUSY;
  4587. netif_carrier_off(netdev);
  4588. /* allocate transmit descriptors */
  4589. err = ixgbe_setup_all_tx_resources(adapter);
  4590. if (err)
  4591. goto err_setup_tx;
  4592. /* allocate receive descriptors */
  4593. err = ixgbe_setup_all_rx_resources(adapter);
  4594. if (err)
  4595. goto err_setup_rx;
  4596. ixgbe_configure(adapter);
  4597. err = ixgbe_request_irq(adapter);
  4598. if (err)
  4599. goto err_req_irq;
  4600. ixgbe_up_complete(adapter);
  4601. return 0;
  4602. err_req_irq:
  4603. err_setup_rx:
  4604. ixgbe_free_all_rx_resources(adapter);
  4605. err_setup_tx:
  4606. ixgbe_free_all_tx_resources(adapter);
  4607. ixgbe_reset(adapter);
  4608. return err;
  4609. }
  4610. /**
  4611. * ixgbe_close - Disables a network interface
  4612. * @netdev: network interface device structure
  4613. *
  4614. * Returns 0, this is not allowed to fail
  4615. *
  4616. * The close entry point is called when an interface is de-activated
  4617. * by the OS. The hardware is still under the drivers control, but
  4618. * needs to be disabled. A global MAC reset is issued to stop the
  4619. * hardware, and all transmit and receive resources are freed.
  4620. **/
  4621. static int ixgbe_close(struct net_device *netdev)
  4622. {
  4623. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  4624. ixgbe_down(adapter);
  4625. ixgbe_free_irq(adapter);
  4626. ixgbe_fdir_filter_exit(adapter);
  4627. ixgbe_free_all_tx_resources(adapter);
  4628. ixgbe_free_all_rx_resources(adapter);
  4629. ixgbe_release_hw_control(adapter);
  4630. return 0;
  4631. }
  4632. #ifdef CONFIG_PM
  4633. static int ixgbe_resume(struct pci_dev *pdev)
  4634. {
  4635. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  4636. struct net_device *netdev = adapter->netdev;
  4637. u32 err;
  4638. pci_set_power_state(pdev, PCI_D0);
  4639. pci_restore_state(pdev);
  4640. /*
  4641. * pci_restore_state clears dev->state_saved so call
  4642. * pci_save_state to restore it.
  4643. */
  4644. pci_save_state(pdev);
  4645. err = pci_enable_device_mem(pdev);
  4646. if (err) {
  4647. e_dev_err("Cannot enable PCI device from suspend\n");
  4648. return err;
  4649. }
  4650. pci_set_master(pdev);
  4651. pci_wake_from_d3(pdev, false);
  4652. err = ixgbe_init_interrupt_scheme(adapter);
  4653. if (err) {
  4654. e_dev_err("Cannot initialize interrupts for device\n");
  4655. return err;
  4656. }
  4657. ixgbe_reset(adapter);
  4658. IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
  4659. if (netif_running(netdev)) {
  4660. err = ixgbe_open(netdev);
  4661. if (err)
  4662. return err;
  4663. }
  4664. netif_device_attach(netdev);
  4665. return 0;
  4666. }
  4667. #endif /* CONFIG_PM */
  4668. static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
  4669. {
  4670. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  4671. struct net_device *netdev = adapter->netdev;
  4672. struct ixgbe_hw *hw = &adapter->hw;
  4673. u32 ctrl, fctrl;
  4674. u32 wufc = adapter->wol;
  4675. #ifdef CONFIG_PM
  4676. int retval = 0;
  4677. #endif
  4678. netif_device_detach(netdev);
  4679. if (netif_running(netdev)) {
  4680. ixgbe_down(adapter);
  4681. ixgbe_free_irq(adapter);
  4682. ixgbe_free_all_tx_resources(adapter);
  4683. ixgbe_free_all_rx_resources(adapter);
  4684. }
  4685. ixgbe_clear_interrupt_scheme(adapter);
  4686. #ifdef CONFIG_DCB
  4687. kfree(adapter->ixgbe_ieee_pfc);
  4688. kfree(adapter->ixgbe_ieee_ets);
  4689. #endif
  4690. #ifdef CONFIG_PM
  4691. retval = pci_save_state(pdev);
  4692. if (retval)
  4693. return retval;
  4694. #endif
  4695. if (wufc) {
  4696. ixgbe_set_rx_mode(netdev);
  4697. /* turn on all-multi mode if wake on multicast is enabled */
  4698. if (wufc & IXGBE_WUFC_MC) {
  4699. fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
  4700. fctrl |= IXGBE_FCTRL_MPE;
  4701. IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
  4702. }
  4703. ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
  4704. ctrl |= IXGBE_CTRL_GIO_DIS;
  4705. IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
  4706. IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
  4707. } else {
  4708. IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
  4709. IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
  4710. }
  4711. switch (hw->mac.type) {
  4712. case ixgbe_mac_82598EB:
  4713. pci_wake_from_d3(pdev, false);
  4714. break;
  4715. case ixgbe_mac_82599EB:
  4716. case ixgbe_mac_X540:
  4717. pci_wake_from_d3(pdev, !!wufc);
  4718. break;
  4719. default:
  4720. break;
  4721. }
  4722. *enable_wake = !!wufc;
  4723. ixgbe_release_hw_control(adapter);
  4724. pci_disable_device(pdev);
  4725. return 0;
  4726. }
  4727. #ifdef CONFIG_PM
  4728. static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
  4729. {
  4730. int retval;
  4731. bool wake;
  4732. retval = __ixgbe_shutdown(pdev, &wake);
  4733. if (retval)
  4734. return retval;
  4735. if (wake) {
  4736. pci_prepare_to_sleep(pdev);
  4737. } else {
  4738. pci_wake_from_d3(pdev, false);
  4739. pci_set_power_state(pdev, PCI_D3hot);
  4740. }
  4741. return 0;
  4742. }
  4743. #endif /* CONFIG_PM */
  4744. static void ixgbe_shutdown(struct pci_dev *pdev)
  4745. {
  4746. bool wake;
  4747. __ixgbe_shutdown(pdev, &wake);
  4748. if (system_state == SYSTEM_POWER_OFF) {
  4749. pci_wake_from_d3(pdev, wake);
  4750. pci_set_power_state(pdev, PCI_D3hot);
  4751. }
  4752. }
  4753. /**
  4754. * ixgbe_update_stats - Update the board statistics counters.
  4755. * @adapter: board private structure
  4756. **/
  4757. void ixgbe_update_stats(struct ixgbe_adapter *adapter)
  4758. {
  4759. struct net_device *netdev = adapter->netdev;
  4760. struct ixgbe_hw *hw = &adapter->hw;
  4761. struct ixgbe_hw_stats *hwstats = &adapter->stats;
  4762. u64 total_mpc = 0;
  4763. u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
  4764. u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
  4765. u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
  4766. u64 bytes = 0, packets = 0;
  4767. #ifdef IXGBE_FCOE
  4768. struct ixgbe_fcoe *fcoe = &adapter->fcoe;
  4769. unsigned int cpu;
  4770. u64 fcoe_noddp_counts_sum = 0, fcoe_noddp_ext_buff_counts_sum = 0;
  4771. #endif /* IXGBE_FCOE */
  4772. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  4773. test_bit(__IXGBE_RESETTING, &adapter->state))
  4774. return;
  4775. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
  4776. u64 rsc_count = 0;
  4777. u64 rsc_flush = 0;
  4778. for (i = 0; i < 16; i++)
  4779. adapter->hw_rx_no_dma_resources +=
  4780. IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
  4781. for (i = 0; i < adapter->num_rx_queues; i++) {
  4782. rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
  4783. rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
  4784. }
  4785. adapter->rsc_total_count = rsc_count;
  4786. adapter->rsc_total_flush = rsc_flush;
  4787. }
  4788. for (i = 0; i < adapter->num_rx_queues; i++) {
  4789. struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
  4790. non_eop_descs += rx_ring->rx_stats.non_eop_descs;
  4791. alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
  4792. alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
  4793. bytes += rx_ring->stats.bytes;
  4794. packets += rx_ring->stats.packets;
  4795. }
  4796. adapter->non_eop_descs = non_eop_descs;
  4797. adapter->alloc_rx_page_failed = alloc_rx_page_failed;
  4798. adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
  4799. netdev->stats.rx_bytes = bytes;
  4800. netdev->stats.rx_packets = packets;
  4801. bytes = 0;
  4802. packets = 0;
  4803. /* gather some stats to the adapter struct that are per queue */
  4804. for (i = 0; i < adapter->num_tx_queues; i++) {
  4805. struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
  4806. restart_queue += tx_ring->tx_stats.restart_queue;
  4807. tx_busy += tx_ring->tx_stats.tx_busy;
  4808. bytes += tx_ring->stats.bytes;
  4809. packets += tx_ring->stats.packets;
  4810. }
  4811. adapter->restart_queue = restart_queue;
  4812. adapter->tx_busy = tx_busy;
  4813. netdev->stats.tx_bytes = bytes;
  4814. netdev->stats.tx_packets = packets;
  4815. hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
  4816. /* 8 register reads */
  4817. for (i = 0; i < 8; i++) {
  4818. /* for packet buffers not used, the register should read 0 */
  4819. mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
  4820. missed_rx += mpc;
  4821. hwstats->mpc[i] += mpc;
  4822. total_mpc += hwstats->mpc[i];
  4823. hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
  4824. hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
  4825. switch (hw->mac.type) {
  4826. case ixgbe_mac_82598EB:
  4827. hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
  4828. hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
  4829. hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
  4830. hwstats->pxonrxc[i] +=
  4831. IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
  4832. break;
  4833. case ixgbe_mac_82599EB:
  4834. case ixgbe_mac_X540:
  4835. hwstats->pxonrxc[i] +=
  4836. IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
  4837. break;
  4838. default:
  4839. break;
  4840. }
  4841. }
  4842. /*16 register reads */
  4843. for (i = 0; i < 16; i++) {
  4844. hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
  4845. hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
  4846. if ((hw->mac.type == ixgbe_mac_82599EB) ||
  4847. (hw->mac.type == ixgbe_mac_X540)) {
  4848. hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
  4849. IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
  4850. hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
  4851. IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
  4852. }
  4853. }
  4854. hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
  4855. /* work around hardware counting issue */
  4856. hwstats->gprc -= missed_rx;
  4857. ixgbe_update_xoff_received(adapter);
  4858. /* 82598 hardware only has a 32 bit counter in the high register */
  4859. switch (hw->mac.type) {
  4860. case ixgbe_mac_82598EB:
  4861. hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
  4862. hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
  4863. hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
  4864. hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
  4865. break;
  4866. case ixgbe_mac_X540:
  4867. /* OS2BMC stats are X540 only*/
  4868. hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
  4869. hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
  4870. hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
  4871. hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
  4872. case ixgbe_mac_82599EB:
  4873. hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
  4874. IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
  4875. hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
  4876. IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
  4877. hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
  4878. IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
  4879. hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
  4880. hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
  4881. hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
  4882. #ifdef IXGBE_FCOE
  4883. hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
  4884. hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
  4885. hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
  4886. hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
  4887. hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
  4888. hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
  4889. /* Add up per cpu counters for total ddp aloc fail */
  4890. if (fcoe->pcpu_noddp && fcoe->pcpu_noddp_ext_buff) {
  4891. for_each_possible_cpu(cpu) {
  4892. fcoe_noddp_counts_sum +=
  4893. *per_cpu_ptr(fcoe->pcpu_noddp, cpu);
  4894. fcoe_noddp_ext_buff_counts_sum +=
  4895. *per_cpu_ptr(fcoe->
  4896. pcpu_noddp_ext_buff, cpu);
  4897. }
  4898. }
  4899. hwstats->fcoe_noddp = fcoe_noddp_counts_sum;
  4900. hwstats->fcoe_noddp_ext_buff = fcoe_noddp_ext_buff_counts_sum;
  4901. #endif /* IXGBE_FCOE */
  4902. break;
  4903. default:
  4904. break;
  4905. }
  4906. bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
  4907. hwstats->bprc += bprc;
  4908. hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
  4909. if (hw->mac.type == ixgbe_mac_82598EB)
  4910. hwstats->mprc -= bprc;
  4911. hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
  4912. hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
  4913. hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
  4914. hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
  4915. hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
  4916. hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
  4917. hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
  4918. hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
  4919. lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
  4920. hwstats->lxontxc += lxon;
  4921. lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
  4922. hwstats->lxofftxc += lxoff;
  4923. hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
  4924. hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
  4925. /*
  4926. * 82598 errata - tx of flow control packets is included in tx counters
  4927. */
  4928. xon_off_tot = lxon + lxoff;
  4929. hwstats->gptc -= xon_off_tot;
  4930. hwstats->mptc -= xon_off_tot;
  4931. hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
  4932. hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
  4933. hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
  4934. hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
  4935. hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
  4936. hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
  4937. hwstats->ptc64 -= xon_off_tot;
  4938. hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
  4939. hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
  4940. hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
  4941. hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
  4942. hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
  4943. hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
  4944. /* Fill out the OS statistics structure */
  4945. netdev->stats.multicast = hwstats->mprc;
  4946. /* Rx Errors */
  4947. netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
  4948. netdev->stats.rx_dropped = 0;
  4949. netdev->stats.rx_length_errors = hwstats->rlec;
  4950. netdev->stats.rx_crc_errors = hwstats->crcerrs;
  4951. netdev->stats.rx_missed_errors = total_mpc;
  4952. }
  4953. /**
  4954. * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
  4955. * @adapter - pointer to the device adapter structure
  4956. **/
  4957. static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
  4958. {
  4959. struct ixgbe_hw *hw = &adapter->hw;
  4960. int i;
  4961. if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
  4962. return;
  4963. adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
  4964. /* if interface is down do nothing */
  4965. if (test_bit(__IXGBE_DOWN, &adapter->state))
  4966. return;
  4967. /* do nothing if we are not using signature filters */
  4968. if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
  4969. return;
  4970. adapter->fdir_overflow++;
  4971. if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
  4972. for (i = 0; i < adapter->num_tx_queues; i++)
  4973. set_bit(__IXGBE_TX_FDIR_INIT_DONE,
  4974. &(adapter->tx_ring[i]->state));
  4975. /* re-enable flow director interrupts */
  4976. IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
  4977. } else {
  4978. e_err(probe, "failed to finish FDIR re-initialization, "
  4979. "ignored adding FDIR ATR filters\n");
  4980. }
  4981. }
  4982. /**
  4983. * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
  4984. * @adapter - pointer to the device adapter structure
  4985. *
  4986. * This function serves two purposes. First it strobes the interrupt lines
  4987. * in order to make certain interrupts are occurring. Secondly it sets the
  4988. * bits needed to check for TX hangs. As a result we should immediately
  4989. * determine if a hang has occurred.
  4990. */
  4991. static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
  4992. {
  4993. struct ixgbe_hw *hw = &adapter->hw;
  4994. u64 eics = 0;
  4995. int i;
  4996. /* If we're down or resetting, just bail */
  4997. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  4998. test_bit(__IXGBE_RESETTING, &adapter->state))
  4999. return;
  5000. /* Force detection of hung controller */
  5001. if (netif_carrier_ok(adapter->netdev)) {
  5002. for (i = 0; i < adapter->num_tx_queues; i++)
  5003. set_check_for_tx_hang(adapter->tx_ring[i]);
  5004. }
  5005. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
  5006. /*
  5007. * for legacy and MSI interrupts don't set any bits
  5008. * that are enabled for EIAM, because this operation
  5009. * would set *both* EIMS and EICS for any bit in EIAM
  5010. */
  5011. IXGBE_WRITE_REG(hw, IXGBE_EICS,
  5012. (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
  5013. } else {
  5014. /* get one bit for every active tx/rx interrupt vector */
  5015. for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
  5016. struct ixgbe_q_vector *qv = adapter->q_vector[i];
  5017. if (qv->rx.ring || qv->tx.ring)
  5018. eics |= ((u64)1 << i);
  5019. }
  5020. }
  5021. /* Cause software interrupt to ensure rings are cleaned */
  5022. ixgbe_irq_rearm_queues(adapter, eics);
  5023. }
  5024. /**
  5025. * ixgbe_watchdog_update_link - update the link status
  5026. * @adapter - pointer to the device adapter structure
  5027. * @link_speed - pointer to a u32 to store the link_speed
  5028. **/
  5029. static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
  5030. {
  5031. struct ixgbe_hw *hw = &adapter->hw;
  5032. u32 link_speed = adapter->link_speed;
  5033. bool link_up = adapter->link_up;
  5034. int i;
  5035. if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
  5036. return;
  5037. if (hw->mac.ops.check_link) {
  5038. hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
  5039. } else {
  5040. /* always assume link is up, if no check link function */
  5041. link_speed = IXGBE_LINK_SPEED_10GB_FULL;
  5042. link_up = true;
  5043. }
  5044. if (link_up) {
  5045. if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
  5046. for (i = 0; i < MAX_TRAFFIC_CLASS; i++)
  5047. hw->mac.ops.fc_enable(hw, i);
  5048. } else {
  5049. hw->mac.ops.fc_enable(hw, 0);
  5050. }
  5051. }
  5052. if (link_up ||
  5053. time_after(jiffies, (adapter->link_check_timeout +
  5054. IXGBE_TRY_LINK_TIMEOUT))) {
  5055. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
  5056. IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
  5057. IXGBE_WRITE_FLUSH(hw);
  5058. }
  5059. adapter->link_up = link_up;
  5060. adapter->link_speed = link_speed;
  5061. }
  5062. /**
  5063. * ixgbe_watchdog_link_is_up - update netif_carrier status and
  5064. * print link up message
  5065. * @adapter - pointer to the device adapter structure
  5066. **/
  5067. static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
  5068. {
  5069. struct net_device *netdev = adapter->netdev;
  5070. struct ixgbe_hw *hw = &adapter->hw;
  5071. u32 link_speed = adapter->link_speed;
  5072. bool flow_rx, flow_tx;
  5073. /* only continue if link was previously down */
  5074. if (netif_carrier_ok(netdev))
  5075. return;
  5076. adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
  5077. switch (hw->mac.type) {
  5078. case ixgbe_mac_82598EB: {
  5079. u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
  5080. u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
  5081. flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
  5082. flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
  5083. }
  5084. break;
  5085. case ixgbe_mac_X540:
  5086. case ixgbe_mac_82599EB: {
  5087. u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
  5088. u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
  5089. flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
  5090. flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
  5091. }
  5092. break;
  5093. default:
  5094. flow_tx = false;
  5095. flow_rx = false;
  5096. break;
  5097. }
  5098. e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
  5099. (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
  5100. "10 Gbps" :
  5101. (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
  5102. "1 Gbps" :
  5103. (link_speed == IXGBE_LINK_SPEED_100_FULL ?
  5104. "100 Mbps" :
  5105. "unknown speed"))),
  5106. ((flow_rx && flow_tx) ? "RX/TX" :
  5107. (flow_rx ? "RX" :
  5108. (flow_tx ? "TX" : "None"))));
  5109. netif_carrier_on(netdev);
  5110. ixgbe_check_vf_rate_limit(adapter);
  5111. }
  5112. /**
  5113. * ixgbe_watchdog_link_is_down - update netif_carrier status and
  5114. * print link down message
  5115. * @adapter - pointer to the adapter structure
  5116. **/
  5117. static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter* adapter)
  5118. {
  5119. struct net_device *netdev = adapter->netdev;
  5120. struct ixgbe_hw *hw = &adapter->hw;
  5121. adapter->link_up = false;
  5122. adapter->link_speed = 0;
  5123. /* only continue if link was up previously */
  5124. if (!netif_carrier_ok(netdev))
  5125. return;
  5126. /* poll for SFP+ cable when link is down */
  5127. if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
  5128. adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
  5129. e_info(drv, "NIC Link is Down\n");
  5130. netif_carrier_off(netdev);
  5131. }
  5132. /**
  5133. * ixgbe_watchdog_flush_tx - flush queues on link down
  5134. * @adapter - pointer to the device adapter structure
  5135. **/
  5136. static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
  5137. {
  5138. int i;
  5139. int some_tx_pending = 0;
  5140. if (!netif_carrier_ok(adapter->netdev)) {
  5141. for (i = 0; i < adapter->num_tx_queues; i++) {
  5142. struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
  5143. if (tx_ring->next_to_use != tx_ring->next_to_clean) {
  5144. some_tx_pending = 1;
  5145. break;
  5146. }
  5147. }
  5148. if (some_tx_pending) {
  5149. /* We've lost link, so the controller stops DMA,
  5150. * but we've got queued Tx work that's never going
  5151. * to get done, so reset controller to flush Tx.
  5152. * (Do the reset outside of interrupt context).
  5153. */
  5154. adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
  5155. }
  5156. }
  5157. }
  5158. static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
  5159. {
  5160. u32 ssvpc;
  5161. /* Do not perform spoof check for 82598 */
  5162. if (adapter->hw.mac.type == ixgbe_mac_82598EB)
  5163. return;
  5164. ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
  5165. /*
  5166. * ssvpc register is cleared on read, if zero then no
  5167. * spoofed packets in the last interval.
  5168. */
  5169. if (!ssvpc)
  5170. return;
  5171. e_warn(drv, "%d Spoofed packets detected\n", ssvpc);
  5172. }
  5173. /**
  5174. * ixgbe_watchdog_subtask - check and bring link up
  5175. * @adapter - pointer to the device adapter structure
  5176. **/
  5177. static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
  5178. {
  5179. /* if interface is down do nothing */
  5180. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  5181. test_bit(__IXGBE_RESETTING, &adapter->state))
  5182. return;
  5183. ixgbe_watchdog_update_link(adapter);
  5184. if (adapter->link_up)
  5185. ixgbe_watchdog_link_is_up(adapter);
  5186. else
  5187. ixgbe_watchdog_link_is_down(adapter);
  5188. ixgbe_spoof_check(adapter);
  5189. ixgbe_update_stats(adapter);
  5190. ixgbe_watchdog_flush_tx(adapter);
  5191. }
  5192. /**
  5193. * ixgbe_sfp_detection_subtask - poll for SFP+ cable
  5194. * @adapter - the ixgbe adapter structure
  5195. **/
  5196. static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
  5197. {
  5198. struct ixgbe_hw *hw = &adapter->hw;
  5199. s32 err;
  5200. /* not searching for SFP so there is nothing to do here */
  5201. if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
  5202. !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
  5203. return;
  5204. /* someone else is in init, wait until next service event */
  5205. if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
  5206. return;
  5207. err = hw->phy.ops.identify_sfp(hw);
  5208. if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
  5209. goto sfp_out;
  5210. if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
  5211. /* If no cable is present, then we need to reset
  5212. * the next time we find a good cable. */
  5213. adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
  5214. }
  5215. /* exit on error */
  5216. if (err)
  5217. goto sfp_out;
  5218. /* exit if reset not needed */
  5219. if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
  5220. goto sfp_out;
  5221. adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
  5222. /*
  5223. * A module may be identified correctly, but the EEPROM may not have
  5224. * support for that module. setup_sfp() will fail in that case, so
  5225. * we should not allow that module to load.
  5226. */
  5227. if (hw->mac.type == ixgbe_mac_82598EB)
  5228. err = hw->phy.ops.reset(hw);
  5229. else
  5230. err = hw->mac.ops.setup_sfp(hw);
  5231. if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
  5232. goto sfp_out;
  5233. adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
  5234. e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
  5235. sfp_out:
  5236. clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
  5237. if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
  5238. (adapter->netdev->reg_state == NETREG_REGISTERED)) {
  5239. e_dev_err("failed to initialize because an unsupported "
  5240. "SFP+ module type was detected.\n");
  5241. e_dev_err("Reload the driver after installing a "
  5242. "supported module.\n");
  5243. unregister_netdev(adapter->netdev);
  5244. }
  5245. }
  5246. /**
  5247. * ixgbe_sfp_link_config_subtask - set up link SFP after module install
  5248. * @adapter - the ixgbe adapter structure
  5249. **/
  5250. static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
  5251. {
  5252. struct ixgbe_hw *hw = &adapter->hw;
  5253. u32 autoneg;
  5254. bool negotiation;
  5255. if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
  5256. return;
  5257. /* someone else is in init, wait until next service event */
  5258. if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
  5259. return;
  5260. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
  5261. autoneg = hw->phy.autoneg_advertised;
  5262. if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
  5263. hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
  5264. if (hw->mac.ops.setup_link)
  5265. hw->mac.ops.setup_link(hw, autoneg, negotiation, true);
  5266. adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
  5267. adapter->link_check_timeout = jiffies;
  5268. clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
  5269. }
  5270. #ifdef CONFIG_PCI_IOV
  5271. static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
  5272. {
  5273. int vf;
  5274. struct ixgbe_hw *hw = &adapter->hw;
  5275. struct net_device *netdev = adapter->netdev;
  5276. u32 gpc;
  5277. u32 ciaa, ciad;
  5278. gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
  5279. if (gpc) /* If incrementing then no need for the check below */
  5280. return;
  5281. /*
  5282. * Check to see if a bad DMA write target from an errant or
  5283. * malicious VF has caused a PCIe error. If so then we can
  5284. * issue a VFLR to the offending VF(s) and then resume without
  5285. * requesting a full slot reset.
  5286. */
  5287. for (vf = 0; vf < adapter->num_vfs; vf++) {
  5288. ciaa = (vf << 16) | 0x80000000;
  5289. /* 32 bit read so align, we really want status at offset 6 */
  5290. ciaa |= PCI_COMMAND;
  5291. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  5292. ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599);
  5293. ciaa &= 0x7FFFFFFF;
  5294. /* disable debug mode asap after reading data */
  5295. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  5296. /* Get the upper 16 bits which will be the PCI status reg */
  5297. ciad >>= 16;
  5298. if (ciad & PCI_STATUS_REC_MASTER_ABORT) {
  5299. netdev_err(netdev, "VF %d Hung DMA\n", vf);
  5300. /* Issue VFLR */
  5301. ciaa = (vf << 16) | 0x80000000;
  5302. ciaa |= 0xA8;
  5303. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  5304. ciad = 0x00008000; /* VFLR */
  5305. IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad);
  5306. ciaa &= 0x7FFFFFFF;
  5307. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  5308. }
  5309. }
  5310. }
  5311. #endif
  5312. /**
  5313. * ixgbe_service_timer - Timer Call-back
  5314. * @data: pointer to adapter cast into an unsigned long
  5315. **/
  5316. static void ixgbe_service_timer(unsigned long data)
  5317. {
  5318. struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
  5319. unsigned long next_event_offset;
  5320. bool ready = true;
  5321. #ifdef CONFIG_PCI_IOV
  5322. ready = false;
  5323. /*
  5324. * don't bother with SR-IOV VF DMA hang check if there are
  5325. * no VFs or the link is down
  5326. */
  5327. if (!adapter->num_vfs ||
  5328. (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) {
  5329. ready = true;
  5330. goto normal_timer_service;
  5331. }
  5332. /* If we have VFs allocated then we must check for DMA hangs */
  5333. ixgbe_check_for_bad_vf(adapter);
  5334. next_event_offset = HZ / 50;
  5335. adapter->timer_event_accumulator++;
  5336. if (adapter->timer_event_accumulator >= 100) {
  5337. ready = true;
  5338. adapter->timer_event_accumulator = 0;
  5339. }
  5340. goto schedule_event;
  5341. normal_timer_service:
  5342. #endif
  5343. /* poll faster when waiting for link */
  5344. if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
  5345. next_event_offset = HZ / 10;
  5346. else
  5347. next_event_offset = HZ * 2;
  5348. #ifdef CONFIG_PCI_IOV
  5349. schedule_event:
  5350. #endif
  5351. /* Reset the timer */
  5352. mod_timer(&adapter->service_timer, next_event_offset + jiffies);
  5353. if (ready)
  5354. ixgbe_service_event_schedule(adapter);
  5355. }
  5356. static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
  5357. {
  5358. if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
  5359. return;
  5360. adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
  5361. /* If we're already down or resetting, just bail */
  5362. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  5363. test_bit(__IXGBE_RESETTING, &adapter->state))
  5364. return;
  5365. ixgbe_dump(adapter);
  5366. netdev_err(adapter->netdev, "Reset adapter\n");
  5367. adapter->tx_timeout_count++;
  5368. ixgbe_reinit_locked(adapter);
  5369. }
  5370. /**
  5371. * ixgbe_service_task - manages and runs subtasks
  5372. * @work: pointer to work_struct containing our data
  5373. **/
  5374. static void ixgbe_service_task(struct work_struct *work)
  5375. {
  5376. struct ixgbe_adapter *adapter = container_of(work,
  5377. struct ixgbe_adapter,
  5378. service_task);
  5379. ixgbe_reset_subtask(adapter);
  5380. ixgbe_sfp_detection_subtask(adapter);
  5381. ixgbe_sfp_link_config_subtask(adapter);
  5382. ixgbe_check_overtemp_subtask(adapter);
  5383. ixgbe_watchdog_subtask(adapter);
  5384. ixgbe_fdir_reinit_subtask(adapter);
  5385. ixgbe_check_hang_subtask(adapter);
  5386. ixgbe_service_event_complete(adapter);
  5387. }
  5388. void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens,
  5389. u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
  5390. {
  5391. struct ixgbe_adv_tx_context_desc *context_desc;
  5392. u16 i = tx_ring->next_to_use;
  5393. context_desc = IXGBE_TX_CTXTDESC_ADV(tx_ring, i);
  5394. i++;
  5395. tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
  5396. /* set bits to identify this as an advanced context descriptor */
  5397. type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
  5398. context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
  5399. context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
  5400. context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
  5401. context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
  5402. }
  5403. static int ixgbe_tso(struct ixgbe_ring *tx_ring, struct sk_buff *skb,
  5404. u32 tx_flags, __be16 protocol, u8 *hdr_len)
  5405. {
  5406. int err;
  5407. u32 vlan_macip_lens, type_tucmd;
  5408. u32 mss_l4len_idx, l4len;
  5409. if (!skb_is_gso(skb))
  5410. return 0;
  5411. if (skb_header_cloned(skb)) {
  5412. err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  5413. if (err)
  5414. return err;
  5415. }
  5416. /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
  5417. type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
  5418. if (protocol == __constant_htons(ETH_P_IP)) {
  5419. struct iphdr *iph = ip_hdr(skb);
  5420. iph->tot_len = 0;
  5421. iph->check = 0;
  5422. tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
  5423. iph->daddr, 0,
  5424. IPPROTO_TCP,
  5425. 0);
  5426. type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
  5427. } else if (skb_is_gso_v6(skb)) {
  5428. ipv6_hdr(skb)->payload_len = 0;
  5429. tcp_hdr(skb)->check =
  5430. ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
  5431. &ipv6_hdr(skb)->daddr,
  5432. 0, IPPROTO_TCP, 0);
  5433. }
  5434. l4len = tcp_hdrlen(skb);
  5435. *hdr_len = skb_transport_offset(skb) + l4len;
  5436. /* mss_l4len_id: use 1 as index for TSO */
  5437. mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
  5438. mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
  5439. mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
  5440. /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
  5441. vlan_macip_lens = skb_network_header_len(skb);
  5442. vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
  5443. vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
  5444. ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
  5445. mss_l4len_idx);
  5446. return 1;
  5447. }
  5448. static bool ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
  5449. struct sk_buff *skb, u32 tx_flags,
  5450. __be16 protocol)
  5451. {
  5452. u32 vlan_macip_lens = 0;
  5453. u32 mss_l4len_idx = 0;
  5454. u32 type_tucmd = 0;
  5455. if (skb->ip_summed != CHECKSUM_PARTIAL) {
  5456. if (!(tx_flags & IXGBE_TX_FLAGS_HW_VLAN) &&
  5457. !(tx_flags & IXGBE_TX_FLAGS_TXSW))
  5458. return false;
  5459. } else {
  5460. u8 l4_hdr = 0;
  5461. switch (protocol) {
  5462. case __constant_htons(ETH_P_IP):
  5463. vlan_macip_lens |= skb_network_header_len(skb);
  5464. type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
  5465. l4_hdr = ip_hdr(skb)->protocol;
  5466. break;
  5467. case __constant_htons(ETH_P_IPV6):
  5468. vlan_macip_lens |= skb_network_header_len(skb);
  5469. l4_hdr = ipv6_hdr(skb)->nexthdr;
  5470. break;
  5471. default:
  5472. if (unlikely(net_ratelimit())) {
  5473. dev_warn(tx_ring->dev,
  5474. "partial checksum but proto=%x!\n",
  5475. skb->protocol);
  5476. }
  5477. break;
  5478. }
  5479. switch (l4_hdr) {
  5480. case IPPROTO_TCP:
  5481. type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
  5482. mss_l4len_idx = tcp_hdrlen(skb) <<
  5483. IXGBE_ADVTXD_L4LEN_SHIFT;
  5484. break;
  5485. case IPPROTO_SCTP:
  5486. type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
  5487. mss_l4len_idx = sizeof(struct sctphdr) <<
  5488. IXGBE_ADVTXD_L4LEN_SHIFT;
  5489. break;
  5490. case IPPROTO_UDP:
  5491. mss_l4len_idx = sizeof(struct udphdr) <<
  5492. IXGBE_ADVTXD_L4LEN_SHIFT;
  5493. break;
  5494. default:
  5495. if (unlikely(net_ratelimit())) {
  5496. dev_warn(tx_ring->dev,
  5497. "partial checksum but l4 proto=%x!\n",
  5498. skb->protocol);
  5499. }
  5500. break;
  5501. }
  5502. }
  5503. vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
  5504. vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
  5505. ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0,
  5506. type_tucmd, mss_l4len_idx);
  5507. return (skb->ip_summed == CHECKSUM_PARTIAL);
  5508. }
  5509. static __le32 ixgbe_tx_cmd_type(u32 tx_flags)
  5510. {
  5511. /* set type for advanced descriptor with frame checksum insertion */
  5512. __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
  5513. IXGBE_ADVTXD_DCMD_IFCS |
  5514. IXGBE_ADVTXD_DCMD_DEXT);
  5515. /* set HW vlan bit if vlan is present */
  5516. if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN)
  5517. cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
  5518. /* set segmentation enable bits for TSO/FSO */
  5519. #ifdef IXGBE_FCOE
  5520. if ((tx_flags & IXGBE_TX_FLAGS_TSO) || (tx_flags & IXGBE_TX_FLAGS_FSO))
  5521. #else
  5522. if (tx_flags & IXGBE_TX_FLAGS_TSO)
  5523. #endif
  5524. cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
  5525. return cmd_type;
  5526. }
  5527. static __le32 ixgbe_tx_olinfo_status(u32 tx_flags, unsigned int paylen)
  5528. {
  5529. __le32 olinfo_status =
  5530. cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
  5531. if (tx_flags & IXGBE_TX_FLAGS_TSO) {
  5532. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM |
  5533. (1 << IXGBE_ADVTXD_IDX_SHIFT));
  5534. /* enble IPv4 checksum for TSO */
  5535. if (tx_flags & IXGBE_TX_FLAGS_IPV4)
  5536. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
  5537. }
  5538. /* enable L4 checksum for TSO and TX checksum offload */
  5539. if (tx_flags & IXGBE_TX_FLAGS_CSUM)
  5540. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
  5541. #ifdef IXGBE_FCOE
  5542. /* use index 1 context for FCOE/FSO */
  5543. if (tx_flags & IXGBE_TX_FLAGS_FCOE)
  5544. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC |
  5545. (1 << IXGBE_ADVTXD_IDX_SHIFT));
  5546. #endif
  5547. /*
  5548. * Check Context must be set if Tx switch is enabled, which it
  5549. * always is for case where virtual functions are running
  5550. */
  5551. if (tx_flags & IXGBE_TX_FLAGS_TXSW)
  5552. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
  5553. return olinfo_status;
  5554. }
  5555. #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \
  5556. IXGBE_TXD_CMD_RS)
  5557. static void ixgbe_tx_map(struct ixgbe_ring *tx_ring,
  5558. struct sk_buff *skb,
  5559. struct ixgbe_tx_buffer *first,
  5560. u32 tx_flags,
  5561. const u8 hdr_len)
  5562. {
  5563. struct device *dev = tx_ring->dev;
  5564. struct ixgbe_tx_buffer *tx_buffer_info;
  5565. union ixgbe_adv_tx_desc *tx_desc;
  5566. dma_addr_t dma;
  5567. __le32 cmd_type, olinfo_status;
  5568. struct skb_frag_struct *frag;
  5569. unsigned int f = 0;
  5570. unsigned int data_len = skb->data_len;
  5571. unsigned int size = skb_headlen(skb);
  5572. u32 offset = 0;
  5573. u32 paylen = skb->len - hdr_len;
  5574. u16 i = tx_ring->next_to_use;
  5575. u16 gso_segs;
  5576. #ifdef IXGBE_FCOE
  5577. if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
  5578. if (data_len >= sizeof(struct fcoe_crc_eof)) {
  5579. data_len -= sizeof(struct fcoe_crc_eof);
  5580. } else {
  5581. size -= sizeof(struct fcoe_crc_eof) - data_len;
  5582. data_len = 0;
  5583. }
  5584. }
  5585. #endif
  5586. dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
  5587. if (dma_mapping_error(dev, dma))
  5588. goto dma_error;
  5589. cmd_type = ixgbe_tx_cmd_type(tx_flags);
  5590. olinfo_status = ixgbe_tx_olinfo_status(tx_flags, paylen);
  5591. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
  5592. for (;;) {
  5593. while (size > IXGBE_MAX_DATA_PER_TXD) {
  5594. tx_desc->read.buffer_addr = cpu_to_le64(dma + offset);
  5595. tx_desc->read.cmd_type_len =
  5596. cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
  5597. tx_desc->read.olinfo_status = olinfo_status;
  5598. offset += IXGBE_MAX_DATA_PER_TXD;
  5599. size -= IXGBE_MAX_DATA_PER_TXD;
  5600. tx_desc++;
  5601. i++;
  5602. if (i == tx_ring->count) {
  5603. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
  5604. i = 0;
  5605. }
  5606. }
  5607. tx_buffer_info = &tx_ring->tx_buffer_info[i];
  5608. tx_buffer_info->length = offset + size;
  5609. tx_buffer_info->tx_flags = tx_flags;
  5610. tx_buffer_info->dma = dma;
  5611. tx_desc->read.buffer_addr = cpu_to_le64(dma + offset);
  5612. tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
  5613. tx_desc->read.olinfo_status = olinfo_status;
  5614. if (!data_len)
  5615. break;
  5616. frag = &skb_shinfo(skb)->frags[f];
  5617. #ifdef IXGBE_FCOE
  5618. size = min_t(unsigned int, data_len, skb_frag_size(frag));
  5619. #else
  5620. size = skb_frag_size(frag);
  5621. #endif
  5622. data_len -= size;
  5623. f++;
  5624. offset = 0;
  5625. tx_flags |= IXGBE_TX_FLAGS_MAPPED_AS_PAGE;
  5626. dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
  5627. if (dma_mapping_error(dev, dma))
  5628. goto dma_error;
  5629. tx_desc++;
  5630. i++;
  5631. if (i == tx_ring->count) {
  5632. tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
  5633. i = 0;
  5634. }
  5635. }
  5636. tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);
  5637. i++;
  5638. if (i == tx_ring->count)
  5639. i = 0;
  5640. tx_ring->next_to_use = i;
  5641. if (tx_flags & IXGBE_TX_FLAGS_TSO)
  5642. gso_segs = skb_shinfo(skb)->gso_segs;
  5643. #ifdef IXGBE_FCOE
  5644. /* adjust for FCoE Sequence Offload */
  5645. else if (tx_flags & IXGBE_TX_FLAGS_FSO)
  5646. gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
  5647. skb_shinfo(skb)->gso_size);
  5648. #endif /* IXGBE_FCOE */
  5649. else
  5650. gso_segs = 1;
  5651. /* multiply data chunks by size of headers */
  5652. tx_buffer_info->bytecount = paylen + (gso_segs * hdr_len);
  5653. tx_buffer_info->gso_segs = gso_segs;
  5654. tx_buffer_info->skb = skb;
  5655. /* set the timestamp */
  5656. first->time_stamp = jiffies;
  5657. /*
  5658. * Force memory writes to complete before letting h/w
  5659. * know there are new descriptors to fetch. (Only
  5660. * applicable for weak-ordered memory model archs,
  5661. * such as IA-64).
  5662. */
  5663. wmb();
  5664. /* set next_to_watch value indicating a packet is present */
  5665. first->next_to_watch = tx_desc;
  5666. /* notify HW of packet */
  5667. writel(i, tx_ring->tail);
  5668. return;
  5669. dma_error:
  5670. dev_err(dev, "TX DMA map failed\n");
  5671. /* clear dma mappings for failed tx_buffer_info map */
  5672. for (;;) {
  5673. tx_buffer_info = &tx_ring->tx_buffer_info[i];
  5674. ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
  5675. if (tx_buffer_info == first)
  5676. break;
  5677. if (i == 0)
  5678. i = tx_ring->count;
  5679. i--;
  5680. }
  5681. dev_kfree_skb_any(skb);
  5682. tx_ring->next_to_use = i;
  5683. }
  5684. static void ixgbe_atr(struct ixgbe_ring *ring, struct sk_buff *skb,
  5685. u32 tx_flags, __be16 protocol)
  5686. {
  5687. struct ixgbe_q_vector *q_vector = ring->q_vector;
  5688. union ixgbe_atr_hash_dword input = { .dword = 0 };
  5689. union ixgbe_atr_hash_dword common = { .dword = 0 };
  5690. union {
  5691. unsigned char *network;
  5692. struct iphdr *ipv4;
  5693. struct ipv6hdr *ipv6;
  5694. } hdr;
  5695. struct tcphdr *th;
  5696. __be16 vlan_id;
  5697. /* if ring doesn't have a interrupt vector, cannot perform ATR */
  5698. if (!q_vector)
  5699. return;
  5700. /* do nothing if sampling is disabled */
  5701. if (!ring->atr_sample_rate)
  5702. return;
  5703. ring->atr_count++;
  5704. /* snag network header to get L4 type and address */
  5705. hdr.network = skb_network_header(skb);
  5706. /* Currently only IPv4/IPv6 with TCP is supported */
  5707. if ((protocol != __constant_htons(ETH_P_IPV6) ||
  5708. hdr.ipv6->nexthdr != IPPROTO_TCP) &&
  5709. (protocol != __constant_htons(ETH_P_IP) ||
  5710. hdr.ipv4->protocol != IPPROTO_TCP))
  5711. return;
  5712. th = tcp_hdr(skb);
  5713. /* skip this packet since it is invalid or the socket is closing */
  5714. if (!th || th->fin)
  5715. return;
  5716. /* sample on all syn packets or once every atr sample count */
  5717. if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
  5718. return;
  5719. /* reset sample count */
  5720. ring->atr_count = 0;
  5721. vlan_id = htons(tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
  5722. /*
  5723. * src and dst are inverted, think how the receiver sees them
  5724. *
  5725. * The input is broken into two sections, a non-compressed section
  5726. * containing vm_pool, vlan_id, and flow_type. The rest of the data
  5727. * is XORed together and stored in the compressed dword.
  5728. */
  5729. input.formatted.vlan_id = vlan_id;
  5730. /*
  5731. * since src port and flex bytes occupy the same word XOR them together
  5732. * and write the value to source port portion of compressed dword
  5733. */
  5734. if (tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
  5735. common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q);
  5736. else
  5737. common.port.src ^= th->dest ^ protocol;
  5738. common.port.dst ^= th->source;
  5739. if (protocol == __constant_htons(ETH_P_IP)) {
  5740. input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
  5741. common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
  5742. } else {
  5743. input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
  5744. common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
  5745. hdr.ipv6->saddr.s6_addr32[1] ^
  5746. hdr.ipv6->saddr.s6_addr32[2] ^
  5747. hdr.ipv6->saddr.s6_addr32[3] ^
  5748. hdr.ipv6->daddr.s6_addr32[0] ^
  5749. hdr.ipv6->daddr.s6_addr32[1] ^
  5750. hdr.ipv6->daddr.s6_addr32[2] ^
  5751. hdr.ipv6->daddr.s6_addr32[3];
  5752. }
  5753. /* This assumes the Rx queue and Tx queue are bound to the same CPU */
  5754. ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
  5755. input, common, ring->queue_index);
  5756. }
  5757. static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
  5758. {
  5759. netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
  5760. /* Herbert's original patch had:
  5761. * smp_mb__after_netif_stop_queue();
  5762. * but since that doesn't exist yet, just open code it. */
  5763. smp_mb();
  5764. /* We need to check again in a case another CPU has just
  5765. * made room available. */
  5766. if (likely(ixgbe_desc_unused(tx_ring) < size))
  5767. return -EBUSY;
  5768. /* A reprieve! - use start_queue because it doesn't call schedule */
  5769. netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
  5770. ++tx_ring->tx_stats.restart_queue;
  5771. return 0;
  5772. }
  5773. static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
  5774. {
  5775. if (likely(ixgbe_desc_unused(tx_ring) >= size))
  5776. return 0;
  5777. return __ixgbe_maybe_stop_tx(tx_ring, size);
  5778. }
  5779. static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb)
  5780. {
  5781. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5782. int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
  5783. smp_processor_id();
  5784. #ifdef IXGBE_FCOE
  5785. __be16 protocol = vlan_get_protocol(skb);
  5786. if (((protocol == htons(ETH_P_FCOE)) ||
  5787. (protocol == htons(ETH_P_FIP))) &&
  5788. (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
  5789. txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1);
  5790. txq += adapter->ring_feature[RING_F_FCOE].mask;
  5791. return txq;
  5792. }
  5793. #endif
  5794. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  5795. while (unlikely(txq >= dev->real_num_tx_queues))
  5796. txq -= dev->real_num_tx_queues;
  5797. return txq;
  5798. }
  5799. return skb_tx_hash(dev, skb);
  5800. }
  5801. netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
  5802. struct ixgbe_adapter *adapter,
  5803. struct ixgbe_ring *tx_ring)
  5804. {
  5805. struct ixgbe_tx_buffer *first;
  5806. int tso;
  5807. u32 tx_flags = 0;
  5808. #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
  5809. unsigned short f;
  5810. #endif
  5811. u16 count = TXD_USE_COUNT(skb_headlen(skb));
  5812. __be16 protocol = skb->protocol;
  5813. u8 hdr_len = 0;
  5814. /*
  5815. * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
  5816. * + 1 desc for skb_head_len/IXGBE_MAX_DATA_PER_TXD,
  5817. * + 2 desc gap to keep tail from touching head,
  5818. * + 1 desc for context descriptor,
  5819. * otherwise try next time
  5820. */
  5821. #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
  5822. for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
  5823. count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
  5824. #else
  5825. count += skb_shinfo(skb)->nr_frags;
  5826. #endif
  5827. if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
  5828. tx_ring->tx_stats.tx_busy++;
  5829. return NETDEV_TX_BUSY;
  5830. }
  5831. #ifdef CONFIG_PCI_IOV
  5832. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  5833. tx_flags |= IXGBE_TX_FLAGS_TXSW;
  5834. #endif
  5835. /* if we have a HW VLAN tag being added default to the HW one */
  5836. if (vlan_tx_tag_present(skb)) {
  5837. tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
  5838. tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
  5839. /* else if it is a SW VLAN check the next protocol and store the tag */
  5840. } else if (protocol == __constant_htons(ETH_P_8021Q)) {
  5841. struct vlan_hdr *vhdr, _vhdr;
  5842. vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
  5843. if (!vhdr)
  5844. goto out_drop;
  5845. protocol = vhdr->h_vlan_encapsulated_proto;
  5846. tx_flags |= ntohs(vhdr->h_vlan_TCI) << IXGBE_TX_FLAGS_VLAN_SHIFT;
  5847. tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
  5848. }
  5849. /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
  5850. if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
  5851. ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
  5852. (skb->priority != TC_PRIO_CONTROL))) {
  5853. tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
  5854. tx_flags |= (skb->priority & 0x7) <<
  5855. IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
  5856. if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
  5857. struct vlan_ethhdr *vhdr;
  5858. if (skb_header_cloned(skb) &&
  5859. pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
  5860. goto out_drop;
  5861. vhdr = (struct vlan_ethhdr *)skb->data;
  5862. vhdr->h_vlan_TCI = htons(tx_flags >>
  5863. IXGBE_TX_FLAGS_VLAN_SHIFT);
  5864. } else {
  5865. tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
  5866. }
  5867. }
  5868. /* record the location of the first descriptor for this packet */
  5869. first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
  5870. #ifdef IXGBE_FCOE
  5871. /* setup tx offload for FCoE */
  5872. if ((protocol == __constant_htons(ETH_P_FCOE)) &&
  5873. (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
  5874. tso = ixgbe_fso(tx_ring, skb, tx_flags, &hdr_len);
  5875. if (tso < 0)
  5876. goto out_drop;
  5877. else if (tso)
  5878. tx_flags |= IXGBE_TX_FLAGS_FSO |
  5879. IXGBE_TX_FLAGS_FCOE;
  5880. else
  5881. tx_flags |= IXGBE_TX_FLAGS_FCOE;
  5882. goto xmit_fcoe;
  5883. }
  5884. #endif /* IXGBE_FCOE */
  5885. /* setup IPv4/IPv6 offloads */
  5886. if (protocol == __constant_htons(ETH_P_IP))
  5887. tx_flags |= IXGBE_TX_FLAGS_IPV4;
  5888. tso = ixgbe_tso(tx_ring, skb, tx_flags, protocol, &hdr_len);
  5889. if (tso < 0)
  5890. goto out_drop;
  5891. else if (tso)
  5892. tx_flags |= IXGBE_TX_FLAGS_TSO;
  5893. else if (ixgbe_tx_csum(tx_ring, skb, tx_flags, protocol))
  5894. tx_flags |= IXGBE_TX_FLAGS_CSUM;
  5895. /* add the ATR filter if ATR is on */
  5896. if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
  5897. ixgbe_atr(tx_ring, skb, tx_flags, protocol);
  5898. #ifdef IXGBE_FCOE
  5899. xmit_fcoe:
  5900. #endif /* IXGBE_FCOE */
  5901. ixgbe_tx_map(tx_ring, skb, first, tx_flags, hdr_len);
  5902. ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
  5903. return NETDEV_TX_OK;
  5904. out_drop:
  5905. dev_kfree_skb_any(skb);
  5906. return NETDEV_TX_OK;
  5907. }
  5908. static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
  5909. {
  5910. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5911. struct ixgbe_ring *tx_ring;
  5912. tx_ring = adapter->tx_ring[skb->queue_mapping];
  5913. return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
  5914. }
  5915. /**
  5916. * ixgbe_set_mac - Change the Ethernet Address of the NIC
  5917. * @netdev: network interface device structure
  5918. * @p: pointer to an address structure
  5919. *
  5920. * Returns 0 on success, negative on failure
  5921. **/
  5922. static int ixgbe_set_mac(struct net_device *netdev, void *p)
  5923. {
  5924. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5925. struct ixgbe_hw *hw = &adapter->hw;
  5926. struct sockaddr *addr = p;
  5927. if (!is_valid_ether_addr(addr->sa_data))
  5928. return -EADDRNOTAVAIL;
  5929. memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
  5930. memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
  5931. hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
  5932. IXGBE_RAH_AV);
  5933. return 0;
  5934. }
  5935. static int
  5936. ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
  5937. {
  5938. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5939. struct ixgbe_hw *hw = &adapter->hw;
  5940. u16 value;
  5941. int rc;
  5942. if (prtad != hw->phy.mdio.prtad)
  5943. return -EINVAL;
  5944. rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
  5945. if (!rc)
  5946. rc = value;
  5947. return rc;
  5948. }
  5949. static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
  5950. u16 addr, u16 value)
  5951. {
  5952. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5953. struct ixgbe_hw *hw = &adapter->hw;
  5954. if (prtad != hw->phy.mdio.prtad)
  5955. return -EINVAL;
  5956. return hw->phy.ops.write_reg(hw, addr, devad, value);
  5957. }
  5958. static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
  5959. {
  5960. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5961. return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
  5962. }
  5963. /**
  5964. * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
  5965. * netdev->dev_addrs
  5966. * @netdev: network interface device structure
  5967. *
  5968. * Returns non-zero on failure
  5969. **/
  5970. static int ixgbe_add_sanmac_netdev(struct net_device *dev)
  5971. {
  5972. int err = 0;
  5973. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5974. struct ixgbe_mac_info *mac = &adapter->hw.mac;
  5975. if (is_valid_ether_addr(mac->san_addr)) {
  5976. rtnl_lock();
  5977. err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
  5978. rtnl_unlock();
  5979. }
  5980. return err;
  5981. }
  5982. /**
  5983. * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
  5984. * netdev->dev_addrs
  5985. * @netdev: network interface device structure
  5986. *
  5987. * Returns non-zero on failure
  5988. **/
  5989. static int ixgbe_del_sanmac_netdev(struct net_device *dev)
  5990. {
  5991. int err = 0;
  5992. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5993. struct ixgbe_mac_info *mac = &adapter->hw.mac;
  5994. if (is_valid_ether_addr(mac->san_addr)) {
  5995. rtnl_lock();
  5996. err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
  5997. rtnl_unlock();
  5998. }
  5999. return err;
  6000. }
  6001. #ifdef CONFIG_NET_POLL_CONTROLLER
  6002. /*
  6003. * Polling 'interrupt' - used by things like netconsole to send skbs
  6004. * without having to re-enable interrupts. It's not called while
  6005. * the interrupt routine is executing.
  6006. */
  6007. static void ixgbe_netpoll(struct net_device *netdev)
  6008. {
  6009. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  6010. int i;
  6011. /* if interface is down do nothing */
  6012. if (test_bit(__IXGBE_DOWN, &adapter->state))
  6013. return;
  6014. adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
  6015. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  6016. int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
  6017. for (i = 0; i < num_q_vectors; i++) {
  6018. struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
  6019. ixgbe_msix_clean_rings(0, q_vector);
  6020. }
  6021. } else {
  6022. ixgbe_intr(adapter->pdev->irq, netdev);
  6023. }
  6024. adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
  6025. }
  6026. #endif
  6027. static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
  6028. struct rtnl_link_stats64 *stats)
  6029. {
  6030. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  6031. int i;
  6032. rcu_read_lock();
  6033. for (i = 0; i < adapter->num_rx_queues; i++) {
  6034. struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
  6035. u64 bytes, packets;
  6036. unsigned int start;
  6037. if (ring) {
  6038. do {
  6039. start = u64_stats_fetch_begin_bh(&ring->syncp);
  6040. packets = ring->stats.packets;
  6041. bytes = ring->stats.bytes;
  6042. } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
  6043. stats->rx_packets += packets;
  6044. stats->rx_bytes += bytes;
  6045. }
  6046. }
  6047. for (i = 0; i < adapter->num_tx_queues; i++) {
  6048. struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
  6049. u64 bytes, packets;
  6050. unsigned int start;
  6051. if (ring) {
  6052. do {
  6053. start = u64_stats_fetch_begin_bh(&ring->syncp);
  6054. packets = ring->stats.packets;
  6055. bytes = ring->stats.bytes;
  6056. } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
  6057. stats->tx_packets += packets;
  6058. stats->tx_bytes += bytes;
  6059. }
  6060. }
  6061. rcu_read_unlock();
  6062. /* following stats updated by ixgbe_watchdog_task() */
  6063. stats->multicast = netdev->stats.multicast;
  6064. stats->rx_errors = netdev->stats.rx_errors;
  6065. stats->rx_length_errors = netdev->stats.rx_length_errors;
  6066. stats->rx_crc_errors = netdev->stats.rx_crc_errors;
  6067. stats->rx_missed_errors = netdev->stats.rx_missed_errors;
  6068. return stats;
  6069. }
  6070. /* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
  6071. * #adapter: pointer to ixgbe_adapter
  6072. * @tc: number of traffic classes currently enabled
  6073. *
  6074. * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
  6075. * 802.1Q priority maps to a packet buffer that exists.
  6076. */
  6077. static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
  6078. {
  6079. struct ixgbe_hw *hw = &adapter->hw;
  6080. u32 reg, rsave;
  6081. int i;
  6082. /* 82598 have a static priority to TC mapping that can not
  6083. * be changed so no validation is needed.
  6084. */
  6085. if (hw->mac.type == ixgbe_mac_82598EB)
  6086. return;
  6087. reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
  6088. rsave = reg;
  6089. for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
  6090. u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
  6091. /* If up2tc is out of bounds default to zero */
  6092. if (up2tc > tc)
  6093. reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
  6094. }
  6095. if (reg != rsave)
  6096. IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
  6097. return;
  6098. }
  6099. /* ixgbe_setup_tc - routine to configure net_device for multiple traffic
  6100. * classes.
  6101. *
  6102. * @netdev: net device to configure
  6103. * @tc: number of traffic classes to enable
  6104. */
  6105. int ixgbe_setup_tc(struct net_device *dev, u8 tc)
  6106. {
  6107. struct ixgbe_adapter *adapter = netdev_priv(dev);
  6108. struct ixgbe_hw *hw = &adapter->hw;
  6109. /* Multiple traffic classes requires multiple queues */
  6110. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
  6111. e_err(drv, "Enable failed, needs MSI-X\n");
  6112. return -EINVAL;
  6113. }
  6114. /* Hardware supports up to 8 traffic classes */
  6115. if (tc > adapter->dcb_cfg.num_tcs.pg_tcs ||
  6116. (hw->mac.type == ixgbe_mac_82598EB && tc < MAX_TRAFFIC_CLASS))
  6117. return -EINVAL;
  6118. /* Hardware has to reinitialize queues and interrupts to
  6119. * match packet buffer alignment. Unfortunately, the
  6120. * hardware is not flexible enough to do this dynamically.
  6121. */
  6122. if (netif_running(dev))
  6123. ixgbe_close(dev);
  6124. ixgbe_clear_interrupt_scheme(adapter);
  6125. if (tc) {
  6126. netdev_set_num_tc(dev, tc);
  6127. adapter->last_lfc_mode = adapter->hw.fc.current_mode;
  6128. adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
  6129. adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
  6130. if (adapter->hw.mac.type == ixgbe_mac_82598EB)
  6131. adapter->hw.fc.requested_mode = ixgbe_fc_none;
  6132. } else {
  6133. netdev_reset_tc(dev);
  6134. adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
  6135. adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
  6136. adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
  6137. adapter->temp_dcb_cfg.pfc_mode_enable = false;
  6138. adapter->dcb_cfg.pfc_mode_enable = false;
  6139. }
  6140. ixgbe_init_interrupt_scheme(adapter);
  6141. ixgbe_validate_rtr(adapter, tc);
  6142. if (netif_running(dev))
  6143. ixgbe_open(dev);
  6144. return 0;
  6145. }
  6146. void ixgbe_do_reset(struct net_device *netdev)
  6147. {
  6148. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  6149. if (netif_running(netdev))
  6150. ixgbe_reinit_locked(adapter);
  6151. else
  6152. ixgbe_reset(adapter);
  6153. }
  6154. static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
  6155. netdev_features_t data)
  6156. {
  6157. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  6158. #ifdef CONFIG_DCB
  6159. if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)
  6160. data &= ~NETIF_F_HW_VLAN_RX;
  6161. #endif
  6162. /* return error if RXHASH is being enabled when RSS is not supported */
  6163. if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
  6164. data &= ~NETIF_F_RXHASH;
  6165. /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
  6166. if (!(data & NETIF_F_RXCSUM))
  6167. data &= ~NETIF_F_LRO;
  6168. /* Turn off LRO if not RSC capable or invalid ITR settings */
  6169. if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) {
  6170. data &= ~NETIF_F_LRO;
  6171. } else if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
  6172. (adapter->rx_itr_setting != 1 &&
  6173. adapter->rx_itr_setting > IXGBE_MAX_RSC_INT_RATE)) {
  6174. data &= ~NETIF_F_LRO;
  6175. e_info(probe, "rx-usecs set too low, not enabling RSC\n");
  6176. }
  6177. return data;
  6178. }
  6179. static int ixgbe_set_features(struct net_device *netdev,
  6180. netdev_features_t data)
  6181. {
  6182. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  6183. bool need_reset = false;
  6184. /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
  6185. if (!(data & NETIF_F_RXCSUM))
  6186. adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED;
  6187. else
  6188. adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
  6189. /* Make sure RSC matches LRO, reset if change */
  6190. if (!!(data & NETIF_F_LRO) !=
  6191. !!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
  6192. adapter->flags2 ^= IXGBE_FLAG2_RSC_ENABLED;
  6193. switch (adapter->hw.mac.type) {
  6194. case ixgbe_mac_X540:
  6195. case ixgbe_mac_82599EB:
  6196. need_reset = true;
  6197. break;
  6198. default:
  6199. break;
  6200. }
  6201. }
  6202. /*
  6203. * Check if Flow Director n-tuple support was enabled or disabled. If
  6204. * the state changed, we need to reset.
  6205. */
  6206. if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) {
  6207. /* turn off ATR, enable perfect filters and reset */
  6208. if (data & NETIF_F_NTUPLE) {
  6209. adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
  6210. adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
  6211. need_reset = true;
  6212. }
  6213. } else if (!(data & NETIF_F_NTUPLE)) {
  6214. /* turn off Flow Director, set ATR and reset */
  6215. adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
  6216. if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
  6217. !(adapter->flags & IXGBE_FLAG_DCB_ENABLED))
  6218. adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
  6219. need_reset = true;
  6220. }
  6221. if (need_reset)
  6222. ixgbe_do_reset(netdev);
  6223. return 0;
  6224. }
  6225. static const struct net_device_ops ixgbe_netdev_ops = {
  6226. .ndo_open = ixgbe_open,
  6227. .ndo_stop = ixgbe_close,
  6228. .ndo_start_xmit = ixgbe_xmit_frame,
  6229. .ndo_select_queue = ixgbe_select_queue,
  6230. .ndo_set_rx_mode = ixgbe_set_rx_mode,
  6231. .ndo_validate_addr = eth_validate_addr,
  6232. .ndo_set_mac_address = ixgbe_set_mac,
  6233. .ndo_change_mtu = ixgbe_change_mtu,
  6234. .ndo_tx_timeout = ixgbe_tx_timeout,
  6235. .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
  6236. .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
  6237. .ndo_do_ioctl = ixgbe_ioctl,
  6238. .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
  6239. .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
  6240. .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw,
  6241. .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
  6242. .ndo_get_vf_config = ixgbe_ndo_get_vf_config,
  6243. .ndo_get_stats64 = ixgbe_get_stats64,
  6244. .ndo_setup_tc = ixgbe_setup_tc,
  6245. #ifdef CONFIG_NET_POLL_CONTROLLER
  6246. .ndo_poll_controller = ixgbe_netpoll,
  6247. #endif
  6248. #ifdef IXGBE_FCOE
  6249. .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
  6250. .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
  6251. .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
  6252. .ndo_fcoe_enable = ixgbe_fcoe_enable,
  6253. .ndo_fcoe_disable = ixgbe_fcoe_disable,
  6254. .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
  6255. #endif /* IXGBE_FCOE */
  6256. .ndo_set_features = ixgbe_set_features,
  6257. .ndo_fix_features = ixgbe_fix_features,
  6258. };
  6259. static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
  6260. const struct ixgbe_info *ii)
  6261. {
  6262. #ifdef CONFIG_PCI_IOV
  6263. struct ixgbe_hw *hw = &adapter->hw;
  6264. if (hw->mac.type == ixgbe_mac_82598EB)
  6265. return;
  6266. /* The 82599 supports up to 64 VFs per physical function
  6267. * but this implementation limits allocation to 63 so that
  6268. * basic networking resources are still available to the
  6269. * physical function
  6270. */
  6271. adapter->num_vfs = (max_vfs > 63) ? 63 : max_vfs;
  6272. ixgbe_enable_sriov(adapter, ii);
  6273. #endif /* CONFIG_PCI_IOV */
  6274. }
  6275. /**
  6276. * ixgbe_probe - Device Initialization Routine
  6277. * @pdev: PCI device information struct
  6278. * @ent: entry in ixgbe_pci_tbl
  6279. *
  6280. * Returns 0 on success, negative on failure
  6281. *
  6282. * ixgbe_probe initializes an adapter identified by a pci_dev structure.
  6283. * The OS initialization, configuring of the adapter private structure,
  6284. * and a hardware reset occur.
  6285. **/
  6286. static int __devinit ixgbe_probe(struct pci_dev *pdev,
  6287. const struct pci_device_id *ent)
  6288. {
  6289. struct net_device *netdev;
  6290. struct ixgbe_adapter *adapter = NULL;
  6291. struct ixgbe_hw *hw;
  6292. const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
  6293. static int cards_found;
  6294. int i, err, pci_using_dac;
  6295. u8 part_str[IXGBE_PBANUM_LENGTH];
  6296. unsigned int indices = num_possible_cpus();
  6297. #ifdef IXGBE_FCOE
  6298. u16 device_caps;
  6299. #endif
  6300. u32 eec;
  6301. u16 wol_cap;
  6302. /* Catch broken hardware that put the wrong VF device ID in
  6303. * the PCIe SR-IOV capability.
  6304. */
  6305. if (pdev->is_virtfn) {
  6306. WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
  6307. pci_name(pdev), pdev->vendor, pdev->device);
  6308. return -EINVAL;
  6309. }
  6310. err = pci_enable_device_mem(pdev);
  6311. if (err)
  6312. return err;
  6313. if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
  6314. !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
  6315. pci_using_dac = 1;
  6316. } else {
  6317. err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
  6318. if (err) {
  6319. err = dma_set_coherent_mask(&pdev->dev,
  6320. DMA_BIT_MASK(32));
  6321. if (err) {
  6322. dev_err(&pdev->dev,
  6323. "No usable DMA configuration, aborting\n");
  6324. goto err_dma;
  6325. }
  6326. }
  6327. pci_using_dac = 0;
  6328. }
  6329. err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
  6330. IORESOURCE_MEM), ixgbe_driver_name);
  6331. if (err) {
  6332. dev_err(&pdev->dev,
  6333. "pci_request_selected_regions failed 0x%x\n", err);
  6334. goto err_pci_reg;
  6335. }
  6336. pci_enable_pcie_error_reporting(pdev);
  6337. pci_set_master(pdev);
  6338. pci_save_state(pdev);
  6339. #ifdef CONFIG_IXGBE_DCB
  6340. indices *= MAX_TRAFFIC_CLASS;
  6341. #endif
  6342. if (ii->mac == ixgbe_mac_82598EB)
  6343. indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES);
  6344. else
  6345. indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES);
  6346. #ifdef IXGBE_FCOE
  6347. indices += min_t(unsigned int, num_possible_cpus(),
  6348. IXGBE_MAX_FCOE_INDICES);
  6349. #endif
  6350. netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
  6351. if (!netdev) {
  6352. err = -ENOMEM;
  6353. goto err_alloc_etherdev;
  6354. }
  6355. SET_NETDEV_DEV(netdev, &pdev->dev);
  6356. adapter = netdev_priv(netdev);
  6357. pci_set_drvdata(pdev, adapter);
  6358. adapter->netdev = netdev;
  6359. adapter->pdev = pdev;
  6360. hw = &adapter->hw;
  6361. hw->back = adapter;
  6362. adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
  6363. hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
  6364. pci_resource_len(pdev, 0));
  6365. if (!hw->hw_addr) {
  6366. err = -EIO;
  6367. goto err_ioremap;
  6368. }
  6369. for (i = 1; i <= 5; i++) {
  6370. if (pci_resource_len(pdev, i) == 0)
  6371. continue;
  6372. }
  6373. netdev->netdev_ops = &ixgbe_netdev_ops;
  6374. ixgbe_set_ethtool_ops(netdev);
  6375. netdev->watchdog_timeo = 5 * HZ;
  6376. strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
  6377. adapter->bd_number = cards_found;
  6378. /* Setup hw api */
  6379. memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
  6380. hw->mac.type = ii->mac;
  6381. /* EEPROM */
  6382. memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops));
  6383. eec = IXGBE_READ_REG(hw, IXGBE_EEC);
  6384. /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
  6385. if (!(eec & (1 << 8)))
  6386. hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
  6387. /* PHY */
  6388. memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops));
  6389. hw->phy.sfp_type = ixgbe_sfp_type_unknown;
  6390. /* ixgbe_identify_phy_generic will set prtad and mmds properly */
  6391. hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
  6392. hw->phy.mdio.mmds = 0;
  6393. hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
  6394. hw->phy.mdio.dev = netdev;
  6395. hw->phy.mdio.mdio_read = ixgbe_mdio_read;
  6396. hw->phy.mdio.mdio_write = ixgbe_mdio_write;
  6397. ii->get_invariants(hw);
  6398. /* setup the private structure */
  6399. err = ixgbe_sw_init(adapter);
  6400. if (err)
  6401. goto err_sw_init;
  6402. /* Make it possible the adapter to be woken up via WOL */
  6403. switch (adapter->hw.mac.type) {
  6404. case ixgbe_mac_82599EB:
  6405. case ixgbe_mac_X540:
  6406. IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
  6407. break;
  6408. default:
  6409. break;
  6410. }
  6411. /*
  6412. * If there is a fan on this device and it has failed log the
  6413. * failure.
  6414. */
  6415. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
  6416. u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
  6417. if (esdp & IXGBE_ESDP_SDP1)
  6418. e_crit(probe, "Fan has stopped, replace the adapter\n");
  6419. }
  6420. /* reset_hw fills in the perm_addr as well */
  6421. hw->phy.reset_if_overtemp = true;
  6422. err = hw->mac.ops.reset_hw(hw);
  6423. hw->phy.reset_if_overtemp = false;
  6424. if (err == IXGBE_ERR_SFP_NOT_PRESENT &&
  6425. hw->mac.type == ixgbe_mac_82598EB) {
  6426. err = 0;
  6427. } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
  6428. e_dev_err("failed to load because an unsupported SFP+ "
  6429. "module type was detected.\n");
  6430. e_dev_err("Reload the driver after installing a supported "
  6431. "module.\n");
  6432. goto err_sw_init;
  6433. } else if (err) {
  6434. e_dev_err("HW Init failed: %d\n", err);
  6435. goto err_sw_init;
  6436. }
  6437. ixgbe_probe_vf(adapter, ii);
  6438. netdev->features = NETIF_F_SG |
  6439. NETIF_F_IP_CSUM |
  6440. NETIF_F_IPV6_CSUM |
  6441. NETIF_F_HW_VLAN_TX |
  6442. NETIF_F_HW_VLAN_RX |
  6443. NETIF_F_HW_VLAN_FILTER |
  6444. NETIF_F_TSO |
  6445. NETIF_F_TSO6 |
  6446. NETIF_F_RXHASH |
  6447. NETIF_F_RXCSUM;
  6448. netdev->hw_features = netdev->features;
  6449. switch (adapter->hw.mac.type) {
  6450. case ixgbe_mac_82599EB:
  6451. case ixgbe_mac_X540:
  6452. netdev->features |= NETIF_F_SCTP_CSUM;
  6453. netdev->hw_features |= NETIF_F_SCTP_CSUM |
  6454. NETIF_F_NTUPLE;
  6455. break;
  6456. default:
  6457. break;
  6458. }
  6459. netdev->vlan_features |= NETIF_F_TSO;
  6460. netdev->vlan_features |= NETIF_F_TSO6;
  6461. netdev->vlan_features |= NETIF_F_IP_CSUM;
  6462. netdev->vlan_features |= NETIF_F_IPV6_CSUM;
  6463. netdev->vlan_features |= NETIF_F_SG;
  6464. netdev->priv_flags |= IFF_UNICAST_FLT;
  6465. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  6466. adapter->flags &= ~(IXGBE_FLAG_RSS_ENABLED |
  6467. IXGBE_FLAG_DCB_ENABLED);
  6468. #ifdef CONFIG_IXGBE_DCB
  6469. netdev->dcbnl_ops = &dcbnl_ops;
  6470. #endif
  6471. #ifdef IXGBE_FCOE
  6472. if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
  6473. if (hw->mac.ops.get_device_caps) {
  6474. hw->mac.ops.get_device_caps(hw, &device_caps);
  6475. if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
  6476. adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
  6477. }
  6478. }
  6479. if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
  6480. netdev->vlan_features |= NETIF_F_FCOE_CRC;
  6481. netdev->vlan_features |= NETIF_F_FSO;
  6482. netdev->vlan_features |= NETIF_F_FCOE_MTU;
  6483. }
  6484. #endif /* IXGBE_FCOE */
  6485. if (pci_using_dac) {
  6486. netdev->features |= NETIF_F_HIGHDMA;
  6487. netdev->vlan_features |= NETIF_F_HIGHDMA;
  6488. }
  6489. if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
  6490. netdev->hw_features |= NETIF_F_LRO;
  6491. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
  6492. netdev->features |= NETIF_F_LRO;
  6493. /* make sure the EEPROM is good */
  6494. if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
  6495. e_dev_err("The EEPROM Checksum Is Not Valid\n");
  6496. err = -EIO;
  6497. goto err_eeprom;
  6498. }
  6499. memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
  6500. memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len);
  6501. if (ixgbe_validate_mac_addr(netdev->perm_addr)) {
  6502. e_dev_err("invalid MAC address\n");
  6503. err = -EIO;
  6504. goto err_eeprom;
  6505. }
  6506. setup_timer(&adapter->service_timer, &ixgbe_service_timer,
  6507. (unsigned long) adapter);
  6508. INIT_WORK(&adapter->service_task, ixgbe_service_task);
  6509. clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
  6510. err = ixgbe_init_interrupt_scheme(adapter);
  6511. if (err)
  6512. goto err_sw_init;
  6513. if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
  6514. netdev->hw_features &= ~NETIF_F_RXHASH;
  6515. netdev->features &= ~NETIF_F_RXHASH;
  6516. }
  6517. /* WOL not supported for all but the following */
  6518. adapter->wol = 0;
  6519. switch (pdev->device) {
  6520. case IXGBE_DEV_ID_82599_SFP:
  6521. /* Only these subdevice supports WOL */
  6522. switch (pdev->subsystem_device) {
  6523. case IXGBE_SUBDEV_ID_82599_560FLR:
  6524. /* only support first port */
  6525. if (hw->bus.func != 0)
  6526. break;
  6527. case IXGBE_SUBDEV_ID_82599_SFP:
  6528. adapter->wol = IXGBE_WUFC_MAG;
  6529. break;
  6530. }
  6531. break;
  6532. case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
  6533. /* All except this subdevice support WOL */
  6534. if (pdev->subsystem_device != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
  6535. adapter->wol = IXGBE_WUFC_MAG;
  6536. break;
  6537. case IXGBE_DEV_ID_82599_KX4:
  6538. adapter->wol = IXGBE_WUFC_MAG;
  6539. break;
  6540. case IXGBE_DEV_ID_X540T:
  6541. /* Check eeprom to see if it is enabled */
  6542. hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
  6543. wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
  6544. if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
  6545. ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
  6546. (hw->bus.func == 0)))
  6547. adapter->wol = IXGBE_WUFC_MAG;
  6548. break;
  6549. }
  6550. device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
  6551. /* save off EEPROM version number */
  6552. hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh);
  6553. hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl);
  6554. /* pick up the PCI bus settings for reporting later */
  6555. hw->mac.ops.get_bus_info(hw);
  6556. /* print bus type/speed/width info */
  6557. e_dev_info("(PCI Express:%s:%s) %pM\n",
  6558. (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" :
  6559. hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" :
  6560. "Unknown"),
  6561. (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
  6562. hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
  6563. hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" :
  6564. "Unknown"),
  6565. netdev->dev_addr);
  6566. err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH);
  6567. if (err)
  6568. strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH);
  6569. if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
  6570. e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
  6571. hw->mac.type, hw->phy.type, hw->phy.sfp_type,
  6572. part_str);
  6573. else
  6574. e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
  6575. hw->mac.type, hw->phy.type, part_str);
  6576. if (hw->bus.width <= ixgbe_bus_width_pcie_x4) {
  6577. e_dev_warn("PCI-Express bandwidth available for this card is "
  6578. "not sufficient for optimal performance.\n");
  6579. e_dev_warn("For optimal performance a x8 PCI-Express slot "
  6580. "is required.\n");
  6581. }
  6582. /* reset the hardware with the new settings */
  6583. err = hw->mac.ops.start_hw(hw);
  6584. if (err == IXGBE_ERR_EEPROM_VERSION) {
  6585. /* We are running on a pre-production device, log a warning */
  6586. e_dev_warn("This device is a pre-production adapter/LOM. "
  6587. "Please be aware there may be issues associated "
  6588. "with your hardware. If you are experiencing "
  6589. "problems please contact your Intel or hardware "
  6590. "representative who provided you with this "
  6591. "hardware.\n");
  6592. }
  6593. strcpy(netdev->name, "eth%d");
  6594. err = register_netdev(netdev);
  6595. if (err)
  6596. goto err_register;
  6597. /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
  6598. if (hw->mac.ops.disable_tx_laser &&
  6599. ((hw->phy.multispeed_fiber) ||
  6600. ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
  6601. (hw->mac.type == ixgbe_mac_82599EB))))
  6602. hw->mac.ops.disable_tx_laser(hw);
  6603. /* carrier off reporting is important to ethtool even BEFORE open */
  6604. netif_carrier_off(netdev);
  6605. #ifdef CONFIG_IXGBE_DCA
  6606. if (dca_add_requester(&pdev->dev) == 0) {
  6607. adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
  6608. ixgbe_setup_dca(adapter);
  6609. }
  6610. #endif
  6611. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  6612. e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
  6613. for (i = 0; i < adapter->num_vfs; i++)
  6614. ixgbe_vf_configuration(pdev, (i | 0x10000000));
  6615. }
  6616. /* firmware requires driver version to be 0xFFFFFFFF
  6617. * since os does not support feature
  6618. */
  6619. if (hw->mac.ops.set_fw_drv_ver)
  6620. hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF,
  6621. 0xFF);
  6622. /* add san mac addr to netdev */
  6623. ixgbe_add_sanmac_netdev(netdev);
  6624. e_dev_info("Intel(R) 10 Gigabit Network Connection\n");
  6625. cards_found++;
  6626. return 0;
  6627. err_register:
  6628. ixgbe_release_hw_control(adapter);
  6629. ixgbe_clear_interrupt_scheme(adapter);
  6630. err_sw_init:
  6631. err_eeprom:
  6632. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  6633. ixgbe_disable_sriov(adapter);
  6634. adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
  6635. iounmap(hw->hw_addr);
  6636. err_ioremap:
  6637. free_netdev(netdev);
  6638. err_alloc_etherdev:
  6639. pci_release_selected_regions(pdev,
  6640. pci_select_bars(pdev, IORESOURCE_MEM));
  6641. err_pci_reg:
  6642. err_dma:
  6643. pci_disable_device(pdev);
  6644. return err;
  6645. }
  6646. /**
  6647. * ixgbe_remove - Device Removal Routine
  6648. * @pdev: PCI device information struct
  6649. *
  6650. * ixgbe_remove is called by the PCI subsystem to alert the driver
  6651. * that it should release a PCI device. The could be caused by a
  6652. * Hot-Plug event, or because the driver is going to be removed from
  6653. * memory.
  6654. **/
  6655. static void __devexit ixgbe_remove(struct pci_dev *pdev)
  6656. {
  6657. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6658. struct net_device *netdev = adapter->netdev;
  6659. set_bit(__IXGBE_DOWN, &adapter->state);
  6660. cancel_work_sync(&adapter->service_task);
  6661. #ifdef CONFIG_IXGBE_DCA
  6662. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
  6663. adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
  6664. dca_remove_requester(&pdev->dev);
  6665. IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
  6666. }
  6667. #endif
  6668. #ifdef IXGBE_FCOE
  6669. if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
  6670. ixgbe_cleanup_fcoe(adapter);
  6671. #endif /* IXGBE_FCOE */
  6672. /* remove the added san mac */
  6673. ixgbe_del_sanmac_netdev(netdev);
  6674. if (netdev->reg_state == NETREG_REGISTERED)
  6675. unregister_netdev(netdev);
  6676. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  6677. if (!(ixgbe_check_vf_assignment(adapter)))
  6678. ixgbe_disable_sriov(adapter);
  6679. else
  6680. e_dev_warn("Unloading driver while VFs are assigned "
  6681. "- VFs will not be deallocated\n");
  6682. }
  6683. ixgbe_clear_interrupt_scheme(adapter);
  6684. ixgbe_release_hw_control(adapter);
  6685. iounmap(adapter->hw.hw_addr);
  6686. pci_release_selected_regions(pdev, pci_select_bars(pdev,
  6687. IORESOURCE_MEM));
  6688. e_dev_info("complete\n");
  6689. free_netdev(netdev);
  6690. pci_disable_pcie_error_reporting(pdev);
  6691. pci_disable_device(pdev);
  6692. }
  6693. /**
  6694. * ixgbe_io_error_detected - called when PCI error is detected
  6695. * @pdev: Pointer to PCI device
  6696. * @state: The current pci connection state
  6697. *
  6698. * This function is called after a PCI bus error affecting
  6699. * this device has been detected.
  6700. */
  6701. static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
  6702. pci_channel_state_t state)
  6703. {
  6704. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6705. struct net_device *netdev = adapter->netdev;
  6706. #ifdef CONFIG_PCI_IOV
  6707. struct pci_dev *bdev, *vfdev;
  6708. u32 dw0, dw1, dw2, dw3;
  6709. int vf, pos;
  6710. u16 req_id, pf_func;
  6711. if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
  6712. adapter->num_vfs == 0)
  6713. goto skip_bad_vf_detection;
  6714. bdev = pdev->bus->self;
  6715. while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT))
  6716. bdev = bdev->bus->self;
  6717. if (!bdev)
  6718. goto skip_bad_vf_detection;
  6719. pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
  6720. if (!pos)
  6721. goto skip_bad_vf_detection;
  6722. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
  6723. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
  6724. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
  6725. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
  6726. req_id = dw1 >> 16;
  6727. /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
  6728. if (!(req_id & 0x0080))
  6729. goto skip_bad_vf_detection;
  6730. pf_func = req_id & 0x01;
  6731. if ((pf_func & 1) == (pdev->devfn & 1)) {
  6732. unsigned int device_id;
  6733. vf = (req_id & 0x7F) >> 1;
  6734. e_dev_err("VF %d has caused a PCIe error\n", vf);
  6735. e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
  6736. "%8.8x\tdw3: %8.8x\n",
  6737. dw0, dw1, dw2, dw3);
  6738. switch (adapter->hw.mac.type) {
  6739. case ixgbe_mac_82599EB:
  6740. device_id = IXGBE_82599_VF_DEVICE_ID;
  6741. break;
  6742. case ixgbe_mac_X540:
  6743. device_id = IXGBE_X540_VF_DEVICE_ID;
  6744. break;
  6745. default:
  6746. device_id = 0;
  6747. break;
  6748. }
  6749. /* Find the pci device of the offending VF */
  6750. vfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID, device_id, NULL);
  6751. while (vfdev) {
  6752. if (vfdev->devfn == (req_id & 0xFF))
  6753. break;
  6754. vfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID,
  6755. device_id, vfdev);
  6756. }
  6757. /*
  6758. * There's a slim chance the VF could have been hot plugged,
  6759. * so if it is no longer present we don't need to issue the
  6760. * VFLR. Just clean up the AER in that case.
  6761. */
  6762. if (vfdev) {
  6763. e_dev_err("Issuing VFLR to VF %d\n", vf);
  6764. pci_write_config_dword(vfdev, 0xA8, 0x00008000);
  6765. }
  6766. pci_cleanup_aer_uncorrect_error_status(pdev);
  6767. }
  6768. /*
  6769. * Even though the error may have occurred on the other port
  6770. * we still need to increment the vf error reference count for
  6771. * both ports because the I/O resume function will be called
  6772. * for both of them.
  6773. */
  6774. adapter->vferr_refcount++;
  6775. return PCI_ERS_RESULT_RECOVERED;
  6776. skip_bad_vf_detection:
  6777. #endif /* CONFIG_PCI_IOV */
  6778. netif_device_detach(netdev);
  6779. if (state == pci_channel_io_perm_failure)
  6780. return PCI_ERS_RESULT_DISCONNECT;
  6781. if (netif_running(netdev))
  6782. ixgbe_down(adapter);
  6783. pci_disable_device(pdev);
  6784. /* Request a slot reset. */
  6785. return PCI_ERS_RESULT_NEED_RESET;
  6786. }
  6787. /**
  6788. * ixgbe_io_slot_reset - called after the pci bus has been reset.
  6789. * @pdev: Pointer to PCI device
  6790. *
  6791. * Restart the card from scratch, as if from a cold-boot.
  6792. */
  6793. static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
  6794. {
  6795. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6796. pci_ers_result_t result;
  6797. int err;
  6798. if (pci_enable_device_mem(pdev)) {
  6799. e_err(probe, "Cannot re-enable PCI device after reset.\n");
  6800. result = PCI_ERS_RESULT_DISCONNECT;
  6801. } else {
  6802. pci_set_master(pdev);
  6803. pci_restore_state(pdev);
  6804. pci_save_state(pdev);
  6805. pci_wake_from_d3(pdev, false);
  6806. ixgbe_reset(adapter);
  6807. IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
  6808. result = PCI_ERS_RESULT_RECOVERED;
  6809. }
  6810. err = pci_cleanup_aer_uncorrect_error_status(pdev);
  6811. if (err) {
  6812. e_dev_err("pci_cleanup_aer_uncorrect_error_status "
  6813. "failed 0x%0x\n", err);
  6814. /* non-fatal, continue */
  6815. }
  6816. return result;
  6817. }
  6818. /**
  6819. * ixgbe_io_resume - called when traffic can start flowing again.
  6820. * @pdev: Pointer to PCI device
  6821. *
  6822. * This callback is called when the error recovery driver tells us that
  6823. * its OK to resume normal operation.
  6824. */
  6825. static void ixgbe_io_resume(struct pci_dev *pdev)
  6826. {
  6827. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6828. struct net_device *netdev = adapter->netdev;
  6829. #ifdef CONFIG_PCI_IOV
  6830. if (adapter->vferr_refcount) {
  6831. e_info(drv, "Resuming after VF err\n");
  6832. adapter->vferr_refcount--;
  6833. return;
  6834. }
  6835. #endif
  6836. if (netif_running(netdev))
  6837. ixgbe_up(adapter);
  6838. netif_device_attach(netdev);
  6839. }
  6840. static struct pci_error_handlers ixgbe_err_handler = {
  6841. .error_detected = ixgbe_io_error_detected,
  6842. .slot_reset = ixgbe_io_slot_reset,
  6843. .resume = ixgbe_io_resume,
  6844. };
  6845. static struct pci_driver ixgbe_driver = {
  6846. .name = ixgbe_driver_name,
  6847. .id_table = ixgbe_pci_tbl,
  6848. .probe = ixgbe_probe,
  6849. .remove = __devexit_p(ixgbe_remove),
  6850. #ifdef CONFIG_PM
  6851. .suspend = ixgbe_suspend,
  6852. .resume = ixgbe_resume,
  6853. #endif
  6854. .shutdown = ixgbe_shutdown,
  6855. .err_handler = &ixgbe_err_handler
  6856. };
  6857. /**
  6858. * ixgbe_init_module - Driver Registration Routine
  6859. *
  6860. * ixgbe_init_module is the first routine called when the driver is
  6861. * loaded. All it does is register with the PCI subsystem.
  6862. **/
  6863. static int __init ixgbe_init_module(void)
  6864. {
  6865. int ret;
  6866. pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
  6867. pr_info("%s\n", ixgbe_copyright);
  6868. #ifdef CONFIG_IXGBE_DCA
  6869. dca_register_notify(&dca_notifier);
  6870. #endif
  6871. ret = pci_register_driver(&ixgbe_driver);
  6872. return ret;
  6873. }
  6874. module_init(ixgbe_init_module);
  6875. /**
  6876. * ixgbe_exit_module - Driver Exit Cleanup Routine
  6877. *
  6878. * ixgbe_exit_module is called just before the driver is removed
  6879. * from memory.
  6880. **/
  6881. static void __exit ixgbe_exit_module(void)
  6882. {
  6883. #ifdef CONFIG_IXGBE_DCA
  6884. dca_unregister_notify(&dca_notifier);
  6885. #endif
  6886. pci_unregister_driver(&ixgbe_driver);
  6887. rcu_barrier(); /* Wait for completion of call_rcu()'s */
  6888. }
  6889. #ifdef CONFIG_IXGBE_DCA
  6890. static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
  6891. void *p)
  6892. {
  6893. int ret_val;
  6894. ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
  6895. __ixgbe_notify_dca);
  6896. return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
  6897. }
  6898. #endif /* CONFIG_IXGBE_DCA */
  6899. module_exit(ixgbe_exit_module);
  6900. /* ixgbe_main.c */