ixgbe_main.c 216 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906
  1. /*******************************************************************************
  2. Intel 10 Gigabit PCI Express Linux driver
  3. Copyright(c) 1999 - 2012 Intel Corporation.
  4. This program is free software; you can redistribute it and/or modify it
  5. under the terms and conditions of the GNU General Public License,
  6. version 2, as published by the Free Software Foundation.
  7. This program is distributed in the hope it will be useful, but WITHOUT
  8. ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  9. FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  10. more details.
  11. You should have received a copy of the GNU General Public License along with
  12. this program; if not, write to the Free Software Foundation, Inc.,
  13. 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  14. The full GNU General Public License is included in this distribution in
  15. the file called "COPYING".
  16. Contact Information:
  17. e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  18. Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  19. *******************************************************************************/
  20. #include <linux/types.h>
  21. #include <linux/module.h>
  22. #include <linux/pci.h>
  23. #include <linux/netdevice.h>
  24. #include <linux/vmalloc.h>
  25. #include <linux/string.h>
  26. #include <linux/in.h>
  27. #include <linux/interrupt.h>
  28. #include <linux/ip.h>
  29. #include <linux/tcp.h>
  30. #include <linux/sctp.h>
  31. #include <linux/pkt_sched.h>
  32. #include <linux/ipv6.h>
  33. #include <linux/slab.h>
  34. #include <net/checksum.h>
  35. #include <net/ip6_checksum.h>
  36. #include <linux/ethtool.h>
  37. #include <linux/if.h>
  38. #include <linux/if_vlan.h>
  39. #include <linux/if_bridge.h>
  40. #include <linux/prefetch.h>
  41. #include <scsi/fc/fc_fcoe.h>
  42. #include "ixgbe.h"
  43. #include "ixgbe_common.h"
  44. #include "ixgbe_dcb_82599.h"
  45. #include "ixgbe_sriov.h"
  46. char ixgbe_driver_name[] = "ixgbe";
  47. static const char ixgbe_driver_string[] =
  48. "Intel(R) 10 Gigabit PCI Express Network Driver";
  49. #ifdef IXGBE_FCOE
  50. char ixgbe_default_device_descr[] =
  51. "Intel(R) 10 Gigabit Network Connection";
  52. #else
  53. static char ixgbe_default_device_descr[] =
  54. "Intel(R) 10 Gigabit Network Connection";
  55. #endif
  56. #define DRV_VERSION "3.11.33-k"
  57. const char ixgbe_driver_version[] = DRV_VERSION;
  58. static const char ixgbe_copyright[] =
  59. "Copyright (c) 1999-2012 Intel Corporation.";
  60. static const struct ixgbe_info *ixgbe_info_tbl[] = {
  61. [board_82598] = &ixgbe_82598_info,
  62. [board_82599] = &ixgbe_82599_info,
  63. [board_X540] = &ixgbe_X540_info,
  64. };
  65. /* ixgbe_pci_tbl - PCI Device ID Table
  66. *
  67. * Wildcard entries (PCI_ANY_ID) should come last
  68. * Last entry must be all 0s
  69. *
  70. * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  71. * Class, Class Mask, private data (not used) }
  72. */
  73. static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
  74. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
  75. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
  76. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
  77. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
  78. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
  79. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
  80. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
  81. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
  82. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
  83. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
  84. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
  85. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
  86. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
  87. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
  88. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
  89. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
  90. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
  91. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
  92. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
  93. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
  94. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
  95. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
  96. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
  97. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
  98. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
  99. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
  100. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
  101. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
  102. {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
  103. /* required last entry */
  104. {0, }
  105. };
  106. MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
  107. #ifdef CONFIG_IXGBE_DCA
  108. static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
  109. void *p);
  110. static struct notifier_block dca_notifier = {
  111. .notifier_call = ixgbe_notify_dca,
  112. .next = NULL,
  113. .priority = 0
  114. };
  115. #endif
  116. #ifdef CONFIG_PCI_IOV
  117. static unsigned int max_vfs;
  118. module_param(max_vfs, uint, 0);
  119. MODULE_PARM_DESC(max_vfs,
  120. "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63");
  121. #endif /* CONFIG_PCI_IOV */
  122. static unsigned int allow_unsupported_sfp;
  123. module_param(allow_unsupported_sfp, uint, 0);
  124. MODULE_PARM_DESC(allow_unsupported_sfp,
  125. "Allow unsupported and untested SFP+ modules on 82599-based adapters");
  126. #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  127. static int debug = -1;
  128. module_param(debug, int, 0);
  129. MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  130. MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  131. MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
  132. MODULE_LICENSE("GPL");
  133. MODULE_VERSION(DRV_VERSION);
  134. static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
  135. {
  136. if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
  137. !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
  138. schedule_work(&adapter->service_task);
  139. }
  140. static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
  141. {
  142. BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
  143. /* flush memory to make sure state is correct before next watchdog */
  144. smp_mb__before_clear_bit();
  145. clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
  146. }
  147. struct ixgbe_reg_info {
  148. u32 ofs;
  149. char *name;
  150. };
  151. static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
  152. /* General Registers */
  153. {IXGBE_CTRL, "CTRL"},
  154. {IXGBE_STATUS, "STATUS"},
  155. {IXGBE_CTRL_EXT, "CTRL_EXT"},
  156. /* Interrupt Registers */
  157. {IXGBE_EICR, "EICR"},
  158. /* RX Registers */
  159. {IXGBE_SRRCTL(0), "SRRCTL"},
  160. {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
  161. {IXGBE_RDLEN(0), "RDLEN"},
  162. {IXGBE_RDH(0), "RDH"},
  163. {IXGBE_RDT(0), "RDT"},
  164. {IXGBE_RXDCTL(0), "RXDCTL"},
  165. {IXGBE_RDBAL(0), "RDBAL"},
  166. {IXGBE_RDBAH(0), "RDBAH"},
  167. /* TX Registers */
  168. {IXGBE_TDBAL(0), "TDBAL"},
  169. {IXGBE_TDBAH(0), "TDBAH"},
  170. {IXGBE_TDLEN(0), "TDLEN"},
  171. {IXGBE_TDH(0), "TDH"},
  172. {IXGBE_TDT(0), "TDT"},
  173. {IXGBE_TXDCTL(0), "TXDCTL"},
  174. /* List Terminator */
  175. {}
  176. };
  177. /*
  178. * ixgbe_regdump - register printout routine
  179. */
  180. static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
  181. {
  182. int i = 0, j = 0;
  183. char rname[16];
  184. u32 regs[64];
  185. switch (reginfo->ofs) {
  186. case IXGBE_SRRCTL(0):
  187. for (i = 0; i < 64; i++)
  188. regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
  189. break;
  190. case IXGBE_DCA_RXCTRL(0):
  191. for (i = 0; i < 64; i++)
  192. regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
  193. break;
  194. case IXGBE_RDLEN(0):
  195. for (i = 0; i < 64; i++)
  196. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
  197. break;
  198. case IXGBE_RDH(0):
  199. for (i = 0; i < 64; i++)
  200. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
  201. break;
  202. case IXGBE_RDT(0):
  203. for (i = 0; i < 64; i++)
  204. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
  205. break;
  206. case IXGBE_RXDCTL(0):
  207. for (i = 0; i < 64; i++)
  208. regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
  209. break;
  210. case IXGBE_RDBAL(0):
  211. for (i = 0; i < 64; i++)
  212. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
  213. break;
  214. case IXGBE_RDBAH(0):
  215. for (i = 0; i < 64; i++)
  216. regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
  217. break;
  218. case IXGBE_TDBAL(0):
  219. for (i = 0; i < 64; i++)
  220. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
  221. break;
  222. case IXGBE_TDBAH(0):
  223. for (i = 0; i < 64; i++)
  224. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
  225. break;
  226. case IXGBE_TDLEN(0):
  227. for (i = 0; i < 64; i++)
  228. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
  229. break;
  230. case IXGBE_TDH(0):
  231. for (i = 0; i < 64; i++)
  232. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
  233. break;
  234. case IXGBE_TDT(0):
  235. for (i = 0; i < 64; i++)
  236. regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
  237. break;
  238. case IXGBE_TXDCTL(0):
  239. for (i = 0; i < 64; i++)
  240. regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
  241. break;
  242. default:
  243. pr_info("%-15s %08x\n", reginfo->name,
  244. IXGBE_READ_REG(hw, reginfo->ofs));
  245. return;
  246. }
  247. for (i = 0; i < 8; i++) {
  248. snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
  249. pr_err("%-15s", rname);
  250. for (j = 0; j < 8; j++)
  251. pr_cont(" %08x", regs[i*8+j]);
  252. pr_cont("\n");
  253. }
  254. }
  255. /*
  256. * ixgbe_dump - Print registers, tx-rings and rx-rings
  257. */
  258. static void ixgbe_dump(struct ixgbe_adapter *adapter)
  259. {
  260. struct net_device *netdev = adapter->netdev;
  261. struct ixgbe_hw *hw = &adapter->hw;
  262. struct ixgbe_reg_info *reginfo;
  263. int n = 0;
  264. struct ixgbe_ring *tx_ring;
  265. struct ixgbe_tx_buffer *tx_buffer;
  266. union ixgbe_adv_tx_desc *tx_desc;
  267. struct my_u0 { u64 a; u64 b; } *u0;
  268. struct ixgbe_ring *rx_ring;
  269. union ixgbe_adv_rx_desc *rx_desc;
  270. struct ixgbe_rx_buffer *rx_buffer_info;
  271. u32 staterr;
  272. int i = 0;
  273. if (!netif_msg_hw(adapter))
  274. return;
  275. /* Print netdevice Info */
  276. if (netdev) {
  277. dev_info(&adapter->pdev->dev, "Net device Info\n");
  278. pr_info("Device Name state "
  279. "trans_start last_rx\n");
  280. pr_info("%-15s %016lX %016lX %016lX\n",
  281. netdev->name,
  282. netdev->state,
  283. netdev->trans_start,
  284. netdev->last_rx);
  285. }
  286. /* Print Registers */
  287. dev_info(&adapter->pdev->dev, "Register Dump\n");
  288. pr_info(" Register Name Value\n");
  289. for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
  290. reginfo->name; reginfo++) {
  291. ixgbe_regdump(hw, reginfo);
  292. }
  293. /* Print TX Ring Summary */
  294. if (!netdev || !netif_running(netdev))
  295. goto exit;
  296. dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
  297. pr_info(" %s %s %s %s\n",
  298. "Queue [NTU] [NTC] [bi(ntc)->dma ]",
  299. "leng", "ntw", "timestamp");
  300. for (n = 0; n < adapter->num_tx_queues; n++) {
  301. tx_ring = adapter->tx_ring[n];
  302. tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
  303. pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
  304. n, tx_ring->next_to_use, tx_ring->next_to_clean,
  305. (u64)dma_unmap_addr(tx_buffer, dma),
  306. dma_unmap_len(tx_buffer, len),
  307. tx_buffer->next_to_watch,
  308. (u64)tx_buffer->time_stamp);
  309. }
  310. /* Print TX Rings */
  311. if (!netif_msg_tx_done(adapter))
  312. goto rx_ring_summary;
  313. dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
  314. /* Transmit Descriptor Formats
  315. *
  316. * 82598 Advanced Transmit Descriptor
  317. * +--------------------------------------------------------------+
  318. * 0 | Buffer Address [63:0] |
  319. * +--------------------------------------------------------------+
  320. * 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
  321. * +--------------------------------------------------------------+
  322. * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
  323. *
  324. * 82598 Advanced Transmit Descriptor (Write-Back Format)
  325. * +--------------------------------------------------------------+
  326. * 0 | RSV [63:0] |
  327. * +--------------------------------------------------------------+
  328. * 8 | RSV | STA | NXTSEQ |
  329. * +--------------------------------------------------------------+
  330. * 63 36 35 32 31 0
  331. *
  332. * 82599+ Advanced Transmit Descriptor
  333. * +--------------------------------------------------------------+
  334. * 0 | Buffer Address [63:0] |
  335. * +--------------------------------------------------------------+
  336. * 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN |
  337. * +--------------------------------------------------------------+
  338. * 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0
  339. *
  340. * 82599+ Advanced Transmit Descriptor (Write-Back Format)
  341. * +--------------------------------------------------------------+
  342. * 0 | RSV [63:0] |
  343. * +--------------------------------------------------------------+
  344. * 8 | RSV | STA | RSV |
  345. * +--------------------------------------------------------------+
  346. * 63 36 35 32 31 0
  347. */
  348. for (n = 0; n < adapter->num_tx_queues; n++) {
  349. tx_ring = adapter->tx_ring[n];
  350. pr_info("------------------------------------\n");
  351. pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
  352. pr_info("------------------------------------\n");
  353. pr_info("%s%s %s %s %s %s\n",
  354. "T [desc] [address 63:0 ] ",
  355. "[PlPOIdStDDt Ln] [bi->dma ] ",
  356. "leng", "ntw", "timestamp", "bi->skb");
  357. for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
  358. tx_desc = IXGBE_TX_DESC(tx_ring, i);
  359. tx_buffer = &tx_ring->tx_buffer_info[i];
  360. u0 = (struct my_u0 *)tx_desc;
  361. if (dma_unmap_len(tx_buffer, len) > 0) {
  362. pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p",
  363. i,
  364. le64_to_cpu(u0->a),
  365. le64_to_cpu(u0->b),
  366. (u64)dma_unmap_addr(tx_buffer, dma),
  367. dma_unmap_len(tx_buffer, len),
  368. tx_buffer->next_to_watch,
  369. (u64)tx_buffer->time_stamp,
  370. tx_buffer->skb);
  371. if (i == tx_ring->next_to_use &&
  372. i == tx_ring->next_to_clean)
  373. pr_cont(" NTC/U\n");
  374. else if (i == tx_ring->next_to_use)
  375. pr_cont(" NTU\n");
  376. else if (i == tx_ring->next_to_clean)
  377. pr_cont(" NTC\n");
  378. else
  379. pr_cont("\n");
  380. if (netif_msg_pktdata(adapter) &&
  381. tx_buffer->skb)
  382. print_hex_dump(KERN_INFO, "",
  383. DUMP_PREFIX_ADDRESS, 16, 1,
  384. tx_buffer->skb->data,
  385. dma_unmap_len(tx_buffer, len),
  386. true);
  387. }
  388. }
  389. }
  390. /* Print RX Rings Summary */
  391. rx_ring_summary:
  392. dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
  393. pr_info("Queue [NTU] [NTC]\n");
  394. for (n = 0; n < adapter->num_rx_queues; n++) {
  395. rx_ring = adapter->rx_ring[n];
  396. pr_info("%5d %5X %5X\n",
  397. n, rx_ring->next_to_use, rx_ring->next_to_clean);
  398. }
  399. /* Print RX Rings */
  400. if (!netif_msg_rx_status(adapter))
  401. goto exit;
  402. dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
  403. /* Receive Descriptor Formats
  404. *
  405. * 82598 Advanced Receive Descriptor (Read) Format
  406. * 63 1 0
  407. * +-----------------------------------------------------+
  408. * 0 | Packet Buffer Address [63:1] |A0/NSE|
  409. * +----------------------------------------------+------+
  410. * 8 | Header Buffer Address [63:1] | DD |
  411. * +-----------------------------------------------------+
  412. *
  413. *
  414. * 82598 Advanced Receive Descriptor (Write-Back) Format
  415. *
  416. * 63 48 47 32 31 30 21 20 16 15 4 3 0
  417. * +------------------------------------------------------+
  418. * 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS |
  419. * | Packet | IP | | | | Type | Type |
  420. * | Checksum | Ident | | | | | |
  421. * +------------------------------------------------------+
  422. * 8 | VLAN Tag | Length | Extended Error | Extended Status |
  423. * +------------------------------------------------------+
  424. * 63 48 47 32 31 20 19 0
  425. *
  426. * 82599+ Advanced Receive Descriptor (Read) Format
  427. * 63 1 0
  428. * +-----------------------------------------------------+
  429. * 0 | Packet Buffer Address [63:1] |A0/NSE|
  430. * +----------------------------------------------+------+
  431. * 8 | Header Buffer Address [63:1] | DD |
  432. * +-----------------------------------------------------+
  433. *
  434. *
  435. * 82599+ Advanced Receive Descriptor (Write-Back) Format
  436. *
  437. * 63 48 47 32 31 30 21 20 17 16 4 3 0
  438. * +------------------------------------------------------+
  439. * 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS |
  440. * |/ RTT / PCoE_PARAM | | | CNT | Type | Type |
  441. * |/ Flow Dir Flt ID | | | | | |
  442. * +------------------------------------------------------+
  443. * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
  444. * +------------------------------------------------------+
  445. * 63 48 47 32 31 20 19 0
  446. */
  447. for (n = 0; n < adapter->num_rx_queues; n++) {
  448. rx_ring = adapter->rx_ring[n];
  449. pr_info("------------------------------------\n");
  450. pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
  451. pr_info("------------------------------------\n");
  452. pr_info("%s%s%s",
  453. "R [desc] [ PktBuf A0] ",
  454. "[ HeadBuf DD] [bi->dma ] [bi->skb ] ",
  455. "<-- Adv Rx Read format\n");
  456. pr_info("%s%s%s",
  457. "RWB[desc] [PcsmIpSHl PtRs] ",
  458. "[vl er S cks ln] ---------------- [bi->skb ] ",
  459. "<-- Adv Rx Write-Back format\n");
  460. for (i = 0; i < rx_ring->count; i++) {
  461. rx_buffer_info = &rx_ring->rx_buffer_info[i];
  462. rx_desc = IXGBE_RX_DESC(rx_ring, i);
  463. u0 = (struct my_u0 *)rx_desc;
  464. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  465. if (staterr & IXGBE_RXD_STAT_DD) {
  466. /* Descriptor Done */
  467. pr_info("RWB[0x%03X] %016llX "
  468. "%016llX ---------------- %p", i,
  469. le64_to_cpu(u0->a),
  470. le64_to_cpu(u0->b),
  471. rx_buffer_info->skb);
  472. } else {
  473. pr_info("R [0x%03X] %016llX "
  474. "%016llX %016llX %p", i,
  475. le64_to_cpu(u0->a),
  476. le64_to_cpu(u0->b),
  477. (u64)rx_buffer_info->dma,
  478. rx_buffer_info->skb);
  479. if (netif_msg_pktdata(adapter) &&
  480. rx_buffer_info->dma) {
  481. print_hex_dump(KERN_INFO, "",
  482. DUMP_PREFIX_ADDRESS, 16, 1,
  483. page_address(rx_buffer_info->page) +
  484. rx_buffer_info->page_offset,
  485. ixgbe_rx_bufsz(rx_ring), true);
  486. }
  487. }
  488. if (i == rx_ring->next_to_use)
  489. pr_cont(" NTU\n");
  490. else if (i == rx_ring->next_to_clean)
  491. pr_cont(" NTC\n");
  492. else
  493. pr_cont("\n");
  494. }
  495. }
  496. exit:
  497. return;
  498. }
  499. static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
  500. {
  501. u32 ctrl_ext;
  502. /* Let firmware take over control of h/w */
  503. ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
  504. IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
  505. ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
  506. }
  507. static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
  508. {
  509. u32 ctrl_ext;
  510. /* Let firmware know the driver has taken over */
  511. ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
  512. IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
  513. ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
  514. }
  515. /**
  516. * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
  517. * @adapter: pointer to adapter struct
  518. * @direction: 0 for Rx, 1 for Tx, -1 for other causes
  519. * @queue: queue to map the corresponding interrupt to
  520. * @msix_vector: the vector to map to the corresponding queue
  521. *
  522. */
  523. static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
  524. u8 queue, u8 msix_vector)
  525. {
  526. u32 ivar, index;
  527. struct ixgbe_hw *hw = &adapter->hw;
  528. switch (hw->mac.type) {
  529. case ixgbe_mac_82598EB:
  530. msix_vector |= IXGBE_IVAR_ALLOC_VAL;
  531. if (direction == -1)
  532. direction = 0;
  533. index = (((direction * 64) + queue) >> 2) & 0x1F;
  534. ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
  535. ivar &= ~(0xFF << (8 * (queue & 0x3)));
  536. ivar |= (msix_vector << (8 * (queue & 0x3)));
  537. IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
  538. break;
  539. case ixgbe_mac_82599EB:
  540. case ixgbe_mac_X540:
  541. if (direction == -1) {
  542. /* other causes */
  543. msix_vector |= IXGBE_IVAR_ALLOC_VAL;
  544. index = ((queue & 1) * 8);
  545. ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
  546. ivar &= ~(0xFF << index);
  547. ivar |= (msix_vector << index);
  548. IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
  549. break;
  550. } else {
  551. /* tx or rx causes */
  552. msix_vector |= IXGBE_IVAR_ALLOC_VAL;
  553. index = ((16 * (queue & 1)) + (8 * direction));
  554. ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
  555. ivar &= ~(0xFF << index);
  556. ivar |= (msix_vector << index);
  557. IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
  558. break;
  559. }
  560. default:
  561. break;
  562. }
  563. }
  564. static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
  565. u64 qmask)
  566. {
  567. u32 mask;
  568. switch (adapter->hw.mac.type) {
  569. case ixgbe_mac_82598EB:
  570. mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
  571. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
  572. break;
  573. case ixgbe_mac_82599EB:
  574. case ixgbe_mac_X540:
  575. mask = (qmask & 0xFFFFFFFF);
  576. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
  577. mask = (qmask >> 32);
  578. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
  579. break;
  580. default:
  581. break;
  582. }
  583. }
  584. void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring,
  585. struct ixgbe_tx_buffer *tx_buffer)
  586. {
  587. if (tx_buffer->skb) {
  588. dev_kfree_skb_any(tx_buffer->skb);
  589. if (dma_unmap_len(tx_buffer, len))
  590. dma_unmap_single(ring->dev,
  591. dma_unmap_addr(tx_buffer, dma),
  592. dma_unmap_len(tx_buffer, len),
  593. DMA_TO_DEVICE);
  594. } else if (dma_unmap_len(tx_buffer, len)) {
  595. dma_unmap_page(ring->dev,
  596. dma_unmap_addr(tx_buffer, dma),
  597. dma_unmap_len(tx_buffer, len),
  598. DMA_TO_DEVICE);
  599. }
  600. tx_buffer->next_to_watch = NULL;
  601. tx_buffer->skb = NULL;
  602. dma_unmap_len_set(tx_buffer, len, 0);
  603. /* tx_buffer must be completely set up in the transmit path */
  604. }
  605. static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
  606. {
  607. struct ixgbe_hw *hw = &adapter->hw;
  608. struct ixgbe_hw_stats *hwstats = &adapter->stats;
  609. int i;
  610. u32 data;
  611. if ((hw->fc.current_mode != ixgbe_fc_full) &&
  612. (hw->fc.current_mode != ixgbe_fc_rx_pause))
  613. return;
  614. switch (hw->mac.type) {
  615. case ixgbe_mac_82598EB:
  616. data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
  617. break;
  618. default:
  619. data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
  620. }
  621. hwstats->lxoffrxc += data;
  622. /* refill credits (no tx hang) if we received xoff */
  623. if (!data)
  624. return;
  625. for (i = 0; i < adapter->num_tx_queues; i++)
  626. clear_bit(__IXGBE_HANG_CHECK_ARMED,
  627. &adapter->tx_ring[i]->state);
  628. }
  629. static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
  630. {
  631. struct ixgbe_hw *hw = &adapter->hw;
  632. struct ixgbe_hw_stats *hwstats = &adapter->stats;
  633. u32 xoff[8] = {0};
  634. u8 tc;
  635. int i;
  636. bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
  637. if (adapter->ixgbe_ieee_pfc)
  638. pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
  639. if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
  640. ixgbe_update_xoff_rx_lfc(adapter);
  641. return;
  642. }
  643. /* update stats for each tc, only valid with PFC enabled */
  644. for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
  645. u32 pxoffrxc;
  646. switch (hw->mac.type) {
  647. case ixgbe_mac_82598EB:
  648. pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
  649. break;
  650. default:
  651. pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
  652. }
  653. hwstats->pxoffrxc[i] += pxoffrxc;
  654. /* Get the TC for given UP */
  655. tc = netdev_get_prio_tc_map(adapter->netdev, i);
  656. xoff[tc] += pxoffrxc;
  657. }
  658. /* disarm tx queues that have received xoff frames */
  659. for (i = 0; i < adapter->num_tx_queues; i++) {
  660. struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
  661. tc = tx_ring->dcb_tc;
  662. if (xoff[tc])
  663. clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
  664. }
  665. }
  666. static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
  667. {
  668. return ring->stats.packets;
  669. }
  670. static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
  671. {
  672. struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
  673. struct ixgbe_hw *hw = &adapter->hw;
  674. u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
  675. u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
  676. if (head != tail)
  677. return (head < tail) ?
  678. tail - head : (tail + ring->count - head);
  679. return 0;
  680. }
  681. static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
  682. {
  683. u32 tx_done = ixgbe_get_tx_completed(tx_ring);
  684. u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
  685. u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
  686. bool ret = false;
  687. clear_check_for_tx_hang(tx_ring);
  688. /*
  689. * Check for a hung queue, but be thorough. This verifies
  690. * that a transmit has been completed since the previous
  691. * check AND there is at least one packet pending. The
  692. * ARMED bit is set to indicate a potential hang. The
  693. * bit is cleared if a pause frame is received to remove
  694. * false hang detection due to PFC or 802.3x frames. By
  695. * requiring this to fail twice we avoid races with
  696. * pfc clearing the ARMED bit and conditions where we
  697. * run the check_tx_hang logic with a transmit completion
  698. * pending but without time to complete it yet.
  699. */
  700. if ((tx_done_old == tx_done) && tx_pending) {
  701. /* make sure it is true for two checks in a row */
  702. ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
  703. &tx_ring->state);
  704. } else {
  705. /* update completed stats and continue */
  706. tx_ring->tx_stats.tx_done_old = tx_done;
  707. /* reset the countdown */
  708. clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
  709. }
  710. return ret;
  711. }
  712. /**
  713. * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
  714. * @adapter: driver private struct
  715. **/
  716. static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
  717. {
  718. /* Do the reset outside of interrupt context */
  719. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  720. adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
  721. ixgbe_service_event_schedule(adapter);
  722. }
  723. }
  724. /**
  725. * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
  726. * @q_vector: structure containing interrupt and ring information
  727. * @tx_ring: tx ring to clean
  728. **/
  729. static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
  730. struct ixgbe_ring *tx_ring)
  731. {
  732. struct ixgbe_adapter *adapter = q_vector->adapter;
  733. struct ixgbe_tx_buffer *tx_buffer;
  734. union ixgbe_adv_tx_desc *tx_desc;
  735. unsigned int total_bytes = 0, total_packets = 0;
  736. unsigned int budget = q_vector->tx.work_limit;
  737. unsigned int i = tx_ring->next_to_clean;
  738. if (test_bit(__IXGBE_DOWN, &adapter->state))
  739. return true;
  740. tx_buffer = &tx_ring->tx_buffer_info[i];
  741. tx_desc = IXGBE_TX_DESC(tx_ring, i);
  742. i -= tx_ring->count;
  743. do {
  744. union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
  745. /* if next_to_watch is not set then there is no work pending */
  746. if (!eop_desc)
  747. break;
  748. /* prevent any other reads prior to eop_desc */
  749. rmb();
  750. /* if DD is not set pending work has not been completed */
  751. if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
  752. break;
  753. /* clear next_to_watch to prevent false hangs */
  754. tx_buffer->next_to_watch = NULL;
  755. /* update the statistics for this packet */
  756. total_bytes += tx_buffer->bytecount;
  757. total_packets += tx_buffer->gso_segs;
  758. if (unlikely(tx_buffer->tx_flags & IXGBE_TX_FLAGS_TSTAMP))
  759. ixgbe_ptp_tx_hwtstamp(q_vector, tx_buffer->skb);
  760. /* free the skb */
  761. dev_kfree_skb_any(tx_buffer->skb);
  762. /* unmap skb header data */
  763. dma_unmap_single(tx_ring->dev,
  764. dma_unmap_addr(tx_buffer, dma),
  765. dma_unmap_len(tx_buffer, len),
  766. DMA_TO_DEVICE);
  767. /* clear tx_buffer data */
  768. tx_buffer->skb = NULL;
  769. dma_unmap_len_set(tx_buffer, len, 0);
  770. /* unmap remaining buffers */
  771. while (tx_desc != eop_desc) {
  772. tx_buffer++;
  773. tx_desc++;
  774. i++;
  775. if (unlikely(!i)) {
  776. i -= tx_ring->count;
  777. tx_buffer = tx_ring->tx_buffer_info;
  778. tx_desc = IXGBE_TX_DESC(tx_ring, 0);
  779. }
  780. /* unmap any remaining paged data */
  781. if (dma_unmap_len(tx_buffer, len)) {
  782. dma_unmap_page(tx_ring->dev,
  783. dma_unmap_addr(tx_buffer, dma),
  784. dma_unmap_len(tx_buffer, len),
  785. DMA_TO_DEVICE);
  786. dma_unmap_len_set(tx_buffer, len, 0);
  787. }
  788. }
  789. /* move us one more past the eop_desc for start of next pkt */
  790. tx_buffer++;
  791. tx_desc++;
  792. i++;
  793. if (unlikely(!i)) {
  794. i -= tx_ring->count;
  795. tx_buffer = tx_ring->tx_buffer_info;
  796. tx_desc = IXGBE_TX_DESC(tx_ring, 0);
  797. }
  798. /* issue prefetch for next Tx descriptor */
  799. prefetch(tx_desc);
  800. /* update budget accounting */
  801. budget--;
  802. } while (likely(budget));
  803. i += tx_ring->count;
  804. tx_ring->next_to_clean = i;
  805. u64_stats_update_begin(&tx_ring->syncp);
  806. tx_ring->stats.bytes += total_bytes;
  807. tx_ring->stats.packets += total_packets;
  808. u64_stats_update_end(&tx_ring->syncp);
  809. q_vector->tx.total_bytes += total_bytes;
  810. q_vector->tx.total_packets += total_packets;
  811. if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
  812. /* schedule immediate reset if we believe we hung */
  813. struct ixgbe_hw *hw = &adapter->hw;
  814. e_err(drv, "Detected Tx Unit Hang\n"
  815. " Tx Queue <%d>\n"
  816. " TDH, TDT <%x>, <%x>\n"
  817. " next_to_use <%x>\n"
  818. " next_to_clean <%x>\n"
  819. "tx_buffer_info[next_to_clean]\n"
  820. " time_stamp <%lx>\n"
  821. " jiffies <%lx>\n",
  822. tx_ring->queue_index,
  823. IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
  824. IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
  825. tx_ring->next_to_use, i,
  826. tx_ring->tx_buffer_info[i].time_stamp, jiffies);
  827. netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
  828. e_info(probe,
  829. "tx hang %d detected on queue %d, resetting adapter\n",
  830. adapter->tx_timeout_count + 1, tx_ring->queue_index);
  831. /* schedule immediate reset if we believe we hung */
  832. ixgbe_tx_timeout_reset(adapter);
  833. /* the adapter is about to reset, no point in enabling stuff */
  834. return true;
  835. }
  836. netdev_tx_completed_queue(txring_txq(tx_ring),
  837. total_packets, total_bytes);
  838. #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
  839. if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
  840. (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
  841. /* Make sure that anybody stopping the queue after this
  842. * sees the new next_to_clean.
  843. */
  844. smp_mb();
  845. if (__netif_subqueue_stopped(tx_ring->netdev,
  846. tx_ring->queue_index)
  847. && !test_bit(__IXGBE_DOWN, &adapter->state)) {
  848. netif_wake_subqueue(tx_ring->netdev,
  849. tx_ring->queue_index);
  850. ++tx_ring->tx_stats.restart_queue;
  851. }
  852. }
  853. return !!budget;
  854. }
  855. #ifdef CONFIG_IXGBE_DCA
  856. static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
  857. struct ixgbe_ring *tx_ring,
  858. int cpu)
  859. {
  860. struct ixgbe_hw *hw = &adapter->hw;
  861. u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
  862. u16 reg_offset;
  863. switch (hw->mac.type) {
  864. case ixgbe_mac_82598EB:
  865. reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
  866. break;
  867. case ixgbe_mac_82599EB:
  868. case ixgbe_mac_X540:
  869. reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
  870. txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
  871. break;
  872. default:
  873. /* for unknown hardware do not write register */
  874. return;
  875. }
  876. /*
  877. * We can enable relaxed ordering for reads, but not writes when
  878. * DCA is enabled. This is due to a known issue in some chipsets
  879. * which will cause the DCA tag to be cleared.
  880. */
  881. txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
  882. IXGBE_DCA_TXCTRL_DATA_RRO_EN |
  883. IXGBE_DCA_TXCTRL_DESC_DCA_EN;
  884. IXGBE_WRITE_REG(hw, reg_offset, txctrl);
  885. }
  886. static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
  887. struct ixgbe_ring *rx_ring,
  888. int cpu)
  889. {
  890. struct ixgbe_hw *hw = &adapter->hw;
  891. u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu);
  892. u8 reg_idx = rx_ring->reg_idx;
  893. switch (hw->mac.type) {
  894. case ixgbe_mac_82599EB:
  895. case ixgbe_mac_X540:
  896. rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
  897. break;
  898. default:
  899. break;
  900. }
  901. /*
  902. * We can enable relaxed ordering for reads, but not writes when
  903. * DCA is enabled. This is due to a known issue in some chipsets
  904. * which will cause the DCA tag to be cleared.
  905. */
  906. rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
  907. IXGBE_DCA_RXCTRL_DESC_DCA_EN;
  908. IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
  909. }
  910. static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
  911. {
  912. struct ixgbe_adapter *adapter = q_vector->adapter;
  913. struct ixgbe_ring *ring;
  914. int cpu = get_cpu();
  915. if (q_vector->cpu == cpu)
  916. goto out_no_update;
  917. ixgbe_for_each_ring(ring, q_vector->tx)
  918. ixgbe_update_tx_dca(adapter, ring, cpu);
  919. ixgbe_for_each_ring(ring, q_vector->rx)
  920. ixgbe_update_rx_dca(adapter, ring, cpu);
  921. q_vector->cpu = cpu;
  922. out_no_update:
  923. put_cpu();
  924. }
  925. static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
  926. {
  927. int i;
  928. if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
  929. return;
  930. /* always use CB2 mode, difference is masked in the CB driver */
  931. IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
  932. for (i = 0; i < adapter->num_q_vectors; i++) {
  933. adapter->q_vector[i]->cpu = -1;
  934. ixgbe_update_dca(adapter->q_vector[i]);
  935. }
  936. }
  937. static int __ixgbe_notify_dca(struct device *dev, void *data)
  938. {
  939. struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
  940. unsigned long event = *(unsigned long *)data;
  941. if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
  942. return 0;
  943. switch (event) {
  944. case DCA_PROVIDER_ADD:
  945. /* if we're already enabled, don't do it again */
  946. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
  947. break;
  948. if (dca_add_requester(dev) == 0) {
  949. adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
  950. ixgbe_setup_dca(adapter);
  951. break;
  952. }
  953. /* Fall Through since DCA is disabled. */
  954. case DCA_PROVIDER_REMOVE:
  955. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
  956. dca_remove_requester(dev);
  957. adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
  958. IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
  959. }
  960. break;
  961. }
  962. return 0;
  963. }
  964. #endif /* CONFIG_IXGBE_DCA */
  965. static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
  966. union ixgbe_adv_rx_desc *rx_desc,
  967. struct sk_buff *skb)
  968. {
  969. if (ring->netdev->features & NETIF_F_RXHASH)
  970. skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
  971. }
  972. #ifdef IXGBE_FCOE
  973. /**
  974. * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
  975. * @ring: structure containing ring specific data
  976. * @rx_desc: advanced rx descriptor
  977. *
  978. * Returns : true if it is FCoE pkt
  979. */
  980. static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
  981. union ixgbe_adv_rx_desc *rx_desc)
  982. {
  983. __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
  984. return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
  985. ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
  986. (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
  987. IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
  988. }
  989. #endif /* IXGBE_FCOE */
  990. /**
  991. * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
  992. * @ring: structure containing ring specific data
  993. * @rx_desc: current Rx descriptor being processed
  994. * @skb: skb currently being received and modified
  995. **/
  996. static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
  997. union ixgbe_adv_rx_desc *rx_desc,
  998. struct sk_buff *skb)
  999. {
  1000. skb_checksum_none_assert(skb);
  1001. /* Rx csum disabled */
  1002. if (!(ring->netdev->features & NETIF_F_RXCSUM))
  1003. return;
  1004. /* if IP and error */
  1005. if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
  1006. ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
  1007. ring->rx_stats.csum_err++;
  1008. return;
  1009. }
  1010. if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
  1011. return;
  1012. if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
  1013. __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
  1014. /*
  1015. * 82599 errata, UDP frames with a 0 checksum can be marked as
  1016. * checksum errors.
  1017. */
  1018. if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
  1019. test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
  1020. return;
  1021. ring->rx_stats.csum_err++;
  1022. return;
  1023. }
  1024. /* It must be a TCP or UDP packet with a valid checksum */
  1025. skb->ip_summed = CHECKSUM_UNNECESSARY;
  1026. }
  1027. static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
  1028. {
  1029. rx_ring->next_to_use = val;
  1030. /* update next to alloc since we have filled the ring */
  1031. rx_ring->next_to_alloc = val;
  1032. /*
  1033. * Force memory writes to complete before letting h/w
  1034. * know there are new descriptors to fetch. (Only
  1035. * applicable for weak-ordered memory model archs,
  1036. * such as IA-64).
  1037. */
  1038. wmb();
  1039. writel(val, rx_ring->tail);
  1040. }
  1041. static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
  1042. struct ixgbe_rx_buffer *bi)
  1043. {
  1044. struct page *page = bi->page;
  1045. dma_addr_t dma = bi->dma;
  1046. /* since we are recycling buffers we should seldom need to alloc */
  1047. if (likely(dma))
  1048. return true;
  1049. /* alloc new page for storage */
  1050. if (likely(!page)) {
  1051. page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
  1052. bi->skb, ixgbe_rx_pg_order(rx_ring));
  1053. if (unlikely(!page)) {
  1054. rx_ring->rx_stats.alloc_rx_page_failed++;
  1055. return false;
  1056. }
  1057. bi->page = page;
  1058. }
  1059. /* map page for use */
  1060. dma = dma_map_page(rx_ring->dev, page, 0,
  1061. ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
  1062. /*
  1063. * if mapping failed free memory back to system since
  1064. * there isn't much point in holding memory we can't use
  1065. */
  1066. if (dma_mapping_error(rx_ring->dev, dma)) {
  1067. __free_pages(page, ixgbe_rx_pg_order(rx_ring));
  1068. bi->page = NULL;
  1069. rx_ring->rx_stats.alloc_rx_page_failed++;
  1070. return false;
  1071. }
  1072. bi->dma = dma;
  1073. bi->page_offset = 0;
  1074. return true;
  1075. }
  1076. /**
  1077. * ixgbe_alloc_rx_buffers - Replace used receive buffers
  1078. * @rx_ring: ring to place buffers on
  1079. * @cleaned_count: number of buffers to replace
  1080. **/
  1081. void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
  1082. {
  1083. union ixgbe_adv_rx_desc *rx_desc;
  1084. struct ixgbe_rx_buffer *bi;
  1085. u16 i = rx_ring->next_to_use;
  1086. /* nothing to do */
  1087. if (!cleaned_count)
  1088. return;
  1089. rx_desc = IXGBE_RX_DESC(rx_ring, i);
  1090. bi = &rx_ring->rx_buffer_info[i];
  1091. i -= rx_ring->count;
  1092. do {
  1093. if (!ixgbe_alloc_mapped_page(rx_ring, bi))
  1094. break;
  1095. /*
  1096. * Refresh the desc even if buffer_addrs didn't change
  1097. * because each write-back erases this info.
  1098. */
  1099. rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
  1100. rx_desc++;
  1101. bi++;
  1102. i++;
  1103. if (unlikely(!i)) {
  1104. rx_desc = IXGBE_RX_DESC(rx_ring, 0);
  1105. bi = rx_ring->rx_buffer_info;
  1106. i -= rx_ring->count;
  1107. }
  1108. /* clear the hdr_addr for the next_to_use descriptor */
  1109. rx_desc->read.hdr_addr = 0;
  1110. cleaned_count--;
  1111. } while (cleaned_count);
  1112. i += rx_ring->count;
  1113. if (rx_ring->next_to_use != i)
  1114. ixgbe_release_rx_desc(rx_ring, i);
  1115. }
  1116. /**
  1117. * ixgbe_get_headlen - determine size of header for RSC/LRO/GRO/FCOE
  1118. * @data: pointer to the start of the headers
  1119. * @max_len: total length of section to find headers in
  1120. *
  1121. * This function is meant to determine the length of headers that will
  1122. * be recognized by hardware for LRO, GRO, and RSC offloads. The main
  1123. * motivation of doing this is to only perform one pull for IPv4 TCP
  1124. * packets so that we can do basic things like calculating the gso_size
  1125. * based on the average data per packet.
  1126. **/
  1127. static unsigned int ixgbe_get_headlen(unsigned char *data,
  1128. unsigned int max_len)
  1129. {
  1130. union {
  1131. unsigned char *network;
  1132. /* l2 headers */
  1133. struct ethhdr *eth;
  1134. struct vlan_hdr *vlan;
  1135. /* l3 headers */
  1136. struct iphdr *ipv4;
  1137. struct ipv6hdr *ipv6;
  1138. } hdr;
  1139. __be16 protocol;
  1140. u8 nexthdr = 0; /* default to not TCP */
  1141. u8 hlen;
  1142. /* this should never happen, but better safe than sorry */
  1143. if (max_len < ETH_HLEN)
  1144. return max_len;
  1145. /* initialize network frame pointer */
  1146. hdr.network = data;
  1147. /* set first protocol and move network header forward */
  1148. protocol = hdr.eth->h_proto;
  1149. hdr.network += ETH_HLEN;
  1150. /* handle any vlan tag if present */
  1151. if (protocol == __constant_htons(ETH_P_8021Q)) {
  1152. if ((hdr.network - data) > (max_len - VLAN_HLEN))
  1153. return max_len;
  1154. protocol = hdr.vlan->h_vlan_encapsulated_proto;
  1155. hdr.network += VLAN_HLEN;
  1156. }
  1157. /* handle L3 protocols */
  1158. if (protocol == __constant_htons(ETH_P_IP)) {
  1159. if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
  1160. return max_len;
  1161. /* access ihl as a u8 to avoid unaligned access on ia64 */
  1162. hlen = (hdr.network[0] & 0x0F) << 2;
  1163. /* verify hlen meets minimum size requirements */
  1164. if (hlen < sizeof(struct iphdr))
  1165. return hdr.network - data;
  1166. /* record next protocol if header is present */
  1167. if (!hdr.ipv4->frag_off)
  1168. nexthdr = hdr.ipv4->protocol;
  1169. } else if (protocol == __constant_htons(ETH_P_IPV6)) {
  1170. if ((hdr.network - data) > (max_len - sizeof(struct ipv6hdr)))
  1171. return max_len;
  1172. /* record next protocol */
  1173. nexthdr = hdr.ipv6->nexthdr;
  1174. hlen = sizeof(struct ipv6hdr);
  1175. #ifdef IXGBE_FCOE
  1176. } else if (protocol == __constant_htons(ETH_P_FCOE)) {
  1177. if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN))
  1178. return max_len;
  1179. hlen = FCOE_HEADER_LEN;
  1180. #endif
  1181. } else {
  1182. return hdr.network - data;
  1183. }
  1184. /* relocate pointer to start of L4 header */
  1185. hdr.network += hlen;
  1186. /* finally sort out TCP/UDP */
  1187. if (nexthdr == IPPROTO_TCP) {
  1188. if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
  1189. return max_len;
  1190. /* access doff as a u8 to avoid unaligned access on ia64 */
  1191. hlen = (hdr.network[12] & 0xF0) >> 2;
  1192. /* verify hlen meets minimum size requirements */
  1193. if (hlen < sizeof(struct tcphdr))
  1194. return hdr.network - data;
  1195. hdr.network += hlen;
  1196. } else if (nexthdr == IPPROTO_UDP) {
  1197. if ((hdr.network - data) > (max_len - sizeof(struct udphdr)))
  1198. return max_len;
  1199. hdr.network += sizeof(struct udphdr);
  1200. }
  1201. /*
  1202. * If everything has gone correctly hdr.network should be the
  1203. * data section of the packet and will be the end of the header.
  1204. * If not then it probably represents the end of the last recognized
  1205. * header.
  1206. */
  1207. if ((hdr.network - data) < max_len)
  1208. return hdr.network - data;
  1209. else
  1210. return max_len;
  1211. }
  1212. static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
  1213. struct sk_buff *skb)
  1214. {
  1215. u16 hdr_len = skb_headlen(skb);
  1216. /* set gso_size to avoid messing up TCP MSS */
  1217. skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
  1218. IXGBE_CB(skb)->append_cnt);
  1219. }
  1220. static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
  1221. struct sk_buff *skb)
  1222. {
  1223. /* if append_cnt is 0 then frame is not RSC */
  1224. if (!IXGBE_CB(skb)->append_cnt)
  1225. return;
  1226. rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
  1227. rx_ring->rx_stats.rsc_flush++;
  1228. ixgbe_set_rsc_gso_size(rx_ring, skb);
  1229. /* gso_size is computed using append_cnt so always clear it last */
  1230. IXGBE_CB(skb)->append_cnt = 0;
  1231. }
  1232. /**
  1233. * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
  1234. * @rx_ring: rx descriptor ring packet is being transacted on
  1235. * @rx_desc: pointer to the EOP Rx descriptor
  1236. * @skb: pointer to current skb being populated
  1237. *
  1238. * This function checks the ring, descriptor, and packet information in
  1239. * order to populate the hash, checksum, VLAN, timestamp, protocol, and
  1240. * other fields within the skb.
  1241. **/
  1242. static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
  1243. union ixgbe_adv_rx_desc *rx_desc,
  1244. struct sk_buff *skb)
  1245. {
  1246. struct net_device *dev = rx_ring->netdev;
  1247. ixgbe_update_rsc_stats(rx_ring, skb);
  1248. ixgbe_rx_hash(rx_ring, rx_desc, skb);
  1249. ixgbe_rx_checksum(rx_ring, rx_desc, skb);
  1250. ixgbe_ptp_rx_hwtstamp(rx_ring->q_vector, rx_desc, skb);
  1251. if ((dev->features & NETIF_F_HW_VLAN_RX) &&
  1252. ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
  1253. u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
  1254. __vlan_hwaccel_put_tag(skb, vid);
  1255. }
  1256. skb_record_rx_queue(skb, rx_ring->queue_index);
  1257. skb->protocol = eth_type_trans(skb, dev);
  1258. }
  1259. static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
  1260. struct sk_buff *skb)
  1261. {
  1262. struct ixgbe_adapter *adapter = q_vector->adapter;
  1263. if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
  1264. napi_gro_receive(&q_vector->napi, skb);
  1265. else
  1266. netif_rx(skb);
  1267. }
  1268. /**
  1269. * ixgbe_is_non_eop - process handling of non-EOP buffers
  1270. * @rx_ring: Rx ring being processed
  1271. * @rx_desc: Rx descriptor for current buffer
  1272. * @skb: Current socket buffer containing buffer in progress
  1273. *
  1274. * This function updates next to clean. If the buffer is an EOP buffer
  1275. * this function exits returning false, otherwise it will place the
  1276. * sk_buff in the next buffer to be chained and return true indicating
  1277. * that this is in fact a non-EOP buffer.
  1278. **/
  1279. static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
  1280. union ixgbe_adv_rx_desc *rx_desc,
  1281. struct sk_buff *skb)
  1282. {
  1283. u32 ntc = rx_ring->next_to_clean + 1;
  1284. /* fetch, update, and store next to clean */
  1285. ntc = (ntc < rx_ring->count) ? ntc : 0;
  1286. rx_ring->next_to_clean = ntc;
  1287. prefetch(IXGBE_RX_DESC(rx_ring, ntc));
  1288. /* update RSC append count if present */
  1289. if (ring_is_rsc_enabled(rx_ring)) {
  1290. __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
  1291. cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
  1292. if (unlikely(rsc_enabled)) {
  1293. u32 rsc_cnt = le32_to_cpu(rsc_enabled);
  1294. rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
  1295. IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
  1296. /* update ntc based on RSC value */
  1297. ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
  1298. ntc &= IXGBE_RXDADV_NEXTP_MASK;
  1299. ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
  1300. }
  1301. }
  1302. /* if we are the last buffer then there is nothing else to do */
  1303. if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
  1304. return false;
  1305. /* place skb in next buffer to be received */
  1306. rx_ring->rx_buffer_info[ntc].skb = skb;
  1307. rx_ring->rx_stats.non_eop_descs++;
  1308. return true;
  1309. }
  1310. /**
  1311. * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
  1312. * @rx_ring: rx descriptor ring packet is being transacted on
  1313. * @skb: pointer to current skb being adjusted
  1314. *
  1315. * This function is an ixgbe specific version of __pskb_pull_tail. The
  1316. * main difference between this version and the original function is that
  1317. * this function can make several assumptions about the state of things
  1318. * that allow for significant optimizations versus the standard function.
  1319. * As a result we can do things like drop a frag and maintain an accurate
  1320. * truesize for the skb.
  1321. */
  1322. static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
  1323. struct sk_buff *skb)
  1324. {
  1325. struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
  1326. unsigned char *va;
  1327. unsigned int pull_len;
  1328. /*
  1329. * it is valid to use page_address instead of kmap since we are
  1330. * working with pages allocated out of the lomem pool per
  1331. * alloc_page(GFP_ATOMIC)
  1332. */
  1333. va = skb_frag_address(frag);
  1334. /*
  1335. * we need the header to contain the greater of either ETH_HLEN or
  1336. * 60 bytes if the skb->len is less than 60 for skb_pad.
  1337. */
  1338. pull_len = ixgbe_get_headlen(va, IXGBE_RX_HDR_SIZE);
  1339. /* align pull length to size of long to optimize memcpy performance */
  1340. skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
  1341. /* update all of the pointers */
  1342. skb_frag_size_sub(frag, pull_len);
  1343. frag->page_offset += pull_len;
  1344. skb->data_len -= pull_len;
  1345. skb->tail += pull_len;
  1346. }
  1347. /**
  1348. * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
  1349. * @rx_ring: rx descriptor ring packet is being transacted on
  1350. * @skb: pointer to current skb being updated
  1351. *
  1352. * This function provides a basic DMA sync up for the first fragment of an
  1353. * skb. The reason for doing this is that the first fragment cannot be
  1354. * unmapped until we have reached the end of packet descriptor for a buffer
  1355. * chain.
  1356. */
  1357. static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
  1358. struct sk_buff *skb)
  1359. {
  1360. /* if the page was released unmap it, else just sync our portion */
  1361. if (unlikely(IXGBE_CB(skb)->page_released)) {
  1362. dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma,
  1363. ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
  1364. IXGBE_CB(skb)->page_released = false;
  1365. } else {
  1366. struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
  1367. dma_sync_single_range_for_cpu(rx_ring->dev,
  1368. IXGBE_CB(skb)->dma,
  1369. frag->page_offset,
  1370. ixgbe_rx_bufsz(rx_ring),
  1371. DMA_FROM_DEVICE);
  1372. }
  1373. IXGBE_CB(skb)->dma = 0;
  1374. }
  1375. /**
  1376. * ixgbe_cleanup_headers - Correct corrupted or empty headers
  1377. * @rx_ring: rx descriptor ring packet is being transacted on
  1378. * @rx_desc: pointer to the EOP Rx descriptor
  1379. * @skb: pointer to current skb being fixed
  1380. *
  1381. * Check for corrupted packet headers caused by senders on the local L2
  1382. * embedded NIC switch not setting up their Tx Descriptors right. These
  1383. * should be very rare.
  1384. *
  1385. * Also address the case where we are pulling data in on pages only
  1386. * and as such no data is present in the skb header.
  1387. *
  1388. * In addition if skb is not at least 60 bytes we need to pad it so that
  1389. * it is large enough to qualify as a valid Ethernet frame.
  1390. *
  1391. * Returns true if an error was encountered and skb was freed.
  1392. **/
  1393. static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
  1394. union ixgbe_adv_rx_desc *rx_desc,
  1395. struct sk_buff *skb)
  1396. {
  1397. struct net_device *netdev = rx_ring->netdev;
  1398. /* verify that the packet does not have any known errors */
  1399. if (unlikely(ixgbe_test_staterr(rx_desc,
  1400. IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
  1401. !(netdev->features & NETIF_F_RXALL))) {
  1402. dev_kfree_skb_any(skb);
  1403. return true;
  1404. }
  1405. /* place header in linear portion of buffer */
  1406. if (skb_is_nonlinear(skb))
  1407. ixgbe_pull_tail(rx_ring, skb);
  1408. #ifdef IXGBE_FCOE
  1409. /* do not attempt to pad FCoE Frames as this will disrupt DDP */
  1410. if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
  1411. return false;
  1412. #endif
  1413. /* if skb_pad returns an error the skb was freed */
  1414. if (unlikely(skb->len < 60)) {
  1415. int pad_len = 60 - skb->len;
  1416. if (skb_pad(skb, pad_len))
  1417. return true;
  1418. __skb_put(skb, pad_len);
  1419. }
  1420. return false;
  1421. }
  1422. /**
  1423. * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
  1424. * @rx_ring: rx descriptor ring to store buffers on
  1425. * @old_buff: donor buffer to have page reused
  1426. *
  1427. * Synchronizes page for reuse by the adapter
  1428. **/
  1429. static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
  1430. struct ixgbe_rx_buffer *old_buff)
  1431. {
  1432. struct ixgbe_rx_buffer *new_buff;
  1433. u16 nta = rx_ring->next_to_alloc;
  1434. new_buff = &rx_ring->rx_buffer_info[nta];
  1435. /* update, and store next to alloc */
  1436. nta++;
  1437. rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
  1438. /* transfer page from old buffer to new buffer */
  1439. new_buff->page = old_buff->page;
  1440. new_buff->dma = old_buff->dma;
  1441. new_buff->page_offset = old_buff->page_offset;
  1442. /* sync the buffer for use by the device */
  1443. dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
  1444. new_buff->page_offset,
  1445. ixgbe_rx_bufsz(rx_ring),
  1446. DMA_FROM_DEVICE);
  1447. }
  1448. /**
  1449. * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
  1450. * @rx_ring: rx descriptor ring to transact packets on
  1451. * @rx_buffer: buffer containing page to add
  1452. * @rx_desc: descriptor containing length of buffer written by hardware
  1453. * @skb: sk_buff to place the data into
  1454. *
  1455. * This function will add the data contained in rx_buffer->page to the skb.
  1456. * This is done either through a direct copy if the data in the buffer is
  1457. * less than the skb header size, otherwise it will just attach the page as
  1458. * a frag to the skb.
  1459. *
  1460. * The function will then update the page offset if necessary and return
  1461. * true if the buffer can be reused by the adapter.
  1462. **/
  1463. static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
  1464. struct ixgbe_rx_buffer *rx_buffer,
  1465. union ixgbe_adv_rx_desc *rx_desc,
  1466. struct sk_buff *skb)
  1467. {
  1468. struct page *page = rx_buffer->page;
  1469. unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
  1470. #if (PAGE_SIZE < 8192)
  1471. unsigned int truesize = ixgbe_rx_bufsz(rx_ring);
  1472. #else
  1473. unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
  1474. unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) -
  1475. ixgbe_rx_bufsz(rx_ring);
  1476. #endif
  1477. if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
  1478. unsigned char *va = page_address(page) + rx_buffer->page_offset;
  1479. memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
  1480. /* we can reuse buffer as-is, just make sure it is local */
  1481. if (likely(page_to_nid(page) == numa_node_id()))
  1482. return true;
  1483. /* this page cannot be reused so discard it */
  1484. put_page(page);
  1485. return false;
  1486. }
  1487. skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
  1488. rx_buffer->page_offset, size, truesize);
  1489. /* avoid re-using remote pages */
  1490. if (unlikely(page_to_nid(page) != numa_node_id()))
  1491. return false;
  1492. #if (PAGE_SIZE < 8192)
  1493. /* if we are only owner of page we can reuse it */
  1494. if (unlikely(page_count(page) != 1))
  1495. return false;
  1496. /* flip page offset to other buffer */
  1497. rx_buffer->page_offset ^= truesize;
  1498. /*
  1499. * since we are the only owner of the page and we need to
  1500. * increment it, just set the value to 2 in order to avoid
  1501. * an unecessary locked operation
  1502. */
  1503. atomic_set(&page->_count, 2);
  1504. #else
  1505. /* move offset up to the next cache line */
  1506. rx_buffer->page_offset += truesize;
  1507. if (rx_buffer->page_offset > last_offset)
  1508. return false;
  1509. /* bump ref count on page before it is given to the stack */
  1510. get_page(page);
  1511. #endif
  1512. return true;
  1513. }
  1514. static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring,
  1515. union ixgbe_adv_rx_desc *rx_desc)
  1516. {
  1517. struct ixgbe_rx_buffer *rx_buffer;
  1518. struct sk_buff *skb;
  1519. struct page *page;
  1520. rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
  1521. page = rx_buffer->page;
  1522. prefetchw(page);
  1523. skb = rx_buffer->skb;
  1524. if (likely(!skb)) {
  1525. void *page_addr = page_address(page) +
  1526. rx_buffer->page_offset;
  1527. /* prefetch first cache line of first page */
  1528. prefetch(page_addr);
  1529. #if L1_CACHE_BYTES < 128
  1530. prefetch(page_addr + L1_CACHE_BYTES);
  1531. #endif
  1532. /* allocate a skb to store the frags */
  1533. skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
  1534. IXGBE_RX_HDR_SIZE);
  1535. if (unlikely(!skb)) {
  1536. rx_ring->rx_stats.alloc_rx_buff_failed++;
  1537. return NULL;
  1538. }
  1539. /*
  1540. * we will be copying header into skb->data in
  1541. * pskb_may_pull so it is in our interest to prefetch
  1542. * it now to avoid a possible cache miss
  1543. */
  1544. prefetchw(skb->data);
  1545. /*
  1546. * Delay unmapping of the first packet. It carries the
  1547. * header information, HW may still access the header
  1548. * after the writeback. Only unmap it when EOP is
  1549. * reached
  1550. */
  1551. if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
  1552. goto dma_sync;
  1553. IXGBE_CB(skb)->dma = rx_buffer->dma;
  1554. } else {
  1555. if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
  1556. ixgbe_dma_sync_frag(rx_ring, skb);
  1557. dma_sync:
  1558. /* we are reusing so sync this buffer for CPU use */
  1559. dma_sync_single_range_for_cpu(rx_ring->dev,
  1560. rx_buffer->dma,
  1561. rx_buffer->page_offset,
  1562. ixgbe_rx_bufsz(rx_ring),
  1563. DMA_FROM_DEVICE);
  1564. }
  1565. /* pull page into skb */
  1566. if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
  1567. /* hand second half of page back to the ring */
  1568. ixgbe_reuse_rx_page(rx_ring, rx_buffer);
  1569. } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) {
  1570. /* the page has been released from the ring */
  1571. IXGBE_CB(skb)->page_released = true;
  1572. } else {
  1573. /* we are not reusing the buffer so unmap it */
  1574. dma_unmap_page(rx_ring->dev, rx_buffer->dma,
  1575. ixgbe_rx_pg_size(rx_ring),
  1576. DMA_FROM_DEVICE);
  1577. }
  1578. /* clear contents of buffer_info */
  1579. rx_buffer->skb = NULL;
  1580. rx_buffer->dma = 0;
  1581. rx_buffer->page = NULL;
  1582. return skb;
  1583. }
  1584. /**
  1585. * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
  1586. * @q_vector: structure containing interrupt and ring information
  1587. * @rx_ring: rx descriptor ring to transact packets on
  1588. * @budget: Total limit on number of packets to process
  1589. *
  1590. * This function provides a "bounce buffer" approach to Rx interrupt
  1591. * processing. The advantage to this is that on systems that have
  1592. * expensive overhead for IOMMU access this provides a means of avoiding
  1593. * it by maintaining the mapping of the page to the syste.
  1594. *
  1595. * Returns true if all work is completed without reaching budget
  1596. **/
  1597. static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
  1598. struct ixgbe_ring *rx_ring,
  1599. const int budget)
  1600. {
  1601. unsigned int total_rx_bytes = 0, total_rx_packets = 0;
  1602. #ifdef IXGBE_FCOE
  1603. struct ixgbe_adapter *adapter = q_vector->adapter;
  1604. int ddp_bytes;
  1605. unsigned int mss = 0;
  1606. #endif /* IXGBE_FCOE */
  1607. u16 cleaned_count = ixgbe_desc_unused(rx_ring);
  1608. do {
  1609. union ixgbe_adv_rx_desc *rx_desc;
  1610. struct sk_buff *skb;
  1611. /* return some buffers to hardware, one at a time is too slow */
  1612. if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
  1613. ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
  1614. cleaned_count = 0;
  1615. }
  1616. rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
  1617. if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
  1618. break;
  1619. /*
  1620. * This memory barrier is needed to keep us from reading
  1621. * any other fields out of the rx_desc until we know the
  1622. * RXD_STAT_DD bit is set
  1623. */
  1624. rmb();
  1625. /* retrieve a buffer from the ring */
  1626. skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc);
  1627. /* exit if we failed to retrieve a buffer */
  1628. if (!skb)
  1629. break;
  1630. cleaned_count++;
  1631. /* place incomplete frames back on ring for completion */
  1632. if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
  1633. continue;
  1634. /* verify the packet layout is correct */
  1635. if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
  1636. continue;
  1637. /* probably a little skewed due to removing CRC */
  1638. total_rx_bytes += skb->len;
  1639. /* populate checksum, timestamp, VLAN, and protocol */
  1640. ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
  1641. #ifdef IXGBE_FCOE
  1642. /* if ddp, not passing to ULD unless for FCP_RSP or error */
  1643. if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
  1644. ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
  1645. /* include DDPed FCoE data */
  1646. if (ddp_bytes > 0) {
  1647. if (!mss) {
  1648. mss = rx_ring->netdev->mtu -
  1649. sizeof(struct fcoe_hdr) -
  1650. sizeof(struct fc_frame_header) -
  1651. sizeof(struct fcoe_crc_eof);
  1652. if (mss > 512)
  1653. mss &= ~511;
  1654. }
  1655. total_rx_bytes += ddp_bytes;
  1656. total_rx_packets += DIV_ROUND_UP(ddp_bytes,
  1657. mss);
  1658. }
  1659. if (!ddp_bytes) {
  1660. dev_kfree_skb_any(skb);
  1661. continue;
  1662. }
  1663. }
  1664. #endif /* IXGBE_FCOE */
  1665. ixgbe_rx_skb(q_vector, skb);
  1666. /* update budget accounting */
  1667. total_rx_packets++;
  1668. } while (likely(total_rx_packets < budget));
  1669. u64_stats_update_begin(&rx_ring->syncp);
  1670. rx_ring->stats.packets += total_rx_packets;
  1671. rx_ring->stats.bytes += total_rx_bytes;
  1672. u64_stats_update_end(&rx_ring->syncp);
  1673. q_vector->rx.total_packets += total_rx_packets;
  1674. q_vector->rx.total_bytes += total_rx_bytes;
  1675. if (cleaned_count)
  1676. ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
  1677. return (total_rx_packets < budget);
  1678. }
  1679. /**
  1680. * ixgbe_configure_msix - Configure MSI-X hardware
  1681. * @adapter: board private structure
  1682. *
  1683. * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
  1684. * interrupts.
  1685. **/
  1686. static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
  1687. {
  1688. struct ixgbe_q_vector *q_vector;
  1689. int v_idx;
  1690. u32 mask;
  1691. /* Populate MSIX to EITR Select */
  1692. if (adapter->num_vfs > 32) {
  1693. u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
  1694. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
  1695. }
  1696. /*
  1697. * Populate the IVAR table and set the ITR values to the
  1698. * corresponding register.
  1699. */
  1700. for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
  1701. struct ixgbe_ring *ring;
  1702. q_vector = adapter->q_vector[v_idx];
  1703. ixgbe_for_each_ring(ring, q_vector->rx)
  1704. ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
  1705. ixgbe_for_each_ring(ring, q_vector->tx)
  1706. ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
  1707. ixgbe_write_eitr(q_vector);
  1708. }
  1709. switch (adapter->hw.mac.type) {
  1710. case ixgbe_mac_82598EB:
  1711. ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
  1712. v_idx);
  1713. break;
  1714. case ixgbe_mac_82599EB:
  1715. case ixgbe_mac_X540:
  1716. ixgbe_set_ivar(adapter, -1, 1, v_idx);
  1717. break;
  1718. default:
  1719. break;
  1720. }
  1721. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
  1722. /* set up to autoclear timer, and the vectors */
  1723. mask = IXGBE_EIMS_ENABLE_MASK;
  1724. mask &= ~(IXGBE_EIMS_OTHER |
  1725. IXGBE_EIMS_MAILBOX |
  1726. IXGBE_EIMS_LSC);
  1727. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
  1728. }
  1729. enum latency_range {
  1730. lowest_latency = 0,
  1731. low_latency = 1,
  1732. bulk_latency = 2,
  1733. latency_invalid = 255
  1734. };
  1735. /**
  1736. * ixgbe_update_itr - update the dynamic ITR value based on statistics
  1737. * @q_vector: structure containing interrupt and ring information
  1738. * @ring_container: structure containing ring performance data
  1739. *
  1740. * Stores a new ITR value based on packets and byte
  1741. * counts during the last interrupt. The advantage of per interrupt
  1742. * computation is faster updates and more accurate ITR for the current
  1743. * traffic pattern. Constants in this function were computed
  1744. * based on theoretical maximum wire speed and thresholds were set based
  1745. * on testing data as well as attempting to minimize response time
  1746. * while increasing bulk throughput.
  1747. * this functionality is controlled by the InterruptThrottleRate module
  1748. * parameter (see ixgbe_param.c)
  1749. **/
  1750. static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
  1751. struct ixgbe_ring_container *ring_container)
  1752. {
  1753. int bytes = ring_container->total_bytes;
  1754. int packets = ring_container->total_packets;
  1755. u32 timepassed_us;
  1756. u64 bytes_perint;
  1757. u8 itr_setting = ring_container->itr;
  1758. if (packets == 0)
  1759. return;
  1760. /* simple throttlerate management
  1761. * 0-10MB/s lowest (100000 ints/s)
  1762. * 10-20MB/s low (20000 ints/s)
  1763. * 20-1249MB/s bulk (8000 ints/s)
  1764. */
  1765. /* what was last interrupt timeslice? */
  1766. timepassed_us = q_vector->itr >> 2;
  1767. bytes_perint = bytes / timepassed_us; /* bytes/usec */
  1768. switch (itr_setting) {
  1769. case lowest_latency:
  1770. if (bytes_perint > 10)
  1771. itr_setting = low_latency;
  1772. break;
  1773. case low_latency:
  1774. if (bytes_perint > 20)
  1775. itr_setting = bulk_latency;
  1776. else if (bytes_perint <= 10)
  1777. itr_setting = lowest_latency;
  1778. break;
  1779. case bulk_latency:
  1780. if (bytes_perint <= 20)
  1781. itr_setting = low_latency;
  1782. break;
  1783. }
  1784. /* clear work counters since we have the values we need */
  1785. ring_container->total_bytes = 0;
  1786. ring_container->total_packets = 0;
  1787. /* write updated itr to ring container */
  1788. ring_container->itr = itr_setting;
  1789. }
  1790. /**
  1791. * ixgbe_write_eitr - write EITR register in hardware specific way
  1792. * @q_vector: structure containing interrupt and ring information
  1793. *
  1794. * This function is made to be called by ethtool and by the driver
  1795. * when it needs to update EITR registers at runtime. Hardware
  1796. * specific quirks/differences are taken care of here.
  1797. */
  1798. void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
  1799. {
  1800. struct ixgbe_adapter *adapter = q_vector->adapter;
  1801. struct ixgbe_hw *hw = &adapter->hw;
  1802. int v_idx = q_vector->v_idx;
  1803. u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
  1804. switch (adapter->hw.mac.type) {
  1805. case ixgbe_mac_82598EB:
  1806. /* must write high and low 16 bits to reset counter */
  1807. itr_reg |= (itr_reg << 16);
  1808. break;
  1809. case ixgbe_mac_82599EB:
  1810. case ixgbe_mac_X540:
  1811. /*
  1812. * set the WDIS bit to not clear the timer bits and cause an
  1813. * immediate assertion of the interrupt
  1814. */
  1815. itr_reg |= IXGBE_EITR_CNT_WDIS;
  1816. break;
  1817. default:
  1818. break;
  1819. }
  1820. IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
  1821. }
  1822. static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
  1823. {
  1824. u32 new_itr = q_vector->itr;
  1825. u8 current_itr;
  1826. ixgbe_update_itr(q_vector, &q_vector->tx);
  1827. ixgbe_update_itr(q_vector, &q_vector->rx);
  1828. current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
  1829. switch (current_itr) {
  1830. /* counts and packets in update_itr are dependent on these numbers */
  1831. case lowest_latency:
  1832. new_itr = IXGBE_100K_ITR;
  1833. break;
  1834. case low_latency:
  1835. new_itr = IXGBE_20K_ITR;
  1836. break;
  1837. case bulk_latency:
  1838. new_itr = IXGBE_8K_ITR;
  1839. break;
  1840. default:
  1841. break;
  1842. }
  1843. if (new_itr != q_vector->itr) {
  1844. /* do an exponential smoothing */
  1845. new_itr = (10 * new_itr * q_vector->itr) /
  1846. ((9 * new_itr) + q_vector->itr);
  1847. /* save the algorithm value here */
  1848. q_vector->itr = new_itr;
  1849. ixgbe_write_eitr(q_vector);
  1850. }
  1851. }
  1852. /**
  1853. * ixgbe_check_overtemp_subtask - check for over temperature
  1854. * @adapter: pointer to adapter
  1855. **/
  1856. static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
  1857. {
  1858. struct ixgbe_hw *hw = &adapter->hw;
  1859. u32 eicr = adapter->interrupt_event;
  1860. if (test_bit(__IXGBE_DOWN, &adapter->state))
  1861. return;
  1862. if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
  1863. !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
  1864. return;
  1865. adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
  1866. switch (hw->device_id) {
  1867. case IXGBE_DEV_ID_82599_T3_LOM:
  1868. /*
  1869. * Since the warning interrupt is for both ports
  1870. * we don't have to check if:
  1871. * - This interrupt wasn't for our port.
  1872. * - We may have missed the interrupt so always have to
  1873. * check if we got a LSC
  1874. */
  1875. if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
  1876. !(eicr & IXGBE_EICR_LSC))
  1877. return;
  1878. if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
  1879. u32 autoneg;
  1880. bool link_up = false;
  1881. hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
  1882. if (link_up)
  1883. return;
  1884. }
  1885. /* Check if this is not due to overtemp */
  1886. if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
  1887. return;
  1888. break;
  1889. default:
  1890. if (!(eicr & IXGBE_EICR_GPI_SDP0))
  1891. return;
  1892. break;
  1893. }
  1894. e_crit(drv,
  1895. "Network adapter has been stopped because it has over heated. "
  1896. "Restart the computer. If the problem persists, "
  1897. "power off the system and replace the adapter\n");
  1898. adapter->interrupt_event = 0;
  1899. }
  1900. static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
  1901. {
  1902. struct ixgbe_hw *hw = &adapter->hw;
  1903. if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
  1904. (eicr & IXGBE_EICR_GPI_SDP1)) {
  1905. e_crit(probe, "Fan has stopped, replace the adapter\n");
  1906. /* write to clear the interrupt */
  1907. IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
  1908. }
  1909. }
  1910. static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
  1911. {
  1912. if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
  1913. return;
  1914. switch (adapter->hw.mac.type) {
  1915. case ixgbe_mac_82599EB:
  1916. /*
  1917. * Need to check link state so complete overtemp check
  1918. * on service task
  1919. */
  1920. if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
  1921. (!test_bit(__IXGBE_DOWN, &adapter->state))) {
  1922. adapter->interrupt_event = eicr;
  1923. adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
  1924. ixgbe_service_event_schedule(adapter);
  1925. return;
  1926. }
  1927. return;
  1928. case ixgbe_mac_X540:
  1929. if (!(eicr & IXGBE_EICR_TS))
  1930. return;
  1931. break;
  1932. default:
  1933. return;
  1934. }
  1935. e_crit(drv,
  1936. "Network adapter has been stopped because it has over heated. "
  1937. "Restart the computer. If the problem persists, "
  1938. "power off the system and replace the adapter\n");
  1939. }
  1940. static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
  1941. {
  1942. struct ixgbe_hw *hw = &adapter->hw;
  1943. if (eicr & IXGBE_EICR_GPI_SDP2) {
  1944. /* Clear the interrupt */
  1945. IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
  1946. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  1947. adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
  1948. ixgbe_service_event_schedule(adapter);
  1949. }
  1950. }
  1951. if (eicr & IXGBE_EICR_GPI_SDP1) {
  1952. /* Clear the interrupt */
  1953. IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
  1954. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  1955. adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
  1956. ixgbe_service_event_schedule(adapter);
  1957. }
  1958. }
  1959. }
  1960. static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
  1961. {
  1962. struct ixgbe_hw *hw = &adapter->hw;
  1963. adapter->lsc_int++;
  1964. adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
  1965. adapter->link_check_timeout = jiffies;
  1966. if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
  1967. IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
  1968. IXGBE_WRITE_FLUSH(hw);
  1969. ixgbe_service_event_schedule(adapter);
  1970. }
  1971. }
  1972. static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
  1973. u64 qmask)
  1974. {
  1975. u32 mask;
  1976. struct ixgbe_hw *hw = &adapter->hw;
  1977. switch (hw->mac.type) {
  1978. case ixgbe_mac_82598EB:
  1979. mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
  1980. IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
  1981. break;
  1982. case ixgbe_mac_82599EB:
  1983. case ixgbe_mac_X540:
  1984. mask = (qmask & 0xFFFFFFFF);
  1985. if (mask)
  1986. IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
  1987. mask = (qmask >> 32);
  1988. if (mask)
  1989. IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
  1990. break;
  1991. default:
  1992. break;
  1993. }
  1994. /* skip the flush */
  1995. }
  1996. static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
  1997. u64 qmask)
  1998. {
  1999. u32 mask;
  2000. struct ixgbe_hw *hw = &adapter->hw;
  2001. switch (hw->mac.type) {
  2002. case ixgbe_mac_82598EB:
  2003. mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
  2004. IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
  2005. break;
  2006. case ixgbe_mac_82599EB:
  2007. case ixgbe_mac_X540:
  2008. mask = (qmask & 0xFFFFFFFF);
  2009. if (mask)
  2010. IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
  2011. mask = (qmask >> 32);
  2012. if (mask)
  2013. IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
  2014. break;
  2015. default:
  2016. break;
  2017. }
  2018. /* skip the flush */
  2019. }
  2020. /**
  2021. * ixgbe_irq_enable - Enable default interrupt generation settings
  2022. * @adapter: board private structure
  2023. **/
  2024. static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
  2025. bool flush)
  2026. {
  2027. u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
  2028. /* don't reenable LSC while waiting for link */
  2029. if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
  2030. mask &= ~IXGBE_EIMS_LSC;
  2031. if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
  2032. switch (adapter->hw.mac.type) {
  2033. case ixgbe_mac_82599EB:
  2034. mask |= IXGBE_EIMS_GPI_SDP0;
  2035. break;
  2036. case ixgbe_mac_X540:
  2037. mask |= IXGBE_EIMS_TS;
  2038. break;
  2039. default:
  2040. break;
  2041. }
  2042. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
  2043. mask |= IXGBE_EIMS_GPI_SDP1;
  2044. switch (adapter->hw.mac.type) {
  2045. case ixgbe_mac_82599EB:
  2046. mask |= IXGBE_EIMS_GPI_SDP1;
  2047. mask |= IXGBE_EIMS_GPI_SDP2;
  2048. case ixgbe_mac_X540:
  2049. mask |= IXGBE_EIMS_ECC;
  2050. mask |= IXGBE_EIMS_MAILBOX;
  2051. break;
  2052. default:
  2053. break;
  2054. }
  2055. if (adapter->hw.mac.type == ixgbe_mac_X540)
  2056. mask |= IXGBE_EIMS_TIMESYNC;
  2057. if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
  2058. !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
  2059. mask |= IXGBE_EIMS_FLOW_DIR;
  2060. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
  2061. if (queues)
  2062. ixgbe_irq_enable_queues(adapter, ~0);
  2063. if (flush)
  2064. IXGBE_WRITE_FLUSH(&adapter->hw);
  2065. }
  2066. static irqreturn_t ixgbe_msix_other(int irq, void *data)
  2067. {
  2068. struct ixgbe_adapter *adapter = data;
  2069. struct ixgbe_hw *hw = &adapter->hw;
  2070. u32 eicr;
  2071. /*
  2072. * Workaround for Silicon errata. Use clear-by-write instead
  2073. * of clear-by-read. Reading with EICS will return the
  2074. * interrupt causes without clearing, which later be done
  2075. * with the write to EICR.
  2076. */
  2077. eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
  2078. IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
  2079. if (eicr & IXGBE_EICR_LSC)
  2080. ixgbe_check_lsc(adapter);
  2081. if (eicr & IXGBE_EICR_MAILBOX)
  2082. ixgbe_msg_task(adapter);
  2083. switch (hw->mac.type) {
  2084. case ixgbe_mac_82599EB:
  2085. case ixgbe_mac_X540:
  2086. if (eicr & IXGBE_EICR_ECC)
  2087. e_info(link, "Received unrecoverable ECC Err, please "
  2088. "reboot\n");
  2089. /* Handle Flow Director Full threshold interrupt */
  2090. if (eicr & IXGBE_EICR_FLOW_DIR) {
  2091. int reinit_count = 0;
  2092. int i;
  2093. for (i = 0; i < adapter->num_tx_queues; i++) {
  2094. struct ixgbe_ring *ring = adapter->tx_ring[i];
  2095. if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
  2096. &ring->state))
  2097. reinit_count++;
  2098. }
  2099. if (reinit_count) {
  2100. /* no more flow director interrupts until after init */
  2101. IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
  2102. adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
  2103. ixgbe_service_event_schedule(adapter);
  2104. }
  2105. }
  2106. ixgbe_check_sfp_event(adapter, eicr);
  2107. ixgbe_check_overtemp_event(adapter, eicr);
  2108. break;
  2109. default:
  2110. break;
  2111. }
  2112. ixgbe_check_fan_failure(adapter, eicr);
  2113. if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
  2114. ixgbe_ptp_check_pps_event(adapter, eicr);
  2115. /* re-enable the original interrupt state, no lsc, no queues */
  2116. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  2117. ixgbe_irq_enable(adapter, false, false);
  2118. return IRQ_HANDLED;
  2119. }
  2120. static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
  2121. {
  2122. struct ixgbe_q_vector *q_vector = data;
  2123. /* EIAM disabled interrupts (on this vector) for us */
  2124. if (q_vector->rx.ring || q_vector->tx.ring)
  2125. napi_schedule(&q_vector->napi);
  2126. return IRQ_HANDLED;
  2127. }
  2128. /**
  2129. * ixgbe_poll - NAPI Rx polling callback
  2130. * @napi: structure for representing this polling device
  2131. * @budget: how many packets driver is allowed to clean
  2132. *
  2133. * This function is used for legacy and MSI, NAPI mode
  2134. **/
  2135. int ixgbe_poll(struct napi_struct *napi, int budget)
  2136. {
  2137. struct ixgbe_q_vector *q_vector =
  2138. container_of(napi, struct ixgbe_q_vector, napi);
  2139. struct ixgbe_adapter *adapter = q_vector->adapter;
  2140. struct ixgbe_ring *ring;
  2141. int per_ring_budget;
  2142. bool clean_complete = true;
  2143. #ifdef CONFIG_IXGBE_DCA
  2144. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
  2145. ixgbe_update_dca(q_vector);
  2146. #endif
  2147. ixgbe_for_each_ring(ring, q_vector->tx)
  2148. clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring);
  2149. /* attempt to distribute budget to each queue fairly, but don't allow
  2150. * the budget to go below 1 because we'll exit polling */
  2151. if (q_vector->rx.count > 1)
  2152. per_ring_budget = max(budget/q_vector->rx.count, 1);
  2153. else
  2154. per_ring_budget = budget;
  2155. ixgbe_for_each_ring(ring, q_vector->rx)
  2156. clean_complete &= ixgbe_clean_rx_irq(q_vector, ring,
  2157. per_ring_budget);
  2158. /* If all work not completed, return budget and keep polling */
  2159. if (!clean_complete)
  2160. return budget;
  2161. /* all work done, exit the polling mode */
  2162. napi_complete(napi);
  2163. if (adapter->rx_itr_setting & 1)
  2164. ixgbe_set_itr(q_vector);
  2165. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  2166. ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));
  2167. return 0;
  2168. }
  2169. /**
  2170. * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
  2171. * @adapter: board private structure
  2172. *
  2173. * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
  2174. * interrupts from the kernel.
  2175. **/
  2176. static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
  2177. {
  2178. struct net_device *netdev = adapter->netdev;
  2179. int vector, err;
  2180. int ri = 0, ti = 0;
  2181. for (vector = 0; vector < adapter->num_q_vectors; vector++) {
  2182. struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
  2183. struct msix_entry *entry = &adapter->msix_entries[vector];
  2184. if (q_vector->tx.ring && q_vector->rx.ring) {
  2185. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  2186. "%s-%s-%d", netdev->name, "TxRx", ri++);
  2187. ti++;
  2188. } else if (q_vector->rx.ring) {
  2189. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  2190. "%s-%s-%d", netdev->name, "rx", ri++);
  2191. } else if (q_vector->tx.ring) {
  2192. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  2193. "%s-%s-%d", netdev->name, "tx", ti++);
  2194. } else {
  2195. /* skip this unused q_vector */
  2196. continue;
  2197. }
  2198. err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
  2199. q_vector->name, q_vector);
  2200. if (err) {
  2201. e_err(probe, "request_irq failed for MSIX interrupt "
  2202. "Error: %d\n", err);
  2203. goto free_queue_irqs;
  2204. }
  2205. /* If Flow Director is enabled, set interrupt affinity */
  2206. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  2207. /* assign the mask for this irq */
  2208. irq_set_affinity_hint(entry->vector,
  2209. &q_vector->affinity_mask);
  2210. }
  2211. }
  2212. err = request_irq(adapter->msix_entries[vector].vector,
  2213. ixgbe_msix_other, 0, netdev->name, adapter);
  2214. if (err) {
  2215. e_err(probe, "request_irq for msix_other failed: %d\n", err);
  2216. goto free_queue_irqs;
  2217. }
  2218. return 0;
  2219. free_queue_irqs:
  2220. while (vector) {
  2221. vector--;
  2222. irq_set_affinity_hint(adapter->msix_entries[vector].vector,
  2223. NULL);
  2224. free_irq(adapter->msix_entries[vector].vector,
  2225. adapter->q_vector[vector]);
  2226. }
  2227. adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
  2228. pci_disable_msix(adapter->pdev);
  2229. kfree(adapter->msix_entries);
  2230. adapter->msix_entries = NULL;
  2231. return err;
  2232. }
  2233. /**
  2234. * ixgbe_intr - legacy mode Interrupt Handler
  2235. * @irq: interrupt number
  2236. * @data: pointer to a network interface device structure
  2237. **/
  2238. static irqreturn_t ixgbe_intr(int irq, void *data)
  2239. {
  2240. struct ixgbe_adapter *adapter = data;
  2241. struct ixgbe_hw *hw = &adapter->hw;
  2242. struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
  2243. u32 eicr;
  2244. /*
  2245. * Workaround for silicon errata #26 on 82598. Mask the interrupt
  2246. * before the read of EICR.
  2247. */
  2248. IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
  2249. /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
  2250. * therefore no explicit interrupt disable is necessary */
  2251. eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
  2252. if (!eicr) {
  2253. /*
  2254. * shared interrupt alert!
  2255. * make sure interrupts are enabled because the read will
  2256. * have disabled interrupts due to EIAM
  2257. * finish the workaround of silicon errata on 82598. Unmask
  2258. * the interrupt that we masked before the EICR read.
  2259. */
  2260. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  2261. ixgbe_irq_enable(adapter, true, true);
  2262. return IRQ_NONE; /* Not our interrupt */
  2263. }
  2264. if (eicr & IXGBE_EICR_LSC)
  2265. ixgbe_check_lsc(adapter);
  2266. switch (hw->mac.type) {
  2267. case ixgbe_mac_82599EB:
  2268. ixgbe_check_sfp_event(adapter, eicr);
  2269. /* Fall through */
  2270. case ixgbe_mac_X540:
  2271. if (eicr & IXGBE_EICR_ECC)
  2272. e_info(link, "Received unrecoverable ECC err, please "
  2273. "reboot\n");
  2274. ixgbe_check_overtemp_event(adapter, eicr);
  2275. break;
  2276. default:
  2277. break;
  2278. }
  2279. ixgbe_check_fan_failure(adapter, eicr);
  2280. if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
  2281. ixgbe_ptp_check_pps_event(adapter, eicr);
  2282. /* would disable interrupts here but EIAM disabled it */
  2283. napi_schedule(&q_vector->napi);
  2284. /*
  2285. * re-enable link(maybe) and non-queue interrupts, no flush.
  2286. * ixgbe_poll will re-enable the queue interrupts
  2287. */
  2288. if (!test_bit(__IXGBE_DOWN, &adapter->state))
  2289. ixgbe_irq_enable(adapter, false, false);
  2290. return IRQ_HANDLED;
  2291. }
  2292. /**
  2293. * ixgbe_request_irq - initialize interrupts
  2294. * @adapter: board private structure
  2295. *
  2296. * Attempts to configure interrupts using the best available
  2297. * capabilities of the hardware and kernel.
  2298. **/
  2299. static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
  2300. {
  2301. struct net_device *netdev = adapter->netdev;
  2302. int err;
  2303. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  2304. err = ixgbe_request_msix_irqs(adapter);
  2305. else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
  2306. err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
  2307. netdev->name, adapter);
  2308. else
  2309. err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
  2310. netdev->name, adapter);
  2311. if (err)
  2312. e_err(probe, "request_irq failed, Error %d\n", err);
  2313. return err;
  2314. }
  2315. static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
  2316. {
  2317. int vector;
  2318. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
  2319. free_irq(adapter->pdev->irq, adapter);
  2320. return;
  2321. }
  2322. for (vector = 0; vector < adapter->num_q_vectors; vector++) {
  2323. struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
  2324. struct msix_entry *entry = &adapter->msix_entries[vector];
  2325. /* free only the irqs that were actually requested */
  2326. if (!q_vector->rx.ring && !q_vector->tx.ring)
  2327. continue;
  2328. /* clear the affinity_mask in the IRQ descriptor */
  2329. irq_set_affinity_hint(entry->vector, NULL);
  2330. free_irq(entry->vector, q_vector);
  2331. }
  2332. free_irq(adapter->msix_entries[vector++].vector, adapter);
  2333. }
  2334. /**
  2335. * ixgbe_irq_disable - Mask off interrupt generation on the NIC
  2336. * @adapter: board private structure
  2337. **/
  2338. static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
  2339. {
  2340. switch (adapter->hw.mac.type) {
  2341. case ixgbe_mac_82598EB:
  2342. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
  2343. break;
  2344. case ixgbe_mac_82599EB:
  2345. case ixgbe_mac_X540:
  2346. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
  2347. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
  2348. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
  2349. break;
  2350. default:
  2351. break;
  2352. }
  2353. IXGBE_WRITE_FLUSH(&adapter->hw);
  2354. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  2355. int vector;
  2356. for (vector = 0; vector < adapter->num_q_vectors; vector++)
  2357. synchronize_irq(adapter->msix_entries[vector].vector);
  2358. synchronize_irq(adapter->msix_entries[vector++].vector);
  2359. } else {
  2360. synchronize_irq(adapter->pdev->irq);
  2361. }
  2362. }
  2363. /**
  2364. * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
  2365. *
  2366. **/
  2367. static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
  2368. {
  2369. struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
  2370. ixgbe_write_eitr(q_vector);
  2371. ixgbe_set_ivar(adapter, 0, 0, 0);
  2372. ixgbe_set_ivar(adapter, 1, 0, 0);
  2373. e_info(hw, "Legacy interrupt IVAR setup done\n");
  2374. }
  2375. /**
  2376. * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
  2377. * @adapter: board private structure
  2378. * @ring: structure containing ring specific data
  2379. *
  2380. * Configure the Tx descriptor ring after a reset.
  2381. **/
  2382. void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
  2383. struct ixgbe_ring *ring)
  2384. {
  2385. struct ixgbe_hw *hw = &adapter->hw;
  2386. u64 tdba = ring->dma;
  2387. int wait_loop = 10;
  2388. u32 txdctl = IXGBE_TXDCTL_ENABLE;
  2389. u8 reg_idx = ring->reg_idx;
  2390. /* disable queue to avoid issues while updating state */
  2391. IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
  2392. IXGBE_WRITE_FLUSH(hw);
  2393. IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
  2394. (tdba & DMA_BIT_MASK(32)));
  2395. IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
  2396. IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
  2397. ring->count * sizeof(union ixgbe_adv_tx_desc));
  2398. IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
  2399. IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
  2400. ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
  2401. /*
  2402. * set WTHRESH to encourage burst writeback, it should not be set
  2403. * higher than 1 when ITR is 0 as it could cause false TX hangs
  2404. *
  2405. * In order to avoid issues WTHRESH + PTHRESH should always be equal
  2406. * to or less than the number of on chip descriptors, which is
  2407. * currently 40.
  2408. */
  2409. if (!ring->q_vector || (ring->q_vector->itr < 8))
  2410. txdctl |= (1 << 16); /* WTHRESH = 1 */
  2411. else
  2412. txdctl |= (8 << 16); /* WTHRESH = 8 */
  2413. /*
  2414. * Setting PTHRESH to 32 both improves performance
  2415. * and avoids a TX hang with DFP enabled
  2416. */
  2417. txdctl |= (1 << 8) | /* HTHRESH = 1 */
  2418. 32; /* PTHRESH = 32 */
  2419. /* reinitialize flowdirector state */
  2420. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  2421. ring->atr_sample_rate = adapter->atr_sample_rate;
  2422. ring->atr_count = 0;
  2423. set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
  2424. } else {
  2425. ring->atr_sample_rate = 0;
  2426. }
  2427. clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
  2428. /* enable queue */
  2429. IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
  2430. /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
  2431. if (hw->mac.type == ixgbe_mac_82598EB &&
  2432. !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
  2433. return;
  2434. /* poll to verify queue is enabled */
  2435. do {
  2436. usleep_range(1000, 2000);
  2437. txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
  2438. } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
  2439. if (!wait_loop)
  2440. e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
  2441. }
  2442. static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
  2443. {
  2444. struct ixgbe_hw *hw = &adapter->hw;
  2445. u32 rttdcs, mtqc;
  2446. u8 tcs = netdev_get_num_tc(adapter->netdev);
  2447. if (hw->mac.type == ixgbe_mac_82598EB)
  2448. return;
  2449. /* disable the arbiter while setting MTQC */
  2450. rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
  2451. rttdcs |= IXGBE_RTTDCS_ARBDIS;
  2452. IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
  2453. /* set transmit pool layout */
  2454. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  2455. mtqc = IXGBE_MTQC_VT_ENA;
  2456. if (tcs > 4)
  2457. mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
  2458. else if (tcs > 1)
  2459. mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
  2460. else if (adapter->ring_feature[RING_F_RSS].indices == 4)
  2461. mtqc |= IXGBE_MTQC_32VF;
  2462. else
  2463. mtqc |= IXGBE_MTQC_64VF;
  2464. } else {
  2465. if (tcs > 4)
  2466. mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
  2467. else if (tcs > 1)
  2468. mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
  2469. else
  2470. mtqc = IXGBE_MTQC_64Q_1PB;
  2471. }
  2472. IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
  2473. /* Enable Security TX Buffer IFG for multiple pb */
  2474. if (tcs) {
  2475. u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
  2476. sectx |= IXGBE_SECTX_DCB;
  2477. IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
  2478. }
  2479. /* re-enable the arbiter */
  2480. rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
  2481. IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
  2482. }
  2483. /**
  2484. * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
  2485. * @adapter: board private structure
  2486. *
  2487. * Configure the Tx unit of the MAC after a reset.
  2488. **/
  2489. static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
  2490. {
  2491. struct ixgbe_hw *hw = &adapter->hw;
  2492. u32 dmatxctl;
  2493. u32 i;
  2494. ixgbe_setup_mtqc(adapter);
  2495. if (hw->mac.type != ixgbe_mac_82598EB) {
  2496. /* DMATXCTL.EN must be before Tx queues are enabled */
  2497. dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
  2498. dmatxctl |= IXGBE_DMATXCTL_TE;
  2499. IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
  2500. }
  2501. /* Setup the HW Tx Head and Tail descriptor pointers */
  2502. for (i = 0; i < adapter->num_tx_queues; i++)
  2503. ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
  2504. }
  2505. static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
  2506. struct ixgbe_ring *ring)
  2507. {
  2508. struct ixgbe_hw *hw = &adapter->hw;
  2509. u8 reg_idx = ring->reg_idx;
  2510. u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
  2511. srrctl |= IXGBE_SRRCTL_DROP_EN;
  2512. IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
  2513. }
  2514. static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
  2515. struct ixgbe_ring *ring)
  2516. {
  2517. struct ixgbe_hw *hw = &adapter->hw;
  2518. u8 reg_idx = ring->reg_idx;
  2519. u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
  2520. srrctl &= ~IXGBE_SRRCTL_DROP_EN;
  2521. IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
  2522. }
  2523. #ifdef CONFIG_IXGBE_DCB
  2524. void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
  2525. #else
  2526. static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
  2527. #endif
  2528. {
  2529. int i;
  2530. bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
  2531. if (adapter->ixgbe_ieee_pfc)
  2532. pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
  2533. /*
  2534. * We should set the drop enable bit if:
  2535. * SR-IOV is enabled
  2536. * or
  2537. * Number of Rx queues > 1 and flow control is disabled
  2538. *
  2539. * This allows us to avoid head of line blocking for security
  2540. * and performance reasons.
  2541. */
  2542. if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
  2543. !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
  2544. for (i = 0; i < adapter->num_rx_queues; i++)
  2545. ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
  2546. } else {
  2547. for (i = 0; i < adapter->num_rx_queues; i++)
  2548. ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
  2549. }
  2550. }
  2551. #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
  2552. static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
  2553. struct ixgbe_ring *rx_ring)
  2554. {
  2555. struct ixgbe_hw *hw = &adapter->hw;
  2556. u32 srrctl;
  2557. u8 reg_idx = rx_ring->reg_idx;
  2558. if (hw->mac.type == ixgbe_mac_82598EB) {
  2559. u16 mask = adapter->ring_feature[RING_F_RSS].mask;
  2560. /*
  2561. * if VMDq is not active we must program one srrctl register
  2562. * per RSS queue since we have enabled RDRXCTL.MVMEN
  2563. */
  2564. reg_idx &= mask;
  2565. }
  2566. /* configure header buffer length, needed for RSC */
  2567. srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
  2568. /* configure the packet buffer length */
  2569. srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
  2570. /* configure descriptor type */
  2571. srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
  2572. IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
  2573. }
  2574. static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
  2575. {
  2576. struct ixgbe_hw *hw = &adapter->hw;
  2577. static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
  2578. 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
  2579. 0x6A3E67EA, 0x14364D17, 0x3BED200D};
  2580. u32 mrqc = 0, reta = 0;
  2581. u32 rxcsum;
  2582. int i, j;
  2583. u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
  2584. /*
  2585. * Program table for at least 2 queues w/ SR-IOV so that VFs can
  2586. * make full use of any rings they may have. We will use the
  2587. * PSRTYPE register to control how many rings we use within the PF.
  2588. */
  2589. if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2))
  2590. rss_i = 2;
  2591. /* Fill out hash function seeds */
  2592. for (i = 0; i < 10; i++)
  2593. IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
  2594. /* Fill out redirection table */
  2595. for (i = 0, j = 0; i < 128; i++, j++) {
  2596. if (j == rss_i)
  2597. j = 0;
  2598. /* reta = 4-byte sliding window of
  2599. * 0x00..(indices-1)(indices-1)00..etc. */
  2600. reta = (reta << 8) | (j * 0x11);
  2601. if ((i & 3) == 3)
  2602. IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
  2603. }
  2604. /* Disable indicating checksum in descriptor, enables RSS hash */
  2605. rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
  2606. rxcsum |= IXGBE_RXCSUM_PCSD;
  2607. IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
  2608. if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
  2609. if (adapter->ring_feature[RING_F_RSS].mask)
  2610. mrqc = IXGBE_MRQC_RSSEN;
  2611. } else {
  2612. u8 tcs = netdev_get_num_tc(adapter->netdev);
  2613. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  2614. if (tcs > 4)
  2615. mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */
  2616. else if (tcs > 1)
  2617. mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */
  2618. else if (adapter->ring_feature[RING_F_RSS].indices == 4)
  2619. mrqc = IXGBE_MRQC_VMDQRSS32EN;
  2620. else
  2621. mrqc = IXGBE_MRQC_VMDQRSS64EN;
  2622. } else {
  2623. if (tcs > 4)
  2624. mrqc = IXGBE_MRQC_RTRSS8TCEN;
  2625. else if (tcs > 1)
  2626. mrqc = IXGBE_MRQC_RTRSS4TCEN;
  2627. else
  2628. mrqc = IXGBE_MRQC_RSSEN;
  2629. }
  2630. }
  2631. /* Perform hash on these packet types */
  2632. mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 |
  2633. IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
  2634. IXGBE_MRQC_RSS_FIELD_IPV6 |
  2635. IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
  2636. if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
  2637. mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
  2638. if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
  2639. mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
  2640. IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
  2641. }
  2642. /**
  2643. * ixgbe_configure_rscctl - enable RSC for the indicated ring
  2644. * @adapter: address of board private structure
  2645. * @index: index of ring to set
  2646. **/
  2647. static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
  2648. struct ixgbe_ring *ring)
  2649. {
  2650. struct ixgbe_hw *hw = &adapter->hw;
  2651. u32 rscctrl;
  2652. u8 reg_idx = ring->reg_idx;
  2653. if (!ring_is_rsc_enabled(ring))
  2654. return;
  2655. rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
  2656. rscctrl |= IXGBE_RSCCTL_RSCEN;
  2657. /*
  2658. * we must limit the number of descriptors so that the
  2659. * total size of max desc * buf_len is not greater
  2660. * than 65536
  2661. */
  2662. rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
  2663. IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
  2664. }
  2665. #define IXGBE_MAX_RX_DESC_POLL 10
  2666. static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
  2667. struct ixgbe_ring *ring)
  2668. {
  2669. struct ixgbe_hw *hw = &adapter->hw;
  2670. int wait_loop = IXGBE_MAX_RX_DESC_POLL;
  2671. u32 rxdctl;
  2672. u8 reg_idx = ring->reg_idx;
  2673. /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
  2674. if (hw->mac.type == ixgbe_mac_82598EB &&
  2675. !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
  2676. return;
  2677. do {
  2678. usleep_range(1000, 2000);
  2679. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2680. } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
  2681. if (!wait_loop) {
  2682. e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
  2683. "the polling period\n", reg_idx);
  2684. }
  2685. }
  2686. void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
  2687. struct ixgbe_ring *ring)
  2688. {
  2689. struct ixgbe_hw *hw = &adapter->hw;
  2690. int wait_loop = IXGBE_MAX_RX_DESC_POLL;
  2691. u32 rxdctl;
  2692. u8 reg_idx = ring->reg_idx;
  2693. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2694. rxdctl &= ~IXGBE_RXDCTL_ENABLE;
  2695. /* write value back with RXDCTL.ENABLE bit cleared */
  2696. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
  2697. if (hw->mac.type == ixgbe_mac_82598EB &&
  2698. !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
  2699. return;
  2700. /* the hardware may take up to 100us to really disable the rx queue */
  2701. do {
  2702. udelay(10);
  2703. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2704. } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
  2705. if (!wait_loop) {
  2706. e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
  2707. "the polling period\n", reg_idx);
  2708. }
  2709. }
  2710. void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
  2711. struct ixgbe_ring *ring)
  2712. {
  2713. struct ixgbe_hw *hw = &adapter->hw;
  2714. u64 rdba = ring->dma;
  2715. u32 rxdctl;
  2716. u8 reg_idx = ring->reg_idx;
  2717. /* disable queue to avoid issues while updating state */
  2718. rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
  2719. ixgbe_disable_rx_queue(adapter, ring);
  2720. IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
  2721. IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
  2722. IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
  2723. ring->count * sizeof(union ixgbe_adv_rx_desc));
  2724. IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
  2725. IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
  2726. ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
  2727. ixgbe_configure_srrctl(adapter, ring);
  2728. ixgbe_configure_rscctl(adapter, ring);
  2729. if (hw->mac.type == ixgbe_mac_82598EB) {
  2730. /*
  2731. * enable cache line friendly hardware writes:
  2732. * PTHRESH=32 descriptors (half the internal cache),
  2733. * this also removes ugly rx_no_buffer_count increment
  2734. * HTHRESH=4 descriptors (to minimize latency on fetch)
  2735. * WTHRESH=8 burst writeback up to two cache lines
  2736. */
  2737. rxdctl &= ~0x3FFFFF;
  2738. rxdctl |= 0x080420;
  2739. }
  2740. /* enable receive descriptor ring */
  2741. rxdctl |= IXGBE_RXDCTL_ENABLE;
  2742. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
  2743. ixgbe_rx_desc_queue_enable(adapter, ring);
  2744. ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
  2745. }
  2746. static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
  2747. {
  2748. struct ixgbe_hw *hw = &adapter->hw;
  2749. int rss_i = adapter->ring_feature[RING_F_RSS].indices;
  2750. int p;
  2751. /* PSRTYPE must be initialized in non 82598 adapters */
  2752. u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
  2753. IXGBE_PSRTYPE_UDPHDR |
  2754. IXGBE_PSRTYPE_IPV4HDR |
  2755. IXGBE_PSRTYPE_L2HDR |
  2756. IXGBE_PSRTYPE_IPV6HDR;
  2757. if (hw->mac.type == ixgbe_mac_82598EB)
  2758. return;
  2759. if (rss_i > 3)
  2760. psrtype |= 2 << 29;
  2761. else if (rss_i > 1)
  2762. psrtype |= 1 << 29;
  2763. for (p = 0; p < adapter->num_rx_pools; p++)
  2764. IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(p)),
  2765. psrtype);
  2766. }
  2767. static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
  2768. {
  2769. struct ixgbe_hw *hw = &adapter->hw;
  2770. u32 reg_offset, vf_shift;
  2771. u32 gcr_ext, vmdctl;
  2772. int i;
  2773. if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
  2774. return;
  2775. vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
  2776. vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
  2777. vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
  2778. vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
  2779. vmdctl |= IXGBE_VT_CTL_REPLEN;
  2780. IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
  2781. vf_shift = VMDQ_P(0) % 32;
  2782. reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
  2783. /* Enable only the PF's pool for Tx/Rx */
  2784. IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (~0) << vf_shift);
  2785. IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
  2786. IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (~0) << vf_shift);
  2787. IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
  2788. if (adapter->flags2 & IXGBE_FLAG2_BRIDGE_MODE_VEB)
  2789. IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
  2790. /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
  2791. hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));
  2792. /*
  2793. * Set up VF register offsets for selected VT Mode,
  2794. * i.e. 32 or 64 VFs for SR-IOV
  2795. */
  2796. switch (adapter->ring_feature[RING_F_VMDQ].mask) {
  2797. case IXGBE_82599_VMDQ_8Q_MASK:
  2798. gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
  2799. break;
  2800. case IXGBE_82599_VMDQ_4Q_MASK:
  2801. gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
  2802. break;
  2803. default:
  2804. gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
  2805. break;
  2806. }
  2807. IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
  2808. /* Enable MAC Anti-Spoofing */
  2809. hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0),
  2810. adapter->num_vfs);
  2811. /* For VFs that have spoof checking turned off */
  2812. for (i = 0; i < adapter->num_vfs; i++) {
  2813. if (!adapter->vfinfo[i].spoofchk_enabled)
  2814. ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
  2815. }
  2816. }
  2817. static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
  2818. {
  2819. struct ixgbe_hw *hw = &adapter->hw;
  2820. struct net_device *netdev = adapter->netdev;
  2821. int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
  2822. struct ixgbe_ring *rx_ring;
  2823. int i;
  2824. u32 mhadd, hlreg0;
  2825. #ifdef IXGBE_FCOE
  2826. /* adjust max frame to be able to do baby jumbo for FCoE */
  2827. if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
  2828. (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
  2829. max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
  2830. #endif /* IXGBE_FCOE */
  2831. /* adjust max frame to be at least the size of a standard frame */
  2832. if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
  2833. max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);
  2834. mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
  2835. if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
  2836. mhadd &= ~IXGBE_MHADD_MFS_MASK;
  2837. mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
  2838. IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
  2839. }
  2840. hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
  2841. /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
  2842. hlreg0 |= IXGBE_HLREG0_JUMBOEN;
  2843. IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
  2844. /*
  2845. * Setup the HW Rx Head and Tail Descriptor Pointers and
  2846. * the Base and Length of the Rx Descriptor Ring
  2847. */
  2848. for (i = 0; i < adapter->num_rx_queues; i++) {
  2849. rx_ring = adapter->rx_ring[i];
  2850. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
  2851. set_ring_rsc_enabled(rx_ring);
  2852. else
  2853. clear_ring_rsc_enabled(rx_ring);
  2854. }
  2855. }
  2856. static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
  2857. {
  2858. struct ixgbe_hw *hw = &adapter->hw;
  2859. u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
  2860. switch (hw->mac.type) {
  2861. case ixgbe_mac_82598EB:
  2862. /*
  2863. * For VMDq support of different descriptor types or
  2864. * buffer sizes through the use of multiple SRRCTL
  2865. * registers, RDRXCTL.MVMEN must be set to 1
  2866. *
  2867. * also, the manual doesn't mention it clearly but DCA hints
  2868. * will only use queue 0's tags unless this bit is set. Side
  2869. * effects of setting this bit are only that SRRCTL must be
  2870. * fully programmed [0..15]
  2871. */
  2872. rdrxctl |= IXGBE_RDRXCTL_MVMEN;
  2873. break;
  2874. case ixgbe_mac_82599EB:
  2875. case ixgbe_mac_X540:
  2876. /* Disable RSC for ACK packets */
  2877. IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
  2878. (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
  2879. rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
  2880. /* hardware requires some bits to be set by default */
  2881. rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
  2882. rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
  2883. break;
  2884. default:
  2885. /* We should do nothing since we don't know this hardware */
  2886. return;
  2887. }
  2888. IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
  2889. }
  2890. /**
  2891. * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
  2892. * @adapter: board private structure
  2893. *
  2894. * Configure the Rx unit of the MAC after a reset.
  2895. **/
  2896. static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
  2897. {
  2898. struct ixgbe_hw *hw = &adapter->hw;
  2899. int i;
  2900. u32 rxctrl;
  2901. /* disable receives while setting up the descriptors */
  2902. rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
  2903. IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
  2904. ixgbe_setup_psrtype(adapter);
  2905. ixgbe_setup_rdrxctl(adapter);
  2906. /* Program registers for the distribution of queues */
  2907. ixgbe_setup_mrqc(adapter);
  2908. /* set_rx_buffer_len must be called before ring initialization */
  2909. ixgbe_set_rx_buffer_len(adapter);
  2910. /*
  2911. * Setup the HW Rx Head and Tail Descriptor Pointers and
  2912. * the Base and Length of the Rx Descriptor Ring
  2913. */
  2914. for (i = 0; i < adapter->num_rx_queues; i++)
  2915. ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
  2916. /* disable drop enable for 82598 parts */
  2917. if (hw->mac.type == ixgbe_mac_82598EB)
  2918. rxctrl |= IXGBE_RXCTRL_DMBYPS;
  2919. /* enable all receives */
  2920. rxctrl |= IXGBE_RXCTRL_RXEN;
  2921. hw->mac.ops.enable_rx_dma(hw, rxctrl);
  2922. }
  2923. static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
  2924. {
  2925. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  2926. struct ixgbe_hw *hw = &adapter->hw;
  2927. /* add VID to filter table */
  2928. hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true);
  2929. set_bit(vid, adapter->active_vlans);
  2930. return 0;
  2931. }
  2932. static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
  2933. {
  2934. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  2935. struct ixgbe_hw *hw = &adapter->hw;
  2936. /* remove VID from filter table */
  2937. hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), false);
  2938. clear_bit(vid, adapter->active_vlans);
  2939. return 0;
  2940. }
  2941. /**
  2942. * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering
  2943. * @adapter: driver data
  2944. */
  2945. static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
  2946. {
  2947. struct ixgbe_hw *hw = &adapter->hw;
  2948. u32 vlnctrl;
  2949. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2950. vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
  2951. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2952. }
  2953. /**
  2954. * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering
  2955. * @adapter: driver data
  2956. */
  2957. static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
  2958. {
  2959. struct ixgbe_hw *hw = &adapter->hw;
  2960. u32 vlnctrl;
  2961. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2962. vlnctrl |= IXGBE_VLNCTRL_VFE;
  2963. vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
  2964. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2965. }
  2966. /**
  2967. * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
  2968. * @adapter: driver data
  2969. */
  2970. static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
  2971. {
  2972. struct ixgbe_hw *hw = &adapter->hw;
  2973. u32 vlnctrl;
  2974. int i, j;
  2975. switch (hw->mac.type) {
  2976. case ixgbe_mac_82598EB:
  2977. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  2978. vlnctrl &= ~IXGBE_VLNCTRL_VME;
  2979. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  2980. break;
  2981. case ixgbe_mac_82599EB:
  2982. case ixgbe_mac_X540:
  2983. for (i = 0; i < adapter->num_rx_queues; i++) {
  2984. j = adapter->rx_ring[i]->reg_idx;
  2985. vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
  2986. vlnctrl &= ~IXGBE_RXDCTL_VME;
  2987. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
  2988. }
  2989. break;
  2990. default:
  2991. break;
  2992. }
  2993. }
  2994. /**
  2995. * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
  2996. * @adapter: driver data
  2997. */
  2998. static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
  2999. {
  3000. struct ixgbe_hw *hw = &adapter->hw;
  3001. u32 vlnctrl;
  3002. int i, j;
  3003. switch (hw->mac.type) {
  3004. case ixgbe_mac_82598EB:
  3005. vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
  3006. vlnctrl |= IXGBE_VLNCTRL_VME;
  3007. IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
  3008. break;
  3009. case ixgbe_mac_82599EB:
  3010. case ixgbe_mac_X540:
  3011. for (i = 0; i < adapter->num_rx_queues; i++) {
  3012. j = adapter->rx_ring[i]->reg_idx;
  3013. vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
  3014. vlnctrl |= IXGBE_RXDCTL_VME;
  3015. IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
  3016. }
  3017. break;
  3018. default:
  3019. break;
  3020. }
  3021. }
  3022. static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
  3023. {
  3024. u16 vid;
  3025. ixgbe_vlan_rx_add_vid(adapter->netdev, 0);
  3026. for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
  3027. ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
  3028. }
  3029. /**
  3030. * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
  3031. * @netdev: network interface device structure
  3032. *
  3033. * Writes unicast address list to the RAR table.
  3034. * Returns: -ENOMEM on failure/insufficient address space
  3035. * 0 on no addresses written
  3036. * X on writing X addresses to the RAR table
  3037. **/
  3038. static int ixgbe_write_uc_addr_list(struct net_device *netdev)
  3039. {
  3040. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  3041. struct ixgbe_hw *hw = &adapter->hw;
  3042. unsigned int rar_entries = hw->mac.num_rar_entries - 1;
  3043. int count = 0;
  3044. /* In SR-IOV mode significantly less RAR entries are available */
  3045. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  3046. rar_entries = IXGBE_MAX_PF_MACVLANS - 1;
  3047. /* return ENOMEM indicating insufficient memory for addresses */
  3048. if (netdev_uc_count(netdev) > rar_entries)
  3049. return -ENOMEM;
  3050. if (!netdev_uc_empty(netdev)) {
  3051. struct netdev_hw_addr *ha;
  3052. /* return error if we do not support writing to RAR table */
  3053. if (!hw->mac.ops.set_rar)
  3054. return -ENOMEM;
  3055. netdev_for_each_uc_addr(ha, netdev) {
  3056. if (!rar_entries)
  3057. break;
  3058. hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
  3059. VMDQ_P(0), IXGBE_RAH_AV);
  3060. count++;
  3061. }
  3062. }
  3063. /* write the addresses in reverse order to avoid write combining */
  3064. for (; rar_entries > 0 ; rar_entries--)
  3065. hw->mac.ops.clear_rar(hw, rar_entries);
  3066. return count;
  3067. }
  3068. /**
  3069. * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
  3070. * @netdev: network interface device structure
  3071. *
  3072. * The set_rx_method entry point is called whenever the unicast/multicast
  3073. * address list or the network interface flags are updated. This routine is
  3074. * responsible for configuring the hardware for proper unicast, multicast and
  3075. * promiscuous mode.
  3076. **/
  3077. void ixgbe_set_rx_mode(struct net_device *netdev)
  3078. {
  3079. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  3080. struct ixgbe_hw *hw = &adapter->hw;
  3081. u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
  3082. int count;
  3083. /* Check for Promiscuous and All Multicast modes */
  3084. fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
  3085. /* set all bits that we expect to always be set */
  3086. fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
  3087. fctrl |= IXGBE_FCTRL_BAM;
  3088. fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
  3089. fctrl |= IXGBE_FCTRL_PMCF;
  3090. /* clear the bits we are changing the status of */
  3091. fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
  3092. if (netdev->flags & IFF_PROMISC) {
  3093. hw->addr_ctrl.user_set_promisc = true;
  3094. fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
  3095. vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
  3096. /* don't hardware filter vlans in promisc mode */
  3097. ixgbe_vlan_filter_disable(adapter);
  3098. } else {
  3099. if (netdev->flags & IFF_ALLMULTI) {
  3100. fctrl |= IXGBE_FCTRL_MPE;
  3101. vmolr |= IXGBE_VMOLR_MPE;
  3102. } else {
  3103. /*
  3104. * Write addresses to the MTA, if the attempt fails
  3105. * then we should just turn on promiscuous mode so
  3106. * that we can at least receive multicast traffic
  3107. */
  3108. hw->mac.ops.update_mc_addr_list(hw, netdev);
  3109. vmolr |= IXGBE_VMOLR_ROMPE;
  3110. }
  3111. ixgbe_vlan_filter_enable(adapter);
  3112. hw->addr_ctrl.user_set_promisc = false;
  3113. }
  3114. /*
  3115. * Write addresses to available RAR registers, if there is not
  3116. * sufficient space to store all the addresses then enable
  3117. * unicast promiscuous mode
  3118. */
  3119. count = ixgbe_write_uc_addr_list(netdev);
  3120. if (count < 0) {
  3121. fctrl |= IXGBE_FCTRL_UPE;
  3122. vmolr |= IXGBE_VMOLR_ROPE;
  3123. }
  3124. if (adapter->num_vfs)
  3125. ixgbe_restore_vf_multicasts(adapter);
  3126. if (hw->mac.type != ixgbe_mac_82598EB) {
  3127. vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
  3128. ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
  3129. IXGBE_VMOLR_ROPE);
  3130. IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
  3131. }
  3132. /* This is useful for sniffing bad packets. */
  3133. if (adapter->netdev->features & NETIF_F_RXALL) {
  3134. /* UPE and MPE will be handled by normal PROMISC logic
  3135. * in e1000e_set_rx_mode */
  3136. fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
  3137. IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
  3138. IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */
  3139. fctrl &= ~(IXGBE_FCTRL_DPF);
  3140. /* NOTE: VLAN filtering is disabled by setting PROMISC */
  3141. }
  3142. IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
  3143. if (netdev->features & NETIF_F_HW_VLAN_RX)
  3144. ixgbe_vlan_strip_enable(adapter);
  3145. else
  3146. ixgbe_vlan_strip_disable(adapter);
  3147. }
  3148. static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
  3149. {
  3150. int q_idx;
  3151. for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
  3152. napi_enable(&adapter->q_vector[q_idx]->napi);
  3153. }
  3154. static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
  3155. {
  3156. int q_idx;
  3157. for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
  3158. napi_disable(&adapter->q_vector[q_idx]->napi);
  3159. }
  3160. #ifdef CONFIG_IXGBE_DCB
  3161. /**
  3162. * ixgbe_configure_dcb - Configure DCB hardware
  3163. * @adapter: ixgbe adapter struct
  3164. *
  3165. * This is called by the driver on open to configure the DCB hardware.
  3166. * This is also called by the gennetlink interface when reconfiguring
  3167. * the DCB state.
  3168. */
  3169. static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
  3170. {
  3171. struct ixgbe_hw *hw = &adapter->hw;
  3172. int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
  3173. if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
  3174. if (hw->mac.type == ixgbe_mac_82598EB)
  3175. netif_set_gso_max_size(adapter->netdev, 65536);
  3176. return;
  3177. }
  3178. if (hw->mac.type == ixgbe_mac_82598EB)
  3179. netif_set_gso_max_size(adapter->netdev, 32768);
  3180. #ifdef IXGBE_FCOE
  3181. if (adapter->netdev->features & NETIF_F_FCOE_MTU)
  3182. max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
  3183. #endif
  3184. /* reconfigure the hardware */
  3185. if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
  3186. ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
  3187. DCB_TX_CONFIG);
  3188. ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
  3189. DCB_RX_CONFIG);
  3190. ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
  3191. } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
  3192. ixgbe_dcb_hw_ets(&adapter->hw,
  3193. adapter->ixgbe_ieee_ets,
  3194. max_frame);
  3195. ixgbe_dcb_hw_pfc_config(&adapter->hw,
  3196. adapter->ixgbe_ieee_pfc->pfc_en,
  3197. adapter->ixgbe_ieee_ets->prio_tc);
  3198. }
  3199. /* Enable RSS Hash per TC */
  3200. if (hw->mac.type != ixgbe_mac_82598EB) {
  3201. u32 msb = 0;
  3202. u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;
  3203. while (rss_i) {
  3204. msb++;
  3205. rss_i >>= 1;
  3206. }
  3207. /* write msb to all 8 TCs in one write */
  3208. IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
  3209. }
  3210. }
  3211. #endif
  3212. /* Additional bittime to account for IXGBE framing */
  3213. #define IXGBE_ETH_FRAMING 20
  3214. /**
  3215. * ixgbe_hpbthresh - calculate high water mark for flow control
  3216. *
  3217. * @adapter: board private structure to calculate for
  3218. * @pb: packet buffer to calculate
  3219. */
  3220. static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
  3221. {
  3222. struct ixgbe_hw *hw = &adapter->hw;
  3223. struct net_device *dev = adapter->netdev;
  3224. int link, tc, kb, marker;
  3225. u32 dv_id, rx_pba;
  3226. /* Calculate max LAN frame size */
  3227. tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
  3228. #ifdef IXGBE_FCOE
  3229. /* FCoE traffic class uses FCOE jumbo frames */
  3230. if ((dev->features & NETIF_F_FCOE_MTU) &&
  3231. (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
  3232. (pb == ixgbe_fcoe_get_tc(adapter)))
  3233. tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
  3234. #endif
  3235. /* Calculate delay value for device */
  3236. switch (hw->mac.type) {
  3237. case ixgbe_mac_X540:
  3238. dv_id = IXGBE_DV_X540(link, tc);
  3239. break;
  3240. default:
  3241. dv_id = IXGBE_DV(link, tc);
  3242. break;
  3243. }
  3244. /* Loopback switch introduces additional latency */
  3245. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  3246. dv_id += IXGBE_B2BT(tc);
  3247. /* Delay value is calculated in bit times convert to KB */
  3248. kb = IXGBE_BT2KB(dv_id);
  3249. rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
  3250. marker = rx_pba - kb;
  3251. /* It is possible that the packet buffer is not large enough
  3252. * to provide required headroom. In this case throw an error
  3253. * to user and a do the best we can.
  3254. */
  3255. if (marker < 0) {
  3256. e_warn(drv, "Packet Buffer(%i) can not provide enough"
  3257. "headroom to support flow control."
  3258. "Decrease MTU or number of traffic classes\n", pb);
  3259. marker = tc + 1;
  3260. }
  3261. return marker;
  3262. }
  3263. /**
  3264. * ixgbe_lpbthresh - calculate low water mark for for flow control
  3265. *
  3266. * @adapter: board private structure to calculate for
  3267. * @pb: packet buffer to calculate
  3268. */
  3269. static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter)
  3270. {
  3271. struct ixgbe_hw *hw = &adapter->hw;
  3272. struct net_device *dev = adapter->netdev;
  3273. int tc;
  3274. u32 dv_id;
  3275. /* Calculate max LAN frame size */
  3276. tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
  3277. /* Calculate delay value for device */
  3278. switch (hw->mac.type) {
  3279. case ixgbe_mac_X540:
  3280. dv_id = IXGBE_LOW_DV_X540(tc);
  3281. break;
  3282. default:
  3283. dv_id = IXGBE_LOW_DV(tc);
  3284. break;
  3285. }
  3286. /* Delay value is calculated in bit times convert to KB */
  3287. return IXGBE_BT2KB(dv_id);
  3288. }
  3289. /*
  3290. * ixgbe_pbthresh_setup - calculate and setup high low water marks
  3291. */
  3292. static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
  3293. {
  3294. struct ixgbe_hw *hw = &adapter->hw;
  3295. int num_tc = netdev_get_num_tc(adapter->netdev);
  3296. int i;
  3297. if (!num_tc)
  3298. num_tc = 1;
  3299. hw->fc.low_water = ixgbe_lpbthresh(adapter);
  3300. for (i = 0; i < num_tc; i++) {
  3301. hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);
  3302. /* Low water marks must not be larger than high water marks */
  3303. if (hw->fc.low_water > hw->fc.high_water[i])
  3304. hw->fc.low_water = 0;
  3305. }
  3306. }
  3307. static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
  3308. {
  3309. struct ixgbe_hw *hw = &adapter->hw;
  3310. int hdrm;
  3311. u8 tc = netdev_get_num_tc(adapter->netdev);
  3312. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
  3313. adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
  3314. hdrm = 32 << adapter->fdir_pballoc;
  3315. else
  3316. hdrm = 0;
  3317. hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
  3318. ixgbe_pbthresh_setup(adapter);
  3319. }
  3320. static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
  3321. {
  3322. struct ixgbe_hw *hw = &adapter->hw;
  3323. struct hlist_node *node, *node2;
  3324. struct ixgbe_fdir_filter *filter;
  3325. spin_lock(&adapter->fdir_perfect_lock);
  3326. if (!hlist_empty(&adapter->fdir_filter_list))
  3327. ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);
  3328. hlist_for_each_entry_safe(filter, node, node2,
  3329. &adapter->fdir_filter_list, fdir_node) {
  3330. ixgbe_fdir_write_perfect_filter_82599(hw,
  3331. &filter->filter,
  3332. filter->sw_idx,
  3333. (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
  3334. IXGBE_FDIR_DROP_QUEUE :
  3335. adapter->rx_ring[filter->action]->reg_idx);
  3336. }
  3337. spin_unlock(&adapter->fdir_perfect_lock);
  3338. }
  3339. static void ixgbe_configure(struct ixgbe_adapter *adapter)
  3340. {
  3341. struct ixgbe_hw *hw = &adapter->hw;
  3342. ixgbe_configure_pb(adapter);
  3343. #ifdef CONFIG_IXGBE_DCB
  3344. ixgbe_configure_dcb(adapter);
  3345. #endif
  3346. /*
  3347. * We must restore virtualization before VLANs or else
  3348. * the VLVF registers will not be populated
  3349. */
  3350. ixgbe_configure_virtualization(adapter);
  3351. ixgbe_set_rx_mode(adapter->netdev);
  3352. ixgbe_restore_vlan(adapter);
  3353. switch (hw->mac.type) {
  3354. case ixgbe_mac_82599EB:
  3355. case ixgbe_mac_X540:
  3356. hw->mac.ops.disable_rx_buff(hw);
  3357. break;
  3358. default:
  3359. break;
  3360. }
  3361. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  3362. ixgbe_init_fdir_signature_82599(&adapter->hw,
  3363. adapter->fdir_pballoc);
  3364. } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
  3365. ixgbe_init_fdir_perfect_82599(&adapter->hw,
  3366. adapter->fdir_pballoc);
  3367. ixgbe_fdir_filter_restore(adapter);
  3368. }
  3369. switch (hw->mac.type) {
  3370. case ixgbe_mac_82599EB:
  3371. case ixgbe_mac_X540:
  3372. hw->mac.ops.enable_rx_buff(hw);
  3373. break;
  3374. default:
  3375. break;
  3376. }
  3377. #ifdef IXGBE_FCOE
  3378. /* configure FCoE L2 filters, redirection table, and Rx control */
  3379. ixgbe_configure_fcoe(adapter);
  3380. #endif /* IXGBE_FCOE */
  3381. ixgbe_configure_tx(adapter);
  3382. ixgbe_configure_rx(adapter);
  3383. }
  3384. static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
  3385. {
  3386. switch (hw->phy.type) {
  3387. case ixgbe_phy_sfp_avago:
  3388. case ixgbe_phy_sfp_ftl:
  3389. case ixgbe_phy_sfp_intel:
  3390. case ixgbe_phy_sfp_unknown:
  3391. case ixgbe_phy_sfp_passive_tyco:
  3392. case ixgbe_phy_sfp_passive_unknown:
  3393. case ixgbe_phy_sfp_active_unknown:
  3394. case ixgbe_phy_sfp_ftl_active:
  3395. return true;
  3396. case ixgbe_phy_nl:
  3397. if (hw->mac.type == ixgbe_mac_82598EB)
  3398. return true;
  3399. default:
  3400. return false;
  3401. }
  3402. }
  3403. /**
  3404. * ixgbe_sfp_link_config - set up SFP+ link
  3405. * @adapter: pointer to private adapter struct
  3406. **/
  3407. static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
  3408. {
  3409. /*
  3410. * We are assuming the worst case scenario here, and that
  3411. * is that an SFP was inserted/removed after the reset
  3412. * but before SFP detection was enabled. As such the best
  3413. * solution is to just start searching as soon as we start
  3414. */
  3415. if (adapter->hw.mac.type == ixgbe_mac_82598EB)
  3416. adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
  3417. adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
  3418. }
  3419. /**
  3420. * ixgbe_non_sfp_link_config - set up non-SFP+ link
  3421. * @hw: pointer to private hardware struct
  3422. *
  3423. * Returns 0 on success, negative on failure
  3424. **/
  3425. static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
  3426. {
  3427. u32 autoneg;
  3428. bool negotiation, link_up = false;
  3429. u32 ret = IXGBE_ERR_LINK_SETUP;
  3430. if (hw->mac.ops.check_link)
  3431. ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
  3432. if (ret)
  3433. goto link_cfg_out;
  3434. autoneg = hw->phy.autoneg_advertised;
  3435. if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
  3436. ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
  3437. &negotiation);
  3438. if (ret)
  3439. goto link_cfg_out;
  3440. if (hw->mac.ops.setup_link)
  3441. ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
  3442. link_cfg_out:
  3443. return ret;
  3444. }
  3445. static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
  3446. {
  3447. struct ixgbe_hw *hw = &adapter->hw;
  3448. u32 gpie = 0;
  3449. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  3450. gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
  3451. IXGBE_GPIE_OCD;
  3452. gpie |= IXGBE_GPIE_EIAME;
  3453. /*
  3454. * use EIAM to auto-mask when MSI-X interrupt is asserted
  3455. * this saves a register write for every interrupt
  3456. */
  3457. switch (hw->mac.type) {
  3458. case ixgbe_mac_82598EB:
  3459. IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
  3460. break;
  3461. case ixgbe_mac_82599EB:
  3462. case ixgbe_mac_X540:
  3463. default:
  3464. IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
  3465. IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
  3466. break;
  3467. }
  3468. } else {
  3469. /* legacy interrupts, use EIAM to auto-mask when reading EICR,
  3470. * specifically only auto mask tx and rx interrupts */
  3471. IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
  3472. }
  3473. /* XXX: to interrupt immediately for EICS writes, enable this */
  3474. /* gpie |= IXGBE_GPIE_EIMEN; */
  3475. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  3476. gpie &= ~IXGBE_GPIE_VTMODE_MASK;
  3477. switch (adapter->ring_feature[RING_F_VMDQ].mask) {
  3478. case IXGBE_82599_VMDQ_8Q_MASK:
  3479. gpie |= IXGBE_GPIE_VTMODE_16;
  3480. break;
  3481. case IXGBE_82599_VMDQ_4Q_MASK:
  3482. gpie |= IXGBE_GPIE_VTMODE_32;
  3483. break;
  3484. default:
  3485. gpie |= IXGBE_GPIE_VTMODE_64;
  3486. break;
  3487. }
  3488. }
  3489. /* Enable Thermal over heat sensor interrupt */
  3490. if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
  3491. switch (adapter->hw.mac.type) {
  3492. case ixgbe_mac_82599EB:
  3493. gpie |= IXGBE_SDP0_GPIEN;
  3494. break;
  3495. case ixgbe_mac_X540:
  3496. gpie |= IXGBE_EIMS_TS;
  3497. break;
  3498. default:
  3499. break;
  3500. }
  3501. }
  3502. /* Enable fan failure interrupt */
  3503. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
  3504. gpie |= IXGBE_SDP1_GPIEN;
  3505. if (hw->mac.type == ixgbe_mac_82599EB) {
  3506. gpie |= IXGBE_SDP1_GPIEN;
  3507. gpie |= IXGBE_SDP2_GPIEN;
  3508. }
  3509. IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
  3510. }
  3511. static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
  3512. {
  3513. struct ixgbe_hw *hw = &adapter->hw;
  3514. int err;
  3515. u32 ctrl_ext;
  3516. ixgbe_get_hw_control(adapter);
  3517. ixgbe_setup_gpie(adapter);
  3518. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
  3519. ixgbe_configure_msix(adapter);
  3520. else
  3521. ixgbe_configure_msi_and_legacy(adapter);
  3522. /* enable the optics for 82599 SFP+ fiber */
  3523. if (hw->mac.ops.enable_tx_laser)
  3524. hw->mac.ops.enable_tx_laser(hw);
  3525. clear_bit(__IXGBE_DOWN, &adapter->state);
  3526. ixgbe_napi_enable_all(adapter);
  3527. if (ixgbe_is_sfp(hw)) {
  3528. ixgbe_sfp_link_config(adapter);
  3529. } else {
  3530. err = ixgbe_non_sfp_link_config(hw);
  3531. if (err)
  3532. e_err(probe, "link_config FAILED %d\n", err);
  3533. }
  3534. /* clear any pending interrupts, may auto mask */
  3535. IXGBE_READ_REG(hw, IXGBE_EICR);
  3536. ixgbe_irq_enable(adapter, true, true);
  3537. /*
  3538. * If this adapter has a fan, check to see if we had a failure
  3539. * before we enabled the interrupt.
  3540. */
  3541. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
  3542. u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
  3543. if (esdp & IXGBE_ESDP_SDP1)
  3544. e_crit(drv, "Fan has stopped, replace the adapter\n");
  3545. }
  3546. /* enable transmits */
  3547. netif_tx_start_all_queues(adapter->netdev);
  3548. /* bring the link up in the watchdog, this could race with our first
  3549. * link up interrupt but shouldn't be a problem */
  3550. adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
  3551. adapter->link_check_timeout = jiffies;
  3552. mod_timer(&adapter->service_timer, jiffies);
  3553. /* Set PF Reset Done bit so PF/VF Mail Ops can work */
  3554. ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
  3555. ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
  3556. IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
  3557. }
  3558. void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
  3559. {
  3560. WARN_ON(in_interrupt());
  3561. /* put off any impending NetWatchDogTimeout */
  3562. adapter->netdev->trans_start = jiffies;
  3563. while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
  3564. usleep_range(1000, 2000);
  3565. ixgbe_down(adapter);
  3566. /*
  3567. * If SR-IOV enabled then wait a bit before bringing the adapter
  3568. * back up to give the VFs time to respond to the reset. The
  3569. * two second wait is based upon the watchdog timer cycle in
  3570. * the VF driver.
  3571. */
  3572. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  3573. msleep(2000);
  3574. ixgbe_up(adapter);
  3575. clear_bit(__IXGBE_RESETTING, &adapter->state);
  3576. }
  3577. void ixgbe_up(struct ixgbe_adapter *adapter)
  3578. {
  3579. /* hardware has been reset, we need to reload some things */
  3580. ixgbe_configure(adapter);
  3581. ixgbe_up_complete(adapter);
  3582. }
  3583. void ixgbe_reset(struct ixgbe_adapter *adapter)
  3584. {
  3585. struct ixgbe_hw *hw = &adapter->hw;
  3586. int err;
  3587. /* lock SFP init bit to prevent race conditions with the watchdog */
  3588. while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
  3589. usleep_range(1000, 2000);
  3590. /* clear all SFP and link config related flags while holding SFP_INIT */
  3591. adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
  3592. IXGBE_FLAG2_SFP_NEEDS_RESET);
  3593. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
  3594. err = hw->mac.ops.init_hw(hw);
  3595. switch (err) {
  3596. case 0:
  3597. case IXGBE_ERR_SFP_NOT_PRESENT:
  3598. case IXGBE_ERR_SFP_NOT_SUPPORTED:
  3599. break;
  3600. case IXGBE_ERR_MASTER_REQUESTS_PENDING:
  3601. e_dev_err("master disable timed out\n");
  3602. break;
  3603. case IXGBE_ERR_EEPROM_VERSION:
  3604. /* We are running on a pre-production device, log a warning */
  3605. e_dev_warn("This device is a pre-production adapter/LOM. "
  3606. "Please be aware there may be issues associated with "
  3607. "your hardware. If you are experiencing problems "
  3608. "please contact your Intel or hardware "
  3609. "representative who provided you with this "
  3610. "hardware.\n");
  3611. break;
  3612. default:
  3613. e_dev_err("Hardware Error: %d\n", err);
  3614. }
  3615. clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
  3616. /* reprogram the RAR[0] in case user changed it. */
  3617. hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV);
  3618. /* update SAN MAC vmdq pool selection */
  3619. if (hw->mac.san_mac_rar_index)
  3620. hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
  3621. if (adapter->flags2 & IXGBE_FLAG2_PTP_ENABLED)
  3622. ixgbe_ptp_reset(adapter);
  3623. }
  3624. /**
  3625. * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
  3626. * @rx_ring: ring to free buffers from
  3627. **/
  3628. static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
  3629. {
  3630. struct device *dev = rx_ring->dev;
  3631. unsigned long size;
  3632. u16 i;
  3633. /* ring already cleared, nothing to do */
  3634. if (!rx_ring->rx_buffer_info)
  3635. return;
  3636. /* Free all the Rx ring sk_buffs */
  3637. for (i = 0; i < rx_ring->count; i++) {
  3638. struct ixgbe_rx_buffer *rx_buffer;
  3639. rx_buffer = &rx_ring->rx_buffer_info[i];
  3640. if (rx_buffer->skb) {
  3641. struct sk_buff *skb = rx_buffer->skb;
  3642. if (IXGBE_CB(skb)->page_released) {
  3643. dma_unmap_page(dev,
  3644. IXGBE_CB(skb)->dma,
  3645. ixgbe_rx_bufsz(rx_ring),
  3646. DMA_FROM_DEVICE);
  3647. IXGBE_CB(skb)->page_released = false;
  3648. }
  3649. dev_kfree_skb(skb);
  3650. }
  3651. rx_buffer->skb = NULL;
  3652. if (rx_buffer->dma)
  3653. dma_unmap_page(dev, rx_buffer->dma,
  3654. ixgbe_rx_pg_size(rx_ring),
  3655. DMA_FROM_DEVICE);
  3656. rx_buffer->dma = 0;
  3657. if (rx_buffer->page)
  3658. __free_pages(rx_buffer->page,
  3659. ixgbe_rx_pg_order(rx_ring));
  3660. rx_buffer->page = NULL;
  3661. }
  3662. size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
  3663. memset(rx_ring->rx_buffer_info, 0, size);
  3664. /* Zero out the descriptor ring */
  3665. memset(rx_ring->desc, 0, rx_ring->size);
  3666. rx_ring->next_to_alloc = 0;
  3667. rx_ring->next_to_clean = 0;
  3668. rx_ring->next_to_use = 0;
  3669. }
  3670. /**
  3671. * ixgbe_clean_tx_ring - Free Tx Buffers
  3672. * @tx_ring: ring to be cleaned
  3673. **/
  3674. static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
  3675. {
  3676. struct ixgbe_tx_buffer *tx_buffer_info;
  3677. unsigned long size;
  3678. u16 i;
  3679. /* ring already cleared, nothing to do */
  3680. if (!tx_ring->tx_buffer_info)
  3681. return;
  3682. /* Free all the Tx ring sk_buffs */
  3683. for (i = 0; i < tx_ring->count; i++) {
  3684. tx_buffer_info = &tx_ring->tx_buffer_info[i];
  3685. ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
  3686. }
  3687. netdev_tx_reset_queue(txring_txq(tx_ring));
  3688. size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
  3689. memset(tx_ring->tx_buffer_info, 0, size);
  3690. /* Zero out the descriptor ring */
  3691. memset(tx_ring->desc, 0, tx_ring->size);
  3692. tx_ring->next_to_use = 0;
  3693. tx_ring->next_to_clean = 0;
  3694. }
  3695. /**
  3696. * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
  3697. * @adapter: board private structure
  3698. **/
  3699. static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
  3700. {
  3701. int i;
  3702. for (i = 0; i < adapter->num_rx_queues; i++)
  3703. ixgbe_clean_rx_ring(adapter->rx_ring[i]);
  3704. }
  3705. /**
  3706. * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
  3707. * @adapter: board private structure
  3708. **/
  3709. static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
  3710. {
  3711. int i;
  3712. for (i = 0; i < adapter->num_tx_queues; i++)
  3713. ixgbe_clean_tx_ring(adapter->tx_ring[i]);
  3714. }
  3715. static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
  3716. {
  3717. struct hlist_node *node, *node2;
  3718. struct ixgbe_fdir_filter *filter;
  3719. spin_lock(&adapter->fdir_perfect_lock);
  3720. hlist_for_each_entry_safe(filter, node, node2,
  3721. &adapter->fdir_filter_list, fdir_node) {
  3722. hlist_del(&filter->fdir_node);
  3723. kfree(filter);
  3724. }
  3725. adapter->fdir_filter_count = 0;
  3726. spin_unlock(&adapter->fdir_perfect_lock);
  3727. }
  3728. void ixgbe_down(struct ixgbe_adapter *adapter)
  3729. {
  3730. struct net_device *netdev = adapter->netdev;
  3731. struct ixgbe_hw *hw = &adapter->hw;
  3732. u32 rxctrl;
  3733. int i;
  3734. /* signal that we are down to the interrupt handler */
  3735. set_bit(__IXGBE_DOWN, &adapter->state);
  3736. /* disable receives */
  3737. rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
  3738. IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
  3739. /* disable all enabled rx queues */
  3740. for (i = 0; i < adapter->num_rx_queues; i++)
  3741. /* this call also flushes the previous write */
  3742. ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
  3743. usleep_range(10000, 20000);
  3744. netif_tx_stop_all_queues(netdev);
  3745. /* call carrier off first to avoid false dev_watchdog timeouts */
  3746. netif_carrier_off(netdev);
  3747. netif_tx_disable(netdev);
  3748. ixgbe_irq_disable(adapter);
  3749. ixgbe_napi_disable_all(adapter);
  3750. adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
  3751. IXGBE_FLAG2_RESET_REQUESTED);
  3752. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
  3753. del_timer_sync(&adapter->service_timer);
  3754. if (adapter->num_vfs) {
  3755. /* Clear EITR Select mapping */
  3756. IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
  3757. /* Mark all the VFs as inactive */
  3758. for (i = 0 ; i < adapter->num_vfs; i++)
  3759. adapter->vfinfo[i].clear_to_send = false;
  3760. /* ping all the active vfs to let them know we are going down */
  3761. ixgbe_ping_all_vfs(adapter);
  3762. /* Disable all VFTE/VFRE TX/RX */
  3763. ixgbe_disable_tx_rx(adapter);
  3764. }
  3765. /* disable transmits in the hardware now that interrupts are off */
  3766. for (i = 0; i < adapter->num_tx_queues; i++) {
  3767. u8 reg_idx = adapter->tx_ring[i]->reg_idx;
  3768. IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
  3769. }
  3770. /* Disable the Tx DMA engine on 82599 and X540 */
  3771. switch (hw->mac.type) {
  3772. case ixgbe_mac_82599EB:
  3773. case ixgbe_mac_X540:
  3774. IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
  3775. (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
  3776. ~IXGBE_DMATXCTL_TE));
  3777. break;
  3778. default:
  3779. break;
  3780. }
  3781. if (!pci_channel_offline(adapter->pdev))
  3782. ixgbe_reset(adapter);
  3783. /* power down the optics for 82599 SFP+ fiber */
  3784. if (hw->mac.ops.disable_tx_laser)
  3785. hw->mac.ops.disable_tx_laser(hw);
  3786. ixgbe_clean_all_tx_rings(adapter);
  3787. ixgbe_clean_all_rx_rings(adapter);
  3788. #ifdef CONFIG_IXGBE_DCA
  3789. /* since we reset the hardware DCA settings were cleared */
  3790. ixgbe_setup_dca(adapter);
  3791. #endif
  3792. }
  3793. /**
  3794. * ixgbe_tx_timeout - Respond to a Tx Hang
  3795. * @netdev: network interface device structure
  3796. **/
  3797. static void ixgbe_tx_timeout(struct net_device *netdev)
  3798. {
  3799. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  3800. /* Do the reset outside of interrupt context */
  3801. ixgbe_tx_timeout_reset(adapter);
  3802. }
  3803. /**
  3804. * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
  3805. * @adapter: board private structure to initialize
  3806. *
  3807. * ixgbe_sw_init initializes the Adapter private data structure.
  3808. * Fields are initialized based on PCI device information and
  3809. * OS network device settings (MTU size).
  3810. **/
  3811. static int ixgbe_sw_init(struct ixgbe_adapter *adapter)
  3812. {
  3813. struct ixgbe_hw *hw = &adapter->hw;
  3814. struct pci_dev *pdev = adapter->pdev;
  3815. unsigned int rss;
  3816. u32 fwsm;
  3817. #ifdef CONFIG_IXGBE_DCB
  3818. int j;
  3819. struct tc_configuration *tc;
  3820. #endif
  3821. /* PCI config space info */
  3822. hw->vendor_id = pdev->vendor;
  3823. hw->device_id = pdev->device;
  3824. hw->revision_id = pdev->revision;
  3825. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  3826. hw->subsystem_device_id = pdev->subsystem_device;
  3827. /* Set capability flags */
  3828. rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus());
  3829. adapter->ring_feature[RING_F_RSS].limit = rss;
  3830. switch (hw->mac.type) {
  3831. case ixgbe_mac_82598EB:
  3832. if (hw->device_id == IXGBE_DEV_ID_82598AT)
  3833. adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
  3834. adapter->max_q_vectors = MAX_Q_VECTORS_82598;
  3835. break;
  3836. case ixgbe_mac_X540:
  3837. fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
  3838. if (fwsm & IXGBE_FWSM_TS_ENABLED)
  3839. adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
  3840. case ixgbe_mac_82599EB:
  3841. adapter->max_q_vectors = MAX_Q_VECTORS_82599;
  3842. adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
  3843. adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
  3844. if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
  3845. adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
  3846. /* Flow Director hash filters enabled */
  3847. adapter->atr_sample_rate = 20;
  3848. adapter->ring_feature[RING_F_FDIR].limit =
  3849. IXGBE_MAX_FDIR_INDICES;
  3850. adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
  3851. #ifdef IXGBE_FCOE
  3852. adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
  3853. adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
  3854. #ifdef CONFIG_IXGBE_DCB
  3855. /* Default traffic class to use for FCoE */
  3856. adapter->fcoe.up = IXGBE_FCOE_DEFTC;
  3857. #endif
  3858. #endif /* IXGBE_FCOE */
  3859. break;
  3860. default:
  3861. break;
  3862. }
  3863. #ifdef IXGBE_FCOE
  3864. /* FCoE support exists, always init the FCoE lock */
  3865. spin_lock_init(&adapter->fcoe.lock);
  3866. #endif
  3867. /* n-tuple support exists, always init our spinlock */
  3868. spin_lock_init(&adapter->fdir_perfect_lock);
  3869. #ifdef CONFIG_IXGBE_DCB
  3870. switch (hw->mac.type) {
  3871. case ixgbe_mac_X540:
  3872. adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
  3873. adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
  3874. break;
  3875. default:
  3876. adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
  3877. adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
  3878. break;
  3879. }
  3880. /* Configure DCB traffic classes */
  3881. for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
  3882. tc = &adapter->dcb_cfg.tc_config[j];
  3883. tc->path[DCB_TX_CONFIG].bwg_id = 0;
  3884. tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
  3885. tc->path[DCB_RX_CONFIG].bwg_id = 0;
  3886. tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
  3887. tc->dcb_pfc = pfc_disabled;
  3888. }
  3889. /* Initialize default user to priority mapping, UPx->TC0 */
  3890. tc = &adapter->dcb_cfg.tc_config[0];
  3891. tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
  3892. tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
  3893. adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
  3894. adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
  3895. adapter->dcb_cfg.pfc_mode_enable = false;
  3896. adapter->dcb_set_bitmap = 0x00;
  3897. adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
  3898. memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
  3899. sizeof(adapter->temp_dcb_cfg));
  3900. #endif
  3901. /* default flow control settings */
  3902. hw->fc.requested_mode = ixgbe_fc_full;
  3903. hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
  3904. ixgbe_pbthresh_setup(adapter);
  3905. hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
  3906. hw->fc.send_xon = true;
  3907. hw->fc.disable_fc_autoneg =
  3908. (ixgbe_device_supports_autoneg_fc(hw) == 0) ? false : true;
  3909. #ifdef CONFIG_PCI_IOV
  3910. /* assign number of SR-IOV VFs */
  3911. if (hw->mac.type != ixgbe_mac_82598EB)
  3912. adapter->num_vfs = (max_vfs > 63) ? 0 : max_vfs;
  3913. #endif
  3914. /* enable itr by default in dynamic mode */
  3915. adapter->rx_itr_setting = 1;
  3916. adapter->tx_itr_setting = 1;
  3917. /* set default ring sizes */
  3918. adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
  3919. adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
  3920. /* set default work limits */
  3921. adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
  3922. /* initialize eeprom parameters */
  3923. if (ixgbe_init_eeprom_params_generic(hw)) {
  3924. e_dev_err("EEPROM initialization failed\n");
  3925. return -EIO;
  3926. }
  3927. set_bit(__IXGBE_DOWN, &adapter->state);
  3928. return 0;
  3929. }
  3930. /**
  3931. * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
  3932. * @tx_ring: tx descriptor ring (for a specific queue) to setup
  3933. *
  3934. * Return 0 on success, negative on failure
  3935. **/
  3936. int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
  3937. {
  3938. struct device *dev = tx_ring->dev;
  3939. int orig_node = dev_to_node(dev);
  3940. int numa_node = -1;
  3941. int size;
  3942. size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
  3943. if (tx_ring->q_vector)
  3944. numa_node = tx_ring->q_vector->numa_node;
  3945. tx_ring->tx_buffer_info = vzalloc_node(size, numa_node);
  3946. if (!tx_ring->tx_buffer_info)
  3947. tx_ring->tx_buffer_info = vzalloc(size);
  3948. if (!tx_ring->tx_buffer_info)
  3949. goto err;
  3950. /* round up to nearest 4K */
  3951. tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
  3952. tx_ring->size = ALIGN(tx_ring->size, 4096);
  3953. set_dev_node(dev, numa_node);
  3954. tx_ring->desc = dma_alloc_coherent(dev,
  3955. tx_ring->size,
  3956. &tx_ring->dma,
  3957. GFP_KERNEL);
  3958. set_dev_node(dev, orig_node);
  3959. if (!tx_ring->desc)
  3960. tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
  3961. &tx_ring->dma, GFP_KERNEL);
  3962. if (!tx_ring->desc)
  3963. goto err;
  3964. tx_ring->next_to_use = 0;
  3965. tx_ring->next_to_clean = 0;
  3966. return 0;
  3967. err:
  3968. vfree(tx_ring->tx_buffer_info);
  3969. tx_ring->tx_buffer_info = NULL;
  3970. dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
  3971. return -ENOMEM;
  3972. }
  3973. /**
  3974. * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
  3975. * @adapter: board private structure
  3976. *
  3977. * If this function returns with an error, then it's possible one or
  3978. * more of the rings is populated (while the rest are not). It is the
  3979. * callers duty to clean those orphaned rings.
  3980. *
  3981. * Return 0 on success, negative on failure
  3982. **/
  3983. static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
  3984. {
  3985. int i, err = 0;
  3986. for (i = 0; i < adapter->num_tx_queues; i++) {
  3987. err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
  3988. if (!err)
  3989. continue;
  3990. e_err(probe, "Allocation for Tx Queue %u failed\n", i);
  3991. goto err_setup_tx;
  3992. }
  3993. return 0;
  3994. err_setup_tx:
  3995. /* rewind the index freeing the rings as we go */
  3996. while (i--)
  3997. ixgbe_free_tx_resources(adapter->tx_ring[i]);
  3998. return err;
  3999. }
  4000. /**
  4001. * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
  4002. * @rx_ring: rx descriptor ring (for a specific queue) to setup
  4003. *
  4004. * Returns 0 on success, negative on failure
  4005. **/
  4006. int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
  4007. {
  4008. struct device *dev = rx_ring->dev;
  4009. int orig_node = dev_to_node(dev);
  4010. int numa_node = -1;
  4011. int size;
  4012. size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
  4013. if (rx_ring->q_vector)
  4014. numa_node = rx_ring->q_vector->numa_node;
  4015. rx_ring->rx_buffer_info = vzalloc_node(size, numa_node);
  4016. if (!rx_ring->rx_buffer_info)
  4017. rx_ring->rx_buffer_info = vzalloc(size);
  4018. if (!rx_ring->rx_buffer_info)
  4019. goto err;
  4020. /* Round up to nearest 4K */
  4021. rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
  4022. rx_ring->size = ALIGN(rx_ring->size, 4096);
  4023. set_dev_node(dev, numa_node);
  4024. rx_ring->desc = dma_alloc_coherent(dev,
  4025. rx_ring->size,
  4026. &rx_ring->dma,
  4027. GFP_KERNEL);
  4028. set_dev_node(dev, orig_node);
  4029. if (!rx_ring->desc)
  4030. rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
  4031. &rx_ring->dma, GFP_KERNEL);
  4032. if (!rx_ring->desc)
  4033. goto err;
  4034. rx_ring->next_to_clean = 0;
  4035. rx_ring->next_to_use = 0;
  4036. return 0;
  4037. err:
  4038. vfree(rx_ring->rx_buffer_info);
  4039. rx_ring->rx_buffer_info = NULL;
  4040. dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
  4041. return -ENOMEM;
  4042. }
  4043. /**
  4044. * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
  4045. * @adapter: board private structure
  4046. *
  4047. * If this function returns with an error, then it's possible one or
  4048. * more of the rings is populated (while the rest are not). It is the
  4049. * callers duty to clean those orphaned rings.
  4050. *
  4051. * Return 0 on success, negative on failure
  4052. **/
  4053. static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
  4054. {
  4055. int i, err = 0;
  4056. for (i = 0; i < adapter->num_rx_queues; i++) {
  4057. err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
  4058. if (!err)
  4059. continue;
  4060. e_err(probe, "Allocation for Rx Queue %u failed\n", i);
  4061. goto err_setup_rx;
  4062. }
  4063. #ifdef IXGBE_FCOE
  4064. err = ixgbe_setup_fcoe_ddp_resources(adapter);
  4065. if (!err)
  4066. #endif
  4067. return 0;
  4068. err_setup_rx:
  4069. /* rewind the index freeing the rings as we go */
  4070. while (i--)
  4071. ixgbe_free_rx_resources(adapter->rx_ring[i]);
  4072. return err;
  4073. }
  4074. /**
  4075. * ixgbe_free_tx_resources - Free Tx Resources per Queue
  4076. * @tx_ring: Tx descriptor ring for a specific queue
  4077. *
  4078. * Free all transmit software resources
  4079. **/
  4080. void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
  4081. {
  4082. ixgbe_clean_tx_ring(tx_ring);
  4083. vfree(tx_ring->tx_buffer_info);
  4084. tx_ring->tx_buffer_info = NULL;
  4085. /* if not set, then don't free */
  4086. if (!tx_ring->desc)
  4087. return;
  4088. dma_free_coherent(tx_ring->dev, tx_ring->size,
  4089. tx_ring->desc, tx_ring->dma);
  4090. tx_ring->desc = NULL;
  4091. }
  4092. /**
  4093. * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
  4094. * @adapter: board private structure
  4095. *
  4096. * Free all transmit software resources
  4097. **/
  4098. static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
  4099. {
  4100. int i;
  4101. for (i = 0; i < adapter->num_tx_queues; i++)
  4102. if (adapter->tx_ring[i]->desc)
  4103. ixgbe_free_tx_resources(adapter->tx_ring[i]);
  4104. }
  4105. /**
  4106. * ixgbe_free_rx_resources - Free Rx Resources
  4107. * @rx_ring: ring to clean the resources from
  4108. *
  4109. * Free all receive software resources
  4110. **/
  4111. void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
  4112. {
  4113. ixgbe_clean_rx_ring(rx_ring);
  4114. vfree(rx_ring->rx_buffer_info);
  4115. rx_ring->rx_buffer_info = NULL;
  4116. /* if not set, then don't free */
  4117. if (!rx_ring->desc)
  4118. return;
  4119. dma_free_coherent(rx_ring->dev, rx_ring->size,
  4120. rx_ring->desc, rx_ring->dma);
  4121. rx_ring->desc = NULL;
  4122. }
  4123. /**
  4124. * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
  4125. * @adapter: board private structure
  4126. *
  4127. * Free all receive software resources
  4128. **/
  4129. static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
  4130. {
  4131. int i;
  4132. #ifdef IXGBE_FCOE
  4133. ixgbe_free_fcoe_ddp_resources(adapter);
  4134. #endif
  4135. for (i = 0; i < adapter->num_rx_queues; i++)
  4136. if (adapter->rx_ring[i]->desc)
  4137. ixgbe_free_rx_resources(adapter->rx_ring[i]);
  4138. }
  4139. /**
  4140. * ixgbe_change_mtu - Change the Maximum Transfer Unit
  4141. * @netdev: network interface device structure
  4142. * @new_mtu: new value for maximum frame size
  4143. *
  4144. * Returns 0 on success, negative on failure
  4145. **/
  4146. static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
  4147. {
  4148. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  4149. int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
  4150. /* MTU < 68 is an error and causes problems on some kernels */
  4151. if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
  4152. return -EINVAL;
  4153. /*
  4154. * For 82599EB we cannot allow legacy VFs to enable their receive
  4155. * paths when MTU greater than 1500 is configured. So display a
  4156. * warning that legacy VFs will be disabled.
  4157. */
  4158. if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
  4159. (adapter->hw.mac.type == ixgbe_mac_82599EB) &&
  4160. (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
  4161. e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n");
  4162. e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
  4163. /* must set new MTU before calling down or up */
  4164. netdev->mtu = new_mtu;
  4165. if (netif_running(netdev))
  4166. ixgbe_reinit_locked(adapter);
  4167. return 0;
  4168. }
  4169. /**
  4170. * ixgbe_open - Called when a network interface is made active
  4171. * @netdev: network interface device structure
  4172. *
  4173. * Returns 0 on success, negative value on failure
  4174. *
  4175. * The open entry point is called when a network interface is made
  4176. * active by the system (IFF_UP). At this point all resources needed
  4177. * for transmit and receive operations are allocated, the interrupt
  4178. * handler is registered with the OS, the watchdog timer is started,
  4179. * and the stack is notified that the interface is ready.
  4180. **/
  4181. static int ixgbe_open(struct net_device *netdev)
  4182. {
  4183. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  4184. int err;
  4185. /* disallow open during test */
  4186. if (test_bit(__IXGBE_TESTING, &adapter->state))
  4187. return -EBUSY;
  4188. netif_carrier_off(netdev);
  4189. /* allocate transmit descriptors */
  4190. err = ixgbe_setup_all_tx_resources(adapter);
  4191. if (err)
  4192. goto err_setup_tx;
  4193. /* allocate receive descriptors */
  4194. err = ixgbe_setup_all_rx_resources(adapter);
  4195. if (err)
  4196. goto err_setup_rx;
  4197. ixgbe_configure(adapter);
  4198. err = ixgbe_request_irq(adapter);
  4199. if (err)
  4200. goto err_req_irq;
  4201. /* Notify the stack of the actual queue counts. */
  4202. err = netif_set_real_num_tx_queues(netdev,
  4203. adapter->num_rx_pools > 1 ? 1 :
  4204. adapter->num_tx_queues);
  4205. if (err)
  4206. goto err_set_queues;
  4207. err = netif_set_real_num_rx_queues(netdev,
  4208. adapter->num_rx_pools > 1 ? 1 :
  4209. adapter->num_rx_queues);
  4210. if (err)
  4211. goto err_set_queues;
  4212. ixgbe_ptp_init(adapter);
  4213. ixgbe_up_complete(adapter);
  4214. return 0;
  4215. err_set_queues:
  4216. ixgbe_free_irq(adapter);
  4217. err_req_irq:
  4218. ixgbe_free_all_rx_resources(adapter);
  4219. err_setup_rx:
  4220. ixgbe_free_all_tx_resources(adapter);
  4221. err_setup_tx:
  4222. ixgbe_reset(adapter);
  4223. return err;
  4224. }
  4225. /**
  4226. * ixgbe_close - Disables a network interface
  4227. * @netdev: network interface device structure
  4228. *
  4229. * Returns 0, this is not allowed to fail
  4230. *
  4231. * The close entry point is called when an interface is de-activated
  4232. * by the OS. The hardware is still under the drivers control, but
  4233. * needs to be disabled. A global MAC reset is issued to stop the
  4234. * hardware, and all transmit and receive resources are freed.
  4235. **/
  4236. static int ixgbe_close(struct net_device *netdev)
  4237. {
  4238. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  4239. ixgbe_ptp_stop(adapter);
  4240. ixgbe_down(adapter);
  4241. ixgbe_free_irq(adapter);
  4242. ixgbe_fdir_filter_exit(adapter);
  4243. ixgbe_free_all_tx_resources(adapter);
  4244. ixgbe_free_all_rx_resources(adapter);
  4245. ixgbe_release_hw_control(adapter);
  4246. return 0;
  4247. }
  4248. #ifdef CONFIG_PM
  4249. static int ixgbe_resume(struct pci_dev *pdev)
  4250. {
  4251. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  4252. struct net_device *netdev = adapter->netdev;
  4253. u32 err;
  4254. pci_set_power_state(pdev, PCI_D0);
  4255. pci_restore_state(pdev);
  4256. /*
  4257. * pci_restore_state clears dev->state_saved so call
  4258. * pci_save_state to restore it.
  4259. */
  4260. pci_save_state(pdev);
  4261. err = pci_enable_device_mem(pdev);
  4262. if (err) {
  4263. e_dev_err("Cannot enable PCI device from suspend\n");
  4264. return err;
  4265. }
  4266. pci_set_master(pdev);
  4267. pci_wake_from_d3(pdev, false);
  4268. ixgbe_reset(adapter);
  4269. IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
  4270. rtnl_lock();
  4271. err = ixgbe_init_interrupt_scheme(adapter);
  4272. if (!err && netif_running(netdev))
  4273. err = ixgbe_open(netdev);
  4274. rtnl_unlock();
  4275. if (err)
  4276. return err;
  4277. netif_device_attach(netdev);
  4278. return 0;
  4279. }
  4280. #endif /* CONFIG_PM */
  4281. static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
  4282. {
  4283. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  4284. struct net_device *netdev = adapter->netdev;
  4285. struct ixgbe_hw *hw = &adapter->hw;
  4286. u32 ctrl, fctrl;
  4287. u32 wufc = adapter->wol;
  4288. #ifdef CONFIG_PM
  4289. int retval = 0;
  4290. #endif
  4291. netif_device_detach(netdev);
  4292. if (netif_running(netdev)) {
  4293. rtnl_lock();
  4294. ixgbe_down(adapter);
  4295. ixgbe_free_irq(adapter);
  4296. ixgbe_free_all_tx_resources(adapter);
  4297. ixgbe_free_all_rx_resources(adapter);
  4298. rtnl_unlock();
  4299. }
  4300. ixgbe_clear_interrupt_scheme(adapter);
  4301. #ifdef CONFIG_PM
  4302. retval = pci_save_state(pdev);
  4303. if (retval)
  4304. return retval;
  4305. #endif
  4306. if (wufc) {
  4307. ixgbe_set_rx_mode(netdev);
  4308. /* enable the optics for 82599 SFP+ fiber as we can WoL */
  4309. if (hw->mac.ops.enable_tx_laser)
  4310. hw->mac.ops.enable_tx_laser(hw);
  4311. /* turn on all-multi mode if wake on multicast is enabled */
  4312. if (wufc & IXGBE_WUFC_MC) {
  4313. fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
  4314. fctrl |= IXGBE_FCTRL_MPE;
  4315. IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
  4316. }
  4317. ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
  4318. ctrl |= IXGBE_CTRL_GIO_DIS;
  4319. IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
  4320. IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
  4321. } else {
  4322. IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
  4323. IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
  4324. }
  4325. switch (hw->mac.type) {
  4326. case ixgbe_mac_82598EB:
  4327. pci_wake_from_d3(pdev, false);
  4328. break;
  4329. case ixgbe_mac_82599EB:
  4330. case ixgbe_mac_X540:
  4331. pci_wake_from_d3(pdev, !!wufc);
  4332. break;
  4333. default:
  4334. break;
  4335. }
  4336. *enable_wake = !!wufc;
  4337. ixgbe_release_hw_control(adapter);
  4338. pci_disable_device(pdev);
  4339. return 0;
  4340. }
  4341. #ifdef CONFIG_PM
  4342. static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
  4343. {
  4344. int retval;
  4345. bool wake;
  4346. retval = __ixgbe_shutdown(pdev, &wake);
  4347. if (retval)
  4348. return retval;
  4349. if (wake) {
  4350. pci_prepare_to_sleep(pdev);
  4351. } else {
  4352. pci_wake_from_d3(pdev, false);
  4353. pci_set_power_state(pdev, PCI_D3hot);
  4354. }
  4355. return 0;
  4356. }
  4357. #endif /* CONFIG_PM */
  4358. static void ixgbe_shutdown(struct pci_dev *pdev)
  4359. {
  4360. bool wake;
  4361. __ixgbe_shutdown(pdev, &wake);
  4362. if (system_state == SYSTEM_POWER_OFF) {
  4363. pci_wake_from_d3(pdev, wake);
  4364. pci_set_power_state(pdev, PCI_D3hot);
  4365. }
  4366. }
  4367. /**
  4368. * ixgbe_update_stats - Update the board statistics counters.
  4369. * @adapter: board private structure
  4370. **/
  4371. void ixgbe_update_stats(struct ixgbe_adapter *adapter)
  4372. {
  4373. struct net_device *netdev = adapter->netdev;
  4374. struct ixgbe_hw *hw = &adapter->hw;
  4375. struct ixgbe_hw_stats *hwstats = &adapter->stats;
  4376. u64 total_mpc = 0;
  4377. u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
  4378. u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
  4379. u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
  4380. u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;
  4381. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  4382. test_bit(__IXGBE_RESETTING, &adapter->state))
  4383. return;
  4384. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
  4385. u64 rsc_count = 0;
  4386. u64 rsc_flush = 0;
  4387. for (i = 0; i < adapter->num_rx_queues; i++) {
  4388. rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
  4389. rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
  4390. }
  4391. adapter->rsc_total_count = rsc_count;
  4392. adapter->rsc_total_flush = rsc_flush;
  4393. }
  4394. for (i = 0; i < adapter->num_rx_queues; i++) {
  4395. struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
  4396. non_eop_descs += rx_ring->rx_stats.non_eop_descs;
  4397. alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
  4398. alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
  4399. hw_csum_rx_error += rx_ring->rx_stats.csum_err;
  4400. bytes += rx_ring->stats.bytes;
  4401. packets += rx_ring->stats.packets;
  4402. }
  4403. adapter->non_eop_descs = non_eop_descs;
  4404. adapter->alloc_rx_page_failed = alloc_rx_page_failed;
  4405. adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
  4406. adapter->hw_csum_rx_error = hw_csum_rx_error;
  4407. netdev->stats.rx_bytes = bytes;
  4408. netdev->stats.rx_packets = packets;
  4409. bytes = 0;
  4410. packets = 0;
  4411. /* gather some stats to the adapter struct that are per queue */
  4412. for (i = 0; i < adapter->num_tx_queues; i++) {
  4413. struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
  4414. restart_queue += tx_ring->tx_stats.restart_queue;
  4415. tx_busy += tx_ring->tx_stats.tx_busy;
  4416. bytes += tx_ring->stats.bytes;
  4417. packets += tx_ring->stats.packets;
  4418. }
  4419. adapter->restart_queue = restart_queue;
  4420. adapter->tx_busy = tx_busy;
  4421. netdev->stats.tx_bytes = bytes;
  4422. netdev->stats.tx_packets = packets;
  4423. hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
  4424. /* 8 register reads */
  4425. for (i = 0; i < 8; i++) {
  4426. /* for packet buffers not used, the register should read 0 */
  4427. mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
  4428. missed_rx += mpc;
  4429. hwstats->mpc[i] += mpc;
  4430. total_mpc += hwstats->mpc[i];
  4431. hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
  4432. hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
  4433. switch (hw->mac.type) {
  4434. case ixgbe_mac_82598EB:
  4435. hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
  4436. hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
  4437. hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
  4438. hwstats->pxonrxc[i] +=
  4439. IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
  4440. break;
  4441. case ixgbe_mac_82599EB:
  4442. case ixgbe_mac_X540:
  4443. hwstats->pxonrxc[i] +=
  4444. IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
  4445. break;
  4446. default:
  4447. break;
  4448. }
  4449. }
  4450. /*16 register reads */
  4451. for (i = 0; i < 16; i++) {
  4452. hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
  4453. hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
  4454. if ((hw->mac.type == ixgbe_mac_82599EB) ||
  4455. (hw->mac.type == ixgbe_mac_X540)) {
  4456. hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
  4457. IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
  4458. hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
  4459. IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
  4460. }
  4461. }
  4462. hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
  4463. /* work around hardware counting issue */
  4464. hwstats->gprc -= missed_rx;
  4465. ixgbe_update_xoff_received(adapter);
  4466. /* 82598 hardware only has a 32 bit counter in the high register */
  4467. switch (hw->mac.type) {
  4468. case ixgbe_mac_82598EB:
  4469. hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
  4470. hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
  4471. hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
  4472. hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
  4473. break;
  4474. case ixgbe_mac_X540:
  4475. /* OS2BMC stats are X540 only*/
  4476. hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
  4477. hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
  4478. hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
  4479. hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
  4480. case ixgbe_mac_82599EB:
  4481. for (i = 0; i < 16; i++)
  4482. adapter->hw_rx_no_dma_resources +=
  4483. IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
  4484. hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
  4485. IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
  4486. hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
  4487. IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
  4488. hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
  4489. IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
  4490. hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
  4491. hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
  4492. hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
  4493. #ifdef IXGBE_FCOE
  4494. hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
  4495. hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
  4496. hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
  4497. hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
  4498. hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
  4499. hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
  4500. /* Add up per cpu counters for total ddp aloc fail */
  4501. if (adapter->fcoe.ddp_pool) {
  4502. struct ixgbe_fcoe *fcoe = &adapter->fcoe;
  4503. struct ixgbe_fcoe_ddp_pool *ddp_pool;
  4504. unsigned int cpu;
  4505. u64 noddp = 0, noddp_ext_buff = 0;
  4506. for_each_possible_cpu(cpu) {
  4507. ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
  4508. noddp += ddp_pool->noddp;
  4509. noddp_ext_buff += ddp_pool->noddp_ext_buff;
  4510. }
  4511. hwstats->fcoe_noddp = noddp;
  4512. hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;
  4513. }
  4514. #endif /* IXGBE_FCOE */
  4515. break;
  4516. default:
  4517. break;
  4518. }
  4519. bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
  4520. hwstats->bprc += bprc;
  4521. hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
  4522. if (hw->mac.type == ixgbe_mac_82598EB)
  4523. hwstats->mprc -= bprc;
  4524. hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
  4525. hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
  4526. hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
  4527. hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
  4528. hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
  4529. hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
  4530. hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
  4531. hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
  4532. lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
  4533. hwstats->lxontxc += lxon;
  4534. lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
  4535. hwstats->lxofftxc += lxoff;
  4536. hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
  4537. hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
  4538. /*
  4539. * 82598 errata - tx of flow control packets is included in tx counters
  4540. */
  4541. xon_off_tot = lxon + lxoff;
  4542. hwstats->gptc -= xon_off_tot;
  4543. hwstats->mptc -= xon_off_tot;
  4544. hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
  4545. hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
  4546. hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
  4547. hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
  4548. hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
  4549. hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
  4550. hwstats->ptc64 -= xon_off_tot;
  4551. hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
  4552. hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
  4553. hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
  4554. hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
  4555. hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
  4556. hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
  4557. /* Fill out the OS statistics structure */
  4558. netdev->stats.multicast = hwstats->mprc;
  4559. /* Rx Errors */
  4560. netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
  4561. netdev->stats.rx_dropped = 0;
  4562. netdev->stats.rx_length_errors = hwstats->rlec;
  4563. netdev->stats.rx_crc_errors = hwstats->crcerrs;
  4564. netdev->stats.rx_missed_errors = total_mpc;
  4565. }
  4566. /**
  4567. * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
  4568. * @adapter: pointer to the device adapter structure
  4569. **/
  4570. static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
  4571. {
  4572. struct ixgbe_hw *hw = &adapter->hw;
  4573. int i;
  4574. if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
  4575. return;
  4576. adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
  4577. /* if interface is down do nothing */
  4578. if (test_bit(__IXGBE_DOWN, &adapter->state))
  4579. return;
  4580. /* do nothing if we are not using signature filters */
  4581. if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
  4582. return;
  4583. adapter->fdir_overflow++;
  4584. if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
  4585. for (i = 0; i < adapter->num_tx_queues; i++)
  4586. set_bit(__IXGBE_TX_FDIR_INIT_DONE,
  4587. &(adapter->tx_ring[i]->state));
  4588. /* re-enable flow director interrupts */
  4589. IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
  4590. } else {
  4591. e_err(probe, "failed to finish FDIR re-initialization, "
  4592. "ignored adding FDIR ATR filters\n");
  4593. }
  4594. }
  4595. /**
  4596. * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
  4597. * @adapter: pointer to the device adapter structure
  4598. *
  4599. * This function serves two purposes. First it strobes the interrupt lines
  4600. * in order to make certain interrupts are occurring. Secondly it sets the
  4601. * bits needed to check for TX hangs. As a result we should immediately
  4602. * determine if a hang has occurred.
  4603. */
  4604. static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
  4605. {
  4606. struct ixgbe_hw *hw = &adapter->hw;
  4607. u64 eics = 0;
  4608. int i;
  4609. /* If we're down or resetting, just bail */
  4610. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  4611. test_bit(__IXGBE_RESETTING, &adapter->state))
  4612. return;
  4613. /* Force detection of hung controller */
  4614. if (netif_carrier_ok(adapter->netdev)) {
  4615. for (i = 0; i < adapter->num_tx_queues; i++)
  4616. set_check_for_tx_hang(adapter->tx_ring[i]);
  4617. }
  4618. if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
  4619. /*
  4620. * for legacy and MSI interrupts don't set any bits
  4621. * that are enabled for EIAM, because this operation
  4622. * would set *both* EIMS and EICS for any bit in EIAM
  4623. */
  4624. IXGBE_WRITE_REG(hw, IXGBE_EICS,
  4625. (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
  4626. } else {
  4627. /* get one bit for every active tx/rx interrupt vector */
  4628. for (i = 0; i < adapter->num_q_vectors; i++) {
  4629. struct ixgbe_q_vector *qv = adapter->q_vector[i];
  4630. if (qv->rx.ring || qv->tx.ring)
  4631. eics |= ((u64)1 << i);
  4632. }
  4633. }
  4634. /* Cause software interrupt to ensure rings are cleaned */
  4635. ixgbe_irq_rearm_queues(adapter, eics);
  4636. }
  4637. /**
  4638. * ixgbe_watchdog_update_link - update the link status
  4639. * @adapter: pointer to the device adapter structure
  4640. * @link_speed: pointer to a u32 to store the link_speed
  4641. **/
  4642. static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
  4643. {
  4644. struct ixgbe_hw *hw = &adapter->hw;
  4645. u32 link_speed = adapter->link_speed;
  4646. bool link_up = adapter->link_up;
  4647. bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
  4648. if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
  4649. return;
  4650. if (hw->mac.ops.check_link) {
  4651. hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
  4652. } else {
  4653. /* always assume link is up, if no check link function */
  4654. link_speed = IXGBE_LINK_SPEED_10GB_FULL;
  4655. link_up = true;
  4656. }
  4657. if (adapter->ixgbe_ieee_pfc)
  4658. pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
  4659. if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
  4660. hw->mac.ops.fc_enable(hw);
  4661. ixgbe_set_rx_drop_en(adapter);
  4662. }
  4663. if (link_up ||
  4664. time_after(jiffies, (adapter->link_check_timeout +
  4665. IXGBE_TRY_LINK_TIMEOUT))) {
  4666. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
  4667. IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
  4668. IXGBE_WRITE_FLUSH(hw);
  4669. }
  4670. adapter->link_up = link_up;
  4671. adapter->link_speed = link_speed;
  4672. }
  4673. static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)
  4674. {
  4675. #ifdef CONFIG_IXGBE_DCB
  4676. struct net_device *netdev = adapter->netdev;
  4677. struct dcb_app app = {
  4678. .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE,
  4679. .protocol = 0,
  4680. };
  4681. u8 up = 0;
  4682. if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)
  4683. up = dcb_ieee_getapp_mask(netdev, &app);
  4684. adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0;
  4685. #endif
  4686. }
  4687. /**
  4688. * ixgbe_watchdog_link_is_up - update netif_carrier status and
  4689. * print link up message
  4690. * @adapter: pointer to the device adapter structure
  4691. **/
  4692. static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
  4693. {
  4694. struct net_device *netdev = adapter->netdev;
  4695. struct ixgbe_hw *hw = &adapter->hw;
  4696. u32 link_speed = adapter->link_speed;
  4697. bool flow_rx, flow_tx;
  4698. /* only continue if link was previously down */
  4699. if (netif_carrier_ok(netdev))
  4700. return;
  4701. adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
  4702. switch (hw->mac.type) {
  4703. case ixgbe_mac_82598EB: {
  4704. u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
  4705. u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
  4706. flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
  4707. flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
  4708. }
  4709. break;
  4710. case ixgbe_mac_X540:
  4711. case ixgbe_mac_82599EB: {
  4712. u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
  4713. u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
  4714. flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
  4715. flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
  4716. }
  4717. break;
  4718. default:
  4719. flow_tx = false;
  4720. flow_rx = false;
  4721. break;
  4722. }
  4723. if (adapter->flags2 & IXGBE_FLAG2_PTP_ENABLED)
  4724. ixgbe_ptp_start_cyclecounter(adapter);
  4725. e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
  4726. (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
  4727. "10 Gbps" :
  4728. (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
  4729. "1 Gbps" :
  4730. (link_speed == IXGBE_LINK_SPEED_100_FULL ?
  4731. "100 Mbps" :
  4732. "unknown speed"))),
  4733. ((flow_rx && flow_tx) ? "RX/TX" :
  4734. (flow_rx ? "RX" :
  4735. (flow_tx ? "TX" : "None"))));
  4736. netif_carrier_on(netdev);
  4737. ixgbe_check_vf_rate_limit(adapter);
  4738. /* update the default user priority for VFs */
  4739. ixgbe_update_default_up(adapter);
  4740. /* ping all the active vfs to let them know link has changed */
  4741. ixgbe_ping_all_vfs(adapter);
  4742. }
  4743. /**
  4744. * ixgbe_watchdog_link_is_down - update netif_carrier status and
  4745. * print link down message
  4746. * @adapter: pointer to the adapter structure
  4747. **/
  4748. static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
  4749. {
  4750. struct net_device *netdev = adapter->netdev;
  4751. struct ixgbe_hw *hw = &adapter->hw;
  4752. adapter->link_up = false;
  4753. adapter->link_speed = 0;
  4754. /* only continue if link was up previously */
  4755. if (!netif_carrier_ok(netdev))
  4756. return;
  4757. /* poll for SFP+ cable when link is down */
  4758. if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
  4759. adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
  4760. if (adapter->flags2 & IXGBE_FLAG2_PTP_ENABLED)
  4761. ixgbe_ptp_start_cyclecounter(adapter);
  4762. e_info(drv, "NIC Link is Down\n");
  4763. netif_carrier_off(netdev);
  4764. /* ping all the active vfs to let them know link has changed */
  4765. ixgbe_ping_all_vfs(adapter);
  4766. }
  4767. /**
  4768. * ixgbe_watchdog_flush_tx - flush queues on link down
  4769. * @adapter: pointer to the device adapter structure
  4770. **/
  4771. static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
  4772. {
  4773. int i;
  4774. int some_tx_pending = 0;
  4775. if (!netif_carrier_ok(adapter->netdev)) {
  4776. for (i = 0; i < adapter->num_tx_queues; i++) {
  4777. struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
  4778. if (tx_ring->next_to_use != tx_ring->next_to_clean) {
  4779. some_tx_pending = 1;
  4780. break;
  4781. }
  4782. }
  4783. if (some_tx_pending) {
  4784. /* We've lost link, so the controller stops DMA,
  4785. * but we've got queued Tx work that's never going
  4786. * to get done, so reset controller to flush Tx.
  4787. * (Do the reset outside of interrupt context).
  4788. */
  4789. adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
  4790. }
  4791. }
  4792. }
  4793. static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
  4794. {
  4795. u32 ssvpc;
  4796. /* Do not perform spoof check for 82598 or if not in IOV mode */
  4797. if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
  4798. adapter->num_vfs == 0)
  4799. return;
  4800. ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
  4801. /*
  4802. * ssvpc register is cleared on read, if zero then no
  4803. * spoofed packets in the last interval.
  4804. */
  4805. if (!ssvpc)
  4806. return;
  4807. e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
  4808. }
  4809. /**
  4810. * ixgbe_watchdog_subtask - check and bring link up
  4811. * @adapter: pointer to the device adapter structure
  4812. **/
  4813. static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
  4814. {
  4815. /* if interface is down do nothing */
  4816. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  4817. test_bit(__IXGBE_RESETTING, &adapter->state))
  4818. return;
  4819. ixgbe_watchdog_update_link(adapter);
  4820. if (adapter->link_up)
  4821. ixgbe_watchdog_link_is_up(adapter);
  4822. else
  4823. ixgbe_watchdog_link_is_down(adapter);
  4824. ixgbe_spoof_check(adapter);
  4825. ixgbe_update_stats(adapter);
  4826. ixgbe_watchdog_flush_tx(adapter);
  4827. }
  4828. /**
  4829. * ixgbe_sfp_detection_subtask - poll for SFP+ cable
  4830. * @adapter: the ixgbe adapter structure
  4831. **/
  4832. static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
  4833. {
  4834. struct ixgbe_hw *hw = &adapter->hw;
  4835. s32 err;
  4836. /* not searching for SFP so there is nothing to do here */
  4837. if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
  4838. !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
  4839. return;
  4840. /* someone else is in init, wait until next service event */
  4841. if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
  4842. return;
  4843. err = hw->phy.ops.identify_sfp(hw);
  4844. if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
  4845. goto sfp_out;
  4846. if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
  4847. /* If no cable is present, then we need to reset
  4848. * the next time we find a good cable. */
  4849. adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
  4850. }
  4851. /* exit on error */
  4852. if (err)
  4853. goto sfp_out;
  4854. /* exit if reset not needed */
  4855. if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
  4856. goto sfp_out;
  4857. adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
  4858. /*
  4859. * A module may be identified correctly, but the EEPROM may not have
  4860. * support for that module. setup_sfp() will fail in that case, so
  4861. * we should not allow that module to load.
  4862. */
  4863. if (hw->mac.type == ixgbe_mac_82598EB)
  4864. err = hw->phy.ops.reset(hw);
  4865. else
  4866. err = hw->mac.ops.setup_sfp(hw);
  4867. if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
  4868. goto sfp_out;
  4869. adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
  4870. e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
  4871. sfp_out:
  4872. clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
  4873. if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
  4874. (adapter->netdev->reg_state == NETREG_REGISTERED)) {
  4875. e_dev_err("failed to initialize because an unsupported "
  4876. "SFP+ module type was detected.\n");
  4877. e_dev_err("Reload the driver after installing a "
  4878. "supported module.\n");
  4879. unregister_netdev(adapter->netdev);
  4880. }
  4881. }
  4882. /**
  4883. * ixgbe_sfp_link_config_subtask - set up link SFP after module install
  4884. * @adapter: the ixgbe adapter structure
  4885. **/
  4886. static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
  4887. {
  4888. struct ixgbe_hw *hw = &adapter->hw;
  4889. u32 autoneg;
  4890. bool negotiation;
  4891. if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
  4892. return;
  4893. /* someone else is in init, wait until next service event */
  4894. if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
  4895. return;
  4896. adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
  4897. autoneg = hw->phy.autoneg_advertised;
  4898. if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
  4899. hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
  4900. if (hw->mac.ops.setup_link)
  4901. hw->mac.ops.setup_link(hw, autoneg, negotiation, true);
  4902. adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
  4903. adapter->link_check_timeout = jiffies;
  4904. clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
  4905. }
  4906. #ifdef CONFIG_PCI_IOV
  4907. static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
  4908. {
  4909. int vf;
  4910. struct ixgbe_hw *hw = &adapter->hw;
  4911. struct net_device *netdev = adapter->netdev;
  4912. u32 gpc;
  4913. u32 ciaa, ciad;
  4914. gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
  4915. if (gpc) /* If incrementing then no need for the check below */
  4916. return;
  4917. /*
  4918. * Check to see if a bad DMA write target from an errant or
  4919. * malicious VF has caused a PCIe error. If so then we can
  4920. * issue a VFLR to the offending VF(s) and then resume without
  4921. * requesting a full slot reset.
  4922. */
  4923. for (vf = 0; vf < adapter->num_vfs; vf++) {
  4924. ciaa = (vf << 16) | 0x80000000;
  4925. /* 32 bit read so align, we really want status at offset 6 */
  4926. ciaa |= PCI_COMMAND;
  4927. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  4928. ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599);
  4929. ciaa &= 0x7FFFFFFF;
  4930. /* disable debug mode asap after reading data */
  4931. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  4932. /* Get the upper 16 bits which will be the PCI status reg */
  4933. ciad >>= 16;
  4934. if (ciad & PCI_STATUS_REC_MASTER_ABORT) {
  4935. netdev_err(netdev, "VF %d Hung DMA\n", vf);
  4936. /* Issue VFLR */
  4937. ciaa = (vf << 16) | 0x80000000;
  4938. ciaa |= 0xA8;
  4939. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  4940. ciad = 0x00008000; /* VFLR */
  4941. IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad);
  4942. ciaa &= 0x7FFFFFFF;
  4943. IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
  4944. }
  4945. }
  4946. }
  4947. #endif
  4948. /**
  4949. * ixgbe_service_timer - Timer Call-back
  4950. * @data: pointer to adapter cast into an unsigned long
  4951. **/
  4952. static void ixgbe_service_timer(unsigned long data)
  4953. {
  4954. struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
  4955. unsigned long next_event_offset;
  4956. bool ready = true;
  4957. /* poll faster when waiting for link */
  4958. if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
  4959. next_event_offset = HZ / 10;
  4960. else
  4961. next_event_offset = HZ * 2;
  4962. #ifdef CONFIG_PCI_IOV
  4963. /*
  4964. * don't bother with SR-IOV VF DMA hang check if there are
  4965. * no VFs or the link is down
  4966. */
  4967. if (!adapter->num_vfs ||
  4968. (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
  4969. goto normal_timer_service;
  4970. /* If we have VFs allocated then we must check for DMA hangs */
  4971. ixgbe_check_for_bad_vf(adapter);
  4972. next_event_offset = HZ / 50;
  4973. adapter->timer_event_accumulator++;
  4974. if (adapter->timer_event_accumulator >= 100)
  4975. adapter->timer_event_accumulator = 0;
  4976. else
  4977. ready = false;
  4978. normal_timer_service:
  4979. #endif
  4980. /* Reset the timer */
  4981. mod_timer(&adapter->service_timer, next_event_offset + jiffies);
  4982. if (ready)
  4983. ixgbe_service_event_schedule(adapter);
  4984. }
  4985. static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
  4986. {
  4987. if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
  4988. return;
  4989. adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
  4990. /* If we're already down or resetting, just bail */
  4991. if (test_bit(__IXGBE_DOWN, &adapter->state) ||
  4992. test_bit(__IXGBE_RESETTING, &adapter->state))
  4993. return;
  4994. ixgbe_dump(adapter);
  4995. netdev_err(adapter->netdev, "Reset adapter\n");
  4996. adapter->tx_timeout_count++;
  4997. ixgbe_reinit_locked(adapter);
  4998. }
  4999. /**
  5000. * ixgbe_service_task - manages and runs subtasks
  5001. * @work: pointer to work_struct containing our data
  5002. **/
  5003. static void ixgbe_service_task(struct work_struct *work)
  5004. {
  5005. struct ixgbe_adapter *adapter = container_of(work,
  5006. struct ixgbe_adapter,
  5007. service_task);
  5008. ixgbe_reset_subtask(adapter);
  5009. ixgbe_sfp_detection_subtask(adapter);
  5010. ixgbe_sfp_link_config_subtask(adapter);
  5011. ixgbe_check_overtemp_subtask(adapter);
  5012. ixgbe_watchdog_subtask(adapter);
  5013. ixgbe_fdir_reinit_subtask(adapter);
  5014. ixgbe_check_hang_subtask(adapter);
  5015. ixgbe_ptp_overflow_check(adapter);
  5016. ixgbe_service_event_complete(adapter);
  5017. }
  5018. static int ixgbe_tso(struct ixgbe_ring *tx_ring,
  5019. struct ixgbe_tx_buffer *first,
  5020. u8 *hdr_len)
  5021. {
  5022. struct sk_buff *skb = first->skb;
  5023. u32 vlan_macip_lens, type_tucmd;
  5024. u32 mss_l4len_idx, l4len;
  5025. if (!skb_is_gso(skb))
  5026. return 0;
  5027. if (skb_header_cloned(skb)) {
  5028. int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  5029. if (err)
  5030. return err;
  5031. }
  5032. /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
  5033. type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
  5034. if (first->protocol == __constant_htons(ETH_P_IP)) {
  5035. struct iphdr *iph = ip_hdr(skb);
  5036. iph->tot_len = 0;
  5037. iph->check = 0;
  5038. tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
  5039. iph->daddr, 0,
  5040. IPPROTO_TCP,
  5041. 0);
  5042. type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
  5043. first->tx_flags |= IXGBE_TX_FLAGS_TSO |
  5044. IXGBE_TX_FLAGS_CSUM |
  5045. IXGBE_TX_FLAGS_IPV4;
  5046. } else if (skb_is_gso_v6(skb)) {
  5047. ipv6_hdr(skb)->payload_len = 0;
  5048. tcp_hdr(skb)->check =
  5049. ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
  5050. &ipv6_hdr(skb)->daddr,
  5051. 0, IPPROTO_TCP, 0);
  5052. first->tx_flags |= IXGBE_TX_FLAGS_TSO |
  5053. IXGBE_TX_FLAGS_CSUM;
  5054. }
  5055. /* compute header lengths */
  5056. l4len = tcp_hdrlen(skb);
  5057. *hdr_len = skb_transport_offset(skb) + l4len;
  5058. /* update gso size and bytecount with header size */
  5059. first->gso_segs = skb_shinfo(skb)->gso_segs;
  5060. first->bytecount += (first->gso_segs - 1) * *hdr_len;
  5061. /* mss_l4len_id: use 1 as index for TSO */
  5062. mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
  5063. mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
  5064. mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
  5065. /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
  5066. vlan_macip_lens = skb_network_header_len(skb);
  5067. vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
  5068. vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
  5069. ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
  5070. mss_l4len_idx);
  5071. return 1;
  5072. }
  5073. static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
  5074. struct ixgbe_tx_buffer *first)
  5075. {
  5076. struct sk_buff *skb = first->skb;
  5077. u32 vlan_macip_lens = 0;
  5078. u32 mss_l4len_idx = 0;
  5079. u32 type_tucmd = 0;
  5080. if (skb->ip_summed != CHECKSUM_PARTIAL) {
  5081. if (!(first->tx_flags & IXGBE_TX_FLAGS_HW_VLAN)) {
  5082. if (unlikely(skb->no_fcs))
  5083. first->tx_flags |= IXGBE_TX_FLAGS_NO_IFCS;
  5084. if (!(first->tx_flags & IXGBE_TX_FLAGS_TXSW))
  5085. return;
  5086. }
  5087. } else {
  5088. u8 l4_hdr = 0;
  5089. switch (first->protocol) {
  5090. case __constant_htons(ETH_P_IP):
  5091. vlan_macip_lens |= skb_network_header_len(skb);
  5092. type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
  5093. l4_hdr = ip_hdr(skb)->protocol;
  5094. break;
  5095. case __constant_htons(ETH_P_IPV6):
  5096. vlan_macip_lens |= skb_network_header_len(skb);
  5097. l4_hdr = ipv6_hdr(skb)->nexthdr;
  5098. break;
  5099. default:
  5100. if (unlikely(net_ratelimit())) {
  5101. dev_warn(tx_ring->dev,
  5102. "partial checksum but proto=%x!\n",
  5103. first->protocol);
  5104. }
  5105. break;
  5106. }
  5107. switch (l4_hdr) {
  5108. case IPPROTO_TCP:
  5109. type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
  5110. mss_l4len_idx = tcp_hdrlen(skb) <<
  5111. IXGBE_ADVTXD_L4LEN_SHIFT;
  5112. break;
  5113. case IPPROTO_SCTP:
  5114. type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
  5115. mss_l4len_idx = sizeof(struct sctphdr) <<
  5116. IXGBE_ADVTXD_L4LEN_SHIFT;
  5117. break;
  5118. case IPPROTO_UDP:
  5119. mss_l4len_idx = sizeof(struct udphdr) <<
  5120. IXGBE_ADVTXD_L4LEN_SHIFT;
  5121. break;
  5122. default:
  5123. if (unlikely(net_ratelimit())) {
  5124. dev_warn(tx_ring->dev,
  5125. "partial checksum but l4 proto=%x!\n",
  5126. l4_hdr);
  5127. }
  5128. break;
  5129. }
  5130. /* update TX checksum flag */
  5131. first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
  5132. }
  5133. /* vlan_macip_lens: MACLEN, VLAN tag */
  5134. vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
  5135. vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
  5136. ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0,
  5137. type_tucmd, mss_l4len_idx);
  5138. }
  5139. static __le32 ixgbe_tx_cmd_type(u32 tx_flags)
  5140. {
  5141. /* set type for advanced descriptor with frame checksum insertion */
  5142. __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
  5143. IXGBE_ADVTXD_DCMD_DEXT);
  5144. /* set HW vlan bit if vlan is present */
  5145. if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN)
  5146. cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
  5147. if (tx_flags & IXGBE_TX_FLAGS_TSTAMP)
  5148. cmd_type |= cpu_to_le32(IXGBE_ADVTXD_MAC_TSTAMP);
  5149. /* set segmentation enable bits for TSO/FSO */
  5150. #ifdef IXGBE_FCOE
  5151. if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_FSO))
  5152. #else
  5153. if (tx_flags & IXGBE_TX_FLAGS_TSO)
  5154. #endif
  5155. cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
  5156. /* insert frame checksum */
  5157. if (!(tx_flags & IXGBE_TX_FLAGS_NO_IFCS))
  5158. cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_IFCS);
  5159. return cmd_type;
  5160. }
  5161. static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
  5162. u32 tx_flags, unsigned int paylen)
  5163. {
  5164. __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
  5165. /* enable L4 checksum for TSO and TX checksum offload */
  5166. if (tx_flags & IXGBE_TX_FLAGS_CSUM)
  5167. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
  5168. /* enble IPv4 checksum for TSO */
  5169. if (tx_flags & IXGBE_TX_FLAGS_IPV4)
  5170. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
  5171. /* use index 1 context for TSO/FSO/FCOE */
  5172. #ifdef IXGBE_FCOE
  5173. if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_FCOE))
  5174. #else
  5175. if (tx_flags & IXGBE_TX_FLAGS_TSO)
  5176. #endif
  5177. olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
  5178. /*
  5179. * Check Context must be set if Tx switch is enabled, which it
  5180. * always is for case where virtual functions are running
  5181. */
  5182. #ifdef IXGBE_FCOE
  5183. if (tx_flags & (IXGBE_TX_FLAGS_TXSW | IXGBE_TX_FLAGS_FCOE))
  5184. #else
  5185. if (tx_flags & IXGBE_TX_FLAGS_TXSW)
  5186. #endif
  5187. olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
  5188. tx_desc->read.olinfo_status = olinfo_status;
  5189. }
  5190. #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \
  5191. IXGBE_TXD_CMD_RS)
  5192. static void ixgbe_tx_map(struct ixgbe_ring *tx_ring,
  5193. struct ixgbe_tx_buffer *first,
  5194. const u8 hdr_len)
  5195. {
  5196. dma_addr_t dma;
  5197. struct sk_buff *skb = first->skb;
  5198. struct ixgbe_tx_buffer *tx_buffer;
  5199. union ixgbe_adv_tx_desc *tx_desc;
  5200. struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
  5201. unsigned int data_len = skb->data_len;
  5202. unsigned int size = skb_headlen(skb);
  5203. unsigned int paylen = skb->len - hdr_len;
  5204. u32 tx_flags = first->tx_flags;
  5205. __le32 cmd_type;
  5206. u16 i = tx_ring->next_to_use;
  5207. tx_desc = IXGBE_TX_DESC(tx_ring, i);
  5208. ixgbe_tx_olinfo_status(tx_desc, tx_flags, paylen);
  5209. cmd_type = ixgbe_tx_cmd_type(tx_flags);
  5210. #ifdef IXGBE_FCOE
  5211. if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
  5212. if (data_len < sizeof(struct fcoe_crc_eof)) {
  5213. size -= sizeof(struct fcoe_crc_eof) - data_len;
  5214. data_len = 0;
  5215. } else {
  5216. data_len -= sizeof(struct fcoe_crc_eof);
  5217. }
  5218. }
  5219. #endif
  5220. dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
  5221. if (dma_mapping_error(tx_ring->dev, dma))
  5222. goto dma_error;
  5223. /* record length, and DMA address */
  5224. dma_unmap_len_set(first, len, size);
  5225. dma_unmap_addr_set(first, dma, dma);
  5226. tx_desc->read.buffer_addr = cpu_to_le64(dma);
  5227. for (;;) {
  5228. while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
  5229. tx_desc->read.cmd_type_len =
  5230. cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
  5231. i++;
  5232. tx_desc++;
  5233. if (i == tx_ring->count) {
  5234. tx_desc = IXGBE_TX_DESC(tx_ring, 0);
  5235. i = 0;
  5236. }
  5237. dma += IXGBE_MAX_DATA_PER_TXD;
  5238. size -= IXGBE_MAX_DATA_PER_TXD;
  5239. tx_desc->read.buffer_addr = cpu_to_le64(dma);
  5240. tx_desc->read.olinfo_status = 0;
  5241. }
  5242. if (likely(!data_len))
  5243. break;
  5244. tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
  5245. i++;
  5246. tx_desc++;
  5247. if (i == tx_ring->count) {
  5248. tx_desc = IXGBE_TX_DESC(tx_ring, 0);
  5249. i = 0;
  5250. }
  5251. #ifdef IXGBE_FCOE
  5252. size = min_t(unsigned int, data_len, skb_frag_size(frag));
  5253. #else
  5254. size = skb_frag_size(frag);
  5255. #endif
  5256. data_len -= size;
  5257. dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
  5258. DMA_TO_DEVICE);
  5259. if (dma_mapping_error(tx_ring->dev, dma))
  5260. goto dma_error;
  5261. tx_buffer = &tx_ring->tx_buffer_info[i];
  5262. dma_unmap_len_set(tx_buffer, len, size);
  5263. dma_unmap_addr_set(tx_buffer, dma, dma);
  5264. tx_desc->read.buffer_addr = cpu_to_le64(dma);
  5265. tx_desc->read.olinfo_status = 0;
  5266. frag++;
  5267. }
  5268. /* write last descriptor with RS and EOP bits */
  5269. cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
  5270. tx_desc->read.cmd_type_len = cmd_type;
  5271. netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
  5272. /* set the timestamp */
  5273. first->time_stamp = jiffies;
  5274. /*
  5275. * Force memory writes to complete before letting h/w know there
  5276. * are new descriptors to fetch. (Only applicable for weak-ordered
  5277. * memory model archs, such as IA-64).
  5278. *
  5279. * We also need this memory barrier to make certain all of the
  5280. * status bits have been updated before next_to_watch is written.
  5281. */
  5282. wmb();
  5283. /* set next_to_watch value indicating a packet is present */
  5284. first->next_to_watch = tx_desc;
  5285. i++;
  5286. if (i == tx_ring->count)
  5287. i = 0;
  5288. tx_ring->next_to_use = i;
  5289. /* notify HW of packet */
  5290. writel(i, tx_ring->tail);
  5291. return;
  5292. dma_error:
  5293. dev_err(tx_ring->dev, "TX DMA map failed\n");
  5294. /* clear dma mappings for failed tx_buffer_info map */
  5295. for (;;) {
  5296. tx_buffer = &tx_ring->tx_buffer_info[i];
  5297. ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer);
  5298. if (tx_buffer == first)
  5299. break;
  5300. if (i == 0)
  5301. i = tx_ring->count;
  5302. i--;
  5303. }
  5304. tx_ring->next_to_use = i;
  5305. }
  5306. static void ixgbe_atr(struct ixgbe_ring *ring,
  5307. struct ixgbe_tx_buffer *first)
  5308. {
  5309. struct ixgbe_q_vector *q_vector = ring->q_vector;
  5310. union ixgbe_atr_hash_dword input = { .dword = 0 };
  5311. union ixgbe_atr_hash_dword common = { .dword = 0 };
  5312. union {
  5313. unsigned char *network;
  5314. struct iphdr *ipv4;
  5315. struct ipv6hdr *ipv6;
  5316. } hdr;
  5317. struct tcphdr *th;
  5318. __be16 vlan_id;
  5319. /* if ring doesn't have a interrupt vector, cannot perform ATR */
  5320. if (!q_vector)
  5321. return;
  5322. /* do nothing if sampling is disabled */
  5323. if (!ring->atr_sample_rate)
  5324. return;
  5325. ring->atr_count++;
  5326. /* snag network header to get L4 type and address */
  5327. hdr.network = skb_network_header(first->skb);
  5328. /* Currently only IPv4/IPv6 with TCP is supported */
  5329. if ((first->protocol != __constant_htons(ETH_P_IPV6) ||
  5330. hdr.ipv6->nexthdr != IPPROTO_TCP) &&
  5331. (first->protocol != __constant_htons(ETH_P_IP) ||
  5332. hdr.ipv4->protocol != IPPROTO_TCP))
  5333. return;
  5334. th = tcp_hdr(first->skb);
  5335. /* skip this packet since it is invalid or the socket is closing */
  5336. if (!th || th->fin)
  5337. return;
  5338. /* sample on all syn packets or once every atr sample count */
  5339. if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
  5340. return;
  5341. /* reset sample count */
  5342. ring->atr_count = 0;
  5343. vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
  5344. /*
  5345. * src and dst are inverted, think how the receiver sees them
  5346. *
  5347. * The input is broken into two sections, a non-compressed section
  5348. * containing vm_pool, vlan_id, and flow_type. The rest of the data
  5349. * is XORed together and stored in the compressed dword.
  5350. */
  5351. input.formatted.vlan_id = vlan_id;
  5352. /*
  5353. * since src port and flex bytes occupy the same word XOR them together
  5354. * and write the value to source port portion of compressed dword
  5355. */
  5356. if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
  5357. common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q);
  5358. else
  5359. common.port.src ^= th->dest ^ first->protocol;
  5360. common.port.dst ^= th->source;
  5361. if (first->protocol == __constant_htons(ETH_P_IP)) {
  5362. input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
  5363. common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
  5364. } else {
  5365. input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
  5366. common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
  5367. hdr.ipv6->saddr.s6_addr32[1] ^
  5368. hdr.ipv6->saddr.s6_addr32[2] ^
  5369. hdr.ipv6->saddr.s6_addr32[3] ^
  5370. hdr.ipv6->daddr.s6_addr32[0] ^
  5371. hdr.ipv6->daddr.s6_addr32[1] ^
  5372. hdr.ipv6->daddr.s6_addr32[2] ^
  5373. hdr.ipv6->daddr.s6_addr32[3];
  5374. }
  5375. /* This assumes the Rx queue and Tx queue are bound to the same CPU */
  5376. ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
  5377. input, common, ring->queue_index);
  5378. }
  5379. static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
  5380. {
  5381. netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
  5382. /* Herbert's original patch had:
  5383. * smp_mb__after_netif_stop_queue();
  5384. * but since that doesn't exist yet, just open code it. */
  5385. smp_mb();
  5386. /* We need to check again in a case another CPU has just
  5387. * made room available. */
  5388. if (likely(ixgbe_desc_unused(tx_ring) < size))
  5389. return -EBUSY;
  5390. /* A reprieve! - use start_queue because it doesn't call schedule */
  5391. netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
  5392. ++tx_ring->tx_stats.restart_queue;
  5393. return 0;
  5394. }
  5395. static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
  5396. {
  5397. if (likely(ixgbe_desc_unused(tx_ring) >= size))
  5398. return 0;
  5399. return __ixgbe_maybe_stop_tx(tx_ring, size);
  5400. }
  5401. static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb)
  5402. {
  5403. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5404. int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
  5405. smp_processor_id();
  5406. #ifdef IXGBE_FCOE
  5407. __be16 protocol = vlan_get_protocol(skb);
  5408. if (((protocol == htons(ETH_P_FCOE)) ||
  5409. (protocol == htons(ETH_P_FIP))) &&
  5410. (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
  5411. struct ixgbe_ring_feature *f;
  5412. f = &adapter->ring_feature[RING_F_FCOE];
  5413. while (txq >= f->indices)
  5414. txq -= f->indices;
  5415. txq += adapter->ring_feature[RING_F_FCOE].offset;
  5416. return txq;
  5417. }
  5418. #endif
  5419. if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
  5420. while (unlikely(txq >= dev->real_num_tx_queues))
  5421. txq -= dev->real_num_tx_queues;
  5422. return txq;
  5423. }
  5424. return skb_tx_hash(dev, skb);
  5425. }
  5426. netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
  5427. struct ixgbe_adapter *adapter,
  5428. struct ixgbe_ring *tx_ring)
  5429. {
  5430. struct ixgbe_tx_buffer *first;
  5431. int tso;
  5432. u32 tx_flags = 0;
  5433. #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
  5434. unsigned short f;
  5435. #endif
  5436. u16 count = TXD_USE_COUNT(skb_headlen(skb));
  5437. __be16 protocol = skb->protocol;
  5438. u8 hdr_len = 0;
  5439. /*
  5440. * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
  5441. * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
  5442. * + 2 desc gap to keep tail from touching head,
  5443. * + 1 desc for context descriptor,
  5444. * otherwise try next time
  5445. */
  5446. #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
  5447. for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
  5448. count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
  5449. #else
  5450. count += skb_shinfo(skb)->nr_frags;
  5451. #endif
  5452. if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
  5453. tx_ring->tx_stats.tx_busy++;
  5454. return NETDEV_TX_BUSY;
  5455. }
  5456. /* record the location of the first descriptor for this packet */
  5457. first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
  5458. first->skb = skb;
  5459. first->bytecount = skb->len;
  5460. first->gso_segs = 1;
  5461. /* if we have a HW VLAN tag being added default to the HW one */
  5462. if (vlan_tx_tag_present(skb)) {
  5463. tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
  5464. tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
  5465. /* else if it is a SW VLAN check the next protocol and store the tag */
  5466. } else if (protocol == __constant_htons(ETH_P_8021Q)) {
  5467. struct vlan_hdr *vhdr, _vhdr;
  5468. vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
  5469. if (!vhdr)
  5470. goto out_drop;
  5471. protocol = vhdr->h_vlan_encapsulated_proto;
  5472. tx_flags |= ntohs(vhdr->h_vlan_TCI) <<
  5473. IXGBE_TX_FLAGS_VLAN_SHIFT;
  5474. tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
  5475. }
  5476. skb_tx_timestamp(skb);
  5477. if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
  5478. skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
  5479. tx_flags |= IXGBE_TX_FLAGS_TSTAMP;
  5480. }
  5481. #ifdef CONFIG_PCI_IOV
  5482. /*
  5483. * Use the l2switch_enable flag - would be false if the DMA
  5484. * Tx switch had been disabled.
  5485. */
  5486. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  5487. tx_flags |= IXGBE_TX_FLAGS_TXSW;
  5488. #endif
  5489. /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
  5490. if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
  5491. ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
  5492. (skb->priority != TC_PRIO_CONTROL))) {
  5493. tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
  5494. tx_flags |= (skb->priority & 0x7) <<
  5495. IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
  5496. if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
  5497. struct vlan_ethhdr *vhdr;
  5498. if (skb_header_cloned(skb) &&
  5499. pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
  5500. goto out_drop;
  5501. vhdr = (struct vlan_ethhdr *)skb->data;
  5502. vhdr->h_vlan_TCI = htons(tx_flags >>
  5503. IXGBE_TX_FLAGS_VLAN_SHIFT);
  5504. } else {
  5505. tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
  5506. }
  5507. }
  5508. /* record initial flags and protocol */
  5509. first->tx_flags = tx_flags;
  5510. first->protocol = protocol;
  5511. #ifdef IXGBE_FCOE
  5512. /* setup tx offload for FCoE */
  5513. if ((protocol == __constant_htons(ETH_P_FCOE)) &&
  5514. (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) {
  5515. tso = ixgbe_fso(tx_ring, first, &hdr_len);
  5516. if (tso < 0)
  5517. goto out_drop;
  5518. goto xmit_fcoe;
  5519. }
  5520. #endif /* IXGBE_FCOE */
  5521. tso = ixgbe_tso(tx_ring, first, &hdr_len);
  5522. if (tso < 0)
  5523. goto out_drop;
  5524. else if (!tso)
  5525. ixgbe_tx_csum(tx_ring, first);
  5526. /* add the ATR filter if ATR is on */
  5527. if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
  5528. ixgbe_atr(tx_ring, first);
  5529. #ifdef IXGBE_FCOE
  5530. xmit_fcoe:
  5531. #endif /* IXGBE_FCOE */
  5532. ixgbe_tx_map(tx_ring, first, hdr_len);
  5533. ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
  5534. return NETDEV_TX_OK;
  5535. out_drop:
  5536. dev_kfree_skb_any(first->skb);
  5537. first->skb = NULL;
  5538. return NETDEV_TX_OK;
  5539. }
  5540. static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
  5541. struct net_device *netdev)
  5542. {
  5543. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5544. struct ixgbe_ring *tx_ring;
  5545. /*
  5546. * The minimum packet size for olinfo paylen is 17 so pad the skb
  5547. * in order to meet this minimum size requirement.
  5548. */
  5549. if (unlikely(skb->len < 17)) {
  5550. if (skb_pad(skb, 17 - skb->len))
  5551. return NETDEV_TX_OK;
  5552. skb->len = 17;
  5553. skb_set_tail_pointer(skb, 17);
  5554. }
  5555. tx_ring = adapter->tx_ring[skb->queue_mapping];
  5556. return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
  5557. }
  5558. /**
  5559. * ixgbe_set_mac - Change the Ethernet Address of the NIC
  5560. * @netdev: network interface device structure
  5561. * @p: pointer to an address structure
  5562. *
  5563. * Returns 0 on success, negative on failure
  5564. **/
  5565. static int ixgbe_set_mac(struct net_device *netdev, void *p)
  5566. {
  5567. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5568. struct ixgbe_hw *hw = &adapter->hw;
  5569. struct sockaddr *addr = p;
  5570. if (!is_valid_ether_addr(addr->sa_data))
  5571. return -EADDRNOTAVAIL;
  5572. memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
  5573. memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
  5574. hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV);
  5575. return 0;
  5576. }
  5577. static int
  5578. ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
  5579. {
  5580. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5581. struct ixgbe_hw *hw = &adapter->hw;
  5582. u16 value;
  5583. int rc;
  5584. if (prtad != hw->phy.mdio.prtad)
  5585. return -EINVAL;
  5586. rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
  5587. if (!rc)
  5588. rc = value;
  5589. return rc;
  5590. }
  5591. static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
  5592. u16 addr, u16 value)
  5593. {
  5594. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5595. struct ixgbe_hw *hw = &adapter->hw;
  5596. if (prtad != hw->phy.mdio.prtad)
  5597. return -EINVAL;
  5598. return hw->phy.ops.write_reg(hw, addr, devad, value);
  5599. }
  5600. static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
  5601. {
  5602. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5603. switch (cmd) {
  5604. case SIOCSHWTSTAMP:
  5605. return ixgbe_ptp_hwtstamp_ioctl(adapter, req, cmd);
  5606. default:
  5607. return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
  5608. }
  5609. }
  5610. /**
  5611. * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
  5612. * netdev->dev_addrs
  5613. * @netdev: network interface device structure
  5614. *
  5615. * Returns non-zero on failure
  5616. **/
  5617. static int ixgbe_add_sanmac_netdev(struct net_device *dev)
  5618. {
  5619. int err = 0;
  5620. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5621. struct ixgbe_hw *hw = &adapter->hw;
  5622. if (is_valid_ether_addr(hw->mac.san_addr)) {
  5623. rtnl_lock();
  5624. err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN);
  5625. rtnl_unlock();
  5626. /* update SAN MAC vmdq pool selection */
  5627. hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
  5628. }
  5629. return err;
  5630. }
  5631. /**
  5632. * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
  5633. * netdev->dev_addrs
  5634. * @netdev: network interface device structure
  5635. *
  5636. * Returns non-zero on failure
  5637. **/
  5638. static int ixgbe_del_sanmac_netdev(struct net_device *dev)
  5639. {
  5640. int err = 0;
  5641. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5642. struct ixgbe_mac_info *mac = &adapter->hw.mac;
  5643. if (is_valid_ether_addr(mac->san_addr)) {
  5644. rtnl_lock();
  5645. err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
  5646. rtnl_unlock();
  5647. }
  5648. return err;
  5649. }
  5650. #ifdef CONFIG_NET_POLL_CONTROLLER
  5651. /*
  5652. * Polling 'interrupt' - used by things like netconsole to send skbs
  5653. * without having to re-enable interrupts. It's not called while
  5654. * the interrupt routine is executing.
  5655. */
  5656. static void ixgbe_netpoll(struct net_device *netdev)
  5657. {
  5658. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5659. int i;
  5660. /* if interface is down do nothing */
  5661. if (test_bit(__IXGBE_DOWN, &adapter->state))
  5662. return;
  5663. adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
  5664. if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
  5665. for (i = 0; i < adapter->num_q_vectors; i++)
  5666. ixgbe_msix_clean_rings(0, adapter->q_vector[i]);
  5667. } else {
  5668. ixgbe_intr(adapter->pdev->irq, netdev);
  5669. }
  5670. adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
  5671. }
  5672. #endif
  5673. static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
  5674. struct rtnl_link_stats64 *stats)
  5675. {
  5676. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5677. int i;
  5678. rcu_read_lock();
  5679. for (i = 0; i < adapter->num_rx_queues; i++) {
  5680. struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
  5681. u64 bytes, packets;
  5682. unsigned int start;
  5683. if (ring) {
  5684. do {
  5685. start = u64_stats_fetch_begin_bh(&ring->syncp);
  5686. packets = ring->stats.packets;
  5687. bytes = ring->stats.bytes;
  5688. } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
  5689. stats->rx_packets += packets;
  5690. stats->rx_bytes += bytes;
  5691. }
  5692. }
  5693. for (i = 0; i < adapter->num_tx_queues; i++) {
  5694. struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
  5695. u64 bytes, packets;
  5696. unsigned int start;
  5697. if (ring) {
  5698. do {
  5699. start = u64_stats_fetch_begin_bh(&ring->syncp);
  5700. packets = ring->stats.packets;
  5701. bytes = ring->stats.bytes;
  5702. } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
  5703. stats->tx_packets += packets;
  5704. stats->tx_bytes += bytes;
  5705. }
  5706. }
  5707. rcu_read_unlock();
  5708. /* following stats updated by ixgbe_watchdog_task() */
  5709. stats->multicast = netdev->stats.multicast;
  5710. stats->rx_errors = netdev->stats.rx_errors;
  5711. stats->rx_length_errors = netdev->stats.rx_length_errors;
  5712. stats->rx_crc_errors = netdev->stats.rx_crc_errors;
  5713. stats->rx_missed_errors = netdev->stats.rx_missed_errors;
  5714. return stats;
  5715. }
  5716. #ifdef CONFIG_IXGBE_DCB
  5717. /**
  5718. * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
  5719. * @adapter: pointer to ixgbe_adapter
  5720. * @tc: number of traffic classes currently enabled
  5721. *
  5722. * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
  5723. * 802.1Q priority maps to a packet buffer that exists.
  5724. */
  5725. static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
  5726. {
  5727. struct ixgbe_hw *hw = &adapter->hw;
  5728. u32 reg, rsave;
  5729. int i;
  5730. /* 82598 have a static priority to TC mapping that can not
  5731. * be changed so no validation is needed.
  5732. */
  5733. if (hw->mac.type == ixgbe_mac_82598EB)
  5734. return;
  5735. reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
  5736. rsave = reg;
  5737. for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
  5738. u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
  5739. /* If up2tc is out of bounds default to zero */
  5740. if (up2tc > tc)
  5741. reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
  5742. }
  5743. if (reg != rsave)
  5744. IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
  5745. return;
  5746. }
  5747. /**
  5748. * ixgbe_set_prio_tc_map - Configure netdev prio tc map
  5749. * @adapter: Pointer to adapter struct
  5750. *
  5751. * Populate the netdev user priority to tc map
  5752. */
  5753. static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
  5754. {
  5755. struct net_device *dev = adapter->netdev;
  5756. struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg;
  5757. struct ieee_ets *ets = adapter->ixgbe_ieee_ets;
  5758. u8 prio;
  5759. for (prio = 0; prio < MAX_USER_PRIORITY; prio++) {
  5760. u8 tc = 0;
  5761. if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE)
  5762. tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio);
  5763. else if (ets)
  5764. tc = ets->prio_tc[prio];
  5765. netdev_set_prio_tc_map(dev, prio, tc);
  5766. }
  5767. }
  5768. /**
  5769. * ixgbe_setup_tc - configure net_device for multiple traffic classes
  5770. *
  5771. * @netdev: net device to configure
  5772. * @tc: number of traffic classes to enable
  5773. */
  5774. int ixgbe_setup_tc(struct net_device *dev, u8 tc)
  5775. {
  5776. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5777. struct ixgbe_hw *hw = &adapter->hw;
  5778. /* Hardware supports up to 8 traffic classes */
  5779. if (tc > adapter->dcb_cfg.num_tcs.pg_tcs ||
  5780. (hw->mac.type == ixgbe_mac_82598EB &&
  5781. tc < MAX_TRAFFIC_CLASS))
  5782. return -EINVAL;
  5783. /* Hardware has to reinitialize queues and interrupts to
  5784. * match packet buffer alignment. Unfortunately, the
  5785. * hardware is not flexible enough to do this dynamically.
  5786. */
  5787. if (netif_running(dev))
  5788. ixgbe_close(dev);
  5789. ixgbe_clear_interrupt_scheme(adapter);
  5790. if (tc) {
  5791. netdev_set_num_tc(dev, tc);
  5792. ixgbe_set_prio_tc_map(adapter);
  5793. adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
  5794. if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
  5795. adapter->last_lfc_mode = adapter->hw.fc.requested_mode;
  5796. adapter->hw.fc.requested_mode = ixgbe_fc_none;
  5797. }
  5798. } else {
  5799. netdev_reset_tc(dev);
  5800. if (adapter->hw.mac.type == ixgbe_mac_82598EB)
  5801. adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
  5802. adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
  5803. adapter->temp_dcb_cfg.pfc_mode_enable = false;
  5804. adapter->dcb_cfg.pfc_mode_enable = false;
  5805. }
  5806. ixgbe_init_interrupt_scheme(adapter);
  5807. ixgbe_validate_rtr(adapter, tc);
  5808. if (netif_running(dev))
  5809. ixgbe_open(dev);
  5810. return 0;
  5811. }
  5812. #endif /* CONFIG_IXGBE_DCB */
  5813. void ixgbe_do_reset(struct net_device *netdev)
  5814. {
  5815. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5816. if (netif_running(netdev))
  5817. ixgbe_reinit_locked(adapter);
  5818. else
  5819. ixgbe_reset(adapter);
  5820. }
  5821. static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
  5822. netdev_features_t features)
  5823. {
  5824. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5825. /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
  5826. if (!(features & NETIF_F_RXCSUM))
  5827. features &= ~NETIF_F_LRO;
  5828. /* Turn off LRO if not RSC capable */
  5829. if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE))
  5830. features &= ~NETIF_F_LRO;
  5831. return features;
  5832. }
  5833. static int ixgbe_set_features(struct net_device *netdev,
  5834. netdev_features_t features)
  5835. {
  5836. struct ixgbe_adapter *adapter = netdev_priv(netdev);
  5837. netdev_features_t changed = netdev->features ^ features;
  5838. bool need_reset = false;
  5839. /* Make sure RSC matches LRO, reset if change */
  5840. if (!(features & NETIF_F_LRO)) {
  5841. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
  5842. need_reset = true;
  5843. adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED;
  5844. } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) &&
  5845. !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
  5846. if (adapter->rx_itr_setting == 1 ||
  5847. adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) {
  5848. adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
  5849. need_reset = true;
  5850. } else if ((changed ^ features) & NETIF_F_LRO) {
  5851. e_info(probe, "rx-usecs set too low, "
  5852. "disabling RSC\n");
  5853. }
  5854. }
  5855. /*
  5856. * Check if Flow Director n-tuple support was enabled or disabled. If
  5857. * the state changed, we need to reset.
  5858. */
  5859. switch (features & NETIF_F_NTUPLE) {
  5860. case NETIF_F_NTUPLE:
  5861. /* turn off ATR, enable perfect filters and reset */
  5862. if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
  5863. need_reset = true;
  5864. adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
  5865. adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
  5866. break;
  5867. default:
  5868. /* turn off perfect filters, enable ATR and reset */
  5869. if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
  5870. need_reset = true;
  5871. adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
  5872. /* We cannot enable ATR if SR-IOV is enabled */
  5873. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  5874. break;
  5875. /* We cannot enable ATR if we have 2 or more traffic classes */
  5876. if (netdev_get_num_tc(netdev) > 1)
  5877. break;
  5878. /* We cannot enable ATR if RSS is disabled */
  5879. if (adapter->ring_feature[RING_F_RSS].limit <= 1)
  5880. break;
  5881. /* A sample rate of 0 indicates ATR disabled */
  5882. if (!adapter->atr_sample_rate)
  5883. break;
  5884. adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
  5885. break;
  5886. }
  5887. if (features & NETIF_F_HW_VLAN_RX)
  5888. ixgbe_vlan_strip_enable(adapter);
  5889. else
  5890. ixgbe_vlan_strip_disable(adapter);
  5891. if (changed & NETIF_F_RXALL)
  5892. need_reset = true;
  5893. netdev->features = features;
  5894. if (need_reset)
  5895. ixgbe_do_reset(netdev);
  5896. return 0;
  5897. }
  5898. static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
  5899. struct net_device *dev,
  5900. const unsigned char *addr,
  5901. u16 flags)
  5902. {
  5903. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5904. int err;
  5905. if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
  5906. return -EOPNOTSUPP;
  5907. /* Hardware does not support aging addresses so if a
  5908. * ndm_state is given only allow permanent addresses
  5909. */
  5910. if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
  5911. pr_info("%s: FDB only supports static addresses\n",
  5912. ixgbe_driver_name);
  5913. return -EINVAL;
  5914. }
  5915. if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
  5916. u32 rar_uc_entries = IXGBE_MAX_PF_MACVLANS;
  5917. if (netdev_uc_count(dev) < rar_uc_entries)
  5918. err = dev_uc_add_excl(dev, addr);
  5919. else
  5920. err = -ENOMEM;
  5921. } else if (is_multicast_ether_addr(addr)) {
  5922. err = dev_mc_add_excl(dev, addr);
  5923. } else {
  5924. err = -EINVAL;
  5925. }
  5926. /* Only return duplicate errors if NLM_F_EXCL is set */
  5927. if (err == -EEXIST && !(flags & NLM_F_EXCL))
  5928. err = 0;
  5929. return err;
  5930. }
  5931. static int ixgbe_ndo_fdb_del(struct ndmsg *ndm,
  5932. struct net_device *dev,
  5933. const unsigned char *addr)
  5934. {
  5935. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5936. int err = -EOPNOTSUPP;
  5937. if (ndm->ndm_state & NUD_PERMANENT) {
  5938. pr_info("%s: FDB only supports static addresses\n",
  5939. ixgbe_driver_name);
  5940. return -EINVAL;
  5941. }
  5942. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  5943. if (is_unicast_ether_addr(addr))
  5944. err = dev_uc_del(dev, addr);
  5945. else if (is_multicast_ether_addr(addr))
  5946. err = dev_mc_del(dev, addr);
  5947. else
  5948. err = -EINVAL;
  5949. }
  5950. return err;
  5951. }
  5952. static int ixgbe_ndo_fdb_dump(struct sk_buff *skb,
  5953. struct netlink_callback *cb,
  5954. struct net_device *dev,
  5955. int idx)
  5956. {
  5957. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5958. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
  5959. idx = ndo_dflt_fdb_dump(skb, cb, dev, idx);
  5960. return idx;
  5961. }
  5962. static int ixgbe_ndo_bridge_setlink(struct net_device *dev,
  5963. struct nlmsghdr *nlh)
  5964. {
  5965. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5966. struct nlattr *attr, *br_spec;
  5967. int rem;
  5968. if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
  5969. return -EOPNOTSUPP;
  5970. br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
  5971. nla_for_each_nested(attr, br_spec, rem) {
  5972. __u16 mode;
  5973. u32 reg = 0;
  5974. if (nla_type(attr) != IFLA_BRIDGE_MODE)
  5975. continue;
  5976. mode = nla_get_u16(attr);
  5977. if (mode == BRIDGE_MODE_VEPA) {
  5978. reg = 0;
  5979. adapter->flags2 &= ~IXGBE_FLAG2_BRIDGE_MODE_VEB;
  5980. } else if (mode == BRIDGE_MODE_VEB) {
  5981. reg = IXGBE_PFDTXGSWC_VT_LBEN;
  5982. adapter->flags2 |= IXGBE_FLAG2_BRIDGE_MODE_VEB;
  5983. } else
  5984. return -EINVAL;
  5985. IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, reg);
  5986. e_info(drv, "enabling bridge mode: %s\n",
  5987. mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
  5988. }
  5989. return 0;
  5990. }
  5991. static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
  5992. struct net_device *dev)
  5993. {
  5994. struct ixgbe_adapter *adapter = netdev_priv(dev);
  5995. u16 mode;
  5996. if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
  5997. return 0;
  5998. if (adapter->flags2 & IXGBE_FLAG2_BRIDGE_MODE_VEB)
  5999. mode = BRIDGE_MODE_VEB;
  6000. else
  6001. mode = BRIDGE_MODE_VEPA;
  6002. return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode);
  6003. }
  6004. static const struct net_device_ops ixgbe_netdev_ops = {
  6005. .ndo_open = ixgbe_open,
  6006. .ndo_stop = ixgbe_close,
  6007. .ndo_start_xmit = ixgbe_xmit_frame,
  6008. .ndo_select_queue = ixgbe_select_queue,
  6009. .ndo_set_rx_mode = ixgbe_set_rx_mode,
  6010. .ndo_validate_addr = eth_validate_addr,
  6011. .ndo_set_mac_address = ixgbe_set_mac,
  6012. .ndo_change_mtu = ixgbe_change_mtu,
  6013. .ndo_tx_timeout = ixgbe_tx_timeout,
  6014. .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
  6015. .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
  6016. .ndo_do_ioctl = ixgbe_ioctl,
  6017. .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
  6018. .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
  6019. .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw,
  6020. .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
  6021. .ndo_get_vf_config = ixgbe_ndo_get_vf_config,
  6022. .ndo_get_stats64 = ixgbe_get_stats64,
  6023. #ifdef CONFIG_IXGBE_DCB
  6024. .ndo_setup_tc = ixgbe_setup_tc,
  6025. #endif
  6026. #ifdef CONFIG_NET_POLL_CONTROLLER
  6027. .ndo_poll_controller = ixgbe_netpoll,
  6028. #endif
  6029. #ifdef IXGBE_FCOE
  6030. .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
  6031. .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
  6032. .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
  6033. .ndo_fcoe_enable = ixgbe_fcoe_enable,
  6034. .ndo_fcoe_disable = ixgbe_fcoe_disable,
  6035. .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
  6036. .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,
  6037. #endif /* IXGBE_FCOE */
  6038. .ndo_set_features = ixgbe_set_features,
  6039. .ndo_fix_features = ixgbe_fix_features,
  6040. .ndo_fdb_add = ixgbe_ndo_fdb_add,
  6041. .ndo_fdb_del = ixgbe_ndo_fdb_del,
  6042. .ndo_fdb_dump = ixgbe_ndo_fdb_dump,
  6043. .ndo_bridge_setlink = ixgbe_ndo_bridge_setlink,
  6044. .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink,
  6045. };
  6046. /**
  6047. * ixgbe_wol_supported - Check whether device supports WoL
  6048. * @hw: hw specific details
  6049. * @device_id: the device ID
  6050. * @subdev_id: the subsystem device ID
  6051. *
  6052. * This function is used by probe and ethtool to determine
  6053. * which devices have WoL support
  6054. *
  6055. **/
  6056. int ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,
  6057. u16 subdevice_id)
  6058. {
  6059. struct ixgbe_hw *hw = &adapter->hw;
  6060. u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
  6061. int is_wol_supported = 0;
  6062. switch (device_id) {
  6063. case IXGBE_DEV_ID_82599_SFP:
  6064. /* Only these subdevices could supports WOL */
  6065. switch (subdevice_id) {
  6066. case IXGBE_SUBDEV_ID_82599_560FLR:
  6067. /* only support first port */
  6068. if (hw->bus.func != 0)
  6069. break;
  6070. case IXGBE_SUBDEV_ID_82599_SFP:
  6071. case IXGBE_SUBDEV_ID_82599_RNDC:
  6072. case IXGBE_SUBDEV_ID_82599_ECNA_DP:
  6073. is_wol_supported = 1;
  6074. break;
  6075. }
  6076. break;
  6077. case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
  6078. /* All except this subdevice support WOL */
  6079. if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
  6080. is_wol_supported = 1;
  6081. break;
  6082. case IXGBE_DEV_ID_82599_KX4:
  6083. is_wol_supported = 1;
  6084. break;
  6085. case IXGBE_DEV_ID_X540T:
  6086. case IXGBE_DEV_ID_X540T1:
  6087. /* check eeprom to see if enabled wol */
  6088. if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
  6089. ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
  6090. (hw->bus.func == 0))) {
  6091. is_wol_supported = 1;
  6092. }
  6093. break;
  6094. }
  6095. return is_wol_supported;
  6096. }
  6097. /**
  6098. * ixgbe_probe - Device Initialization Routine
  6099. * @pdev: PCI device information struct
  6100. * @ent: entry in ixgbe_pci_tbl
  6101. *
  6102. * Returns 0 on success, negative on failure
  6103. *
  6104. * ixgbe_probe initializes an adapter identified by a pci_dev structure.
  6105. * The OS initialization, configuring of the adapter private structure,
  6106. * and a hardware reset occur.
  6107. **/
  6108. static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  6109. {
  6110. struct net_device *netdev;
  6111. struct ixgbe_adapter *adapter = NULL;
  6112. struct ixgbe_hw *hw;
  6113. const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
  6114. static int cards_found;
  6115. int i, err, pci_using_dac;
  6116. u8 part_str[IXGBE_PBANUM_LENGTH];
  6117. unsigned int indices = num_possible_cpus();
  6118. unsigned int dcb_max = 0;
  6119. #ifdef IXGBE_FCOE
  6120. u16 device_caps;
  6121. #endif
  6122. u32 eec;
  6123. /* Catch broken hardware that put the wrong VF device ID in
  6124. * the PCIe SR-IOV capability.
  6125. */
  6126. if (pdev->is_virtfn) {
  6127. WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
  6128. pci_name(pdev), pdev->vendor, pdev->device);
  6129. return -EINVAL;
  6130. }
  6131. err = pci_enable_device_mem(pdev);
  6132. if (err)
  6133. return err;
  6134. if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
  6135. !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
  6136. pci_using_dac = 1;
  6137. } else {
  6138. err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
  6139. if (err) {
  6140. err = dma_set_coherent_mask(&pdev->dev,
  6141. DMA_BIT_MASK(32));
  6142. if (err) {
  6143. dev_err(&pdev->dev,
  6144. "No usable DMA configuration, aborting\n");
  6145. goto err_dma;
  6146. }
  6147. }
  6148. pci_using_dac = 0;
  6149. }
  6150. err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
  6151. IORESOURCE_MEM), ixgbe_driver_name);
  6152. if (err) {
  6153. dev_err(&pdev->dev,
  6154. "pci_request_selected_regions failed 0x%x\n", err);
  6155. goto err_pci_reg;
  6156. }
  6157. pci_enable_pcie_error_reporting(pdev);
  6158. pci_set_master(pdev);
  6159. pci_save_state(pdev);
  6160. #ifdef CONFIG_IXGBE_DCB
  6161. if (ii->mac == ixgbe_mac_82598EB)
  6162. dcb_max = min_t(unsigned int, indices * MAX_TRAFFIC_CLASS,
  6163. IXGBE_MAX_RSS_INDICES);
  6164. else
  6165. dcb_max = min_t(unsigned int, indices * MAX_TRAFFIC_CLASS,
  6166. IXGBE_MAX_FDIR_INDICES);
  6167. #endif
  6168. if (ii->mac == ixgbe_mac_82598EB)
  6169. indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES);
  6170. else
  6171. indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES);
  6172. #ifdef IXGBE_FCOE
  6173. indices += min_t(unsigned int, num_possible_cpus(),
  6174. IXGBE_MAX_FCOE_INDICES);
  6175. #endif
  6176. indices = max_t(unsigned int, dcb_max, indices);
  6177. netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
  6178. if (!netdev) {
  6179. err = -ENOMEM;
  6180. goto err_alloc_etherdev;
  6181. }
  6182. SET_NETDEV_DEV(netdev, &pdev->dev);
  6183. adapter = netdev_priv(netdev);
  6184. pci_set_drvdata(pdev, adapter);
  6185. adapter->netdev = netdev;
  6186. adapter->pdev = pdev;
  6187. hw = &adapter->hw;
  6188. hw->back = adapter;
  6189. adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
  6190. hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
  6191. pci_resource_len(pdev, 0));
  6192. if (!hw->hw_addr) {
  6193. err = -EIO;
  6194. goto err_ioremap;
  6195. }
  6196. netdev->netdev_ops = &ixgbe_netdev_ops;
  6197. ixgbe_set_ethtool_ops(netdev);
  6198. netdev->watchdog_timeo = 5 * HZ;
  6199. strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
  6200. adapter->bd_number = cards_found;
  6201. /* Setup hw api */
  6202. memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
  6203. hw->mac.type = ii->mac;
  6204. /* EEPROM */
  6205. memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops));
  6206. eec = IXGBE_READ_REG(hw, IXGBE_EEC);
  6207. /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
  6208. if (!(eec & (1 << 8)))
  6209. hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
  6210. /* PHY */
  6211. memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops));
  6212. hw->phy.sfp_type = ixgbe_sfp_type_unknown;
  6213. /* ixgbe_identify_phy_generic will set prtad and mmds properly */
  6214. hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
  6215. hw->phy.mdio.mmds = 0;
  6216. hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
  6217. hw->phy.mdio.dev = netdev;
  6218. hw->phy.mdio.mdio_read = ixgbe_mdio_read;
  6219. hw->phy.mdio.mdio_write = ixgbe_mdio_write;
  6220. ii->get_invariants(hw);
  6221. /* setup the private structure */
  6222. err = ixgbe_sw_init(adapter);
  6223. if (err)
  6224. goto err_sw_init;
  6225. /* Make it possible the adapter to be woken up via WOL */
  6226. switch (adapter->hw.mac.type) {
  6227. case ixgbe_mac_82599EB:
  6228. case ixgbe_mac_X540:
  6229. IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
  6230. break;
  6231. default:
  6232. break;
  6233. }
  6234. /*
  6235. * If there is a fan on this device and it has failed log the
  6236. * failure.
  6237. */
  6238. if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
  6239. u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
  6240. if (esdp & IXGBE_ESDP_SDP1)
  6241. e_crit(probe, "Fan has stopped, replace the adapter\n");
  6242. }
  6243. if (allow_unsupported_sfp)
  6244. hw->allow_unsupported_sfp = allow_unsupported_sfp;
  6245. /* reset_hw fills in the perm_addr as well */
  6246. hw->phy.reset_if_overtemp = true;
  6247. err = hw->mac.ops.reset_hw(hw);
  6248. hw->phy.reset_if_overtemp = false;
  6249. if (err == IXGBE_ERR_SFP_NOT_PRESENT &&
  6250. hw->mac.type == ixgbe_mac_82598EB) {
  6251. err = 0;
  6252. } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
  6253. e_dev_err("failed to load because an unsupported SFP+ "
  6254. "module type was detected.\n");
  6255. e_dev_err("Reload the driver after installing a supported "
  6256. "module.\n");
  6257. goto err_sw_init;
  6258. } else if (err) {
  6259. e_dev_err("HW Init failed: %d\n", err);
  6260. goto err_sw_init;
  6261. }
  6262. #ifdef CONFIG_PCI_IOV
  6263. ixgbe_enable_sriov(adapter, ii);
  6264. #endif
  6265. netdev->features = NETIF_F_SG |
  6266. NETIF_F_IP_CSUM |
  6267. NETIF_F_IPV6_CSUM |
  6268. NETIF_F_HW_VLAN_TX |
  6269. NETIF_F_HW_VLAN_RX |
  6270. NETIF_F_HW_VLAN_FILTER |
  6271. NETIF_F_TSO |
  6272. NETIF_F_TSO6 |
  6273. NETIF_F_RXHASH |
  6274. NETIF_F_RXCSUM;
  6275. netdev->hw_features = netdev->features;
  6276. switch (adapter->hw.mac.type) {
  6277. case ixgbe_mac_82599EB:
  6278. case ixgbe_mac_X540:
  6279. netdev->features |= NETIF_F_SCTP_CSUM;
  6280. netdev->hw_features |= NETIF_F_SCTP_CSUM |
  6281. NETIF_F_NTUPLE;
  6282. break;
  6283. default:
  6284. break;
  6285. }
  6286. netdev->hw_features |= NETIF_F_RXALL;
  6287. netdev->vlan_features |= NETIF_F_TSO;
  6288. netdev->vlan_features |= NETIF_F_TSO6;
  6289. netdev->vlan_features |= NETIF_F_IP_CSUM;
  6290. netdev->vlan_features |= NETIF_F_IPV6_CSUM;
  6291. netdev->vlan_features |= NETIF_F_SG;
  6292. netdev->priv_flags |= IFF_UNICAST_FLT;
  6293. netdev->priv_flags |= IFF_SUPP_NOFCS;
  6294. #ifdef CONFIG_IXGBE_DCB
  6295. netdev->dcbnl_ops = &dcbnl_ops;
  6296. #endif
  6297. #ifdef IXGBE_FCOE
  6298. if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
  6299. if (hw->mac.ops.get_device_caps) {
  6300. hw->mac.ops.get_device_caps(hw, &device_caps);
  6301. if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
  6302. adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
  6303. }
  6304. adapter->ring_feature[RING_F_FCOE].limit = IXGBE_FCRETA_SIZE;
  6305. netdev->features |= NETIF_F_FSO |
  6306. NETIF_F_FCOE_CRC;
  6307. netdev->vlan_features |= NETIF_F_FSO |
  6308. NETIF_F_FCOE_CRC |
  6309. NETIF_F_FCOE_MTU;
  6310. }
  6311. #endif /* IXGBE_FCOE */
  6312. if (pci_using_dac) {
  6313. netdev->features |= NETIF_F_HIGHDMA;
  6314. netdev->vlan_features |= NETIF_F_HIGHDMA;
  6315. }
  6316. if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
  6317. netdev->hw_features |= NETIF_F_LRO;
  6318. if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
  6319. netdev->features |= NETIF_F_LRO;
  6320. /* make sure the EEPROM is good */
  6321. if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
  6322. e_dev_err("The EEPROM Checksum Is Not Valid\n");
  6323. err = -EIO;
  6324. goto err_sw_init;
  6325. }
  6326. memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
  6327. memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len);
  6328. if (!is_valid_ether_addr(netdev->perm_addr)) {
  6329. e_dev_err("invalid MAC address\n");
  6330. err = -EIO;
  6331. goto err_sw_init;
  6332. }
  6333. setup_timer(&adapter->service_timer, &ixgbe_service_timer,
  6334. (unsigned long) adapter);
  6335. INIT_WORK(&adapter->service_task, ixgbe_service_task);
  6336. clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
  6337. err = ixgbe_init_interrupt_scheme(adapter);
  6338. if (err)
  6339. goto err_sw_init;
  6340. /* WOL not supported for all devices */
  6341. adapter->wol = 0;
  6342. hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
  6343. if (ixgbe_wol_supported(adapter, pdev->device, pdev->subsystem_device))
  6344. adapter->wol = IXGBE_WUFC_MAG;
  6345. device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
  6346. /* save off EEPROM version number */
  6347. hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh);
  6348. hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl);
  6349. /* pick up the PCI bus settings for reporting later */
  6350. hw->mac.ops.get_bus_info(hw);
  6351. /* print bus type/speed/width info */
  6352. e_dev_info("(PCI Express:%s:%s) %pM\n",
  6353. (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" :
  6354. hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" :
  6355. "Unknown"),
  6356. (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
  6357. hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
  6358. hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" :
  6359. "Unknown"),
  6360. netdev->dev_addr);
  6361. err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH);
  6362. if (err)
  6363. strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH);
  6364. if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
  6365. e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
  6366. hw->mac.type, hw->phy.type, hw->phy.sfp_type,
  6367. part_str);
  6368. else
  6369. e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
  6370. hw->mac.type, hw->phy.type, part_str);
  6371. if (hw->bus.width <= ixgbe_bus_width_pcie_x4) {
  6372. e_dev_warn("PCI-Express bandwidth available for this card is "
  6373. "not sufficient for optimal performance.\n");
  6374. e_dev_warn("For optimal performance a x8 PCI-Express slot "
  6375. "is required.\n");
  6376. }
  6377. /* reset the hardware with the new settings */
  6378. err = hw->mac.ops.start_hw(hw);
  6379. if (err == IXGBE_ERR_EEPROM_VERSION) {
  6380. /* We are running on a pre-production device, log a warning */
  6381. e_dev_warn("This device is a pre-production adapter/LOM. "
  6382. "Please be aware there may be issues associated "
  6383. "with your hardware. If you are experiencing "
  6384. "problems please contact your Intel or hardware "
  6385. "representative who provided you with this "
  6386. "hardware.\n");
  6387. }
  6388. strcpy(netdev->name, "eth%d");
  6389. err = register_netdev(netdev);
  6390. if (err)
  6391. goto err_register;
  6392. /* power down the optics for 82599 SFP+ fiber */
  6393. if (hw->mac.ops.disable_tx_laser)
  6394. hw->mac.ops.disable_tx_laser(hw);
  6395. /* carrier off reporting is important to ethtool even BEFORE open */
  6396. netif_carrier_off(netdev);
  6397. #ifdef CONFIG_IXGBE_DCA
  6398. if (dca_add_requester(&pdev->dev) == 0) {
  6399. adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
  6400. ixgbe_setup_dca(adapter);
  6401. }
  6402. #endif
  6403. if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
  6404. e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
  6405. for (i = 0; i < adapter->num_vfs; i++)
  6406. ixgbe_vf_configuration(pdev, (i | 0x10000000));
  6407. }
  6408. /* firmware requires driver version to be 0xFFFFFFFF
  6409. * since os does not support feature
  6410. */
  6411. if (hw->mac.ops.set_fw_drv_ver)
  6412. hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF,
  6413. 0xFF);
  6414. /* add san mac addr to netdev */
  6415. ixgbe_add_sanmac_netdev(netdev);
  6416. e_dev_info("%s\n", ixgbe_default_device_descr);
  6417. cards_found++;
  6418. #ifdef CONFIG_IXGBE_HWMON
  6419. if (ixgbe_sysfs_init(adapter))
  6420. e_err(probe, "failed to allocate sysfs resources\n");
  6421. #endif /* CONFIG_IXGBE_HWMON */
  6422. #ifdef CONFIG_DEBUG_FS
  6423. ixgbe_dbg_adapter_init(adapter);
  6424. #endif /* CONFIG_DEBUG_FS */
  6425. return 0;
  6426. err_register:
  6427. ixgbe_release_hw_control(adapter);
  6428. ixgbe_clear_interrupt_scheme(adapter);
  6429. err_sw_init:
  6430. ixgbe_disable_sriov(adapter);
  6431. adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
  6432. iounmap(hw->hw_addr);
  6433. err_ioremap:
  6434. free_netdev(netdev);
  6435. err_alloc_etherdev:
  6436. pci_release_selected_regions(pdev,
  6437. pci_select_bars(pdev, IORESOURCE_MEM));
  6438. err_pci_reg:
  6439. err_dma:
  6440. pci_disable_device(pdev);
  6441. return err;
  6442. }
  6443. /**
  6444. * ixgbe_remove - Device Removal Routine
  6445. * @pdev: PCI device information struct
  6446. *
  6447. * ixgbe_remove is called by the PCI subsystem to alert the driver
  6448. * that it should release a PCI device. The could be caused by a
  6449. * Hot-Plug event, or because the driver is going to be removed from
  6450. * memory.
  6451. **/
  6452. static void ixgbe_remove(struct pci_dev *pdev)
  6453. {
  6454. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6455. struct net_device *netdev = adapter->netdev;
  6456. #ifdef CONFIG_DEBUG_FS
  6457. ixgbe_dbg_adapter_exit(adapter);
  6458. #endif /*CONFIG_DEBUG_FS */
  6459. set_bit(__IXGBE_DOWN, &adapter->state);
  6460. cancel_work_sync(&adapter->service_task);
  6461. #ifdef CONFIG_IXGBE_DCA
  6462. if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
  6463. adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
  6464. dca_remove_requester(&pdev->dev);
  6465. IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
  6466. }
  6467. #endif
  6468. #ifdef CONFIG_IXGBE_HWMON
  6469. ixgbe_sysfs_exit(adapter);
  6470. #endif /* CONFIG_IXGBE_HWMON */
  6471. /* remove the added san mac */
  6472. ixgbe_del_sanmac_netdev(netdev);
  6473. if (netdev->reg_state == NETREG_REGISTERED)
  6474. unregister_netdev(netdev);
  6475. ixgbe_disable_sriov(adapter);
  6476. ixgbe_clear_interrupt_scheme(adapter);
  6477. ixgbe_release_hw_control(adapter);
  6478. #ifdef CONFIG_DCB
  6479. kfree(adapter->ixgbe_ieee_pfc);
  6480. kfree(adapter->ixgbe_ieee_ets);
  6481. #endif
  6482. iounmap(adapter->hw.hw_addr);
  6483. pci_release_selected_regions(pdev, pci_select_bars(pdev,
  6484. IORESOURCE_MEM));
  6485. e_dev_info("complete\n");
  6486. free_netdev(netdev);
  6487. pci_disable_pcie_error_reporting(pdev);
  6488. pci_disable_device(pdev);
  6489. }
  6490. /**
  6491. * ixgbe_io_error_detected - called when PCI error is detected
  6492. * @pdev: Pointer to PCI device
  6493. * @state: The current pci connection state
  6494. *
  6495. * This function is called after a PCI bus error affecting
  6496. * this device has been detected.
  6497. */
  6498. static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
  6499. pci_channel_state_t state)
  6500. {
  6501. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6502. struct net_device *netdev = adapter->netdev;
  6503. #ifdef CONFIG_PCI_IOV
  6504. struct pci_dev *bdev, *vfdev;
  6505. u32 dw0, dw1, dw2, dw3;
  6506. int vf, pos;
  6507. u16 req_id, pf_func;
  6508. if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
  6509. adapter->num_vfs == 0)
  6510. goto skip_bad_vf_detection;
  6511. bdev = pdev->bus->self;
  6512. while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT))
  6513. bdev = bdev->bus->self;
  6514. if (!bdev)
  6515. goto skip_bad_vf_detection;
  6516. pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
  6517. if (!pos)
  6518. goto skip_bad_vf_detection;
  6519. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
  6520. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
  6521. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
  6522. pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
  6523. req_id = dw1 >> 16;
  6524. /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
  6525. if (!(req_id & 0x0080))
  6526. goto skip_bad_vf_detection;
  6527. pf_func = req_id & 0x01;
  6528. if ((pf_func & 1) == (pdev->devfn & 1)) {
  6529. unsigned int device_id;
  6530. vf = (req_id & 0x7F) >> 1;
  6531. e_dev_err("VF %d has caused a PCIe error\n", vf);
  6532. e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
  6533. "%8.8x\tdw3: %8.8x\n",
  6534. dw0, dw1, dw2, dw3);
  6535. switch (adapter->hw.mac.type) {
  6536. case ixgbe_mac_82599EB:
  6537. device_id = IXGBE_82599_VF_DEVICE_ID;
  6538. break;
  6539. case ixgbe_mac_X540:
  6540. device_id = IXGBE_X540_VF_DEVICE_ID;
  6541. break;
  6542. default:
  6543. device_id = 0;
  6544. break;
  6545. }
  6546. /* Find the pci device of the offending VF */
  6547. vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL);
  6548. while (vfdev) {
  6549. if (vfdev->devfn == (req_id & 0xFF))
  6550. break;
  6551. vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
  6552. device_id, vfdev);
  6553. }
  6554. /*
  6555. * There's a slim chance the VF could have been hot plugged,
  6556. * so if it is no longer present we don't need to issue the
  6557. * VFLR. Just clean up the AER in that case.
  6558. */
  6559. if (vfdev) {
  6560. e_dev_err("Issuing VFLR to VF %d\n", vf);
  6561. pci_write_config_dword(vfdev, 0xA8, 0x00008000);
  6562. }
  6563. pci_cleanup_aer_uncorrect_error_status(pdev);
  6564. }
  6565. /*
  6566. * Even though the error may have occurred on the other port
  6567. * we still need to increment the vf error reference count for
  6568. * both ports because the I/O resume function will be called
  6569. * for both of them.
  6570. */
  6571. adapter->vferr_refcount++;
  6572. return PCI_ERS_RESULT_RECOVERED;
  6573. skip_bad_vf_detection:
  6574. #endif /* CONFIG_PCI_IOV */
  6575. netif_device_detach(netdev);
  6576. if (state == pci_channel_io_perm_failure)
  6577. return PCI_ERS_RESULT_DISCONNECT;
  6578. if (netif_running(netdev))
  6579. ixgbe_down(adapter);
  6580. pci_disable_device(pdev);
  6581. /* Request a slot reset. */
  6582. return PCI_ERS_RESULT_NEED_RESET;
  6583. }
  6584. /**
  6585. * ixgbe_io_slot_reset - called after the pci bus has been reset.
  6586. * @pdev: Pointer to PCI device
  6587. *
  6588. * Restart the card from scratch, as if from a cold-boot.
  6589. */
  6590. static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
  6591. {
  6592. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6593. pci_ers_result_t result;
  6594. int err;
  6595. if (pci_enable_device_mem(pdev)) {
  6596. e_err(probe, "Cannot re-enable PCI device after reset.\n");
  6597. result = PCI_ERS_RESULT_DISCONNECT;
  6598. } else {
  6599. pci_set_master(pdev);
  6600. pci_restore_state(pdev);
  6601. pci_save_state(pdev);
  6602. pci_wake_from_d3(pdev, false);
  6603. ixgbe_reset(adapter);
  6604. IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
  6605. result = PCI_ERS_RESULT_RECOVERED;
  6606. }
  6607. err = pci_cleanup_aer_uncorrect_error_status(pdev);
  6608. if (err) {
  6609. e_dev_err("pci_cleanup_aer_uncorrect_error_status "
  6610. "failed 0x%0x\n", err);
  6611. /* non-fatal, continue */
  6612. }
  6613. return result;
  6614. }
  6615. /**
  6616. * ixgbe_io_resume - called when traffic can start flowing again.
  6617. * @pdev: Pointer to PCI device
  6618. *
  6619. * This callback is called when the error recovery driver tells us that
  6620. * its OK to resume normal operation.
  6621. */
  6622. static void ixgbe_io_resume(struct pci_dev *pdev)
  6623. {
  6624. struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
  6625. struct net_device *netdev = adapter->netdev;
  6626. #ifdef CONFIG_PCI_IOV
  6627. if (adapter->vferr_refcount) {
  6628. e_info(drv, "Resuming after VF err\n");
  6629. adapter->vferr_refcount--;
  6630. return;
  6631. }
  6632. #endif
  6633. if (netif_running(netdev))
  6634. ixgbe_up(adapter);
  6635. netif_device_attach(netdev);
  6636. }
  6637. static const struct pci_error_handlers ixgbe_err_handler = {
  6638. .error_detected = ixgbe_io_error_detected,
  6639. .slot_reset = ixgbe_io_slot_reset,
  6640. .resume = ixgbe_io_resume,
  6641. };
  6642. static struct pci_driver ixgbe_driver = {
  6643. .name = ixgbe_driver_name,
  6644. .id_table = ixgbe_pci_tbl,
  6645. .probe = ixgbe_probe,
  6646. .remove = ixgbe_remove,
  6647. #ifdef CONFIG_PM
  6648. .suspend = ixgbe_suspend,
  6649. .resume = ixgbe_resume,
  6650. #endif
  6651. .shutdown = ixgbe_shutdown,
  6652. .err_handler = &ixgbe_err_handler
  6653. };
  6654. /**
  6655. * ixgbe_init_module - Driver Registration Routine
  6656. *
  6657. * ixgbe_init_module is the first routine called when the driver is
  6658. * loaded. All it does is register with the PCI subsystem.
  6659. **/
  6660. static int __init ixgbe_init_module(void)
  6661. {
  6662. int ret;
  6663. pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
  6664. pr_info("%s\n", ixgbe_copyright);
  6665. #ifdef CONFIG_DEBUG_FS
  6666. ixgbe_dbg_init();
  6667. #endif /* CONFIG_DEBUG_FS */
  6668. #ifdef CONFIG_IXGBE_DCA
  6669. dca_register_notify(&dca_notifier);
  6670. #endif
  6671. ret = pci_register_driver(&ixgbe_driver);
  6672. return ret;
  6673. }
  6674. module_init(ixgbe_init_module);
  6675. /**
  6676. * ixgbe_exit_module - Driver Exit Cleanup Routine
  6677. *
  6678. * ixgbe_exit_module is called just before the driver is removed
  6679. * from memory.
  6680. **/
  6681. static void __exit ixgbe_exit_module(void)
  6682. {
  6683. #ifdef CONFIG_IXGBE_DCA
  6684. dca_unregister_notify(&dca_notifier);
  6685. #endif
  6686. pci_unregister_driver(&ixgbe_driver);
  6687. #ifdef CONFIG_DEBUG_FS
  6688. ixgbe_dbg_exit();
  6689. #endif /* CONFIG_DEBUG_FS */
  6690. rcu_barrier(); /* Wait for completion of call_rcu()'s */
  6691. }
  6692. #ifdef CONFIG_IXGBE_DCA
  6693. static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
  6694. void *p)
  6695. {
  6696. int ret_val;
  6697. ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
  6698. __ixgbe_notify_dca);
  6699. return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
  6700. }
  6701. #endif /* CONFIG_IXGBE_DCA */
  6702. module_exit(ixgbe_exit_module);
  6703. /* ixgbe_main.c */