igb_main.c 133 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884
  1. /*******************************************************************************
  2. Intel(R) Gigabit Ethernet Linux driver
  3. Copyright(c) 2007-2009 Intel Corporation.
  4. This program is free software; you can redistribute it and/or modify it
  5. under the terms and conditions of the GNU General Public License,
  6. version 2, as published by the Free Software Foundation.
  7. This program is distributed in the hope it will be useful, but WITHOUT
  8. ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  9. FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  10. more details.
  11. You should have received a copy of the GNU General Public License along with
  12. this program; if not, write to the Free Software Foundation, Inc.,
  13. 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  14. The full GNU General Public License is included in this distribution in
  15. the file called "COPYING".
  16. Contact Information:
  17. e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  18. Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  19. *******************************************************************************/
  20. #include <linux/module.h>
  21. #include <linux/types.h>
  22. #include <linux/init.h>
  23. #include <linux/vmalloc.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/netdevice.h>
  26. #include <linux/ipv6.h>
  27. #include <net/checksum.h>
  28. #include <net/ip6_checksum.h>
  29. #include <linux/net_tstamp.h>
  30. #include <linux/mii.h>
  31. #include <linux/ethtool.h>
  32. #include <linux/if_vlan.h>
  33. #include <linux/pci.h>
  34. #include <linux/pci-aspm.h>
  35. #include <linux/delay.h>
  36. #include <linux/interrupt.h>
  37. #include <linux/if_ether.h>
  38. #include <linux/aer.h>
  39. #ifdef CONFIG_IGB_DCA
  40. #include <linux/dca.h>
  41. #endif
  42. #include "igb.h"
  43. #define DRV_VERSION "1.3.16-k2"
  44. char igb_driver_name[] = "igb";
  45. char igb_driver_version[] = DRV_VERSION;
  46. static const char igb_driver_string[] =
  47. "Intel(R) Gigabit Ethernet Network Driver";
  48. static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
  49. static const struct e1000_info *igb_info_tbl[] = {
  50. [board_82575] = &e1000_82575_info,
  51. };
  52. static struct pci_device_id igb_pci_tbl[] = {
  53. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  54. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  55. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  56. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  57. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  58. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  59. /* required last entry */
  60. {0, }
  61. };
  62. MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  63. void igb_reset(struct igb_adapter *);
  64. static int igb_setup_all_tx_resources(struct igb_adapter *);
  65. static int igb_setup_all_rx_resources(struct igb_adapter *);
  66. static void igb_free_all_tx_resources(struct igb_adapter *);
  67. static void igb_free_all_rx_resources(struct igb_adapter *);
  68. void igb_update_stats(struct igb_adapter *);
  69. static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  70. static void __devexit igb_remove(struct pci_dev *pdev);
  71. static int igb_sw_init(struct igb_adapter *);
  72. static int igb_open(struct net_device *);
  73. static int igb_close(struct net_device *);
  74. static void igb_configure_tx(struct igb_adapter *);
  75. static void igb_configure_rx(struct igb_adapter *);
  76. static void igb_setup_rctl(struct igb_adapter *);
  77. static void igb_clean_all_tx_rings(struct igb_adapter *);
  78. static void igb_clean_all_rx_rings(struct igb_adapter *);
  79. static void igb_clean_tx_ring(struct igb_ring *);
  80. static void igb_clean_rx_ring(struct igb_ring *);
  81. static void igb_set_multi(struct net_device *);
  82. static void igb_update_phy_info(unsigned long);
  83. static void igb_watchdog(unsigned long);
  84. static void igb_watchdog_task(struct work_struct *);
  85. static int igb_xmit_frame_ring_adv(struct sk_buff *, struct net_device *,
  86. struct igb_ring *);
  87. static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
  88. static struct net_device_stats *igb_get_stats(struct net_device *);
  89. static int igb_change_mtu(struct net_device *, int);
  90. static int igb_set_mac(struct net_device *, void *);
  91. static irqreturn_t igb_intr(int irq, void *);
  92. static irqreturn_t igb_intr_msi(int irq, void *);
  93. static irqreturn_t igb_msix_other(int irq, void *);
  94. static irqreturn_t igb_msix_rx(int irq, void *);
  95. static irqreturn_t igb_msix_tx(int irq, void *);
  96. static int igb_clean_rx_ring_msix(struct napi_struct *, int);
  97. #ifdef CONFIG_IGB_DCA
  98. static void igb_update_rx_dca(struct igb_ring *);
  99. static void igb_update_tx_dca(struct igb_ring *);
  100. static void igb_setup_dca(struct igb_adapter *);
  101. #endif /* CONFIG_IGB_DCA */
  102. static bool igb_clean_tx_irq(struct igb_ring *);
  103. static int igb_poll(struct napi_struct *, int);
  104. static bool igb_clean_rx_irq_adv(struct igb_ring *, int *, int);
  105. static void igb_alloc_rx_buffers_adv(struct igb_ring *, int);
  106. static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
  107. static void igb_tx_timeout(struct net_device *);
  108. static void igb_reset_task(struct work_struct *);
  109. static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
  110. static void igb_vlan_rx_add_vid(struct net_device *, u16);
  111. static void igb_vlan_rx_kill_vid(struct net_device *, u16);
  112. static void igb_restore_vlan(struct igb_adapter *);
  113. static int igb_suspend(struct pci_dev *, pm_message_t);
  114. #ifdef CONFIG_PM
  115. static int igb_resume(struct pci_dev *);
  116. #endif
  117. static void igb_shutdown(struct pci_dev *);
  118. #ifdef CONFIG_IGB_DCA
  119. static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
  120. static struct notifier_block dca_notifier = {
  121. .notifier_call = igb_notify_dca,
  122. .next = NULL,
  123. .priority = 0
  124. };
  125. #endif
  126. #ifdef CONFIG_NET_POLL_CONTROLLER
  127. /* for netdump / net console */
  128. static void igb_netpoll(struct net_device *);
  129. #endif
  130. static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
  131. pci_channel_state_t);
  132. static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
  133. static void igb_io_resume(struct pci_dev *);
  134. static struct pci_error_handlers igb_err_handler = {
  135. .error_detected = igb_io_error_detected,
  136. .slot_reset = igb_io_slot_reset,
  137. .resume = igb_io_resume,
  138. };
  139. static struct pci_driver igb_driver = {
  140. .name = igb_driver_name,
  141. .id_table = igb_pci_tbl,
  142. .probe = igb_probe,
  143. .remove = __devexit_p(igb_remove),
  144. #ifdef CONFIG_PM
  145. /* Power Managment Hooks */
  146. .suspend = igb_suspend,
  147. .resume = igb_resume,
  148. #endif
  149. .shutdown = igb_shutdown,
  150. .err_handler = &igb_err_handler
  151. };
  152. static int global_quad_port_a; /* global quad port a indication */
  153. MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  154. MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
  155. MODULE_LICENSE("GPL");
  156. MODULE_VERSION(DRV_VERSION);
  157. /**
  158. * Scale the NIC clock cycle by a large factor so that
  159. * relatively small clock corrections can be added or
  160. * substracted at each clock tick. The drawbacks of a
  161. * large factor are a) that the clock register overflows
  162. * more quickly (not such a big deal) and b) that the
  163. * increment per tick has to fit into 24 bits.
  164. *
  165. * Note that
  166. * TIMINCA = IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS *
  167. * IGB_TSYNC_SCALE
  168. * TIMINCA += TIMINCA * adjustment [ppm] / 1e9
  169. *
  170. * The base scale factor is intentionally a power of two
  171. * so that the division in %struct timecounter can be done with
  172. * a shift.
  173. */
  174. #define IGB_TSYNC_SHIFT (19)
  175. #define IGB_TSYNC_SCALE (1<<IGB_TSYNC_SHIFT)
  176. /**
  177. * The duration of one clock cycle of the NIC.
  178. *
  179. * @todo This hard-coded value is part of the specification and might change
  180. * in future hardware revisions. Add revision check.
  181. */
  182. #define IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS 16
  183. #if (IGB_TSYNC_SCALE * IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS) >= (1<<24)
  184. # error IGB_TSYNC_SCALE and/or IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS are too large to fit into TIMINCA
  185. #endif
  186. /**
  187. * igb_read_clock - read raw cycle counter (to be used by time counter)
  188. */
  189. static cycle_t igb_read_clock(const struct cyclecounter *tc)
  190. {
  191. struct igb_adapter *adapter =
  192. container_of(tc, struct igb_adapter, cycles);
  193. struct e1000_hw *hw = &adapter->hw;
  194. u64 stamp;
  195. stamp = rd32(E1000_SYSTIML);
  196. stamp |= (u64)rd32(E1000_SYSTIMH) << 32ULL;
  197. return stamp;
  198. }
  199. #ifdef DEBUG
  200. /**
  201. * igb_get_hw_dev_name - return device name string
  202. * used by hardware layer to print debugging information
  203. **/
  204. char *igb_get_hw_dev_name(struct e1000_hw *hw)
  205. {
  206. struct igb_adapter *adapter = hw->back;
  207. return adapter->netdev->name;
  208. }
  209. /**
  210. * igb_get_time_str - format current NIC and system time as string
  211. */
  212. static char *igb_get_time_str(struct igb_adapter *adapter,
  213. char buffer[160])
  214. {
  215. cycle_t hw = adapter->cycles.read(&adapter->cycles);
  216. struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
  217. struct timespec sys;
  218. struct timespec delta;
  219. getnstimeofday(&sys);
  220. delta = timespec_sub(nic, sys);
  221. sprintf(buffer,
  222. "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
  223. hw,
  224. (long)nic.tv_sec, nic.tv_nsec,
  225. (long)sys.tv_sec, sys.tv_nsec,
  226. (long)delta.tv_sec, delta.tv_nsec);
  227. return buffer;
  228. }
  229. #endif
  230. /**
  231. * igb_init_module - Driver Registration Routine
  232. *
  233. * igb_init_module is the first routine called when the driver is
  234. * loaded. All it does is register with the PCI subsystem.
  235. **/
  236. static int __init igb_init_module(void)
  237. {
  238. int ret;
  239. printk(KERN_INFO "%s - version %s\n",
  240. igb_driver_string, igb_driver_version);
  241. printk(KERN_INFO "%s\n", igb_copyright);
  242. global_quad_port_a = 0;
  243. #ifdef CONFIG_IGB_DCA
  244. dca_register_notify(&dca_notifier);
  245. #endif
  246. ret = pci_register_driver(&igb_driver);
  247. return ret;
  248. }
  249. module_init(igb_init_module);
  250. /**
  251. * igb_exit_module - Driver Exit Cleanup Routine
  252. *
  253. * igb_exit_module is called just before the driver is removed
  254. * from memory.
  255. **/
  256. static void __exit igb_exit_module(void)
  257. {
  258. #ifdef CONFIG_IGB_DCA
  259. dca_unregister_notify(&dca_notifier);
  260. #endif
  261. pci_unregister_driver(&igb_driver);
  262. }
  263. module_exit(igb_exit_module);
  264. #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
  265. /**
  266. * igb_cache_ring_register - Descriptor ring to register mapping
  267. * @adapter: board private structure to initialize
  268. *
  269. * Once we know the feature-set enabled for the device, we'll cache
  270. * the register offset the descriptor ring is assigned to.
  271. **/
  272. static void igb_cache_ring_register(struct igb_adapter *adapter)
  273. {
  274. int i;
  275. switch (adapter->hw.mac.type) {
  276. case e1000_82576:
  277. /* The queues are allocated for virtualization such that VF 0
  278. * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
  279. * In order to avoid collision we start at the first free queue
  280. * and continue consuming queues in the same sequence
  281. */
  282. for (i = 0; i < adapter->num_rx_queues; i++)
  283. adapter->rx_ring[i].reg_idx = Q_IDX_82576(i);
  284. for (i = 0; i < adapter->num_tx_queues; i++)
  285. adapter->tx_ring[i].reg_idx = Q_IDX_82576(i);
  286. break;
  287. case e1000_82575:
  288. default:
  289. for (i = 0; i < adapter->num_rx_queues; i++)
  290. adapter->rx_ring[i].reg_idx = i;
  291. for (i = 0; i < adapter->num_tx_queues; i++)
  292. adapter->tx_ring[i].reg_idx = i;
  293. break;
  294. }
  295. }
  296. /**
  297. * igb_alloc_queues - Allocate memory for all rings
  298. * @adapter: board private structure to initialize
  299. *
  300. * We allocate one ring per queue at run-time since we don't know the
  301. * number of queues at compile-time.
  302. **/
  303. static int igb_alloc_queues(struct igb_adapter *adapter)
  304. {
  305. int i;
  306. adapter->tx_ring = kcalloc(adapter->num_tx_queues,
  307. sizeof(struct igb_ring), GFP_KERNEL);
  308. if (!adapter->tx_ring)
  309. return -ENOMEM;
  310. adapter->rx_ring = kcalloc(adapter->num_rx_queues,
  311. sizeof(struct igb_ring), GFP_KERNEL);
  312. if (!adapter->rx_ring) {
  313. kfree(adapter->tx_ring);
  314. return -ENOMEM;
  315. }
  316. adapter->rx_ring->buddy = adapter->tx_ring;
  317. for (i = 0; i < adapter->num_tx_queues; i++) {
  318. struct igb_ring *ring = &(adapter->tx_ring[i]);
  319. ring->count = adapter->tx_ring_count;
  320. ring->adapter = adapter;
  321. ring->queue_index = i;
  322. }
  323. for (i = 0; i < adapter->num_rx_queues; i++) {
  324. struct igb_ring *ring = &(adapter->rx_ring[i]);
  325. ring->count = adapter->rx_ring_count;
  326. ring->adapter = adapter;
  327. ring->queue_index = i;
  328. ring->itr_register = E1000_ITR;
  329. /* set a default napi handler for each rx_ring */
  330. netif_napi_add(adapter->netdev, &ring->napi, igb_poll, 64);
  331. }
  332. igb_cache_ring_register(adapter);
  333. return 0;
  334. }
  335. static void igb_free_queues(struct igb_adapter *adapter)
  336. {
  337. int i;
  338. for (i = 0; i < adapter->num_rx_queues; i++)
  339. netif_napi_del(&adapter->rx_ring[i].napi);
  340. kfree(adapter->tx_ring);
  341. kfree(adapter->rx_ring);
  342. }
  343. #define IGB_N0_QUEUE -1
  344. static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue,
  345. int tx_queue, int msix_vector)
  346. {
  347. u32 msixbm = 0;
  348. struct e1000_hw *hw = &adapter->hw;
  349. u32 ivar, index;
  350. switch (hw->mac.type) {
  351. case e1000_82575:
  352. /* The 82575 assigns vectors using a bitmask, which matches the
  353. bitmask for the EICR/EIMS/EIMC registers. To assign one
  354. or more queues to a vector, we write the appropriate bits
  355. into the MSIXBM register for that vector. */
  356. if (rx_queue > IGB_N0_QUEUE) {
  357. msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
  358. adapter->rx_ring[rx_queue].eims_value = msixbm;
  359. }
  360. if (tx_queue > IGB_N0_QUEUE) {
  361. msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
  362. adapter->tx_ring[tx_queue].eims_value =
  363. E1000_EICR_TX_QUEUE0 << tx_queue;
  364. }
  365. array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
  366. break;
  367. case e1000_82576:
  368. /* 82576 uses a table-based method for assigning vectors.
  369. Each queue has a single entry in the table to which we write
  370. a vector number along with a "valid" bit. Sadly, the layout
  371. of the table is somewhat counterintuitive. */
  372. if (rx_queue > IGB_N0_QUEUE) {
  373. index = (rx_queue >> 1);
  374. ivar = array_rd32(E1000_IVAR0, index);
  375. if (rx_queue & 0x1) {
  376. /* vector goes into third byte of register */
  377. ivar = ivar & 0xFF00FFFF;
  378. ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
  379. } else {
  380. /* vector goes into low byte of register */
  381. ivar = ivar & 0xFFFFFF00;
  382. ivar |= msix_vector | E1000_IVAR_VALID;
  383. }
  384. adapter->rx_ring[rx_queue].eims_value= 1 << msix_vector;
  385. array_wr32(E1000_IVAR0, index, ivar);
  386. }
  387. if (tx_queue > IGB_N0_QUEUE) {
  388. index = (tx_queue >> 1);
  389. ivar = array_rd32(E1000_IVAR0, index);
  390. if (tx_queue & 0x1) {
  391. /* vector goes into high byte of register */
  392. ivar = ivar & 0x00FFFFFF;
  393. ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
  394. } else {
  395. /* vector goes into second byte of register */
  396. ivar = ivar & 0xFFFF00FF;
  397. ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
  398. }
  399. adapter->tx_ring[tx_queue].eims_value= 1 << msix_vector;
  400. array_wr32(E1000_IVAR0, index, ivar);
  401. }
  402. break;
  403. default:
  404. BUG();
  405. break;
  406. }
  407. }
  408. /**
  409. * igb_configure_msix - Configure MSI-X hardware
  410. *
  411. * igb_configure_msix sets up the hardware to properly
  412. * generate MSI-X interrupts.
  413. **/
  414. static void igb_configure_msix(struct igb_adapter *adapter)
  415. {
  416. u32 tmp;
  417. int i, vector = 0;
  418. struct e1000_hw *hw = &adapter->hw;
  419. adapter->eims_enable_mask = 0;
  420. if (hw->mac.type == e1000_82576)
  421. /* Turn on MSI-X capability first, or our settings
  422. * won't stick. And it will take days to debug. */
  423. wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
  424. E1000_GPIE_PBA | E1000_GPIE_EIAME |
  425. E1000_GPIE_NSICR);
  426. for (i = 0; i < adapter->num_tx_queues; i++) {
  427. struct igb_ring *tx_ring = &adapter->tx_ring[i];
  428. igb_assign_vector(adapter, IGB_N0_QUEUE, i, vector++);
  429. adapter->eims_enable_mask |= tx_ring->eims_value;
  430. if (tx_ring->itr_val)
  431. writel(tx_ring->itr_val,
  432. hw->hw_addr + tx_ring->itr_register);
  433. else
  434. writel(1, hw->hw_addr + tx_ring->itr_register);
  435. }
  436. for (i = 0; i < adapter->num_rx_queues; i++) {
  437. struct igb_ring *rx_ring = &adapter->rx_ring[i];
  438. rx_ring->buddy = NULL;
  439. igb_assign_vector(adapter, i, IGB_N0_QUEUE, vector++);
  440. adapter->eims_enable_mask |= rx_ring->eims_value;
  441. if (rx_ring->itr_val)
  442. writel(rx_ring->itr_val,
  443. hw->hw_addr + rx_ring->itr_register);
  444. else
  445. writel(1, hw->hw_addr + rx_ring->itr_register);
  446. }
  447. /* set vector for other causes, i.e. link changes */
  448. switch (hw->mac.type) {
  449. case e1000_82575:
  450. array_wr32(E1000_MSIXBM(0), vector++,
  451. E1000_EIMS_OTHER);
  452. tmp = rd32(E1000_CTRL_EXT);
  453. /* enable MSI-X PBA support*/
  454. tmp |= E1000_CTRL_EXT_PBA_CLR;
  455. /* Auto-Mask interrupts upon ICR read. */
  456. tmp |= E1000_CTRL_EXT_EIAME;
  457. tmp |= E1000_CTRL_EXT_IRCA;
  458. wr32(E1000_CTRL_EXT, tmp);
  459. adapter->eims_enable_mask |= E1000_EIMS_OTHER;
  460. adapter->eims_other = E1000_EIMS_OTHER;
  461. break;
  462. case e1000_82576:
  463. tmp = (vector++ | E1000_IVAR_VALID) << 8;
  464. wr32(E1000_IVAR_MISC, tmp);
  465. adapter->eims_enable_mask = (1 << (vector)) - 1;
  466. adapter->eims_other = 1 << (vector - 1);
  467. break;
  468. default:
  469. /* do nothing, since nothing else supports MSI-X */
  470. break;
  471. } /* switch (hw->mac.type) */
  472. wrfl();
  473. }
  474. /**
  475. * igb_request_msix - Initialize MSI-X interrupts
  476. *
  477. * igb_request_msix allocates MSI-X vectors and requests interrupts from the
  478. * kernel.
  479. **/
  480. static int igb_request_msix(struct igb_adapter *adapter)
  481. {
  482. struct net_device *netdev = adapter->netdev;
  483. int i, err = 0, vector = 0;
  484. vector = 0;
  485. for (i = 0; i < adapter->num_tx_queues; i++) {
  486. struct igb_ring *ring = &(adapter->tx_ring[i]);
  487. sprintf(ring->name, "%s-tx-%d", netdev->name, i);
  488. err = request_irq(adapter->msix_entries[vector].vector,
  489. &igb_msix_tx, 0, ring->name,
  490. &(adapter->tx_ring[i]));
  491. if (err)
  492. goto out;
  493. ring->itr_register = E1000_EITR(0) + (vector << 2);
  494. ring->itr_val = 976; /* ~4000 ints/sec */
  495. vector++;
  496. }
  497. for (i = 0; i < adapter->num_rx_queues; i++) {
  498. struct igb_ring *ring = &(adapter->rx_ring[i]);
  499. if (strlen(netdev->name) < (IFNAMSIZ - 5))
  500. sprintf(ring->name, "%s-rx-%d", netdev->name, i);
  501. else
  502. memcpy(ring->name, netdev->name, IFNAMSIZ);
  503. err = request_irq(adapter->msix_entries[vector].vector,
  504. &igb_msix_rx, 0, ring->name,
  505. &(adapter->rx_ring[i]));
  506. if (err)
  507. goto out;
  508. ring->itr_register = E1000_EITR(0) + (vector << 2);
  509. ring->itr_val = adapter->itr;
  510. /* overwrite the poll routine for MSIX, we've already done
  511. * netif_napi_add */
  512. ring->napi.poll = &igb_clean_rx_ring_msix;
  513. vector++;
  514. }
  515. err = request_irq(adapter->msix_entries[vector].vector,
  516. &igb_msix_other, 0, netdev->name, netdev);
  517. if (err)
  518. goto out;
  519. igb_configure_msix(adapter);
  520. return 0;
  521. out:
  522. return err;
  523. }
  524. static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
  525. {
  526. if (adapter->msix_entries) {
  527. pci_disable_msix(adapter->pdev);
  528. kfree(adapter->msix_entries);
  529. adapter->msix_entries = NULL;
  530. } else if (adapter->flags & IGB_FLAG_HAS_MSI)
  531. pci_disable_msi(adapter->pdev);
  532. return;
  533. }
  534. /**
  535. * igb_set_interrupt_capability - set MSI or MSI-X if supported
  536. *
  537. * Attempt to configure interrupts using the best available
  538. * capabilities of the hardware and kernel.
  539. **/
  540. static void igb_set_interrupt_capability(struct igb_adapter *adapter)
  541. {
  542. int err;
  543. int numvecs, i;
  544. /* Number of supported queues. */
  545. /* Having more queues than CPUs doesn't make sense. */
  546. adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
  547. adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
  548. numvecs = adapter->num_tx_queues + adapter->num_rx_queues + 1;
  549. adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
  550. GFP_KERNEL);
  551. if (!adapter->msix_entries)
  552. goto msi_only;
  553. for (i = 0; i < numvecs; i++)
  554. adapter->msix_entries[i].entry = i;
  555. err = pci_enable_msix(adapter->pdev,
  556. adapter->msix_entries,
  557. numvecs);
  558. if (err == 0)
  559. goto out;
  560. igb_reset_interrupt_capability(adapter);
  561. /* If we can't do MSI-X, try MSI */
  562. msi_only:
  563. adapter->num_rx_queues = 1;
  564. adapter->num_tx_queues = 1;
  565. if (!pci_enable_msi(adapter->pdev))
  566. adapter->flags |= IGB_FLAG_HAS_MSI;
  567. out:
  568. /* Notify the stack of the (possibly) reduced Tx Queue count. */
  569. adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
  570. return;
  571. }
  572. /**
  573. * igb_request_irq - initialize interrupts
  574. *
  575. * Attempts to configure interrupts using the best available
  576. * capabilities of the hardware and kernel.
  577. **/
  578. static int igb_request_irq(struct igb_adapter *adapter)
  579. {
  580. struct net_device *netdev = adapter->netdev;
  581. struct e1000_hw *hw = &adapter->hw;
  582. int err = 0;
  583. if (adapter->msix_entries) {
  584. err = igb_request_msix(adapter);
  585. if (!err)
  586. goto request_done;
  587. /* fall back to MSI */
  588. igb_reset_interrupt_capability(adapter);
  589. if (!pci_enable_msi(adapter->pdev))
  590. adapter->flags |= IGB_FLAG_HAS_MSI;
  591. igb_free_all_tx_resources(adapter);
  592. igb_free_all_rx_resources(adapter);
  593. adapter->num_rx_queues = 1;
  594. igb_alloc_queues(adapter);
  595. } else {
  596. switch (hw->mac.type) {
  597. case e1000_82575:
  598. wr32(E1000_MSIXBM(0),
  599. (E1000_EICR_RX_QUEUE0 | E1000_EIMS_OTHER));
  600. break;
  601. case e1000_82576:
  602. wr32(E1000_IVAR0, E1000_IVAR_VALID);
  603. break;
  604. default:
  605. break;
  606. }
  607. }
  608. if (adapter->flags & IGB_FLAG_HAS_MSI) {
  609. err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
  610. netdev->name, netdev);
  611. if (!err)
  612. goto request_done;
  613. /* fall back to legacy interrupts */
  614. igb_reset_interrupt_capability(adapter);
  615. adapter->flags &= ~IGB_FLAG_HAS_MSI;
  616. }
  617. err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
  618. netdev->name, netdev);
  619. if (err)
  620. dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
  621. err);
  622. request_done:
  623. return err;
  624. }
  625. static void igb_free_irq(struct igb_adapter *adapter)
  626. {
  627. struct net_device *netdev = adapter->netdev;
  628. if (adapter->msix_entries) {
  629. int vector = 0, i;
  630. for (i = 0; i < adapter->num_tx_queues; i++)
  631. free_irq(adapter->msix_entries[vector++].vector,
  632. &(adapter->tx_ring[i]));
  633. for (i = 0; i < adapter->num_rx_queues; i++)
  634. free_irq(adapter->msix_entries[vector++].vector,
  635. &(adapter->rx_ring[i]));
  636. free_irq(adapter->msix_entries[vector++].vector, netdev);
  637. return;
  638. }
  639. free_irq(adapter->pdev->irq, netdev);
  640. }
  641. /**
  642. * igb_irq_disable - Mask off interrupt generation on the NIC
  643. * @adapter: board private structure
  644. **/
  645. static void igb_irq_disable(struct igb_adapter *adapter)
  646. {
  647. struct e1000_hw *hw = &adapter->hw;
  648. if (adapter->msix_entries) {
  649. wr32(E1000_EIAM, 0);
  650. wr32(E1000_EIMC, ~0);
  651. wr32(E1000_EIAC, 0);
  652. }
  653. wr32(E1000_IAM, 0);
  654. wr32(E1000_IMC, ~0);
  655. wrfl();
  656. synchronize_irq(adapter->pdev->irq);
  657. }
  658. /**
  659. * igb_irq_enable - Enable default interrupt generation settings
  660. * @adapter: board private structure
  661. **/
  662. static void igb_irq_enable(struct igb_adapter *adapter)
  663. {
  664. struct e1000_hw *hw = &adapter->hw;
  665. if (adapter->msix_entries) {
  666. wr32(E1000_EIAC, adapter->eims_enable_mask);
  667. wr32(E1000_EIAM, adapter->eims_enable_mask);
  668. wr32(E1000_EIMS, adapter->eims_enable_mask);
  669. wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
  670. } else {
  671. wr32(E1000_IMS, IMS_ENABLE_MASK);
  672. wr32(E1000_IAM, IMS_ENABLE_MASK);
  673. }
  674. }
  675. static void igb_update_mng_vlan(struct igb_adapter *adapter)
  676. {
  677. struct net_device *netdev = adapter->netdev;
  678. u16 vid = adapter->hw.mng_cookie.vlan_id;
  679. u16 old_vid = adapter->mng_vlan_id;
  680. if (adapter->vlgrp) {
  681. if (!vlan_group_get_device(adapter->vlgrp, vid)) {
  682. if (adapter->hw.mng_cookie.status &
  683. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
  684. igb_vlan_rx_add_vid(netdev, vid);
  685. adapter->mng_vlan_id = vid;
  686. } else
  687. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  688. if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
  689. (vid != old_vid) &&
  690. !vlan_group_get_device(adapter->vlgrp, old_vid))
  691. igb_vlan_rx_kill_vid(netdev, old_vid);
  692. } else
  693. adapter->mng_vlan_id = vid;
  694. }
  695. }
  696. /**
  697. * igb_release_hw_control - release control of the h/w to f/w
  698. * @adapter: address of board private structure
  699. *
  700. * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
  701. * For ASF and Pass Through versions of f/w this means that the
  702. * driver is no longer loaded.
  703. *
  704. **/
  705. static void igb_release_hw_control(struct igb_adapter *adapter)
  706. {
  707. struct e1000_hw *hw = &adapter->hw;
  708. u32 ctrl_ext;
  709. /* Let firmware take over control of h/w */
  710. ctrl_ext = rd32(E1000_CTRL_EXT);
  711. wr32(E1000_CTRL_EXT,
  712. ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
  713. }
  714. /**
  715. * igb_get_hw_control - get control of the h/w from f/w
  716. * @adapter: address of board private structure
  717. *
  718. * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
  719. * For ASF and Pass Through versions of f/w this means that
  720. * the driver is loaded.
  721. *
  722. **/
  723. static void igb_get_hw_control(struct igb_adapter *adapter)
  724. {
  725. struct e1000_hw *hw = &adapter->hw;
  726. u32 ctrl_ext;
  727. /* Let firmware know the driver has taken over */
  728. ctrl_ext = rd32(E1000_CTRL_EXT);
  729. wr32(E1000_CTRL_EXT,
  730. ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
  731. }
  732. /**
  733. * igb_configure - configure the hardware for RX and TX
  734. * @adapter: private board structure
  735. **/
  736. static void igb_configure(struct igb_adapter *adapter)
  737. {
  738. struct net_device *netdev = adapter->netdev;
  739. int i;
  740. igb_get_hw_control(adapter);
  741. igb_set_multi(netdev);
  742. igb_restore_vlan(adapter);
  743. igb_configure_tx(adapter);
  744. igb_setup_rctl(adapter);
  745. igb_configure_rx(adapter);
  746. igb_rx_fifo_flush_82575(&adapter->hw);
  747. /* call IGB_DESC_UNUSED which always leaves
  748. * at least 1 descriptor unused to make sure
  749. * next_to_use != next_to_clean */
  750. for (i = 0; i < adapter->num_rx_queues; i++) {
  751. struct igb_ring *ring = &adapter->rx_ring[i];
  752. igb_alloc_rx_buffers_adv(ring, IGB_DESC_UNUSED(ring));
  753. }
  754. adapter->tx_queue_len = netdev->tx_queue_len;
  755. }
  756. /**
  757. * igb_up - Open the interface and prepare it to handle traffic
  758. * @adapter: board private structure
  759. **/
  760. int igb_up(struct igb_adapter *adapter)
  761. {
  762. struct e1000_hw *hw = &adapter->hw;
  763. int i;
  764. /* hardware has been reset, we need to reload some things */
  765. igb_configure(adapter);
  766. clear_bit(__IGB_DOWN, &adapter->state);
  767. for (i = 0; i < adapter->num_rx_queues; i++)
  768. napi_enable(&adapter->rx_ring[i].napi);
  769. if (adapter->msix_entries)
  770. igb_configure_msix(adapter);
  771. /* Clear any pending interrupts. */
  772. rd32(E1000_ICR);
  773. igb_irq_enable(adapter);
  774. /* Fire a link change interrupt to start the watchdog. */
  775. wr32(E1000_ICS, E1000_ICS_LSC);
  776. return 0;
  777. }
  778. void igb_down(struct igb_adapter *adapter)
  779. {
  780. struct e1000_hw *hw = &adapter->hw;
  781. struct net_device *netdev = adapter->netdev;
  782. u32 tctl, rctl;
  783. int i;
  784. /* signal that we're down so the interrupt handler does not
  785. * reschedule our watchdog timer */
  786. set_bit(__IGB_DOWN, &adapter->state);
  787. /* disable receives in the hardware */
  788. rctl = rd32(E1000_RCTL);
  789. wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
  790. /* flush and sleep below */
  791. netif_tx_stop_all_queues(netdev);
  792. /* disable transmits in the hardware */
  793. tctl = rd32(E1000_TCTL);
  794. tctl &= ~E1000_TCTL_EN;
  795. wr32(E1000_TCTL, tctl);
  796. /* flush both disables and wait for them to finish */
  797. wrfl();
  798. msleep(10);
  799. for (i = 0; i < adapter->num_rx_queues; i++)
  800. napi_disable(&adapter->rx_ring[i].napi);
  801. igb_irq_disable(adapter);
  802. del_timer_sync(&adapter->watchdog_timer);
  803. del_timer_sync(&adapter->phy_info_timer);
  804. netdev->tx_queue_len = adapter->tx_queue_len;
  805. netif_carrier_off(netdev);
  806. /* record the stats before reset*/
  807. igb_update_stats(adapter);
  808. adapter->link_speed = 0;
  809. adapter->link_duplex = 0;
  810. if (!pci_channel_offline(adapter->pdev))
  811. igb_reset(adapter);
  812. igb_clean_all_tx_rings(adapter);
  813. igb_clean_all_rx_rings(adapter);
  814. }
  815. void igb_reinit_locked(struct igb_adapter *adapter)
  816. {
  817. WARN_ON(in_interrupt());
  818. while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
  819. msleep(1);
  820. igb_down(adapter);
  821. igb_up(adapter);
  822. clear_bit(__IGB_RESETTING, &adapter->state);
  823. }
  824. void igb_reset(struct igb_adapter *adapter)
  825. {
  826. struct e1000_hw *hw = &adapter->hw;
  827. struct e1000_mac_info *mac = &hw->mac;
  828. struct e1000_fc_info *fc = &hw->fc;
  829. u32 pba = 0, tx_space, min_tx_space, min_rx_space;
  830. u16 hwm;
  831. /* Repartition Pba for greater than 9k mtu
  832. * To take effect CTRL.RST is required.
  833. */
  834. switch (mac->type) {
  835. case e1000_82576:
  836. pba = E1000_PBA_64K;
  837. break;
  838. case e1000_82575:
  839. default:
  840. pba = E1000_PBA_34K;
  841. break;
  842. }
  843. if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
  844. (mac->type < e1000_82576)) {
  845. /* adjust PBA for jumbo frames */
  846. wr32(E1000_PBA, pba);
  847. /* To maintain wire speed transmits, the Tx FIFO should be
  848. * large enough to accommodate two full transmit packets,
  849. * rounded up to the next 1KB and expressed in KB. Likewise,
  850. * the Rx FIFO should be large enough to accommodate at least
  851. * one full receive packet and is similarly rounded up and
  852. * expressed in KB. */
  853. pba = rd32(E1000_PBA);
  854. /* upper 16 bits has Tx packet buffer allocation size in KB */
  855. tx_space = pba >> 16;
  856. /* lower 16 bits has Rx packet buffer allocation size in KB */
  857. pba &= 0xffff;
  858. /* the tx fifo also stores 16 bytes of information about the tx
  859. * but don't include ethernet FCS because hardware appends it */
  860. min_tx_space = (adapter->max_frame_size +
  861. sizeof(struct e1000_tx_desc) -
  862. ETH_FCS_LEN) * 2;
  863. min_tx_space = ALIGN(min_tx_space, 1024);
  864. min_tx_space >>= 10;
  865. /* software strips receive CRC, so leave room for it */
  866. min_rx_space = adapter->max_frame_size;
  867. min_rx_space = ALIGN(min_rx_space, 1024);
  868. min_rx_space >>= 10;
  869. /* If current Tx allocation is less than the min Tx FIFO size,
  870. * and the min Tx FIFO size is less than the current Rx FIFO
  871. * allocation, take space away from current Rx allocation */
  872. if (tx_space < min_tx_space &&
  873. ((min_tx_space - tx_space) < pba)) {
  874. pba = pba - (min_tx_space - tx_space);
  875. /* if short on rx space, rx wins and must trump tx
  876. * adjustment */
  877. if (pba < min_rx_space)
  878. pba = min_rx_space;
  879. }
  880. wr32(E1000_PBA, pba);
  881. }
  882. /* flow control settings */
  883. /* The high water mark must be low enough to fit one full frame
  884. * (or the size used for early receive) above it in the Rx FIFO.
  885. * Set it to the lower of:
  886. * - 90% of the Rx FIFO size, or
  887. * - the full Rx FIFO size minus one full frame */
  888. hwm = min(((pba << 10) * 9 / 10),
  889. ((pba << 10) - 2 * adapter->max_frame_size));
  890. if (mac->type < e1000_82576) {
  891. fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
  892. fc->low_water = fc->high_water - 8;
  893. } else {
  894. fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
  895. fc->low_water = fc->high_water - 16;
  896. }
  897. fc->pause_time = 0xFFFF;
  898. fc->send_xon = 1;
  899. fc->type = fc->original_type;
  900. /* Allow time for pending master requests to run */
  901. adapter->hw.mac.ops.reset_hw(&adapter->hw);
  902. wr32(E1000_WUC, 0);
  903. if (adapter->hw.mac.ops.init_hw(&adapter->hw))
  904. dev_err(&adapter->pdev->dev, "Hardware Error\n");
  905. igb_update_mng_vlan(adapter);
  906. /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
  907. wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
  908. igb_reset_adaptive(&adapter->hw);
  909. igb_get_phy_info(&adapter->hw);
  910. }
  911. static const struct net_device_ops igb_netdev_ops = {
  912. .ndo_open = igb_open,
  913. .ndo_stop = igb_close,
  914. .ndo_start_xmit = igb_xmit_frame_adv,
  915. .ndo_get_stats = igb_get_stats,
  916. .ndo_set_multicast_list = igb_set_multi,
  917. .ndo_set_mac_address = igb_set_mac,
  918. .ndo_change_mtu = igb_change_mtu,
  919. .ndo_do_ioctl = igb_ioctl,
  920. .ndo_tx_timeout = igb_tx_timeout,
  921. .ndo_validate_addr = eth_validate_addr,
  922. .ndo_vlan_rx_register = igb_vlan_rx_register,
  923. .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
  924. .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
  925. #ifdef CONFIG_NET_POLL_CONTROLLER
  926. .ndo_poll_controller = igb_netpoll,
  927. #endif
  928. };
  929. /**
  930. * igb_probe - Device Initialization Routine
  931. * @pdev: PCI device information struct
  932. * @ent: entry in igb_pci_tbl
  933. *
  934. * Returns 0 on success, negative on failure
  935. *
  936. * igb_probe initializes an adapter identified by a pci_dev structure.
  937. * The OS initialization, configuring of the adapter private structure,
  938. * and a hardware reset occur.
  939. **/
  940. static int __devinit igb_probe(struct pci_dev *pdev,
  941. const struct pci_device_id *ent)
  942. {
  943. struct net_device *netdev;
  944. struct igb_adapter *adapter;
  945. struct e1000_hw *hw;
  946. struct pci_dev *us_dev;
  947. const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
  948. unsigned long mmio_start, mmio_len;
  949. int err, pci_using_dac, pos;
  950. u16 eeprom_data = 0, state = 0;
  951. u16 eeprom_apme_mask = IGB_EEPROM_APME;
  952. u32 part_num;
  953. err = pci_enable_device_mem(pdev);
  954. if (err)
  955. return err;
  956. pci_using_dac = 0;
  957. err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
  958. if (!err) {
  959. err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
  960. if (!err)
  961. pci_using_dac = 1;
  962. } else {
  963. err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
  964. if (err) {
  965. err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
  966. if (err) {
  967. dev_err(&pdev->dev, "No usable DMA "
  968. "configuration, aborting\n");
  969. goto err_dma;
  970. }
  971. }
  972. }
  973. /* 82575 requires that the pci-e link partner disable the L0s state */
  974. switch (pdev->device) {
  975. case E1000_DEV_ID_82575EB_COPPER:
  976. case E1000_DEV_ID_82575EB_FIBER_SERDES:
  977. case E1000_DEV_ID_82575GB_QUAD_COPPER:
  978. us_dev = pdev->bus->self;
  979. pos = pci_find_capability(us_dev, PCI_CAP_ID_EXP);
  980. if (pos) {
  981. pci_read_config_word(us_dev, pos + PCI_EXP_LNKCTL,
  982. &state);
  983. state &= ~PCIE_LINK_STATE_L0S;
  984. pci_write_config_word(us_dev, pos + PCI_EXP_LNKCTL,
  985. state);
  986. dev_info(&pdev->dev,
  987. "Disabling ASPM L0s upstream switch port %s\n",
  988. pci_name(us_dev));
  989. }
  990. default:
  991. break;
  992. }
  993. err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
  994. IORESOURCE_MEM),
  995. igb_driver_name);
  996. if (err)
  997. goto err_pci_reg;
  998. err = pci_enable_pcie_error_reporting(pdev);
  999. if (err) {
  1000. dev_err(&pdev->dev, "pci_enable_pcie_error_reporting failed "
  1001. "0x%x\n", err);
  1002. /* non-fatal, continue */
  1003. }
  1004. pci_set_master(pdev);
  1005. pci_save_state(pdev);
  1006. err = -ENOMEM;
  1007. netdev = alloc_etherdev_mq(sizeof(struct igb_adapter), IGB_MAX_TX_QUEUES);
  1008. if (!netdev)
  1009. goto err_alloc_etherdev;
  1010. SET_NETDEV_DEV(netdev, &pdev->dev);
  1011. pci_set_drvdata(pdev, netdev);
  1012. adapter = netdev_priv(netdev);
  1013. adapter->netdev = netdev;
  1014. adapter->pdev = pdev;
  1015. hw = &adapter->hw;
  1016. hw->back = adapter;
  1017. adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
  1018. mmio_start = pci_resource_start(pdev, 0);
  1019. mmio_len = pci_resource_len(pdev, 0);
  1020. err = -EIO;
  1021. hw->hw_addr = ioremap(mmio_start, mmio_len);
  1022. if (!hw->hw_addr)
  1023. goto err_ioremap;
  1024. netdev->netdev_ops = &igb_netdev_ops;
  1025. igb_set_ethtool_ops(netdev);
  1026. netdev->watchdog_timeo = 5 * HZ;
  1027. strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
  1028. netdev->mem_start = mmio_start;
  1029. netdev->mem_end = mmio_start + mmio_len;
  1030. /* PCI config space info */
  1031. hw->vendor_id = pdev->vendor;
  1032. hw->device_id = pdev->device;
  1033. hw->revision_id = pdev->revision;
  1034. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  1035. hw->subsystem_device_id = pdev->subsystem_device;
  1036. /* setup the private structure */
  1037. hw->back = adapter;
  1038. /* Copy the default MAC, PHY and NVM function pointers */
  1039. memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
  1040. memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
  1041. memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
  1042. /* Initialize skew-specific constants */
  1043. err = ei->get_invariants(hw);
  1044. if (err)
  1045. goto err_sw_init;
  1046. /* setup the private structure */
  1047. err = igb_sw_init(adapter);
  1048. if (err)
  1049. goto err_sw_init;
  1050. igb_get_bus_info_pcie(hw);
  1051. /* set flags */
  1052. switch (hw->mac.type) {
  1053. case e1000_82575:
  1054. adapter->flags |= IGB_FLAG_NEED_CTX_IDX;
  1055. break;
  1056. case e1000_82576:
  1057. default:
  1058. break;
  1059. }
  1060. hw->phy.autoneg_wait_to_complete = false;
  1061. hw->mac.adaptive_ifs = true;
  1062. /* Copper options */
  1063. if (hw->phy.media_type == e1000_media_type_copper) {
  1064. hw->phy.mdix = AUTO_ALL_MODES;
  1065. hw->phy.disable_polarity_correction = false;
  1066. hw->phy.ms_type = e1000_ms_hw_default;
  1067. }
  1068. if (igb_check_reset_block(hw))
  1069. dev_info(&pdev->dev,
  1070. "PHY reset is blocked due to SOL/IDER session.\n");
  1071. netdev->features = NETIF_F_SG |
  1072. NETIF_F_IP_CSUM |
  1073. NETIF_F_HW_VLAN_TX |
  1074. NETIF_F_HW_VLAN_RX |
  1075. NETIF_F_HW_VLAN_FILTER;
  1076. netdev->features |= NETIF_F_IPV6_CSUM;
  1077. netdev->features |= NETIF_F_TSO;
  1078. netdev->features |= NETIF_F_TSO6;
  1079. #ifdef CONFIG_IGB_LRO
  1080. netdev->features |= NETIF_F_GRO;
  1081. #endif
  1082. netdev->vlan_features |= NETIF_F_TSO;
  1083. netdev->vlan_features |= NETIF_F_TSO6;
  1084. netdev->vlan_features |= NETIF_F_IP_CSUM;
  1085. netdev->vlan_features |= NETIF_F_SG;
  1086. if (pci_using_dac)
  1087. netdev->features |= NETIF_F_HIGHDMA;
  1088. adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
  1089. /* before reading the NVM, reset the controller to put the device in a
  1090. * known good starting state */
  1091. hw->mac.ops.reset_hw(hw);
  1092. /* make sure the NVM is good */
  1093. if (igb_validate_nvm_checksum(hw) < 0) {
  1094. dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
  1095. err = -EIO;
  1096. goto err_eeprom;
  1097. }
  1098. /* copy the MAC address out of the NVM */
  1099. if (hw->mac.ops.read_mac_addr(hw))
  1100. dev_err(&pdev->dev, "NVM Read Error\n");
  1101. memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
  1102. memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
  1103. if (!is_valid_ether_addr(netdev->perm_addr)) {
  1104. dev_err(&pdev->dev, "Invalid MAC Address\n");
  1105. err = -EIO;
  1106. goto err_eeprom;
  1107. }
  1108. init_timer(&adapter->watchdog_timer);
  1109. adapter->watchdog_timer.function = &igb_watchdog;
  1110. adapter->watchdog_timer.data = (unsigned long) adapter;
  1111. init_timer(&adapter->phy_info_timer);
  1112. adapter->phy_info_timer.function = &igb_update_phy_info;
  1113. adapter->phy_info_timer.data = (unsigned long) adapter;
  1114. INIT_WORK(&adapter->reset_task, igb_reset_task);
  1115. INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
  1116. /* Initialize link properties that are user-changeable */
  1117. adapter->fc_autoneg = true;
  1118. hw->mac.autoneg = true;
  1119. hw->phy.autoneg_advertised = 0x2f;
  1120. hw->fc.original_type = e1000_fc_default;
  1121. hw->fc.type = e1000_fc_default;
  1122. adapter->itr_setting = 3;
  1123. adapter->itr = IGB_START_ITR;
  1124. igb_validate_mdi_setting(hw);
  1125. adapter->rx_csum = 1;
  1126. /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
  1127. * enable the ACPI Magic Packet filter
  1128. */
  1129. if (hw->bus.func == 0 ||
  1130. hw->device_id == E1000_DEV_ID_82575EB_COPPER)
  1131. hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
  1132. if (eeprom_data & eeprom_apme_mask)
  1133. adapter->eeprom_wol |= E1000_WUFC_MAG;
  1134. /* now that we have the eeprom settings, apply the special cases where
  1135. * the eeprom may be wrong or the board simply won't support wake on
  1136. * lan on a particular port */
  1137. switch (pdev->device) {
  1138. case E1000_DEV_ID_82575GB_QUAD_COPPER:
  1139. adapter->eeprom_wol = 0;
  1140. break;
  1141. case E1000_DEV_ID_82575EB_FIBER_SERDES:
  1142. case E1000_DEV_ID_82576_FIBER:
  1143. case E1000_DEV_ID_82576_SERDES:
  1144. /* Wake events only supported on port A for dual fiber
  1145. * regardless of eeprom setting */
  1146. if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
  1147. adapter->eeprom_wol = 0;
  1148. break;
  1149. }
  1150. /* initialize the wol settings based on the eeprom settings */
  1151. adapter->wol = adapter->eeprom_wol;
  1152. device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
  1153. /* reset the hardware with the new settings */
  1154. igb_reset(adapter);
  1155. /* let the f/w know that the h/w is now under the control of the
  1156. * driver. */
  1157. igb_get_hw_control(adapter);
  1158. /* tell the stack to leave us alone until igb_open() is called */
  1159. netif_carrier_off(netdev);
  1160. netif_tx_stop_all_queues(netdev);
  1161. strcpy(netdev->name, "eth%d");
  1162. err = register_netdev(netdev);
  1163. if (err)
  1164. goto err_register;
  1165. #ifdef CONFIG_IGB_DCA
  1166. if (dca_add_requester(&pdev->dev) == 0) {
  1167. adapter->flags |= IGB_FLAG_DCA_ENABLED;
  1168. dev_info(&pdev->dev, "DCA enabled\n");
  1169. /* Always use CB2 mode, difference is masked
  1170. * in the CB driver. */
  1171. wr32(E1000_DCA_CTRL, 2);
  1172. igb_setup_dca(adapter);
  1173. }
  1174. #endif
  1175. /*
  1176. * Initialize hardware timer: we keep it running just in case
  1177. * that some program needs it later on.
  1178. */
  1179. memset(&adapter->cycles, 0, sizeof(adapter->cycles));
  1180. adapter->cycles.read = igb_read_clock;
  1181. adapter->cycles.mask = CLOCKSOURCE_MASK(64);
  1182. adapter->cycles.mult = 1;
  1183. adapter->cycles.shift = IGB_TSYNC_SHIFT;
  1184. wr32(E1000_TIMINCA,
  1185. (1<<24) |
  1186. IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS * IGB_TSYNC_SCALE);
  1187. #if 0
  1188. /*
  1189. * Avoid rollover while we initialize by resetting the time counter.
  1190. */
  1191. wr32(E1000_SYSTIML, 0x00000000);
  1192. wr32(E1000_SYSTIMH, 0x00000000);
  1193. #else
  1194. /*
  1195. * Set registers so that rollover occurs soon to test this.
  1196. */
  1197. wr32(E1000_SYSTIML, 0x00000000);
  1198. wr32(E1000_SYSTIMH, 0xFF800000);
  1199. #endif
  1200. wrfl();
  1201. timecounter_init(&adapter->clock,
  1202. &adapter->cycles,
  1203. ktime_to_ns(ktime_get_real()));
  1204. /*
  1205. * Synchronize our NIC clock against system wall clock. NIC
  1206. * time stamp reading requires ~3us per sample, each sample
  1207. * was pretty stable even under load => only require 10
  1208. * samples for each offset comparison.
  1209. */
  1210. memset(&adapter->compare, 0, sizeof(adapter->compare));
  1211. adapter->compare.source = &adapter->clock;
  1212. adapter->compare.target = ktime_get_real;
  1213. adapter->compare.num_samples = 10;
  1214. timecompare_update(&adapter->compare, 0);
  1215. #ifdef DEBUG
  1216. {
  1217. char buffer[160];
  1218. printk(KERN_DEBUG
  1219. "igb: %s: hw %p initialized timer\n",
  1220. igb_get_time_str(adapter, buffer),
  1221. &adapter->hw);
  1222. }
  1223. #endif
  1224. dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
  1225. /* print bus type/speed/width info */
  1226. dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
  1227. netdev->name,
  1228. ((hw->bus.speed == e1000_bus_speed_2500)
  1229. ? "2.5Gb/s" : "unknown"),
  1230. ((hw->bus.width == e1000_bus_width_pcie_x4)
  1231. ? "Width x4" : (hw->bus.width == e1000_bus_width_pcie_x1)
  1232. ? "Width x1" : "unknown"),
  1233. netdev->dev_addr);
  1234. igb_read_part_num(hw, &part_num);
  1235. dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
  1236. (part_num >> 8), (part_num & 0xff));
  1237. dev_info(&pdev->dev,
  1238. "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
  1239. adapter->msix_entries ? "MSI-X" :
  1240. (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
  1241. adapter->num_rx_queues, adapter->num_tx_queues);
  1242. return 0;
  1243. err_register:
  1244. igb_release_hw_control(adapter);
  1245. err_eeprom:
  1246. if (!igb_check_reset_block(hw))
  1247. igb_reset_phy(hw);
  1248. if (hw->flash_address)
  1249. iounmap(hw->flash_address);
  1250. igb_free_queues(adapter);
  1251. err_sw_init:
  1252. iounmap(hw->hw_addr);
  1253. err_ioremap:
  1254. free_netdev(netdev);
  1255. err_alloc_etherdev:
  1256. pci_release_selected_regions(pdev, pci_select_bars(pdev,
  1257. IORESOURCE_MEM));
  1258. err_pci_reg:
  1259. err_dma:
  1260. pci_disable_device(pdev);
  1261. return err;
  1262. }
  1263. /**
  1264. * igb_remove - Device Removal Routine
  1265. * @pdev: PCI device information struct
  1266. *
  1267. * igb_remove is called by the PCI subsystem to alert the driver
  1268. * that it should release a PCI device. The could be caused by a
  1269. * Hot-Plug event, or because the driver is going to be removed from
  1270. * memory.
  1271. **/
  1272. static void __devexit igb_remove(struct pci_dev *pdev)
  1273. {
  1274. struct net_device *netdev = pci_get_drvdata(pdev);
  1275. struct igb_adapter *adapter = netdev_priv(netdev);
  1276. struct e1000_hw *hw = &adapter->hw;
  1277. int err;
  1278. /* flush_scheduled work may reschedule our watchdog task, so
  1279. * explicitly disable watchdog tasks from being rescheduled */
  1280. set_bit(__IGB_DOWN, &adapter->state);
  1281. del_timer_sync(&adapter->watchdog_timer);
  1282. del_timer_sync(&adapter->phy_info_timer);
  1283. flush_scheduled_work();
  1284. #ifdef CONFIG_IGB_DCA
  1285. if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
  1286. dev_info(&pdev->dev, "DCA disabled\n");
  1287. dca_remove_requester(&pdev->dev);
  1288. adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
  1289. wr32(E1000_DCA_CTRL, 1);
  1290. }
  1291. #endif
  1292. /* Release control of h/w to f/w. If f/w is AMT enabled, this
  1293. * would have already happened in close and is redundant. */
  1294. igb_release_hw_control(adapter);
  1295. unregister_netdev(netdev);
  1296. if (!igb_check_reset_block(&adapter->hw))
  1297. igb_reset_phy(&adapter->hw);
  1298. igb_reset_interrupt_capability(adapter);
  1299. igb_free_queues(adapter);
  1300. iounmap(hw->hw_addr);
  1301. if (hw->flash_address)
  1302. iounmap(hw->flash_address);
  1303. pci_release_selected_regions(pdev, pci_select_bars(pdev,
  1304. IORESOURCE_MEM));
  1305. free_netdev(netdev);
  1306. err = pci_disable_pcie_error_reporting(pdev);
  1307. if (err)
  1308. dev_err(&pdev->dev,
  1309. "pci_disable_pcie_error_reporting failed 0x%x\n", err);
  1310. pci_disable_device(pdev);
  1311. }
  1312. /**
  1313. * igb_sw_init - Initialize general software structures (struct igb_adapter)
  1314. * @adapter: board private structure to initialize
  1315. *
  1316. * igb_sw_init initializes the Adapter private data structure.
  1317. * Fields are initialized based on PCI device information and
  1318. * OS network device settings (MTU size).
  1319. **/
  1320. static int __devinit igb_sw_init(struct igb_adapter *adapter)
  1321. {
  1322. struct e1000_hw *hw = &adapter->hw;
  1323. struct net_device *netdev = adapter->netdev;
  1324. struct pci_dev *pdev = adapter->pdev;
  1325. pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
  1326. adapter->tx_ring_count = IGB_DEFAULT_TXD;
  1327. adapter->rx_ring_count = IGB_DEFAULT_RXD;
  1328. adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
  1329. adapter->rx_ps_hdr_size = 0; /* disable packet split */
  1330. adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
  1331. adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
  1332. /* This call may decrease the number of queues depending on
  1333. * interrupt mode. */
  1334. igb_set_interrupt_capability(adapter);
  1335. if (igb_alloc_queues(adapter)) {
  1336. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  1337. return -ENOMEM;
  1338. }
  1339. /* Explicitly disable IRQ since the NIC can be in any state. */
  1340. igb_irq_disable(adapter);
  1341. set_bit(__IGB_DOWN, &adapter->state);
  1342. return 0;
  1343. }
  1344. /**
  1345. * igb_open - Called when a network interface is made active
  1346. * @netdev: network interface device structure
  1347. *
  1348. * Returns 0 on success, negative value on failure
  1349. *
  1350. * The open entry point is called when a network interface is made
  1351. * active by the system (IFF_UP). At this point all resources needed
  1352. * for transmit and receive operations are allocated, the interrupt
  1353. * handler is registered with the OS, the watchdog timer is started,
  1354. * and the stack is notified that the interface is ready.
  1355. **/
  1356. static int igb_open(struct net_device *netdev)
  1357. {
  1358. struct igb_adapter *adapter = netdev_priv(netdev);
  1359. struct e1000_hw *hw = &adapter->hw;
  1360. int err;
  1361. int i;
  1362. /* disallow open during test */
  1363. if (test_bit(__IGB_TESTING, &adapter->state))
  1364. return -EBUSY;
  1365. /* allocate transmit descriptors */
  1366. err = igb_setup_all_tx_resources(adapter);
  1367. if (err)
  1368. goto err_setup_tx;
  1369. /* allocate receive descriptors */
  1370. err = igb_setup_all_rx_resources(adapter);
  1371. if (err)
  1372. goto err_setup_rx;
  1373. /* e1000_power_up_phy(adapter); */
  1374. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  1375. if ((adapter->hw.mng_cookie.status &
  1376. E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
  1377. igb_update_mng_vlan(adapter);
  1378. /* before we allocate an interrupt, we must be ready to handle it.
  1379. * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
  1380. * as soon as we call pci_request_irq, so we have to setup our
  1381. * clean_rx handler before we do so. */
  1382. igb_configure(adapter);
  1383. err = igb_request_irq(adapter);
  1384. if (err)
  1385. goto err_req_irq;
  1386. /* From here on the code is the same as igb_up() */
  1387. clear_bit(__IGB_DOWN, &adapter->state);
  1388. for (i = 0; i < adapter->num_rx_queues; i++)
  1389. napi_enable(&adapter->rx_ring[i].napi);
  1390. /* Clear any pending interrupts. */
  1391. rd32(E1000_ICR);
  1392. igb_irq_enable(adapter);
  1393. netif_tx_start_all_queues(netdev);
  1394. /* Fire a link status change interrupt to start the watchdog. */
  1395. wr32(E1000_ICS, E1000_ICS_LSC);
  1396. return 0;
  1397. err_req_irq:
  1398. igb_release_hw_control(adapter);
  1399. /* e1000_power_down_phy(adapter); */
  1400. igb_free_all_rx_resources(adapter);
  1401. err_setup_rx:
  1402. igb_free_all_tx_resources(adapter);
  1403. err_setup_tx:
  1404. igb_reset(adapter);
  1405. return err;
  1406. }
  1407. /**
  1408. * igb_close - Disables a network interface
  1409. * @netdev: network interface device structure
  1410. *
  1411. * Returns 0, this is not allowed to fail
  1412. *
  1413. * The close entry point is called when an interface is de-activated
  1414. * by the OS. The hardware is still under the driver's control, but
  1415. * needs to be disabled. A global MAC reset is issued to stop the
  1416. * hardware, and all transmit and receive resources are freed.
  1417. **/
  1418. static int igb_close(struct net_device *netdev)
  1419. {
  1420. struct igb_adapter *adapter = netdev_priv(netdev);
  1421. WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
  1422. igb_down(adapter);
  1423. igb_free_irq(adapter);
  1424. igb_free_all_tx_resources(adapter);
  1425. igb_free_all_rx_resources(adapter);
  1426. /* kill manageability vlan ID if supported, but not if a vlan with
  1427. * the same ID is registered on the host OS (let 8021q kill it) */
  1428. if ((adapter->hw.mng_cookie.status &
  1429. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
  1430. !(adapter->vlgrp &&
  1431. vlan_group_get_device(adapter->vlgrp, adapter->mng_vlan_id)))
  1432. igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
  1433. return 0;
  1434. }
  1435. /**
  1436. * igb_setup_tx_resources - allocate Tx resources (Descriptors)
  1437. * @adapter: board private structure
  1438. * @tx_ring: tx descriptor ring (for a specific queue) to setup
  1439. *
  1440. * Return 0 on success, negative on failure
  1441. **/
  1442. int igb_setup_tx_resources(struct igb_adapter *adapter,
  1443. struct igb_ring *tx_ring)
  1444. {
  1445. struct pci_dev *pdev = adapter->pdev;
  1446. int size;
  1447. size = sizeof(struct igb_buffer) * tx_ring->count;
  1448. tx_ring->buffer_info = vmalloc(size);
  1449. if (!tx_ring->buffer_info)
  1450. goto err;
  1451. memset(tx_ring->buffer_info, 0, size);
  1452. /* round up to nearest 4K */
  1453. tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
  1454. tx_ring->size = ALIGN(tx_ring->size, 4096);
  1455. tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
  1456. &tx_ring->dma);
  1457. if (!tx_ring->desc)
  1458. goto err;
  1459. tx_ring->adapter = adapter;
  1460. tx_ring->next_to_use = 0;
  1461. tx_ring->next_to_clean = 0;
  1462. return 0;
  1463. err:
  1464. vfree(tx_ring->buffer_info);
  1465. dev_err(&adapter->pdev->dev,
  1466. "Unable to allocate memory for the transmit descriptor ring\n");
  1467. return -ENOMEM;
  1468. }
  1469. /**
  1470. * igb_setup_all_tx_resources - wrapper to allocate Tx resources
  1471. * (Descriptors) for all queues
  1472. * @adapter: board private structure
  1473. *
  1474. * Return 0 on success, negative on failure
  1475. **/
  1476. static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
  1477. {
  1478. int i, err = 0;
  1479. int r_idx;
  1480. for (i = 0; i < adapter->num_tx_queues; i++) {
  1481. err = igb_setup_tx_resources(adapter, &adapter->tx_ring[i]);
  1482. if (err) {
  1483. dev_err(&adapter->pdev->dev,
  1484. "Allocation for Tx Queue %u failed\n", i);
  1485. for (i--; i >= 0; i--)
  1486. igb_free_tx_resources(&adapter->tx_ring[i]);
  1487. break;
  1488. }
  1489. }
  1490. for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
  1491. r_idx = i % adapter->num_tx_queues;
  1492. adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
  1493. }
  1494. return err;
  1495. }
  1496. /**
  1497. * igb_configure_tx - Configure transmit Unit after Reset
  1498. * @adapter: board private structure
  1499. *
  1500. * Configure the Tx unit of the MAC after a reset.
  1501. **/
  1502. static void igb_configure_tx(struct igb_adapter *adapter)
  1503. {
  1504. u64 tdba;
  1505. struct e1000_hw *hw = &adapter->hw;
  1506. u32 tctl;
  1507. u32 txdctl, txctrl;
  1508. int i, j;
  1509. for (i = 0; i < adapter->num_tx_queues; i++) {
  1510. struct igb_ring *ring = &(adapter->tx_ring[i]);
  1511. j = ring->reg_idx;
  1512. wr32(E1000_TDLEN(j),
  1513. ring->count * sizeof(struct e1000_tx_desc));
  1514. tdba = ring->dma;
  1515. wr32(E1000_TDBAL(j),
  1516. tdba & 0x00000000ffffffffULL);
  1517. wr32(E1000_TDBAH(j), tdba >> 32);
  1518. ring->head = E1000_TDH(j);
  1519. ring->tail = E1000_TDT(j);
  1520. writel(0, hw->hw_addr + ring->tail);
  1521. writel(0, hw->hw_addr + ring->head);
  1522. txdctl = rd32(E1000_TXDCTL(j));
  1523. txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
  1524. wr32(E1000_TXDCTL(j), txdctl);
  1525. /* Turn off Relaxed Ordering on head write-backs. The
  1526. * writebacks MUST be delivered in order or it will
  1527. * completely screw up our bookeeping.
  1528. */
  1529. txctrl = rd32(E1000_DCA_TXCTRL(j));
  1530. txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
  1531. wr32(E1000_DCA_TXCTRL(j), txctrl);
  1532. }
  1533. /* Use the default values for the Tx Inter Packet Gap (IPG) timer */
  1534. /* Program the Transmit Control Register */
  1535. tctl = rd32(E1000_TCTL);
  1536. tctl &= ~E1000_TCTL_CT;
  1537. tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
  1538. (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
  1539. igb_config_collision_dist(hw);
  1540. /* Setup Transmit Descriptor Settings for eop descriptor */
  1541. adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS;
  1542. /* Enable transmits */
  1543. tctl |= E1000_TCTL_EN;
  1544. wr32(E1000_TCTL, tctl);
  1545. }
  1546. /**
  1547. * igb_setup_rx_resources - allocate Rx resources (Descriptors)
  1548. * @adapter: board private structure
  1549. * @rx_ring: rx descriptor ring (for a specific queue) to setup
  1550. *
  1551. * Returns 0 on success, negative on failure
  1552. **/
  1553. int igb_setup_rx_resources(struct igb_adapter *adapter,
  1554. struct igb_ring *rx_ring)
  1555. {
  1556. struct pci_dev *pdev = adapter->pdev;
  1557. int size, desc_len;
  1558. size = sizeof(struct igb_buffer) * rx_ring->count;
  1559. rx_ring->buffer_info = vmalloc(size);
  1560. if (!rx_ring->buffer_info)
  1561. goto err;
  1562. memset(rx_ring->buffer_info, 0, size);
  1563. desc_len = sizeof(union e1000_adv_rx_desc);
  1564. /* Round up to nearest 4K */
  1565. rx_ring->size = rx_ring->count * desc_len;
  1566. rx_ring->size = ALIGN(rx_ring->size, 4096);
  1567. rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
  1568. &rx_ring->dma);
  1569. if (!rx_ring->desc)
  1570. goto err;
  1571. rx_ring->next_to_clean = 0;
  1572. rx_ring->next_to_use = 0;
  1573. rx_ring->adapter = adapter;
  1574. return 0;
  1575. err:
  1576. vfree(rx_ring->buffer_info);
  1577. dev_err(&adapter->pdev->dev, "Unable to allocate memory for "
  1578. "the receive descriptor ring\n");
  1579. return -ENOMEM;
  1580. }
  1581. /**
  1582. * igb_setup_all_rx_resources - wrapper to allocate Rx resources
  1583. * (Descriptors) for all queues
  1584. * @adapter: board private structure
  1585. *
  1586. * Return 0 on success, negative on failure
  1587. **/
  1588. static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
  1589. {
  1590. int i, err = 0;
  1591. for (i = 0; i < adapter->num_rx_queues; i++) {
  1592. err = igb_setup_rx_resources(adapter, &adapter->rx_ring[i]);
  1593. if (err) {
  1594. dev_err(&adapter->pdev->dev,
  1595. "Allocation for Rx Queue %u failed\n", i);
  1596. for (i--; i >= 0; i--)
  1597. igb_free_rx_resources(&adapter->rx_ring[i]);
  1598. break;
  1599. }
  1600. }
  1601. return err;
  1602. }
  1603. /**
  1604. * igb_setup_rctl - configure the receive control registers
  1605. * @adapter: Board private structure
  1606. **/
  1607. static void igb_setup_rctl(struct igb_adapter *adapter)
  1608. {
  1609. struct e1000_hw *hw = &adapter->hw;
  1610. u32 rctl;
  1611. u32 srrctl = 0;
  1612. int i, j;
  1613. rctl = rd32(E1000_RCTL);
  1614. rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
  1615. rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
  1616. rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
  1617. (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
  1618. /*
  1619. * enable stripping of CRC. It's unlikely this will break BMC
  1620. * redirection as it did with e1000. Newer features require
  1621. * that the HW strips the CRC.
  1622. */
  1623. rctl |= E1000_RCTL_SECRC;
  1624. /*
  1625. * disable store bad packets and clear size bits.
  1626. */
  1627. rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
  1628. /* enable LPE when to prevent packets larger than max_frame_size */
  1629. rctl |= E1000_RCTL_LPE;
  1630. /* Setup buffer sizes */
  1631. switch (adapter->rx_buffer_len) {
  1632. case IGB_RXBUFFER_256:
  1633. rctl |= E1000_RCTL_SZ_256;
  1634. break;
  1635. case IGB_RXBUFFER_512:
  1636. rctl |= E1000_RCTL_SZ_512;
  1637. break;
  1638. default:
  1639. srrctl = ALIGN(adapter->rx_buffer_len, 1024)
  1640. >> E1000_SRRCTL_BSIZEPKT_SHIFT;
  1641. break;
  1642. }
  1643. /* 82575 and greater support packet-split where the protocol
  1644. * header is placed in skb->data and the packet data is
  1645. * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
  1646. * In the case of a non-split, skb->data is linearly filled,
  1647. * followed by the page buffers. Therefore, skb->data is
  1648. * sized to hold the largest protocol header.
  1649. */
  1650. /* allocations using alloc_page take too long for regular MTU
  1651. * so only enable packet split for jumbo frames */
  1652. if (adapter->netdev->mtu > ETH_DATA_LEN) {
  1653. adapter->rx_ps_hdr_size = IGB_RXBUFFER_128;
  1654. srrctl |= adapter->rx_ps_hdr_size <<
  1655. E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
  1656. srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
  1657. } else {
  1658. adapter->rx_ps_hdr_size = 0;
  1659. srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
  1660. }
  1661. for (i = 0; i < adapter->num_rx_queues; i++) {
  1662. j = adapter->rx_ring[i].reg_idx;
  1663. wr32(E1000_SRRCTL(j), srrctl);
  1664. }
  1665. wr32(E1000_RCTL, rctl);
  1666. }
  1667. /**
  1668. * igb_configure_rx - Configure receive Unit after Reset
  1669. * @adapter: board private structure
  1670. *
  1671. * Configure the Rx unit of the MAC after a reset.
  1672. **/
  1673. static void igb_configure_rx(struct igb_adapter *adapter)
  1674. {
  1675. u64 rdba;
  1676. struct e1000_hw *hw = &adapter->hw;
  1677. u32 rctl, rxcsum;
  1678. u32 rxdctl;
  1679. int i, j;
  1680. /* disable receives while setting up the descriptors */
  1681. rctl = rd32(E1000_RCTL);
  1682. wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
  1683. wrfl();
  1684. mdelay(10);
  1685. if (adapter->itr_setting > 3)
  1686. wr32(E1000_ITR, adapter->itr);
  1687. /* Setup the HW Rx Head and Tail Descriptor Pointers and
  1688. * the Base and Length of the Rx Descriptor Ring */
  1689. for (i = 0; i < adapter->num_rx_queues; i++) {
  1690. struct igb_ring *ring = &(adapter->rx_ring[i]);
  1691. j = ring->reg_idx;
  1692. rdba = ring->dma;
  1693. wr32(E1000_RDBAL(j),
  1694. rdba & 0x00000000ffffffffULL);
  1695. wr32(E1000_RDBAH(j), rdba >> 32);
  1696. wr32(E1000_RDLEN(j),
  1697. ring->count * sizeof(union e1000_adv_rx_desc));
  1698. ring->head = E1000_RDH(j);
  1699. ring->tail = E1000_RDT(j);
  1700. writel(0, hw->hw_addr + ring->tail);
  1701. writel(0, hw->hw_addr + ring->head);
  1702. rxdctl = rd32(E1000_RXDCTL(j));
  1703. rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
  1704. rxdctl &= 0xFFF00000;
  1705. rxdctl |= IGB_RX_PTHRESH;
  1706. rxdctl |= IGB_RX_HTHRESH << 8;
  1707. rxdctl |= IGB_RX_WTHRESH << 16;
  1708. wr32(E1000_RXDCTL(j), rxdctl);
  1709. }
  1710. if (adapter->num_rx_queues > 1) {
  1711. u32 random[10];
  1712. u32 mrqc;
  1713. u32 j, shift;
  1714. union e1000_reta {
  1715. u32 dword;
  1716. u8 bytes[4];
  1717. } reta;
  1718. get_random_bytes(&random[0], 40);
  1719. if (hw->mac.type >= e1000_82576)
  1720. shift = 0;
  1721. else
  1722. shift = 6;
  1723. for (j = 0; j < (32 * 4); j++) {
  1724. reta.bytes[j & 3] =
  1725. adapter->rx_ring[(j % adapter->num_rx_queues)].reg_idx << shift;
  1726. if ((j & 3) == 3)
  1727. writel(reta.dword,
  1728. hw->hw_addr + E1000_RETA(0) + (j & ~3));
  1729. }
  1730. mrqc = E1000_MRQC_ENABLE_RSS_4Q;
  1731. /* Fill out hash function seeds */
  1732. for (j = 0; j < 10; j++)
  1733. array_wr32(E1000_RSSRK(0), j, random[j]);
  1734. mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
  1735. E1000_MRQC_RSS_FIELD_IPV4_TCP);
  1736. mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
  1737. E1000_MRQC_RSS_FIELD_IPV6_TCP);
  1738. mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
  1739. E1000_MRQC_RSS_FIELD_IPV6_UDP);
  1740. mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
  1741. E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
  1742. wr32(E1000_MRQC, mrqc);
  1743. /* Multiqueue and raw packet checksumming are mutually
  1744. * exclusive. Note that this not the same as TCP/IP
  1745. * checksumming, which works fine. */
  1746. rxcsum = rd32(E1000_RXCSUM);
  1747. rxcsum |= E1000_RXCSUM_PCSD;
  1748. wr32(E1000_RXCSUM, rxcsum);
  1749. } else {
  1750. /* Enable Receive Checksum Offload for TCP and UDP */
  1751. rxcsum = rd32(E1000_RXCSUM);
  1752. if (adapter->rx_csum) {
  1753. rxcsum |= E1000_RXCSUM_TUOFL;
  1754. /* Enable IPv4 payload checksum for UDP fragments
  1755. * Must be used in conjunction with packet-split. */
  1756. if (adapter->rx_ps_hdr_size)
  1757. rxcsum |= E1000_RXCSUM_IPPCSE;
  1758. } else {
  1759. rxcsum &= ~E1000_RXCSUM_TUOFL;
  1760. /* don't need to clear IPPCSE as it defaults to 0 */
  1761. }
  1762. wr32(E1000_RXCSUM, rxcsum);
  1763. }
  1764. if (adapter->vlgrp)
  1765. wr32(E1000_RLPML,
  1766. adapter->max_frame_size + VLAN_TAG_SIZE);
  1767. else
  1768. wr32(E1000_RLPML, adapter->max_frame_size);
  1769. /* Enable Receives */
  1770. wr32(E1000_RCTL, rctl);
  1771. }
  1772. /**
  1773. * igb_free_tx_resources - Free Tx Resources per Queue
  1774. * @tx_ring: Tx descriptor ring for a specific queue
  1775. *
  1776. * Free all transmit software resources
  1777. **/
  1778. void igb_free_tx_resources(struct igb_ring *tx_ring)
  1779. {
  1780. struct pci_dev *pdev = tx_ring->adapter->pdev;
  1781. igb_clean_tx_ring(tx_ring);
  1782. vfree(tx_ring->buffer_info);
  1783. tx_ring->buffer_info = NULL;
  1784. pci_free_consistent(pdev, tx_ring->size, tx_ring->desc, tx_ring->dma);
  1785. tx_ring->desc = NULL;
  1786. }
  1787. /**
  1788. * igb_free_all_tx_resources - Free Tx Resources for All Queues
  1789. * @adapter: board private structure
  1790. *
  1791. * Free all transmit software resources
  1792. **/
  1793. static void igb_free_all_tx_resources(struct igb_adapter *adapter)
  1794. {
  1795. int i;
  1796. for (i = 0; i < adapter->num_tx_queues; i++)
  1797. igb_free_tx_resources(&adapter->tx_ring[i]);
  1798. }
  1799. static void igb_unmap_and_free_tx_resource(struct igb_adapter *adapter,
  1800. struct igb_buffer *buffer_info)
  1801. {
  1802. if (buffer_info->dma) {
  1803. pci_unmap_page(adapter->pdev,
  1804. buffer_info->dma,
  1805. buffer_info->length,
  1806. PCI_DMA_TODEVICE);
  1807. buffer_info->dma = 0;
  1808. }
  1809. if (buffer_info->skb) {
  1810. dev_kfree_skb_any(buffer_info->skb);
  1811. buffer_info->skb = NULL;
  1812. }
  1813. buffer_info->time_stamp = 0;
  1814. /* buffer_info must be completely set up in the transmit path */
  1815. }
  1816. /**
  1817. * igb_clean_tx_ring - Free Tx Buffers
  1818. * @tx_ring: ring to be cleaned
  1819. **/
  1820. static void igb_clean_tx_ring(struct igb_ring *tx_ring)
  1821. {
  1822. struct igb_adapter *adapter = tx_ring->adapter;
  1823. struct igb_buffer *buffer_info;
  1824. unsigned long size;
  1825. unsigned int i;
  1826. if (!tx_ring->buffer_info)
  1827. return;
  1828. /* Free all the Tx ring sk_buffs */
  1829. for (i = 0; i < tx_ring->count; i++) {
  1830. buffer_info = &tx_ring->buffer_info[i];
  1831. igb_unmap_and_free_tx_resource(adapter, buffer_info);
  1832. }
  1833. size = sizeof(struct igb_buffer) * tx_ring->count;
  1834. memset(tx_ring->buffer_info, 0, size);
  1835. /* Zero out the descriptor ring */
  1836. memset(tx_ring->desc, 0, tx_ring->size);
  1837. tx_ring->next_to_use = 0;
  1838. tx_ring->next_to_clean = 0;
  1839. writel(0, adapter->hw.hw_addr + tx_ring->head);
  1840. writel(0, adapter->hw.hw_addr + tx_ring->tail);
  1841. }
  1842. /**
  1843. * igb_clean_all_tx_rings - Free Tx Buffers for all queues
  1844. * @adapter: board private structure
  1845. **/
  1846. static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
  1847. {
  1848. int i;
  1849. for (i = 0; i < adapter->num_tx_queues; i++)
  1850. igb_clean_tx_ring(&adapter->tx_ring[i]);
  1851. }
  1852. /**
  1853. * igb_free_rx_resources - Free Rx Resources
  1854. * @rx_ring: ring to clean the resources from
  1855. *
  1856. * Free all receive software resources
  1857. **/
  1858. void igb_free_rx_resources(struct igb_ring *rx_ring)
  1859. {
  1860. struct pci_dev *pdev = rx_ring->adapter->pdev;
  1861. igb_clean_rx_ring(rx_ring);
  1862. vfree(rx_ring->buffer_info);
  1863. rx_ring->buffer_info = NULL;
  1864. pci_free_consistent(pdev, rx_ring->size, rx_ring->desc, rx_ring->dma);
  1865. rx_ring->desc = NULL;
  1866. }
  1867. /**
  1868. * igb_free_all_rx_resources - Free Rx Resources for All Queues
  1869. * @adapter: board private structure
  1870. *
  1871. * Free all receive software resources
  1872. **/
  1873. static void igb_free_all_rx_resources(struct igb_adapter *adapter)
  1874. {
  1875. int i;
  1876. for (i = 0; i < adapter->num_rx_queues; i++)
  1877. igb_free_rx_resources(&adapter->rx_ring[i]);
  1878. }
  1879. /**
  1880. * igb_clean_rx_ring - Free Rx Buffers per Queue
  1881. * @rx_ring: ring to free buffers from
  1882. **/
  1883. static void igb_clean_rx_ring(struct igb_ring *rx_ring)
  1884. {
  1885. struct igb_adapter *adapter = rx_ring->adapter;
  1886. struct igb_buffer *buffer_info;
  1887. struct pci_dev *pdev = adapter->pdev;
  1888. unsigned long size;
  1889. unsigned int i;
  1890. if (!rx_ring->buffer_info)
  1891. return;
  1892. /* Free all the Rx ring sk_buffs */
  1893. for (i = 0; i < rx_ring->count; i++) {
  1894. buffer_info = &rx_ring->buffer_info[i];
  1895. if (buffer_info->dma) {
  1896. if (adapter->rx_ps_hdr_size)
  1897. pci_unmap_single(pdev, buffer_info->dma,
  1898. adapter->rx_ps_hdr_size,
  1899. PCI_DMA_FROMDEVICE);
  1900. else
  1901. pci_unmap_single(pdev, buffer_info->dma,
  1902. adapter->rx_buffer_len,
  1903. PCI_DMA_FROMDEVICE);
  1904. buffer_info->dma = 0;
  1905. }
  1906. if (buffer_info->skb) {
  1907. dev_kfree_skb(buffer_info->skb);
  1908. buffer_info->skb = NULL;
  1909. }
  1910. if (buffer_info->page) {
  1911. if (buffer_info->page_dma)
  1912. pci_unmap_page(pdev, buffer_info->page_dma,
  1913. PAGE_SIZE / 2,
  1914. PCI_DMA_FROMDEVICE);
  1915. put_page(buffer_info->page);
  1916. buffer_info->page = NULL;
  1917. buffer_info->page_dma = 0;
  1918. buffer_info->page_offset = 0;
  1919. }
  1920. }
  1921. size = sizeof(struct igb_buffer) * rx_ring->count;
  1922. memset(rx_ring->buffer_info, 0, size);
  1923. /* Zero out the descriptor ring */
  1924. memset(rx_ring->desc, 0, rx_ring->size);
  1925. rx_ring->next_to_clean = 0;
  1926. rx_ring->next_to_use = 0;
  1927. writel(0, adapter->hw.hw_addr + rx_ring->head);
  1928. writel(0, adapter->hw.hw_addr + rx_ring->tail);
  1929. }
  1930. /**
  1931. * igb_clean_all_rx_rings - Free Rx Buffers for all queues
  1932. * @adapter: board private structure
  1933. **/
  1934. static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
  1935. {
  1936. int i;
  1937. for (i = 0; i < adapter->num_rx_queues; i++)
  1938. igb_clean_rx_ring(&adapter->rx_ring[i]);
  1939. }
  1940. /**
  1941. * igb_set_mac - Change the Ethernet Address of the NIC
  1942. * @netdev: network interface device structure
  1943. * @p: pointer to an address structure
  1944. *
  1945. * Returns 0 on success, negative on failure
  1946. **/
  1947. static int igb_set_mac(struct net_device *netdev, void *p)
  1948. {
  1949. struct igb_adapter *adapter = netdev_priv(netdev);
  1950. struct e1000_hw *hw = &adapter->hw;
  1951. struct sockaddr *addr = p;
  1952. if (!is_valid_ether_addr(addr->sa_data))
  1953. return -EADDRNOTAVAIL;
  1954. memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
  1955. memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
  1956. hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
  1957. return 0;
  1958. }
  1959. /**
  1960. * igb_set_multi - Multicast and Promiscuous mode set
  1961. * @netdev: network interface device structure
  1962. *
  1963. * The set_multi entry point is called whenever the multicast address
  1964. * list or the network interface flags are updated. This routine is
  1965. * responsible for configuring the hardware for proper multicast,
  1966. * promiscuous mode, and all-multi behavior.
  1967. **/
  1968. static void igb_set_multi(struct net_device *netdev)
  1969. {
  1970. struct igb_adapter *adapter = netdev_priv(netdev);
  1971. struct e1000_hw *hw = &adapter->hw;
  1972. struct e1000_mac_info *mac = &hw->mac;
  1973. struct dev_mc_list *mc_ptr;
  1974. u8 *mta_list;
  1975. u32 rctl;
  1976. int i;
  1977. /* Check for Promiscuous and All Multicast modes */
  1978. rctl = rd32(E1000_RCTL);
  1979. if (netdev->flags & IFF_PROMISC) {
  1980. rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
  1981. rctl &= ~E1000_RCTL_VFE;
  1982. } else {
  1983. if (netdev->flags & IFF_ALLMULTI) {
  1984. rctl |= E1000_RCTL_MPE;
  1985. rctl &= ~E1000_RCTL_UPE;
  1986. } else
  1987. rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
  1988. rctl |= E1000_RCTL_VFE;
  1989. }
  1990. wr32(E1000_RCTL, rctl);
  1991. if (!netdev->mc_count) {
  1992. /* nothing to program, so clear mc list */
  1993. igb_update_mc_addr_list(hw, NULL, 0, 1,
  1994. mac->rar_entry_count);
  1995. return;
  1996. }
  1997. mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
  1998. if (!mta_list)
  1999. return;
  2000. /* The shared function expects a packed array of only addresses. */
  2001. mc_ptr = netdev->mc_list;
  2002. for (i = 0; i < netdev->mc_count; i++) {
  2003. if (!mc_ptr)
  2004. break;
  2005. memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
  2006. mc_ptr = mc_ptr->next;
  2007. }
  2008. igb_update_mc_addr_list(hw, mta_list, i, 1, mac->rar_entry_count);
  2009. kfree(mta_list);
  2010. }
  2011. /* Need to wait a few seconds after link up to get diagnostic information from
  2012. * the phy */
  2013. static void igb_update_phy_info(unsigned long data)
  2014. {
  2015. struct igb_adapter *adapter = (struct igb_adapter *) data;
  2016. igb_get_phy_info(&adapter->hw);
  2017. }
  2018. /**
  2019. * igb_has_link - check shared code for link and determine up/down
  2020. * @adapter: pointer to driver private info
  2021. **/
  2022. static bool igb_has_link(struct igb_adapter *adapter)
  2023. {
  2024. struct e1000_hw *hw = &adapter->hw;
  2025. bool link_active = false;
  2026. s32 ret_val = 0;
  2027. /* get_link_status is set on LSC (link status) interrupt or
  2028. * rx sequence error interrupt. get_link_status will stay
  2029. * false until the e1000_check_for_link establishes link
  2030. * for copper adapters ONLY
  2031. */
  2032. switch (hw->phy.media_type) {
  2033. case e1000_media_type_copper:
  2034. if (hw->mac.get_link_status) {
  2035. ret_val = hw->mac.ops.check_for_link(hw);
  2036. link_active = !hw->mac.get_link_status;
  2037. } else {
  2038. link_active = true;
  2039. }
  2040. break;
  2041. case e1000_media_type_fiber:
  2042. ret_val = hw->mac.ops.check_for_link(hw);
  2043. link_active = !!(rd32(E1000_STATUS) & E1000_STATUS_LU);
  2044. break;
  2045. case e1000_media_type_internal_serdes:
  2046. ret_val = hw->mac.ops.check_for_link(hw);
  2047. link_active = hw->mac.serdes_has_link;
  2048. break;
  2049. default:
  2050. case e1000_media_type_unknown:
  2051. break;
  2052. }
  2053. return link_active;
  2054. }
  2055. /**
  2056. * igb_watchdog - Timer Call-back
  2057. * @data: pointer to adapter cast into an unsigned long
  2058. **/
  2059. static void igb_watchdog(unsigned long data)
  2060. {
  2061. struct igb_adapter *adapter = (struct igb_adapter *)data;
  2062. /* Do the rest outside of interrupt context */
  2063. schedule_work(&adapter->watchdog_task);
  2064. }
  2065. static void igb_watchdog_task(struct work_struct *work)
  2066. {
  2067. struct igb_adapter *adapter = container_of(work,
  2068. struct igb_adapter, watchdog_task);
  2069. struct e1000_hw *hw = &adapter->hw;
  2070. struct net_device *netdev = adapter->netdev;
  2071. struct igb_ring *tx_ring = adapter->tx_ring;
  2072. u32 link;
  2073. u32 eics = 0;
  2074. int i;
  2075. link = igb_has_link(adapter);
  2076. if ((netif_carrier_ok(netdev)) && link)
  2077. goto link_up;
  2078. if (link) {
  2079. if (!netif_carrier_ok(netdev)) {
  2080. u32 ctrl;
  2081. hw->mac.ops.get_speed_and_duplex(&adapter->hw,
  2082. &adapter->link_speed,
  2083. &adapter->link_duplex);
  2084. ctrl = rd32(E1000_CTRL);
  2085. /* Links status message must follow this format */
  2086. printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
  2087. "Flow Control: %s\n",
  2088. netdev->name,
  2089. adapter->link_speed,
  2090. adapter->link_duplex == FULL_DUPLEX ?
  2091. "Full Duplex" : "Half Duplex",
  2092. ((ctrl & E1000_CTRL_TFCE) && (ctrl &
  2093. E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
  2094. E1000_CTRL_RFCE) ? "RX" : ((ctrl &
  2095. E1000_CTRL_TFCE) ? "TX" : "None")));
  2096. /* tweak tx_queue_len according to speed/duplex and
  2097. * adjust the timeout factor */
  2098. netdev->tx_queue_len = adapter->tx_queue_len;
  2099. adapter->tx_timeout_factor = 1;
  2100. switch (adapter->link_speed) {
  2101. case SPEED_10:
  2102. netdev->tx_queue_len = 10;
  2103. adapter->tx_timeout_factor = 14;
  2104. break;
  2105. case SPEED_100:
  2106. netdev->tx_queue_len = 100;
  2107. /* maybe add some timeout factor ? */
  2108. break;
  2109. }
  2110. netif_carrier_on(netdev);
  2111. netif_tx_wake_all_queues(netdev);
  2112. /* link state has changed, schedule phy info update */
  2113. if (!test_bit(__IGB_DOWN, &adapter->state))
  2114. mod_timer(&adapter->phy_info_timer,
  2115. round_jiffies(jiffies + 2 * HZ));
  2116. }
  2117. } else {
  2118. if (netif_carrier_ok(netdev)) {
  2119. adapter->link_speed = 0;
  2120. adapter->link_duplex = 0;
  2121. /* Links status message must follow this format */
  2122. printk(KERN_INFO "igb: %s NIC Link is Down\n",
  2123. netdev->name);
  2124. netif_carrier_off(netdev);
  2125. netif_tx_stop_all_queues(netdev);
  2126. /* link state has changed, schedule phy info update */
  2127. if (!test_bit(__IGB_DOWN, &adapter->state))
  2128. mod_timer(&adapter->phy_info_timer,
  2129. round_jiffies(jiffies + 2 * HZ));
  2130. }
  2131. }
  2132. link_up:
  2133. igb_update_stats(adapter);
  2134. hw->mac.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
  2135. adapter->tpt_old = adapter->stats.tpt;
  2136. hw->mac.collision_delta = adapter->stats.colc - adapter->colc_old;
  2137. adapter->colc_old = adapter->stats.colc;
  2138. adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
  2139. adapter->gorc_old = adapter->stats.gorc;
  2140. adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
  2141. adapter->gotc_old = adapter->stats.gotc;
  2142. igb_update_adaptive(&adapter->hw);
  2143. if (!netif_carrier_ok(netdev)) {
  2144. if (IGB_DESC_UNUSED(tx_ring) + 1 < tx_ring->count) {
  2145. /* We've lost link, so the controller stops DMA,
  2146. * but we've got queued Tx work that's never going
  2147. * to get done, so reset controller to flush Tx.
  2148. * (Do the reset outside of interrupt context). */
  2149. adapter->tx_timeout_count++;
  2150. schedule_work(&adapter->reset_task);
  2151. }
  2152. }
  2153. /* Cause software interrupt to ensure rx ring is cleaned */
  2154. if (adapter->msix_entries) {
  2155. for (i = 0; i < adapter->num_rx_queues; i++)
  2156. eics |= adapter->rx_ring[i].eims_value;
  2157. wr32(E1000_EICS, eics);
  2158. } else {
  2159. wr32(E1000_ICS, E1000_ICS_RXDMT0);
  2160. }
  2161. /* Force detection of hung controller every watchdog period */
  2162. tx_ring->detect_tx_hung = true;
  2163. /* Reset the timer */
  2164. if (!test_bit(__IGB_DOWN, &adapter->state))
  2165. mod_timer(&adapter->watchdog_timer,
  2166. round_jiffies(jiffies + 2 * HZ));
  2167. }
  2168. enum latency_range {
  2169. lowest_latency = 0,
  2170. low_latency = 1,
  2171. bulk_latency = 2,
  2172. latency_invalid = 255
  2173. };
  2174. /**
  2175. * igb_update_ring_itr - update the dynamic ITR value based on packet size
  2176. *
  2177. * Stores a new ITR value based on strictly on packet size. This
  2178. * algorithm is less sophisticated than that used in igb_update_itr,
  2179. * due to the difficulty of synchronizing statistics across multiple
  2180. * receive rings. The divisors and thresholds used by this fuction
  2181. * were determined based on theoretical maximum wire speed and testing
  2182. * data, in order to minimize response time while increasing bulk
  2183. * throughput.
  2184. * This functionality is controlled by the InterruptThrottleRate module
  2185. * parameter (see igb_param.c)
  2186. * NOTE: This function is called only when operating in a multiqueue
  2187. * receive environment.
  2188. * @rx_ring: pointer to ring
  2189. **/
  2190. static void igb_update_ring_itr(struct igb_ring *rx_ring)
  2191. {
  2192. int new_val = rx_ring->itr_val;
  2193. int avg_wire_size = 0;
  2194. struct igb_adapter *adapter = rx_ring->adapter;
  2195. if (!rx_ring->total_packets)
  2196. goto clear_counts; /* no packets, so don't do anything */
  2197. /* For non-gigabit speeds, just fix the interrupt rate at 4000
  2198. * ints/sec - ITR timer value of 120 ticks.
  2199. */
  2200. if (adapter->link_speed != SPEED_1000) {
  2201. new_val = 120;
  2202. goto set_itr_val;
  2203. }
  2204. avg_wire_size = rx_ring->total_bytes / rx_ring->total_packets;
  2205. /* Add 24 bytes to size to account for CRC, preamble, and gap */
  2206. avg_wire_size += 24;
  2207. /* Don't starve jumbo frames */
  2208. avg_wire_size = min(avg_wire_size, 3000);
  2209. /* Give a little boost to mid-size frames */
  2210. if ((avg_wire_size > 300) && (avg_wire_size < 1200))
  2211. new_val = avg_wire_size / 3;
  2212. else
  2213. new_val = avg_wire_size / 2;
  2214. set_itr_val:
  2215. if (new_val != rx_ring->itr_val) {
  2216. rx_ring->itr_val = new_val;
  2217. rx_ring->set_itr = 1;
  2218. }
  2219. clear_counts:
  2220. rx_ring->total_bytes = 0;
  2221. rx_ring->total_packets = 0;
  2222. }
  2223. /**
  2224. * igb_update_itr - update the dynamic ITR value based on statistics
  2225. * Stores a new ITR value based on packets and byte
  2226. * counts during the last interrupt. The advantage of per interrupt
  2227. * computation is faster updates and more accurate ITR for the current
  2228. * traffic pattern. Constants in this function were computed
  2229. * based on theoretical maximum wire speed and thresholds were set based
  2230. * on testing data as well as attempting to minimize response time
  2231. * while increasing bulk throughput.
  2232. * this functionality is controlled by the InterruptThrottleRate module
  2233. * parameter (see igb_param.c)
  2234. * NOTE: These calculations are only valid when operating in a single-
  2235. * queue environment.
  2236. * @adapter: pointer to adapter
  2237. * @itr_setting: current adapter->itr
  2238. * @packets: the number of packets during this measurement interval
  2239. * @bytes: the number of bytes during this measurement interval
  2240. **/
  2241. static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
  2242. int packets, int bytes)
  2243. {
  2244. unsigned int retval = itr_setting;
  2245. if (packets == 0)
  2246. goto update_itr_done;
  2247. switch (itr_setting) {
  2248. case lowest_latency:
  2249. /* handle TSO and jumbo frames */
  2250. if (bytes/packets > 8000)
  2251. retval = bulk_latency;
  2252. else if ((packets < 5) && (bytes > 512))
  2253. retval = low_latency;
  2254. break;
  2255. case low_latency: /* 50 usec aka 20000 ints/s */
  2256. if (bytes > 10000) {
  2257. /* this if handles the TSO accounting */
  2258. if (bytes/packets > 8000) {
  2259. retval = bulk_latency;
  2260. } else if ((packets < 10) || ((bytes/packets) > 1200)) {
  2261. retval = bulk_latency;
  2262. } else if ((packets > 35)) {
  2263. retval = lowest_latency;
  2264. }
  2265. } else if (bytes/packets > 2000) {
  2266. retval = bulk_latency;
  2267. } else if (packets <= 2 && bytes < 512) {
  2268. retval = lowest_latency;
  2269. }
  2270. break;
  2271. case bulk_latency: /* 250 usec aka 4000 ints/s */
  2272. if (bytes > 25000) {
  2273. if (packets > 35)
  2274. retval = low_latency;
  2275. } else if (bytes < 6000) {
  2276. retval = low_latency;
  2277. }
  2278. break;
  2279. }
  2280. update_itr_done:
  2281. return retval;
  2282. }
  2283. static void igb_set_itr(struct igb_adapter *adapter)
  2284. {
  2285. u16 current_itr;
  2286. u32 new_itr = adapter->itr;
  2287. /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
  2288. if (adapter->link_speed != SPEED_1000) {
  2289. current_itr = 0;
  2290. new_itr = 4000;
  2291. goto set_itr_now;
  2292. }
  2293. adapter->rx_itr = igb_update_itr(adapter,
  2294. adapter->rx_itr,
  2295. adapter->rx_ring->total_packets,
  2296. adapter->rx_ring->total_bytes);
  2297. if (adapter->rx_ring->buddy) {
  2298. adapter->tx_itr = igb_update_itr(adapter,
  2299. adapter->tx_itr,
  2300. adapter->tx_ring->total_packets,
  2301. adapter->tx_ring->total_bytes);
  2302. current_itr = max(adapter->rx_itr, adapter->tx_itr);
  2303. } else {
  2304. current_itr = adapter->rx_itr;
  2305. }
  2306. /* conservative mode (itr 3) eliminates the lowest_latency setting */
  2307. if (adapter->itr_setting == 3 &&
  2308. current_itr == lowest_latency)
  2309. current_itr = low_latency;
  2310. switch (current_itr) {
  2311. /* counts and packets in update_itr are dependent on these numbers */
  2312. case lowest_latency:
  2313. new_itr = 70000;
  2314. break;
  2315. case low_latency:
  2316. new_itr = 20000; /* aka hwitr = ~200 */
  2317. break;
  2318. case bulk_latency:
  2319. new_itr = 4000;
  2320. break;
  2321. default:
  2322. break;
  2323. }
  2324. set_itr_now:
  2325. adapter->rx_ring->total_bytes = 0;
  2326. adapter->rx_ring->total_packets = 0;
  2327. if (adapter->rx_ring->buddy) {
  2328. adapter->rx_ring->buddy->total_bytes = 0;
  2329. adapter->rx_ring->buddy->total_packets = 0;
  2330. }
  2331. if (new_itr != adapter->itr) {
  2332. /* this attempts to bias the interrupt rate towards Bulk
  2333. * by adding intermediate steps when interrupt rate is
  2334. * increasing */
  2335. new_itr = new_itr > adapter->itr ?
  2336. min(adapter->itr + (new_itr >> 2), new_itr) :
  2337. new_itr;
  2338. /* Don't write the value here; it resets the adapter's
  2339. * internal timer, and causes us to delay far longer than
  2340. * we should between interrupts. Instead, we write the ITR
  2341. * value at the beginning of the next interrupt so the timing
  2342. * ends up being correct.
  2343. */
  2344. adapter->itr = new_itr;
  2345. adapter->rx_ring->itr_val = 1000000000 / (new_itr * 256);
  2346. adapter->rx_ring->set_itr = 1;
  2347. }
  2348. return;
  2349. }
  2350. #define IGB_TX_FLAGS_CSUM 0x00000001
  2351. #define IGB_TX_FLAGS_VLAN 0x00000002
  2352. #define IGB_TX_FLAGS_TSO 0x00000004
  2353. #define IGB_TX_FLAGS_IPV4 0x00000008
  2354. #define IGB_TX_FLAGS_TSTAMP 0x00000010
  2355. #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
  2356. #define IGB_TX_FLAGS_VLAN_SHIFT 16
  2357. static inline int igb_tso_adv(struct igb_adapter *adapter,
  2358. struct igb_ring *tx_ring,
  2359. struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
  2360. {
  2361. struct e1000_adv_tx_context_desc *context_desc;
  2362. unsigned int i;
  2363. int err;
  2364. struct igb_buffer *buffer_info;
  2365. u32 info = 0, tu_cmd = 0;
  2366. u32 mss_l4len_idx, l4len;
  2367. *hdr_len = 0;
  2368. if (skb_header_cloned(skb)) {
  2369. err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  2370. if (err)
  2371. return err;
  2372. }
  2373. l4len = tcp_hdrlen(skb);
  2374. *hdr_len += l4len;
  2375. if (skb->protocol == htons(ETH_P_IP)) {
  2376. struct iphdr *iph = ip_hdr(skb);
  2377. iph->tot_len = 0;
  2378. iph->check = 0;
  2379. tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
  2380. iph->daddr, 0,
  2381. IPPROTO_TCP,
  2382. 0);
  2383. } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
  2384. ipv6_hdr(skb)->payload_len = 0;
  2385. tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
  2386. &ipv6_hdr(skb)->daddr,
  2387. 0, IPPROTO_TCP, 0);
  2388. }
  2389. i = tx_ring->next_to_use;
  2390. buffer_info = &tx_ring->buffer_info[i];
  2391. context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
  2392. /* VLAN MACLEN IPLEN */
  2393. if (tx_flags & IGB_TX_FLAGS_VLAN)
  2394. info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
  2395. info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
  2396. *hdr_len += skb_network_offset(skb);
  2397. info |= skb_network_header_len(skb);
  2398. *hdr_len += skb_network_header_len(skb);
  2399. context_desc->vlan_macip_lens = cpu_to_le32(info);
  2400. /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
  2401. tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
  2402. if (skb->protocol == htons(ETH_P_IP))
  2403. tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
  2404. tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
  2405. context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
  2406. /* MSS L4LEN IDX */
  2407. mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
  2408. mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
  2409. /* Context index must be unique per ring. */
  2410. if (adapter->flags & IGB_FLAG_NEED_CTX_IDX)
  2411. mss_l4len_idx |= tx_ring->queue_index << 4;
  2412. context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
  2413. context_desc->seqnum_seed = 0;
  2414. buffer_info->time_stamp = jiffies;
  2415. buffer_info->next_to_watch = i;
  2416. buffer_info->dma = 0;
  2417. i++;
  2418. if (i == tx_ring->count)
  2419. i = 0;
  2420. tx_ring->next_to_use = i;
  2421. return true;
  2422. }
  2423. static inline bool igb_tx_csum_adv(struct igb_adapter *adapter,
  2424. struct igb_ring *tx_ring,
  2425. struct sk_buff *skb, u32 tx_flags)
  2426. {
  2427. struct e1000_adv_tx_context_desc *context_desc;
  2428. unsigned int i;
  2429. struct igb_buffer *buffer_info;
  2430. u32 info = 0, tu_cmd = 0;
  2431. if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
  2432. (tx_flags & IGB_TX_FLAGS_VLAN)) {
  2433. i = tx_ring->next_to_use;
  2434. buffer_info = &tx_ring->buffer_info[i];
  2435. context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
  2436. if (tx_flags & IGB_TX_FLAGS_VLAN)
  2437. info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
  2438. info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
  2439. if (skb->ip_summed == CHECKSUM_PARTIAL)
  2440. info |= skb_network_header_len(skb);
  2441. context_desc->vlan_macip_lens = cpu_to_le32(info);
  2442. tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
  2443. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  2444. switch (skb->protocol) {
  2445. case cpu_to_be16(ETH_P_IP):
  2446. tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
  2447. if (ip_hdr(skb)->protocol == IPPROTO_TCP)
  2448. tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
  2449. break;
  2450. case cpu_to_be16(ETH_P_IPV6):
  2451. /* XXX what about other V6 headers?? */
  2452. if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
  2453. tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
  2454. break;
  2455. default:
  2456. if (unlikely(net_ratelimit()))
  2457. dev_warn(&adapter->pdev->dev,
  2458. "partial checksum but proto=%x!\n",
  2459. skb->protocol);
  2460. break;
  2461. }
  2462. }
  2463. context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
  2464. context_desc->seqnum_seed = 0;
  2465. if (adapter->flags & IGB_FLAG_NEED_CTX_IDX)
  2466. context_desc->mss_l4len_idx =
  2467. cpu_to_le32(tx_ring->queue_index << 4);
  2468. else
  2469. context_desc->mss_l4len_idx = 0;
  2470. buffer_info->time_stamp = jiffies;
  2471. buffer_info->next_to_watch = i;
  2472. buffer_info->dma = 0;
  2473. i++;
  2474. if (i == tx_ring->count)
  2475. i = 0;
  2476. tx_ring->next_to_use = i;
  2477. return true;
  2478. }
  2479. return false;
  2480. }
  2481. #define IGB_MAX_TXD_PWR 16
  2482. #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
  2483. static inline int igb_tx_map_adv(struct igb_adapter *adapter,
  2484. struct igb_ring *tx_ring, struct sk_buff *skb,
  2485. unsigned int first)
  2486. {
  2487. struct igb_buffer *buffer_info;
  2488. unsigned int len = skb_headlen(skb);
  2489. unsigned int count = 0, i;
  2490. unsigned int f;
  2491. i = tx_ring->next_to_use;
  2492. buffer_info = &tx_ring->buffer_info[i];
  2493. BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
  2494. buffer_info->length = len;
  2495. /* set time_stamp *before* dma to help avoid a possible race */
  2496. buffer_info->time_stamp = jiffies;
  2497. buffer_info->next_to_watch = i;
  2498. buffer_info->dma = pci_map_single(adapter->pdev, skb->data, len,
  2499. PCI_DMA_TODEVICE);
  2500. count++;
  2501. i++;
  2502. if (i == tx_ring->count)
  2503. i = 0;
  2504. for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
  2505. struct skb_frag_struct *frag;
  2506. frag = &skb_shinfo(skb)->frags[f];
  2507. len = frag->size;
  2508. buffer_info = &tx_ring->buffer_info[i];
  2509. BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
  2510. buffer_info->length = len;
  2511. buffer_info->time_stamp = jiffies;
  2512. buffer_info->next_to_watch = i;
  2513. buffer_info->dma = pci_map_page(adapter->pdev,
  2514. frag->page,
  2515. frag->page_offset,
  2516. len,
  2517. PCI_DMA_TODEVICE);
  2518. count++;
  2519. i++;
  2520. if (i == tx_ring->count)
  2521. i = 0;
  2522. }
  2523. i = ((i == 0) ? tx_ring->count - 1 : i - 1);
  2524. tx_ring->buffer_info[i].skb = skb;
  2525. tx_ring->buffer_info[first].next_to_watch = i;
  2526. return count;
  2527. }
  2528. static inline void igb_tx_queue_adv(struct igb_adapter *adapter,
  2529. struct igb_ring *tx_ring,
  2530. int tx_flags, int count, u32 paylen,
  2531. u8 hdr_len)
  2532. {
  2533. union e1000_adv_tx_desc *tx_desc = NULL;
  2534. struct igb_buffer *buffer_info;
  2535. u32 olinfo_status = 0, cmd_type_len;
  2536. unsigned int i;
  2537. cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
  2538. E1000_ADVTXD_DCMD_DEXT);
  2539. if (tx_flags & IGB_TX_FLAGS_VLAN)
  2540. cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
  2541. if (tx_flags & IGB_TX_FLAGS_TSTAMP)
  2542. cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
  2543. if (tx_flags & IGB_TX_FLAGS_TSO) {
  2544. cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
  2545. /* insert tcp checksum */
  2546. olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
  2547. /* insert ip checksum */
  2548. if (tx_flags & IGB_TX_FLAGS_IPV4)
  2549. olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
  2550. } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
  2551. olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
  2552. }
  2553. if ((adapter->flags & IGB_FLAG_NEED_CTX_IDX) &&
  2554. (tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_TSO |
  2555. IGB_TX_FLAGS_VLAN)))
  2556. olinfo_status |= tx_ring->queue_index << 4;
  2557. olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
  2558. i = tx_ring->next_to_use;
  2559. while (count--) {
  2560. buffer_info = &tx_ring->buffer_info[i];
  2561. tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
  2562. tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
  2563. tx_desc->read.cmd_type_len =
  2564. cpu_to_le32(cmd_type_len | buffer_info->length);
  2565. tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
  2566. i++;
  2567. if (i == tx_ring->count)
  2568. i = 0;
  2569. }
  2570. tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
  2571. /* Force memory writes to complete before letting h/w
  2572. * know there are new descriptors to fetch. (Only
  2573. * applicable for weak-ordered memory model archs,
  2574. * such as IA-64). */
  2575. wmb();
  2576. tx_ring->next_to_use = i;
  2577. writel(i, adapter->hw.hw_addr + tx_ring->tail);
  2578. /* we need this if more than one processor can write to our tail
  2579. * at a time, it syncronizes IO on IA64/Altix systems */
  2580. mmiowb();
  2581. }
  2582. static int __igb_maybe_stop_tx(struct net_device *netdev,
  2583. struct igb_ring *tx_ring, int size)
  2584. {
  2585. struct igb_adapter *adapter = netdev_priv(netdev);
  2586. netif_stop_subqueue(netdev, tx_ring->queue_index);
  2587. /* Herbert's original patch had:
  2588. * smp_mb__after_netif_stop_queue();
  2589. * but since that doesn't exist yet, just open code it. */
  2590. smp_mb();
  2591. /* We need to check again in a case another CPU has just
  2592. * made room available. */
  2593. if (IGB_DESC_UNUSED(tx_ring) < size)
  2594. return -EBUSY;
  2595. /* A reprieve! */
  2596. netif_wake_subqueue(netdev, tx_ring->queue_index);
  2597. ++adapter->restart_queue;
  2598. return 0;
  2599. }
  2600. static int igb_maybe_stop_tx(struct net_device *netdev,
  2601. struct igb_ring *tx_ring, int size)
  2602. {
  2603. if (IGB_DESC_UNUSED(tx_ring) >= size)
  2604. return 0;
  2605. return __igb_maybe_stop_tx(netdev, tx_ring, size);
  2606. }
  2607. #define TXD_USE_COUNT(S) (((S) >> (IGB_MAX_TXD_PWR)) + 1)
  2608. static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
  2609. struct net_device *netdev,
  2610. struct igb_ring *tx_ring)
  2611. {
  2612. struct igb_adapter *adapter = netdev_priv(netdev);
  2613. unsigned int first;
  2614. unsigned int tx_flags = 0;
  2615. u8 hdr_len = 0;
  2616. int tso = 0;
  2617. union skb_shared_tx *shtx;
  2618. if (test_bit(__IGB_DOWN, &adapter->state)) {
  2619. dev_kfree_skb_any(skb);
  2620. return NETDEV_TX_OK;
  2621. }
  2622. if (skb->len <= 0) {
  2623. dev_kfree_skb_any(skb);
  2624. return NETDEV_TX_OK;
  2625. }
  2626. /* need: 1 descriptor per page,
  2627. * + 2 desc gap to keep tail from touching head,
  2628. * + 1 desc for skb->data,
  2629. * + 1 desc for context descriptor,
  2630. * otherwise try next time */
  2631. if (igb_maybe_stop_tx(netdev, tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
  2632. /* this is a hard error */
  2633. return NETDEV_TX_BUSY;
  2634. }
  2635. /*
  2636. * TODO: check that there currently is no other packet with
  2637. * time stamping in the queue
  2638. *
  2639. * When doing time stamping, keep the connection to the socket
  2640. * a while longer: it is still needed by skb_hwtstamp_tx(),
  2641. * called either in igb_tx_hwtstamp() or by our caller when
  2642. * doing software time stamping.
  2643. */
  2644. shtx = skb_tx(skb);
  2645. if (unlikely(shtx->hardware)) {
  2646. shtx->in_progress = 1;
  2647. tx_flags |= IGB_TX_FLAGS_TSTAMP;
  2648. } else if (likely(!shtx->software)) {
  2649. /*
  2650. * TODO: can this be solved in dev.c:dev_hard_start_xmit()?
  2651. * There are probably unmodified driver which do something
  2652. * like this and thus don't work in combination with
  2653. * SOF_TIMESTAMPING_TX_SOFTWARE.
  2654. */
  2655. skb_orphan(skb);
  2656. }
  2657. if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
  2658. tx_flags |= IGB_TX_FLAGS_VLAN;
  2659. tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
  2660. }
  2661. if (skb->protocol == htons(ETH_P_IP))
  2662. tx_flags |= IGB_TX_FLAGS_IPV4;
  2663. first = tx_ring->next_to_use;
  2664. tso = skb_is_gso(skb) ? igb_tso_adv(adapter, tx_ring, skb, tx_flags,
  2665. &hdr_len) : 0;
  2666. if (tso < 0) {
  2667. dev_kfree_skb_any(skb);
  2668. return NETDEV_TX_OK;
  2669. }
  2670. if (tso)
  2671. tx_flags |= IGB_TX_FLAGS_TSO;
  2672. else if (igb_tx_csum_adv(adapter, tx_ring, skb, tx_flags))
  2673. if (skb->ip_summed == CHECKSUM_PARTIAL)
  2674. tx_flags |= IGB_TX_FLAGS_CSUM;
  2675. igb_tx_queue_adv(adapter, tx_ring, tx_flags,
  2676. igb_tx_map_adv(adapter, tx_ring, skb, first),
  2677. skb->len, hdr_len);
  2678. netdev->trans_start = jiffies;
  2679. /* Make sure there is space in the ring for the next send. */
  2680. igb_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 4);
  2681. return NETDEV_TX_OK;
  2682. }
  2683. static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *netdev)
  2684. {
  2685. struct igb_adapter *adapter = netdev_priv(netdev);
  2686. struct igb_ring *tx_ring;
  2687. int r_idx = 0;
  2688. r_idx = skb->queue_mapping & (IGB_MAX_TX_QUEUES - 1);
  2689. tx_ring = adapter->multi_tx_table[r_idx];
  2690. /* This goes back to the question of how to logically map a tx queue
  2691. * to a flow. Right now, performance is impacted slightly negatively
  2692. * if using multiple tx queues. If the stack breaks away from a
  2693. * single qdisc implementation, we can look at this again. */
  2694. return (igb_xmit_frame_ring_adv(skb, netdev, tx_ring));
  2695. }
  2696. /**
  2697. * igb_tx_timeout - Respond to a Tx Hang
  2698. * @netdev: network interface device structure
  2699. **/
  2700. static void igb_tx_timeout(struct net_device *netdev)
  2701. {
  2702. struct igb_adapter *adapter = netdev_priv(netdev);
  2703. struct e1000_hw *hw = &adapter->hw;
  2704. /* Do the reset outside of interrupt context */
  2705. adapter->tx_timeout_count++;
  2706. schedule_work(&adapter->reset_task);
  2707. wr32(E1000_EICS,
  2708. (adapter->eims_enable_mask & ~adapter->eims_other));
  2709. }
  2710. static void igb_reset_task(struct work_struct *work)
  2711. {
  2712. struct igb_adapter *adapter;
  2713. adapter = container_of(work, struct igb_adapter, reset_task);
  2714. igb_reinit_locked(adapter);
  2715. }
  2716. /**
  2717. * igb_get_stats - Get System Network Statistics
  2718. * @netdev: network interface device structure
  2719. *
  2720. * Returns the address of the device statistics structure.
  2721. * The statistics are actually updated from the timer callback.
  2722. **/
  2723. static struct net_device_stats *
  2724. igb_get_stats(struct net_device *netdev)
  2725. {
  2726. struct igb_adapter *adapter = netdev_priv(netdev);
  2727. /* only return the current stats */
  2728. return &adapter->net_stats;
  2729. }
  2730. /**
  2731. * igb_change_mtu - Change the Maximum Transfer Unit
  2732. * @netdev: network interface device structure
  2733. * @new_mtu: new value for maximum frame size
  2734. *
  2735. * Returns 0 on success, negative on failure
  2736. **/
  2737. static int igb_change_mtu(struct net_device *netdev, int new_mtu)
  2738. {
  2739. struct igb_adapter *adapter = netdev_priv(netdev);
  2740. int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
  2741. if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
  2742. (max_frame > MAX_JUMBO_FRAME_SIZE)) {
  2743. dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
  2744. return -EINVAL;
  2745. }
  2746. #define MAX_STD_JUMBO_FRAME_SIZE 9234
  2747. if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
  2748. dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
  2749. return -EINVAL;
  2750. }
  2751. while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
  2752. msleep(1);
  2753. /* igb_down has a dependency on max_frame_size */
  2754. adapter->max_frame_size = max_frame;
  2755. if (netif_running(netdev))
  2756. igb_down(adapter);
  2757. /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
  2758. * means we reserve 2 more, this pushes us to allocate from the next
  2759. * larger slab size.
  2760. * i.e. RXBUFFER_2048 --> size-4096 slab
  2761. */
  2762. if (max_frame <= IGB_RXBUFFER_256)
  2763. adapter->rx_buffer_len = IGB_RXBUFFER_256;
  2764. else if (max_frame <= IGB_RXBUFFER_512)
  2765. adapter->rx_buffer_len = IGB_RXBUFFER_512;
  2766. else if (max_frame <= IGB_RXBUFFER_1024)
  2767. adapter->rx_buffer_len = IGB_RXBUFFER_1024;
  2768. else if (max_frame <= IGB_RXBUFFER_2048)
  2769. adapter->rx_buffer_len = IGB_RXBUFFER_2048;
  2770. else
  2771. #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
  2772. adapter->rx_buffer_len = IGB_RXBUFFER_16384;
  2773. #else
  2774. adapter->rx_buffer_len = PAGE_SIZE / 2;
  2775. #endif
  2776. /* adjust allocation if LPE protects us, and we aren't using SBP */
  2777. if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
  2778. (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE))
  2779. adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
  2780. dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
  2781. netdev->mtu, new_mtu);
  2782. netdev->mtu = new_mtu;
  2783. if (netif_running(netdev))
  2784. igb_up(adapter);
  2785. else
  2786. igb_reset(adapter);
  2787. clear_bit(__IGB_RESETTING, &adapter->state);
  2788. return 0;
  2789. }
  2790. /**
  2791. * igb_update_stats - Update the board statistics counters
  2792. * @adapter: board private structure
  2793. **/
  2794. void igb_update_stats(struct igb_adapter *adapter)
  2795. {
  2796. struct e1000_hw *hw = &adapter->hw;
  2797. struct pci_dev *pdev = adapter->pdev;
  2798. u16 phy_tmp;
  2799. #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
  2800. /*
  2801. * Prevent stats update while adapter is being reset, or if the pci
  2802. * connection is down.
  2803. */
  2804. if (adapter->link_speed == 0)
  2805. return;
  2806. if (pci_channel_offline(pdev))
  2807. return;
  2808. adapter->stats.crcerrs += rd32(E1000_CRCERRS);
  2809. adapter->stats.gprc += rd32(E1000_GPRC);
  2810. adapter->stats.gorc += rd32(E1000_GORCL);
  2811. rd32(E1000_GORCH); /* clear GORCL */
  2812. adapter->stats.bprc += rd32(E1000_BPRC);
  2813. adapter->stats.mprc += rd32(E1000_MPRC);
  2814. adapter->stats.roc += rd32(E1000_ROC);
  2815. adapter->stats.prc64 += rd32(E1000_PRC64);
  2816. adapter->stats.prc127 += rd32(E1000_PRC127);
  2817. adapter->stats.prc255 += rd32(E1000_PRC255);
  2818. adapter->stats.prc511 += rd32(E1000_PRC511);
  2819. adapter->stats.prc1023 += rd32(E1000_PRC1023);
  2820. adapter->stats.prc1522 += rd32(E1000_PRC1522);
  2821. adapter->stats.symerrs += rd32(E1000_SYMERRS);
  2822. adapter->stats.sec += rd32(E1000_SEC);
  2823. adapter->stats.mpc += rd32(E1000_MPC);
  2824. adapter->stats.scc += rd32(E1000_SCC);
  2825. adapter->stats.ecol += rd32(E1000_ECOL);
  2826. adapter->stats.mcc += rd32(E1000_MCC);
  2827. adapter->stats.latecol += rd32(E1000_LATECOL);
  2828. adapter->stats.dc += rd32(E1000_DC);
  2829. adapter->stats.rlec += rd32(E1000_RLEC);
  2830. adapter->stats.xonrxc += rd32(E1000_XONRXC);
  2831. adapter->stats.xontxc += rd32(E1000_XONTXC);
  2832. adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
  2833. adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
  2834. adapter->stats.fcruc += rd32(E1000_FCRUC);
  2835. adapter->stats.gptc += rd32(E1000_GPTC);
  2836. adapter->stats.gotc += rd32(E1000_GOTCL);
  2837. rd32(E1000_GOTCH); /* clear GOTCL */
  2838. adapter->stats.rnbc += rd32(E1000_RNBC);
  2839. adapter->stats.ruc += rd32(E1000_RUC);
  2840. adapter->stats.rfc += rd32(E1000_RFC);
  2841. adapter->stats.rjc += rd32(E1000_RJC);
  2842. adapter->stats.tor += rd32(E1000_TORH);
  2843. adapter->stats.tot += rd32(E1000_TOTH);
  2844. adapter->stats.tpr += rd32(E1000_TPR);
  2845. adapter->stats.ptc64 += rd32(E1000_PTC64);
  2846. adapter->stats.ptc127 += rd32(E1000_PTC127);
  2847. adapter->stats.ptc255 += rd32(E1000_PTC255);
  2848. adapter->stats.ptc511 += rd32(E1000_PTC511);
  2849. adapter->stats.ptc1023 += rd32(E1000_PTC1023);
  2850. adapter->stats.ptc1522 += rd32(E1000_PTC1522);
  2851. adapter->stats.mptc += rd32(E1000_MPTC);
  2852. adapter->stats.bptc += rd32(E1000_BPTC);
  2853. /* used for adaptive IFS */
  2854. hw->mac.tx_packet_delta = rd32(E1000_TPT);
  2855. adapter->stats.tpt += hw->mac.tx_packet_delta;
  2856. hw->mac.collision_delta = rd32(E1000_COLC);
  2857. adapter->stats.colc += hw->mac.collision_delta;
  2858. adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
  2859. adapter->stats.rxerrc += rd32(E1000_RXERRC);
  2860. adapter->stats.tncrs += rd32(E1000_TNCRS);
  2861. adapter->stats.tsctc += rd32(E1000_TSCTC);
  2862. adapter->stats.tsctfc += rd32(E1000_TSCTFC);
  2863. adapter->stats.iac += rd32(E1000_IAC);
  2864. adapter->stats.icrxoc += rd32(E1000_ICRXOC);
  2865. adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
  2866. adapter->stats.icrxatc += rd32(E1000_ICRXATC);
  2867. adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
  2868. adapter->stats.ictxatc += rd32(E1000_ICTXATC);
  2869. adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
  2870. adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
  2871. adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
  2872. /* Fill out the OS statistics structure */
  2873. adapter->net_stats.multicast = adapter->stats.mprc;
  2874. adapter->net_stats.collisions = adapter->stats.colc;
  2875. /* Rx Errors */
  2876. /* RLEC on some newer hardware can be incorrect so build
  2877. * our own version based on RUC and ROC */
  2878. adapter->net_stats.rx_errors = adapter->stats.rxerrc +
  2879. adapter->stats.crcerrs + adapter->stats.algnerrc +
  2880. adapter->stats.ruc + adapter->stats.roc +
  2881. adapter->stats.cexterr;
  2882. adapter->net_stats.rx_length_errors = adapter->stats.ruc +
  2883. adapter->stats.roc;
  2884. adapter->net_stats.rx_crc_errors = adapter->stats.crcerrs;
  2885. adapter->net_stats.rx_frame_errors = adapter->stats.algnerrc;
  2886. adapter->net_stats.rx_missed_errors = adapter->stats.mpc;
  2887. /* Tx Errors */
  2888. adapter->net_stats.tx_errors = adapter->stats.ecol +
  2889. adapter->stats.latecol;
  2890. adapter->net_stats.tx_aborted_errors = adapter->stats.ecol;
  2891. adapter->net_stats.tx_window_errors = adapter->stats.latecol;
  2892. adapter->net_stats.tx_carrier_errors = adapter->stats.tncrs;
  2893. /* Tx Dropped needs to be maintained elsewhere */
  2894. /* Phy Stats */
  2895. if (hw->phy.media_type == e1000_media_type_copper) {
  2896. if ((adapter->link_speed == SPEED_1000) &&
  2897. (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
  2898. &phy_tmp))) {
  2899. phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
  2900. adapter->phy_stats.idle_errors += phy_tmp;
  2901. }
  2902. }
  2903. /* Management Stats */
  2904. adapter->stats.mgptc += rd32(E1000_MGTPTC);
  2905. adapter->stats.mgprc += rd32(E1000_MGTPRC);
  2906. adapter->stats.mgpdc += rd32(E1000_MGTPDC);
  2907. }
  2908. static irqreturn_t igb_msix_other(int irq, void *data)
  2909. {
  2910. struct net_device *netdev = data;
  2911. struct igb_adapter *adapter = netdev_priv(netdev);
  2912. struct e1000_hw *hw = &adapter->hw;
  2913. u32 icr = rd32(E1000_ICR);
  2914. /* reading ICR causes bit 31 of EICR to be cleared */
  2915. if(icr & E1000_ICR_DOUTSYNC) {
  2916. /* HW is reporting DMA is out of sync */
  2917. adapter->stats.doosync++;
  2918. }
  2919. if (!(icr & E1000_ICR_LSC))
  2920. goto no_link_interrupt;
  2921. hw->mac.get_link_status = 1;
  2922. /* guard against interrupt when we're going down */
  2923. if (!test_bit(__IGB_DOWN, &adapter->state))
  2924. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  2925. no_link_interrupt:
  2926. wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
  2927. wr32(E1000_EIMS, adapter->eims_other);
  2928. return IRQ_HANDLED;
  2929. }
  2930. static irqreturn_t igb_msix_tx(int irq, void *data)
  2931. {
  2932. struct igb_ring *tx_ring = data;
  2933. struct igb_adapter *adapter = tx_ring->adapter;
  2934. struct e1000_hw *hw = &adapter->hw;
  2935. #ifdef CONFIG_IGB_DCA
  2936. if (adapter->flags & IGB_FLAG_DCA_ENABLED)
  2937. igb_update_tx_dca(tx_ring);
  2938. #endif
  2939. tx_ring->total_bytes = 0;
  2940. tx_ring->total_packets = 0;
  2941. /* auto mask will automatically reenable the interrupt when we write
  2942. * EICS */
  2943. if (!igb_clean_tx_irq(tx_ring))
  2944. /* Ring was not completely cleaned, so fire another interrupt */
  2945. wr32(E1000_EICS, tx_ring->eims_value);
  2946. else
  2947. wr32(E1000_EIMS, tx_ring->eims_value);
  2948. return IRQ_HANDLED;
  2949. }
  2950. static void igb_write_itr(struct igb_ring *ring)
  2951. {
  2952. struct e1000_hw *hw = &ring->adapter->hw;
  2953. if ((ring->adapter->itr_setting & 3) && ring->set_itr) {
  2954. switch (hw->mac.type) {
  2955. case e1000_82576:
  2956. wr32(ring->itr_register,
  2957. ring->itr_val |
  2958. 0x80000000);
  2959. break;
  2960. default:
  2961. wr32(ring->itr_register,
  2962. ring->itr_val |
  2963. (ring->itr_val << 16));
  2964. break;
  2965. }
  2966. ring->set_itr = 0;
  2967. }
  2968. }
  2969. static irqreturn_t igb_msix_rx(int irq, void *data)
  2970. {
  2971. struct igb_ring *rx_ring = data;
  2972. /* Write the ITR value calculated at the end of the
  2973. * previous interrupt.
  2974. */
  2975. igb_write_itr(rx_ring);
  2976. if (napi_schedule_prep(&rx_ring->napi))
  2977. __napi_schedule(&rx_ring->napi);
  2978. #ifdef CONFIG_IGB_DCA
  2979. if (rx_ring->adapter->flags & IGB_FLAG_DCA_ENABLED)
  2980. igb_update_rx_dca(rx_ring);
  2981. #endif
  2982. return IRQ_HANDLED;
  2983. }
  2984. #ifdef CONFIG_IGB_DCA
  2985. static void igb_update_rx_dca(struct igb_ring *rx_ring)
  2986. {
  2987. u32 dca_rxctrl;
  2988. struct igb_adapter *adapter = rx_ring->adapter;
  2989. struct e1000_hw *hw = &adapter->hw;
  2990. int cpu = get_cpu();
  2991. int q = rx_ring->reg_idx;
  2992. if (rx_ring->cpu != cpu) {
  2993. dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
  2994. if (hw->mac.type == e1000_82576) {
  2995. dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
  2996. dca_rxctrl |= dca_get_tag(cpu) <<
  2997. E1000_DCA_RXCTRL_CPUID_SHIFT;
  2998. } else {
  2999. dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
  3000. dca_rxctrl |= dca_get_tag(cpu);
  3001. }
  3002. dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
  3003. dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
  3004. dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
  3005. wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
  3006. rx_ring->cpu = cpu;
  3007. }
  3008. put_cpu();
  3009. }
  3010. static void igb_update_tx_dca(struct igb_ring *tx_ring)
  3011. {
  3012. u32 dca_txctrl;
  3013. struct igb_adapter *adapter = tx_ring->adapter;
  3014. struct e1000_hw *hw = &adapter->hw;
  3015. int cpu = get_cpu();
  3016. int q = tx_ring->reg_idx;
  3017. if (tx_ring->cpu != cpu) {
  3018. dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
  3019. if (hw->mac.type == e1000_82576) {
  3020. dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
  3021. dca_txctrl |= dca_get_tag(cpu) <<
  3022. E1000_DCA_TXCTRL_CPUID_SHIFT;
  3023. } else {
  3024. dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
  3025. dca_txctrl |= dca_get_tag(cpu);
  3026. }
  3027. dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
  3028. wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
  3029. tx_ring->cpu = cpu;
  3030. }
  3031. put_cpu();
  3032. }
  3033. static void igb_setup_dca(struct igb_adapter *adapter)
  3034. {
  3035. int i;
  3036. if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
  3037. return;
  3038. for (i = 0; i < adapter->num_tx_queues; i++) {
  3039. adapter->tx_ring[i].cpu = -1;
  3040. igb_update_tx_dca(&adapter->tx_ring[i]);
  3041. }
  3042. for (i = 0; i < adapter->num_rx_queues; i++) {
  3043. adapter->rx_ring[i].cpu = -1;
  3044. igb_update_rx_dca(&adapter->rx_ring[i]);
  3045. }
  3046. }
  3047. static int __igb_notify_dca(struct device *dev, void *data)
  3048. {
  3049. struct net_device *netdev = dev_get_drvdata(dev);
  3050. struct igb_adapter *adapter = netdev_priv(netdev);
  3051. struct e1000_hw *hw = &adapter->hw;
  3052. unsigned long event = *(unsigned long *)data;
  3053. switch (event) {
  3054. case DCA_PROVIDER_ADD:
  3055. /* if already enabled, don't do it again */
  3056. if (adapter->flags & IGB_FLAG_DCA_ENABLED)
  3057. break;
  3058. /* Always use CB2 mode, difference is masked
  3059. * in the CB driver. */
  3060. wr32(E1000_DCA_CTRL, 2);
  3061. if (dca_add_requester(dev) == 0) {
  3062. adapter->flags |= IGB_FLAG_DCA_ENABLED;
  3063. dev_info(&adapter->pdev->dev, "DCA enabled\n");
  3064. igb_setup_dca(adapter);
  3065. break;
  3066. }
  3067. /* Fall Through since DCA is disabled. */
  3068. case DCA_PROVIDER_REMOVE:
  3069. if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
  3070. /* without this a class_device is left
  3071. * hanging around in the sysfs model */
  3072. dca_remove_requester(dev);
  3073. dev_info(&adapter->pdev->dev, "DCA disabled\n");
  3074. adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
  3075. wr32(E1000_DCA_CTRL, 1);
  3076. }
  3077. break;
  3078. }
  3079. return 0;
  3080. }
  3081. static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
  3082. void *p)
  3083. {
  3084. int ret_val;
  3085. ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
  3086. __igb_notify_dca);
  3087. return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
  3088. }
  3089. #endif /* CONFIG_IGB_DCA */
  3090. /**
  3091. * igb_intr_msi - Interrupt Handler
  3092. * @irq: interrupt number
  3093. * @data: pointer to a network interface device structure
  3094. **/
  3095. static irqreturn_t igb_intr_msi(int irq, void *data)
  3096. {
  3097. struct net_device *netdev = data;
  3098. struct igb_adapter *adapter = netdev_priv(netdev);
  3099. struct e1000_hw *hw = &adapter->hw;
  3100. /* read ICR disables interrupts using IAM */
  3101. u32 icr = rd32(E1000_ICR);
  3102. igb_write_itr(adapter->rx_ring);
  3103. if(icr & E1000_ICR_DOUTSYNC) {
  3104. /* HW is reporting DMA is out of sync */
  3105. adapter->stats.doosync++;
  3106. }
  3107. if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
  3108. hw->mac.get_link_status = 1;
  3109. if (!test_bit(__IGB_DOWN, &adapter->state))
  3110. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  3111. }
  3112. napi_schedule(&adapter->rx_ring[0].napi);
  3113. return IRQ_HANDLED;
  3114. }
  3115. /**
  3116. * igb_intr - Legacy Interrupt Handler
  3117. * @irq: interrupt number
  3118. * @data: pointer to a network interface device structure
  3119. **/
  3120. static irqreturn_t igb_intr(int irq, void *data)
  3121. {
  3122. struct net_device *netdev = data;
  3123. struct igb_adapter *adapter = netdev_priv(netdev);
  3124. struct e1000_hw *hw = &adapter->hw;
  3125. /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
  3126. * need for the IMC write */
  3127. u32 icr = rd32(E1000_ICR);
  3128. if (!icr)
  3129. return IRQ_NONE; /* Not our interrupt */
  3130. igb_write_itr(adapter->rx_ring);
  3131. /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
  3132. * not set, then the adapter didn't send an interrupt */
  3133. if (!(icr & E1000_ICR_INT_ASSERTED))
  3134. return IRQ_NONE;
  3135. if(icr & E1000_ICR_DOUTSYNC) {
  3136. /* HW is reporting DMA is out of sync */
  3137. adapter->stats.doosync++;
  3138. }
  3139. if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
  3140. hw->mac.get_link_status = 1;
  3141. /* guard against interrupt when we're going down */
  3142. if (!test_bit(__IGB_DOWN, &adapter->state))
  3143. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  3144. }
  3145. napi_schedule(&adapter->rx_ring[0].napi);
  3146. return IRQ_HANDLED;
  3147. }
  3148. /**
  3149. * igb_poll - NAPI Rx polling callback
  3150. * @napi: napi polling structure
  3151. * @budget: count of how many packets we should handle
  3152. **/
  3153. static int igb_poll(struct napi_struct *napi, int budget)
  3154. {
  3155. struct igb_ring *rx_ring = container_of(napi, struct igb_ring, napi);
  3156. struct igb_adapter *adapter = rx_ring->adapter;
  3157. struct net_device *netdev = adapter->netdev;
  3158. int tx_clean_complete, work_done = 0;
  3159. /* this poll routine only supports one tx and one rx queue */
  3160. #ifdef CONFIG_IGB_DCA
  3161. if (adapter->flags & IGB_FLAG_DCA_ENABLED)
  3162. igb_update_tx_dca(&adapter->tx_ring[0]);
  3163. #endif
  3164. tx_clean_complete = igb_clean_tx_irq(&adapter->tx_ring[0]);
  3165. #ifdef CONFIG_IGB_DCA
  3166. if (adapter->flags & IGB_FLAG_DCA_ENABLED)
  3167. igb_update_rx_dca(&adapter->rx_ring[0]);
  3168. #endif
  3169. igb_clean_rx_irq_adv(&adapter->rx_ring[0], &work_done, budget);
  3170. /* If no Tx and not enough Rx work done, exit the polling mode */
  3171. if ((tx_clean_complete && (work_done < budget)) ||
  3172. !netif_running(netdev)) {
  3173. if (adapter->itr_setting & 3)
  3174. igb_set_itr(adapter);
  3175. napi_complete(napi);
  3176. if (!test_bit(__IGB_DOWN, &adapter->state))
  3177. igb_irq_enable(adapter);
  3178. return 0;
  3179. }
  3180. return 1;
  3181. }
  3182. static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
  3183. {
  3184. struct igb_ring *rx_ring = container_of(napi, struct igb_ring, napi);
  3185. struct igb_adapter *adapter = rx_ring->adapter;
  3186. struct e1000_hw *hw = &adapter->hw;
  3187. struct net_device *netdev = adapter->netdev;
  3188. int work_done = 0;
  3189. #ifdef CONFIG_IGB_DCA
  3190. if (adapter->flags & IGB_FLAG_DCA_ENABLED)
  3191. igb_update_rx_dca(rx_ring);
  3192. #endif
  3193. igb_clean_rx_irq_adv(rx_ring, &work_done, budget);
  3194. /* If not enough Rx work done, exit the polling mode */
  3195. if ((work_done == 0) || !netif_running(netdev)) {
  3196. napi_complete(napi);
  3197. if (adapter->itr_setting & 3) {
  3198. if (adapter->num_rx_queues == 1)
  3199. igb_set_itr(adapter);
  3200. else
  3201. igb_update_ring_itr(rx_ring);
  3202. }
  3203. if (!test_bit(__IGB_DOWN, &adapter->state))
  3204. wr32(E1000_EIMS, rx_ring->eims_value);
  3205. return 0;
  3206. }
  3207. return 1;
  3208. }
  3209. /**
  3210. * igb_hwtstamp - utility function which checks for TX time stamp
  3211. * @adapter: board private structure
  3212. * @skb: packet that was just sent
  3213. *
  3214. * If we were asked to do hardware stamping and such a time stamp is
  3215. * available, then it must have been for this skb here because we only
  3216. * allow only one such packet into the queue.
  3217. */
  3218. static void igb_tx_hwtstamp(struct igb_adapter *adapter, struct sk_buff *skb)
  3219. {
  3220. union skb_shared_tx *shtx = skb_tx(skb);
  3221. struct e1000_hw *hw = &adapter->hw;
  3222. if (unlikely(shtx->hardware)) {
  3223. u32 valid = rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID;
  3224. if (valid) {
  3225. u64 regval = rd32(E1000_TXSTMPL);
  3226. u64 ns;
  3227. struct skb_shared_hwtstamps shhwtstamps;
  3228. memset(&shhwtstamps, 0, sizeof(shhwtstamps));
  3229. regval |= (u64)rd32(E1000_TXSTMPH) << 32;
  3230. ns = timecounter_cyc2time(&adapter->clock,
  3231. regval);
  3232. timecompare_update(&adapter->compare, ns);
  3233. shhwtstamps.hwtstamp = ns_to_ktime(ns);
  3234. shhwtstamps.syststamp =
  3235. timecompare_transform(&adapter->compare, ns);
  3236. skb_tstamp_tx(skb, &shhwtstamps);
  3237. }
  3238. /* delayed orphaning: skb_tstamp_tx() needs the socket */
  3239. skb_orphan(skb);
  3240. }
  3241. }
  3242. /**
  3243. * igb_clean_tx_irq - Reclaim resources after transmit completes
  3244. * @adapter: board private structure
  3245. * returns true if ring is completely cleaned
  3246. **/
  3247. static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
  3248. {
  3249. struct igb_adapter *adapter = tx_ring->adapter;
  3250. struct net_device *netdev = adapter->netdev;
  3251. struct e1000_hw *hw = &adapter->hw;
  3252. struct igb_buffer *buffer_info;
  3253. struct sk_buff *skb;
  3254. union e1000_adv_tx_desc *tx_desc, *eop_desc;
  3255. unsigned int total_bytes = 0, total_packets = 0;
  3256. unsigned int i, eop, count = 0;
  3257. bool cleaned = false;
  3258. i = tx_ring->next_to_clean;
  3259. eop = tx_ring->buffer_info[i].next_to_watch;
  3260. eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
  3261. while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
  3262. (count < tx_ring->count)) {
  3263. for (cleaned = false; !cleaned; count++) {
  3264. tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
  3265. buffer_info = &tx_ring->buffer_info[i];
  3266. cleaned = (i == eop);
  3267. skb = buffer_info->skb;
  3268. if (skb) {
  3269. unsigned int segs, bytecount;
  3270. /* gso_segs is currently only valid for tcp */
  3271. segs = skb_shinfo(skb)->gso_segs ?: 1;
  3272. /* multiply data chunks by size of headers */
  3273. bytecount = ((segs - 1) * skb_headlen(skb)) +
  3274. skb->len;
  3275. total_packets += segs;
  3276. total_bytes += bytecount;
  3277. igb_tx_hwtstamp(adapter, skb);
  3278. }
  3279. igb_unmap_and_free_tx_resource(adapter, buffer_info);
  3280. tx_desc->wb.status = 0;
  3281. i++;
  3282. if (i == tx_ring->count)
  3283. i = 0;
  3284. }
  3285. eop = tx_ring->buffer_info[i].next_to_watch;
  3286. eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
  3287. }
  3288. tx_ring->next_to_clean = i;
  3289. if (unlikely(count &&
  3290. netif_carrier_ok(netdev) &&
  3291. IGB_DESC_UNUSED(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
  3292. /* Make sure that anybody stopping the queue after this
  3293. * sees the new next_to_clean.
  3294. */
  3295. smp_mb();
  3296. if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
  3297. !(test_bit(__IGB_DOWN, &adapter->state))) {
  3298. netif_wake_subqueue(netdev, tx_ring->queue_index);
  3299. ++adapter->restart_queue;
  3300. }
  3301. }
  3302. if (tx_ring->detect_tx_hung) {
  3303. /* Detect a transmit hang in hardware, this serializes the
  3304. * check with the clearing of time_stamp and movement of i */
  3305. tx_ring->detect_tx_hung = false;
  3306. if (tx_ring->buffer_info[i].time_stamp &&
  3307. time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
  3308. (adapter->tx_timeout_factor * HZ))
  3309. && !(rd32(E1000_STATUS) &
  3310. E1000_STATUS_TXOFF)) {
  3311. /* detected Tx unit hang */
  3312. dev_err(&adapter->pdev->dev,
  3313. "Detected Tx Unit Hang\n"
  3314. " Tx Queue <%d>\n"
  3315. " TDH <%x>\n"
  3316. " TDT <%x>\n"
  3317. " next_to_use <%x>\n"
  3318. " next_to_clean <%x>\n"
  3319. "buffer_info[next_to_clean]\n"
  3320. " time_stamp <%lx>\n"
  3321. " next_to_watch <%x>\n"
  3322. " jiffies <%lx>\n"
  3323. " desc.status <%x>\n",
  3324. tx_ring->queue_index,
  3325. readl(adapter->hw.hw_addr + tx_ring->head),
  3326. readl(adapter->hw.hw_addr + tx_ring->tail),
  3327. tx_ring->next_to_use,
  3328. tx_ring->next_to_clean,
  3329. tx_ring->buffer_info[i].time_stamp,
  3330. eop,
  3331. jiffies,
  3332. eop_desc->wb.status);
  3333. netif_stop_subqueue(netdev, tx_ring->queue_index);
  3334. }
  3335. }
  3336. tx_ring->total_bytes += total_bytes;
  3337. tx_ring->total_packets += total_packets;
  3338. tx_ring->tx_stats.bytes += total_bytes;
  3339. tx_ring->tx_stats.packets += total_packets;
  3340. adapter->net_stats.tx_bytes += total_bytes;
  3341. adapter->net_stats.tx_packets += total_packets;
  3342. return (count < tx_ring->count);
  3343. }
  3344. /**
  3345. * igb_receive_skb - helper function to handle rx indications
  3346. * @ring: pointer to receive ring receving this packet
  3347. * @status: descriptor status field as written by hardware
  3348. * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
  3349. * @skb: pointer to sk_buff to be indicated to stack
  3350. **/
  3351. static void igb_receive_skb(struct igb_ring *ring, u8 status,
  3352. union e1000_adv_rx_desc * rx_desc,
  3353. struct sk_buff *skb)
  3354. {
  3355. struct igb_adapter * adapter = ring->adapter;
  3356. bool vlan_extracted = (adapter->vlgrp && (status & E1000_RXD_STAT_VP));
  3357. skb_record_rx_queue(skb, ring->queue_index);
  3358. if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
  3359. if (vlan_extracted)
  3360. vlan_gro_receive(&ring->napi, adapter->vlgrp,
  3361. le16_to_cpu(rx_desc->wb.upper.vlan),
  3362. skb);
  3363. else
  3364. napi_gro_receive(&ring->napi, skb);
  3365. } else {
  3366. if (vlan_extracted)
  3367. vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
  3368. le16_to_cpu(rx_desc->wb.upper.vlan));
  3369. else
  3370. netif_receive_skb(skb);
  3371. }
  3372. }
  3373. static inline void igb_rx_checksum_adv(struct igb_adapter *adapter,
  3374. u32 status_err, struct sk_buff *skb)
  3375. {
  3376. skb->ip_summed = CHECKSUM_NONE;
  3377. /* Ignore Checksum bit is set or checksum is disabled through ethtool */
  3378. if ((status_err & E1000_RXD_STAT_IXSM) || !adapter->rx_csum)
  3379. return;
  3380. /* TCP/UDP checksum error bit is set */
  3381. if (status_err &
  3382. (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
  3383. /* let the stack verify checksum errors */
  3384. adapter->hw_csum_err++;
  3385. return;
  3386. }
  3387. /* It must be a TCP or UDP packet with a valid checksum */
  3388. if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
  3389. skb->ip_summed = CHECKSUM_UNNECESSARY;
  3390. adapter->hw_csum_good++;
  3391. }
  3392. static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring,
  3393. int *work_done, int budget)
  3394. {
  3395. struct igb_adapter *adapter = rx_ring->adapter;
  3396. struct net_device *netdev = adapter->netdev;
  3397. struct e1000_hw *hw = &adapter->hw;
  3398. struct pci_dev *pdev = adapter->pdev;
  3399. union e1000_adv_rx_desc *rx_desc , *next_rxd;
  3400. struct igb_buffer *buffer_info , *next_buffer;
  3401. struct sk_buff *skb;
  3402. unsigned int i;
  3403. u32 length, hlen, staterr;
  3404. bool cleaned = false;
  3405. int cleaned_count = 0;
  3406. unsigned int total_bytes = 0, total_packets = 0;
  3407. i = rx_ring->next_to_clean;
  3408. buffer_info = &rx_ring->buffer_info[i];
  3409. rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
  3410. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  3411. while (staterr & E1000_RXD_STAT_DD) {
  3412. if (*work_done >= budget)
  3413. break;
  3414. (*work_done)++;
  3415. skb = buffer_info->skb;
  3416. prefetch(skb->data - NET_IP_ALIGN);
  3417. buffer_info->skb = NULL;
  3418. i++;
  3419. if (i == rx_ring->count)
  3420. i = 0;
  3421. next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
  3422. prefetch(next_rxd);
  3423. next_buffer = &rx_ring->buffer_info[i];
  3424. length = le16_to_cpu(rx_desc->wb.upper.length);
  3425. cleaned = true;
  3426. cleaned_count++;
  3427. if (!adapter->rx_ps_hdr_size) {
  3428. pci_unmap_single(pdev, buffer_info->dma,
  3429. adapter->rx_buffer_len +
  3430. NET_IP_ALIGN,
  3431. PCI_DMA_FROMDEVICE);
  3432. skb_put(skb, length);
  3433. goto send_up;
  3434. }
  3435. /* HW will not DMA in data larger than the given buffer, even
  3436. * if it parses the (NFS, of course) header to be larger. In
  3437. * that case, it fills the header buffer and spills the rest
  3438. * into the page.
  3439. */
  3440. hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
  3441. E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
  3442. if (hlen > adapter->rx_ps_hdr_size)
  3443. hlen = adapter->rx_ps_hdr_size;
  3444. if (!skb_shinfo(skb)->nr_frags) {
  3445. pci_unmap_single(pdev, buffer_info->dma,
  3446. adapter->rx_ps_hdr_size +
  3447. NET_IP_ALIGN,
  3448. PCI_DMA_FROMDEVICE);
  3449. skb_put(skb, hlen);
  3450. }
  3451. if (length) {
  3452. pci_unmap_page(pdev, buffer_info->page_dma,
  3453. PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
  3454. buffer_info->page_dma = 0;
  3455. skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
  3456. buffer_info->page,
  3457. buffer_info->page_offset,
  3458. length);
  3459. if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
  3460. (page_count(buffer_info->page) != 1))
  3461. buffer_info->page = NULL;
  3462. else
  3463. get_page(buffer_info->page);
  3464. skb->len += length;
  3465. skb->data_len += length;
  3466. skb->truesize += length;
  3467. }
  3468. if (!(staterr & E1000_RXD_STAT_EOP)) {
  3469. buffer_info->skb = next_buffer->skb;
  3470. buffer_info->dma = next_buffer->dma;
  3471. next_buffer->skb = skb;
  3472. next_buffer->dma = 0;
  3473. goto next_desc;
  3474. }
  3475. send_up:
  3476. /*
  3477. * If this bit is set, then the RX registers contain
  3478. * the time stamp. No other packet will be time
  3479. * stamped until we read these registers, so read the
  3480. * registers to make them available again. Because
  3481. * only one packet can be time stamped at a time, we
  3482. * know that the register values must belong to this
  3483. * one here and therefore we don't need to compare
  3484. * any of the additional attributes stored for it.
  3485. *
  3486. * If nothing went wrong, then it should have a
  3487. * skb_shared_tx that we can turn into a
  3488. * skb_shared_hwtstamps.
  3489. *
  3490. * TODO: can time stamping be triggered (thus locking
  3491. * the registers) without the packet reaching this point
  3492. * here? In that case RX time stamping would get stuck.
  3493. *
  3494. * TODO: in "time stamp all packets" mode this bit is
  3495. * not set. Need a global flag for this mode and then
  3496. * always read the registers. Cannot be done without
  3497. * a race condition.
  3498. */
  3499. if (unlikely(staterr & E1000_RXD_STAT_TS)) {
  3500. u64 regval;
  3501. u64 ns;
  3502. struct skb_shared_hwtstamps *shhwtstamps =
  3503. skb_hwtstamps(skb);
  3504. WARN(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID),
  3505. "igb: no RX time stamp available for time stamped packet");
  3506. regval = rd32(E1000_RXSTMPL);
  3507. regval |= (u64)rd32(E1000_RXSTMPH) << 32;
  3508. ns = timecounter_cyc2time(&adapter->clock, regval);
  3509. timecompare_update(&adapter->compare, ns);
  3510. memset(shhwtstamps, 0, sizeof(*shhwtstamps));
  3511. shhwtstamps->hwtstamp = ns_to_ktime(ns);
  3512. shhwtstamps->syststamp =
  3513. timecompare_transform(&adapter->compare, ns);
  3514. }
  3515. if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
  3516. dev_kfree_skb_irq(skb);
  3517. goto next_desc;
  3518. }
  3519. total_bytes += skb->len;
  3520. total_packets++;
  3521. igb_rx_checksum_adv(adapter, staterr, skb);
  3522. skb->protocol = eth_type_trans(skb, netdev);
  3523. igb_receive_skb(rx_ring, staterr, rx_desc, skb);
  3524. next_desc:
  3525. rx_desc->wb.upper.status_error = 0;
  3526. /* return some buffers to hardware, one at a time is too slow */
  3527. if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
  3528. igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
  3529. cleaned_count = 0;
  3530. }
  3531. /* use prefetched values */
  3532. rx_desc = next_rxd;
  3533. buffer_info = next_buffer;
  3534. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  3535. }
  3536. rx_ring->next_to_clean = i;
  3537. cleaned_count = IGB_DESC_UNUSED(rx_ring);
  3538. if (cleaned_count)
  3539. igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
  3540. rx_ring->total_packets += total_packets;
  3541. rx_ring->total_bytes += total_bytes;
  3542. rx_ring->rx_stats.packets += total_packets;
  3543. rx_ring->rx_stats.bytes += total_bytes;
  3544. adapter->net_stats.rx_bytes += total_bytes;
  3545. adapter->net_stats.rx_packets += total_packets;
  3546. return cleaned;
  3547. }
  3548. /**
  3549. * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
  3550. * @adapter: address of board private structure
  3551. **/
  3552. static void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring,
  3553. int cleaned_count)
  3554. {
  3555. struct igb_adapter *adapter = rx_ring->adapter;
  3556. struct net_device *netdev = adapter->netdev;
  3557. struct pci_dev *pdev = adapter->pdev;
  3558. union e1000_adv_rx_desc *rx_desc;
  3559. struct igb_buffer *buffer_info;
  3560. struct sk_buff *skb;
  3561. unsigned int i;
  3562. int bufsz;
  3563. i = rx_ring->next_to_use;
  3564. buffer_info = &rx_ring->buffer_info[i];
  3565. if (adapter->rx_ps_hdr_size)
  3566. bufsz = adapter->rx_ps_hdr_size;
  3567. else
  3568. bufsz = adapter->rx_buffer_len;
  3569. bufsz += NET_IP_ALIGN;
  3570. while (cleaned_count--) {
  3571. rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
  3572. if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
  3573. if (!buffer_info->page) {
  3574. buffer_info->page = alloc_page(GFP_ATOMIC);
  3575. if (!buffer_info->page) {
  3576. adapter->alloc_rx_buff_failed++;
  3577. goto no_buffers;
  3578. }
  3579. buffer_info->page_offset = 0;
  3580. } else {
  3581. buffer_info->page_offset ^= PAGE_SIZE / 2;
  3582. }
  3583. buffer_info->page_dma =
  3584. pci_map_page(pdev, buffer_info->page,
  3585. buffer_info->page_offset,
  3586. PAGE_SIZE / 2,
  3587. PCI_DMA_FROMDEVICE);
  3588. }
  3589. if (!buffer_info->skb) {
  3590. skb = netdev_alloc_skb(netdev, bufsz);
  3591. if (!skb) {
  3592. adapter->alloc_rx_buff_failed++;
  3593. goto no_buffers;
  3594. }
  3595. /* Make buffer alignment 2 beyond a 16 byte boundary
  3596. * this will result in a 16 byte aligned IP header after
  3597. * the 14 byte MAC header is removed
  3598. */
  3599. skb_reserve(skb, NET_IP_ALIGN);
  3600. buffer_info->skb = skb;
  3601. buffer_info->dma = pci_map_single(pdev, skb->data,
  3602. bufsz,
  3603. PCI_DMA_FROMDEVICE);
  3604. }
  3605. /* Refresh the desc even if buffer_addrs didn't change because
  3606. * each write-back erases this info. */
  3607. if (adapter->rx_ps_hdr_size) {
  3608. rx_desc->read.pkt_addr =
  3609. cpu_to_le64(buffer_info->page_dma);
  3610. rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
  3611. } else {
  3612. rx_desc->read.pkt_addr =
  3613. cpu_to_le64(buffer_info->dma);
  3614. rx_desc->read.hdr_addr = 0;
  3615. }
  3616. i++;
  3617. if (i == rx_ring->count)
  3618. i = 0;
  3619. buffer_info = &rx_ring->buffer_info[i];
  3620. }
  3621. no_buffers:
  3622. if (rx_ring->next_to_use != i) {
  3623. rx_ring->next_to_use = i;
  3624. if (i == 0)
  3625. i = (rx_ring->count - 1);
  3626. else
  3627. i--;
  3628. /* Force memory writes to complete before letting h/w
  3629. * know there are new descriptors to fetch. (Only
  3630. * applicable for weak-ordered memory model archs,
  3631. * such as IA-64). */
  3632. wmb();
  3633. writel(i, adapter->hw.hw_addr + rx_ring->tail);
  3634. }
  3635. }
  3636. /**
  3637. * igb_mii_ioctl -
  3638. * @netdev:
  3639. * @ifreq:
  3640. * @cmd:
  3641. **/
  3642. static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  3643. {
  3644. struct igb_adapter *adapter = netdev_priv(netdev);
  3645. struct mii_ioctl_data *data = if_mii(ifr);
  3646. if (adapter->hw.phy.media_type != e1000_media_type_copper)
  3647. return -EOPNOTSUPP;
  3648. switch (cmd) {
  3649. case SIOCGMIIPHY:
  3650. data->phy_id = adapter->hw.phy.addr;
  3651. break;
  3652. case SIOCGMIIREG:
  3653. if (!capable(CAP_NET_ADMIN))
  3654. return -EPERM;
  3655. if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
  3656. &data->val_out))
  3657. return -EIO;
  3658. break;
  3659. case SIOCSMIIREG:
  3660. default:
  3661. return -EOPNOTSUPP;
  3662. }
  3663. return 0;
  3664. }
  3665. /**
  3666. * igb_hwtstamp_ioctl - control hardware time stamping
  3667. * @netdev:
  3668. * @ifreq:
  3669. * @cmd:
  3670. *
  3671. * Outgoing time stamping can be enabled and disabled. Play nice and
  3672. * disable it when requested, although it shouldn't case any overhead
  3673. * when no packet needs it. At most one packet in the queue may be
  3674. * marked for time stamping, otherwise it would be impossible to tell
  3675. * for sure to which packet the hardware time stamp belongs.
  3676. *
  3677. * Incoming time stamping has to be configured via the hardware
  3678. * filters. Not all combinations are supported, in particular event
  3679. * type has to be specified. Matching the kind of event packet is
  3680. * not supported, with the exception of "all V2 events regardless of
  3681. * level 2 or 4".
  3682. *
  3683. **/
  3684. static int igb_hwtstamp_ioctl(struct net_device *netdev,
  3685. struct ifreq *ifr, int cmd)
  3686. {
  3687. struct igb_adapter *adapter = netdev_priv(netdev);
  3688. struct e1000_hw *hw = &adapter->hw;
  3689. struct hwtstamp_config config;
  3690. u32 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
  3691. u32 tsync_rx_ctl_bit = E1000_TSYNCRXCTL_ENABLED;
  3692. u32 tsync_rx_ctl_type = 0;
  3693. u32 tsync_rx_cfg = 0;
  3694. int is_l4 = 0;
  3695. int is_l2 = 0;
  3696. short port = 319; /* PTP */
  3697. u32 regval;
  3698. if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
  3699. return -EFAULT;
  3700. /* reserved for future extensions */
  3701. if (config.flags)
  3702. return -EINVAL;
  3703. switch (config.tx_type) {
  3704. case HWTSTAMP_TX_OFF:
  3705. tsync_tx_ctl_bit = 0;
  3706. break;
  3707. case HWTSTAMP_TX_ON:
  3708. tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
  3709. break;
  3710. default:
  3711. return -ERANGE;
  3712. }
  3713. switch (config.rx_filter) {
  3714. case HWTSTAMP_FILTER_NONE:
  3715. tsync_rx_ctl_bit = 0;
  3716. break;
  3717. case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
  3718. case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
  3719. case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
  3720. case HWTSTAMP_FILTER_ALL:
  3721. /*
  3722. * register TSYNCRXCFG must be set, therefore it is not
  3723. * possible to time stamp both Sync and Delay_Req messages
  3724. * => fall back to time stamping all packets
  3725. */
  3726. tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_ALL;
  3727. config.rx_filter = HWTSTAMP_FILTER_ALL;
  3728. break;
  3729. case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
  3730. tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
  3731. tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
  3732. is_l4 = 1;
  3733. break;
  3734. case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
  3735. tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
  3736. tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
  3737. is_l4 = 1;
  3738. break;
  3739. case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
  3740. case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
  3741. tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
  3742. tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
  3743. is_l2 = 1;
  3744. is_l4 = 1;
  3745. config.rx_filter = HWTSTAMP_FILTER_SOME;
  3746. break;
  3747. case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
  3748. case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
  3749. tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
  3750. tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
  3751. is_l2 = 1;
  3752. is_l4 = 1;
  3753. config.rx_filter = HWTSTAMP_FILTER_SOME;
  3754. break;
  3755. case HWTSTAMP_FILTER_PTP_V2_EVENT:
  3756. case HWTSTAMP_FILTER_PTP_V2_SYNC:
  3757. case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
  3758. tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_EVENT_V2;
  3759. config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
  3760. is_l2 = 1;
  3761. break;
  3762. default:
  3763. return -ERANGE;
  3764. }
  3765. /* enable/disable TX */
  3766. regval = rd32(E1000_TSYNCTXCTL);
  3767. regval = (regval & ~E1000_TSYNCTXCTL_ENABLED) | tsync_tx_ctl_bit;
  3768. wr32(E1000_TSYNCTXCTL, regval);
  3769. /* enable/disable RX, define which PTP packets are time stamped */
  3770. regval = rd32(E1000_TSYNCRXCTL);
  3771. regval = (regval & ~E1000_TSYNCRXCTL_ENABLED) | tsync_rx_ctl_bit;
  3772. regval = (regval & ~0xE) | tsync_rx_ctl_type;
  3773. wr32(E1000_TSYNCRXCTL, regval);
  3774. wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
  3775. /*
  3776. * Ethertype Filter Queue Filter[0][15:0] = 0x88F7
  3777. * (Ethertype to filter on)
  3778. * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter)
  3779. * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable Timestamping)
  3780. */
  3781. wr32(E1000_ETQF0, is_l2 ? 0x440088f7 : 0);
  3782. /* L4 Queue Filter[0]: only filter by source and destination port */
  3783. wr32(E1000_SPQF0, htons(port));
  3784. wr32(E1000_IMIREXT(0), is_l4 ?
  3785. ((1<<12) | (1<<19) /* bypass size and control flags */) : 0);
  3786. wr32(E1000_IMIR(0), is_l4 ?
  3787. (htons(port)
  3788. | (0<<16) /* immediate interrupt disabled */
  3789. | 0 /* (1<<17) bit cleared: do not bypass
  3790. destination port check */)
  3791. : 0);
  3792. wr32(E1000_FTQF0, is_l4 ?
  3793. (0x11 /* UDP */
  3794. | (1<<15) /* VF not compared */
  3795. | (1<<27) /* Enable Timestamping */
  3796. | (7<<28) /* only source port filter enabled,
  3797. source/target address and protocol
  3798. masked */)
  3799. : ((1<<15) | (15<<28) /* all mask bits set = filter not
  3800. enabled */));
  3801. wrfl();
  3802. adapter->hwtstamp_config = config;
  3803. /* clear TX/RX time stamp registers, just to be sure */
  3804. regval = rd32(E1000_TXSTMPH);
  3805. regval = rd32(E1000_RXSTMPH);
  3806. return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
  3807. -EFAULT : 0;
  3808. }
  3809. /**
  3810. * igb_ioctl -
  3811. * @netdev:
  3812. * @ifreq:
  3813. * @cmd:
  3814. **/
  3815. static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  3816. {
  3817. switch (cmd) {
  3818. case SIOCGMIIPHY:
  3819. case SIOCGMIIREG:
  3820. case SIOCSMIIREG:
  3821. return igb_mii_ioctl(netdev, ifr, cmd);
  3822. case SIOCSHWTSTAMP:
  3823. return igb_hwtstamp_ioctl(netdev, ifr, cmd);
  3824. default:
  3825. return -EOPNOTSUPP;
  3826. }
  3827. }
  3828. static void igb_vlan_rx_register(struct net_device *netdev,
  3829. struct vlan_group *grp)
  3830. {
  3831. struct igb_adapter *adapter = netdev_priv(netdev);
  3832. struct e1000_hw *hw = &adapter->hw;
  3833. u32 ctrl, rctl;
  3834. igb_irq_disable(adapter);
  3835. adapter->vlgrp = grp;
  3836. if (grp) {
  3837. /* enable VLAN tag insert/strip */
  3838. ctrl = rd32(E1000_CTRL);
  3839. ctrl |= E1000_CTRL_VME;
  3840. wr32(E1000_CTRL, ctrl);
  3841. /* enable VLAN receive filtering */
  3842. rctl = rd32(E1000_RCTL);
  3843. rctl &= ~E1000_RCTL_CFIEN;
  3844. wr32(E1000_RCTL, rctl);
  3845. igb_update_mng_vlan(adapter);
  3846. wr32(E1000_RLPML,
  3847. adapter->max_frame_size + VLAN_TAG_SIZE);
  3848. } else {
  3849. /* disable VLAN tag insert/strip */
  3850. ctrl = rd32(E1000_CTRL);
  3851. ctrl &= ~E1000_CTRL_VME;
  3852. wr32(E1000_CTRL, ctrl);
  3853. if (adapter->mng_vlan_id != (u16)IGB_MNG_VLAN_NONE) {
  3854. igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
  3855. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  3856. }
  3857. wr32(E1000_RLPML,
  3858. adapter->max_frame_size);
  3859. }
  3860. if (!test_bit(__IGB_DOWN, &adapter->state))
  3861. igb_irq_enable(adapter);
  3862. }
  3863. static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
  3864. {
  3865. struct igb_adapter *adapter = netdev_priv(netdev);
  3866. struct e1000_hw *hw = &adapter->hw;
  3867. u32 vfta, index;
  3868. if ((hw->mng_cookie.status &
  3869. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
  3870. (vid == adapter->mng_vlan_id))
  3871. return;
  3872. /* add VID to filter table */
  3873. index = (vid >> 5) & 0x7F;
  3874. vfta = array_rd32(E1000_VFTA, index);
  3875. vfta |= (1 << (vid & 0x1F));
  3876. igb_write_vfta(&adapter->hw, index, vfta);
  3877. }
  3878. static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
  3879. {
  3880. struct igb_adapter *adapter = netdev_priv(netdev);
  3881. struct e1000_hw *hw = &adapter->hw;
  3882. u32 vfta, index;
  3883. igb_irq_disable(adapter);
  3884. vlan_group_set_device(adapter->vlgrp, vid, NULL);
  3885. if (!test_bit(__IGB_DOWN, &adapter->state))
  3886. igb_irq_enable(adapter);
  3887. if ((adapter->hw.mng_cookie.status &
  3888. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
  3889. (vid == adapter->mng_vlan_id)) {
  3890. /* release control to f/w */
  3891. igb_release_hw_control(adapter);
  3892. return;
  3893. }
  3894. /* remove VID from filter table */
  3895. index = (vid >> 5) & 0x7F;
  3896. vfta = array_rd32(E1000_VFTA, index);
  3897. vfta &= ~(1 << (vid & 0x1F));
  3898. igb_write_vfta(&adapter->hw, index, vfta);
  3899. }
  3900. static void igb_restore_vlan(struct igb_adapter *adapter)
  3901. {
  3902. igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
  3903. if (adapter->vlgrp) {
  3904. u16 vid;
  3905. for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
  3906. if (!vlan_group_get_device(adapter->vlgrp, vid))
  3907. continue;
  3908. igb_vlan_rx_add_vid(adapter->netdev, vid);
  3909. }
  3910. }
  3911. }
  3912. int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
  3913. {
  3914. struct e1000_mac_info *mac = &adapter->hw.mac;
  3915. mac->autoneg = 0;
  3916. /* Fiber NICs only allow 1000 gbps Full duplex */
  3917. if ((adapter->hw.phy.media_type == e1000_media_type_fiber) &&
  3918. spddplx != (SPEED_1000 + DUPLEX_FULL)) {
  3919. dev_err(&adapter->pdev->dev,
  3920. "Unsupported Speed/Duplex configuration\n");
  3921. return -EINVAL;
  3922. }
  3923. switch (spddplx) {
  3924. case SPEED_10 + DUPLEX_HALF:
  3925. mac->forced_speed_duplex = ADVERTISE_10_HALF;
  3926. break;
  3927. case SPEED_10 + DUPLEX_FULL:
  3928. mac->forced_speed_duplex = ADVERTISE_10_FULL;
  3929. break;
  3930. case SPEED_100 + DUPLEX_HALF:
  3931. mac->forced_speed_duplex = ADVERTISE_100_HALF;
  3932. break;
  3933. case SPEED_100 + DUPLEX_FULL:
  3934. mac->forced_speed_duplex = ADVERTISE_100_FULL;
  3935. break;
  3936. case SPEED_1000 + DUPLEX_FULL:
  3937. mac->autoneg = 1;
  3938. adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
  3939. break;
  3940. case SPEED_1000 + DUPLEX_HALF: /* not supported */
  3941. default:
  3942. dev_err(&adapter->pdev->dev,
  3943. "Unsupported Speed/Duplex configuration\n");
  3944. return -EINVAL;
  3945. }
  3946. return 0;
  3947. }
  3948. static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
  3949. {
  3950. struct net_device *netdev = pci_get_drvdata(pdev);
  3951. struct igb_adapter *adapter = netdev_priv(netdev);
  3952. struct e1000_hw *hw = &adapter->hw;
  3953. u32 ctrl, rctl, status;
  3954. u32 wufc = adapter->wol;
  3955. #ifdef CONFIG_PM
  3956. int retval = 0;
  3957. #endif
  3958. netif_device_detach(netdev);
  3959. if (netif_running(netdev))
  3960. igb_close(netdev);
  3961. igb_reset_interrupt_capability(adapter);
  3962. igb_free_queues(adapter);
  3963. #ifdef CONFIG_PM
  3964. retval = pci_save_state(pdev);
  3965. if (retval)
  3966. return retval;
  3967. #endif
  3968. status = rd32(E1000_STATUS);
  3969. if (status & E1000_STATUS_LU)
  3970. wufc &= ~E1000_WUFC_LNKC;
  3971. if (wufc) {
  3972. igb_setup_rctl(adapter);
  3973. igb_set_multi(netdev);
  3974. /* turn on all-multi mode if wake on multicast is enabled */
  3975. if (wufc & E1000_WUFC_MC) {
  3976. rctl = rd32(E1000_RCTL);
  3977. rctl |= E1000_RCTL_MPE;
  3978. wr32(E1000_RCTL, rctl);
  3979. }
  3980. ctrl = rd32(E1000_CTRL);
  3981. /* advertise wake from D3Cold */
  3982. #define E1000_CTRL_ADVD3WUC 0x00100000
  3983. /* phy power management enable */
  3984. #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
  3985. ctrl |= E1000_CTRL_ADVD3WUC;
  3986. wr32(E1000_CTRL, ctrl);
  3987. /* Allow time for pending master requests to run */
  3988. igb_disable_pcie_master(&adapter->hw);
  3989. wr32(E1000_WUC, E1000_WUC_PME_EN);
  3990. wr32(E1000_WUFC, wufc);
  3991. } else {
  3992. wr32(E1000_WUC, 0);
  3993. wr32(E1000_WUFC, 0);
  3994. }
  3995. /* make sure adapter isn't asleep if manageability/wol is enabled */
  3996. if (wufc || adapter->en_mng_pt) {
  3997. pci_enable_wake(pdev, PCI_D3hot, 1);
  3998. pci_enable_wake(pdev, PCI_D3cold, 1);
  3999. } else {
  4000. igb_shutdown_fiber_serdes_link_82575(hw);
  4001. pci_enable_wake(pdev, PCI_D3hot, 0);
  4002. pci_enable_wake(pdev, PCI_D3cold, 0);
  4003. }
  4004. /* Release control of h/w to f/w. If f/w is AMT enabled, this
  4005. * would have already happened in close and is redundant. */
  4006. igb_release_hw_control(adapter);
  4007. pci_disable_device(pdev);
  4008. pci_set_power_state(pdev, pci_choose_state(pdev, state));
  4009. return 0;
  4010. }
  4011. #ifdef CONFIG_PM
  4012. static int igb_resume(struct pci_dev *pdev)
  4013. {
  4014. struct net_device *netdev = pci_get_drvdata(pdev);
  4015. struct igb_adapter *adapter = netdev_priv(netdev);
  4016. struct e1000_hw *hw = &adapter->hw;
  4017. u32 err;
  4018. pci_set_power_state(pdev, PCI_D0);
  4019. pci_restore_state(pdev);
  4020. err = pci_enable_device_mem(pdev);
  4021. if (err) {
  4022. dev_err(&pdev->dev,
  4023. "igb: Cannot enable PCI device from suspend\n");
  4024. return err;
  4025. }
  4026. pci_set_master(pdev);
  4027. pci_enable_wake(pdev, PCI_D3hot, 0);
  4028. pci_enable_wake(pdev, PCI_D3cold, 0);
  4029. igb_set_interrupt_capability(adapter);
  4030. if (igb_alloc_queues(adapter)) {
  4031. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  4032. return -ENOMEM;
  4033. }
  4034. /* e1000_power_up_phy(adapter); */
  4035. igb_reset(adapter);
  4036. /* let the f/w know that the h/w is now under the control of the
  4037. * driver. */
  4038. igb_get_hw_control(adapter);
  4039. wr32(E1000_WUS, ~0);
  4040. if (netif_running(netdev)) {
  4041. err = igb_open(netdev);
  4042. if (err)
  4043. return err;
  4044. }
  4045. netif_device_attach(netdev);
  4046. return 0;
  4047. }
  4048. #endif
  4049. static void igb_shutdown(struct pci_dev *pdev)
  4050. {
  4051. igb_suspend(pdev, PMSG_SUSPEND);
  4052. }
  4053. #ifdef CONFIG_NET_POLL_CONTROLLER
  4054. /*
  4055. * Polling 'interrupt' - used by things like netconsole to send skbs
  4056. * without having to re-enable interrupts. It's not called while
  4057. * the interrupt routine is executing.
  4058. */
  4059. static void igb_netpoll(struct net_device *netdev)
  4060. {
  4061. struct igb_adapter *adapter = netdev_priv(netdev);
  4062. struct e1000_hw *hw = &adapter->hw;
  4063. int i;
  4064. if (!adapter->msix_entries) {
  4065. igb_irq_disable(adapter);
  4066. napi_schedule(&adapter->rx_ring[0].napi);
  4067. return;
  4068. }
  4069. for (i = 0; i < adapter->num_tx_queues; i++) {
  4070. struct igb_ring *tx_ring = &adapter->tx_ring[i];
  4071. wr32(E1000_EIMC, tx_ring->eims_value);
  4072. igb_clean_tx_irq(tx_ring);
  4073. wr32(E1000_EIMS, tx_ring->eims_value);
  4074. }
  4075. for (i = 0; i < adapter->num_rx_queues; i++) {
  4076. struct igb_ring *rx_ring = &adapter->rx_ring[i];
  4077. wr32(E1000_EIMC, rx_ring->eims_value);
  4078. napi_schedule(&rx_ring->napi);
  4079. }
  4080. }
  4081. #endif /* CONFIG_NET_POLL_CONTROLLER */
  4082. /**
  4083. * igb_io_error_detected - called when PCI error is detected
  4084. * @pdev: Pointer to PCI device
  4085. * @state: The current pci connection state
  4086. *
  4087. * This function is called after a PCI bus error affecting
  4088. * this device has been detected.
  4089. */
  4090. static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
  4091. pci_channel_state_t state)
  4092. {
  4093. struct net_device *netdev = pci_get_drvdata(pdev);
  4094. struct igb_adapter *adapter = netdev_priv(netdev);
  4095. netif_device_detach(netdev);
  4096. if (netif_running(netdev))
  4097. igb_down(adapter);
  4098. pci_disable_device(pdev);
  4099. /* Request a slot slot reset. */
  4100. return PCI_ERS_RESULT_NEED_RESET;
  4101. }
  4102. /**
  4103. * igb_io_slot_reset - called after the pci bus has been reset.
  4104. * @pdev: Pointer to PCI device
  4105. *
  4106. * Restart the card from scratch, as if from a cold-boot. Implementation
  4107. * resembles the first-half of the igb_resume routine.
  4108. */
  4109. static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
  4110. {
  4111. struct net_device *netdev = pci_get_drvdata(pdev);
  4112. struct igb_adapter *adapter = netdev_priv(netdev);
  4113. struct e1000_hw *hw = &adapter->hw;
  4114. pci_ers_result_t result;
  4115. int err;
  4116. if (pci_enable_device_mem(pdev)) {
  4117. dev_err(&pdev->dev,
  4118. "Cannot re-enable PCI device after reset.\n");
  4119. result = PCI_ERS_RESULT_DISCONNECT;
  4120. } else {
  4121. pci_set_master(pdev);
  4122. pci_restore_state(pdev);
  4123. pci_enable_wake(pdev, PCI_D3hot, 0);
  4124. pci_enable_wake(pdev, PCI_D3cold, 0);
  4125. igb_reset(adapter);
  4126. wr32(E1000_WUS, ~0);
  4127. result = PCI_ERS_RESULT_RECOVERED;
  4128. }
  4129. err = pci_cleanup_aer_uncorrect_error_status(pdev);
  4130. if (err) {
  4131. dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
  4132. "failed 0x%0x\n", err);
  4133. /* non-fatal, continue */
  4134. }
  4135. return result;
  4136. }
  4137. /**
  4138. * igb_io_resume - called when traffic can start flowing again.
  4139. * @pdev: Pointer to PCI device
  4140. *
  4141. * This callback is called when the error recovery driver tells us that
  4142. * its OK to resume normal operation. Implementation resembles the
  4143. * second-half of the igb_resume routine.
  4144. */
  4145. static void igb_io_resume(struct pci_dev *pdev)
  4146. {
  4147. struct net_device *netdev = pci_get_drvdata(pdev);
  4148. struct igb_adapter *adapter = netdev_priv(netdev);
  4149. if (netif_running(netdev)) {
  4150. if (igb_up(adapter)) {
  4151. dev_err(&pdev->dev, "igb_up failed after reset\n");
  4152. return;
  4153. }
  4154. }
  4155. netif_device_attach(netdev);
  4156. /* let the f/w know that the h/w is now under the control of the
  4157. * driver. */
  4158. igb_get_hw_control(adapter);
  4159. }
  4160. /* igb_main.c */