igb_main.c 111 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124
  1. /*******************************************************************************
  2. Intel(R) Gigabit Ethernet Linux driver
  3. Copyright(c) 2007 Intel Corporation.
  4. This program is free software; you can redistribute it and/or modify it
  5. under the terms and conditions of the GNU General Public License,
  6. version 2, as published by the Free Software Foundation.
  7. This program is distributed in the hope it will be useful, but WITHOUT
  8. ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  9. FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  10. more details.
  11. You should have received a copy of the GNU General Public License along with
  12. this program; if not, write to the Free Software Foundation, Inc.,
  13. 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  14. The full GNU General Public License is included in this distribution in
  15. the file called "COPYING".
  16. Contact Information:
  17. e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  18. Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  19. *******************************************************************************/
  20. #include <linux/module.h>
  21. #include <linux/types.h>
  22. #include <linux/init.h>
  23. #include <linux/vmalloc.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/netdevice.h>
  26. #include <linux/ipv6.h>
  27. #include <net/checksum.h>
  28. #include <net/ip6_checksum.h>
  29. #include <linux/mii.h>
  30. #include <linux/ethtool.h>
  31. #include <linux/if_vlan.h>
  32. #include <linux/pci.h>
  33. #include <linux/delay.h>
  34. #include <linux/interrupt.h>
  35. #include <linux/if_ether.h>
  36. #include "igb.h"
  37. #define DRV_VERSION "1.0.8-k2"
  38. char igb_driver_name[] = "igb";
  39. char igb_driver_version[] = DRV_VERSION;
  40. static const char igb_driver_string[] =
  41. "Intel(R) Gigabit Ethernet Network Driver";
  42. static const char igb_copyright[] = "Copyright (c) 2007 Intel Corporation.";
  43. static const struct e1000_info *igb_info_tbl[] = {
  44. [board_82575] = &e1000_82575_info,
  45. };
  46. static struct pci_device_id igb_pci_tbl[] = {
  47. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  48. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  49. { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  50. /* required last entry */
  51. {0, }
  52. };
  53. MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  54. void igb_reset(struct igb_adapter *);
  55. static int igb_setup_all_tx_resources(struct igb_adapter *);
  56. static int igb_setup_all_rx_resources(struct igb_adapter *);
  57. static void igb_free_all_tx_resources(struct igb_adapter *);
  58. static void igb_free_all_rx_resources(struct igb_adapter *);
  59. static void igb_free_tx_resources(struct igb_adapter *, struct igb_ring *);
  60. static void igb_free_rx_resources(struct igb_adapter *, struct igb_ring *);
  61. void igb_update_stats(struct igb_adapter *);
  62. static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  63. static void __devexit igb_remove(struct pci_dev *pdev);
  64. static int igb_sw_init(struct igb_adapter *);
  65. static int igb_open(struct net_device *);
  66. static int igb_close(struct net_device *);
  67. static void igb_configure_tx(struct igb_adapter *);
  68. static void igb_configure_rx(struct igb_adapter *);
  69. static void igb_setup_rctl(struct igb_adapter *);
  70. static void igb_clean_all_tx_rings(struct igb_adapter *);
  71. static void igb_clean_all_rx_rings(struct igb_adapter *);
  72. static void igb_clean_tx_ring(struct igb_adapter *, struct igb_ring *);
  73. static void igb_clean_rx_ring(struct igb_adapter *, struct igb_ring *);
  74. static void igb_set_multi(struct net_device *);
  75. static void igb_update_phy_info(unsigned long);
  76. static void igb_watchdog(unsigned long);
  77. static void igb_watchdog_task(struct work_struct *);
  78. static int igb_xmit_frame_ring_adv(struct sk_buff *, struct net_device *,
  79. struct igb_ring *);
  80. static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
  81. static struct net_device_stats *igb_get_stats(struct net_device *);
  82. static int igb_change_mtu(struct net_device *, int);
  83. static int igb_set_mac(struct net_device *, void *);
  84. static irqreturn_t igb_intr(int irq, void *);
  85. static irqreturn_t igb_intr_msi(int irq, void *);
  86. static irqreturn_t igb_msix_other(int irq, void *);
  87. static irqreturn_t igb_msix_rx(int irq, void *);
  88. static irqreturn_t igb_msix_tx(int irq, void *);
  89. static int igb_clean_rx_ring_msix(struct napi_struct *, int);
  90. static bool igb_clean_tx_irq(struct igb_adapter *, struct igb_ring *);
  91. static int igb_clean(struct napi_struct *, int);
  92. static bool igb_clean_rx_irq_adv(struct igb_adapter *,
  93. struct igb_ring *, int *, int);
  94. static void igb_alloc_rx_buffers_adv(struct igb_adapter *,
  95. struct igb_ring *, int);
  96. static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
  97. static void igb_tx_timeout(struct net_device *);
  98. static void igb_reset_task(struct work_struct *);
  99. static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
  100. static void igb_vlan_rx_add_vid(struct net_device *, u16);
  101. static void igb_vlan_rx_kill_vid(struct net_device *, u16);
  102. static void igb_restore_vlan(struct igb_adapter *);
  103. static int igb_suspend(struct pci_dev *, pm_message_t);
  104. #ifdef CONFIG_PM
  105. static int igb_resume(struct pci_dev *);
  106. #endif
  107. static void igb_shutdown(struct pci_dev *);
  108. #ifdef CONFIG_NET_POLL_CONTROLLER
  109. /* for netdump / net console */
  110. static void igb_netpoll(struct net_device *);
  111. #endif
  112. static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
  113. pci_channel_state_t);
  114. static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
  115. static void igb_io_resume(struct pci_dev *);
  116. static struct pci_error_handlers igb_err_handler = {
  117. .error_detected = igb_io_error_detected,
  118. .slot_reset = igb_io_slot_reset,
  119. .resume = igb_io_resume,
  120. };
  121. static struct pci_driver igb_driver = {
  122. .name = igb_driver_name,
  123. .id_table = igb_pci_tbl,
  124. .probe = igb_probe,
  125. .remove = __devexit_p(igb_remove),
  126. #ifdef CONFIG_PM
  127. /* Power Managment Hooks */
  128. .suspend = igb_suspend,
  129. .resume = igb_resume,
  130. #endif
  131. .shutdown = igb_shutdown,
  132. .err_handler = &igb_err_handler
  133. };
  134. MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  135. MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
  136. MODULE_LICENSE("GPL");
  137. MODULE_VERSION(DRV_VERSION);
  138. #ifdef DEBUG
  139. /**
  140. * igb_get_hw_dev_name - return device name string
  141. * used by hardware layer to print debugging information
  142. **/
  143. char *igb_get_hw_dev_name(struct e1000_hw *hw)
  144. {
  145. struct igb_adapter *adapter = hw->back;
  146. return adapter->netdev->name;
  147. }
  148. #endif
  149. /**
  150. * igb_init_module - Driver Registration Routine
  151. *
  152. * igb_init_module is the first routine called when the driver is
  153. * loaded. All it does is register with the PCI subsystem.
  154. **/
  155. static int __init igb_init_module(void)
  156. {
  157. int ret;
  158. printk(KERN_INFO "%s - version %s\n",
  159. igb_driver_string, igb_driver_version);
  160. printk(KERN_INFO "%s\n", igb_copyright);
  161. ret = pci_register_driver(&igb_driver);
  162. return ret;
  163. }
  164. module_init(igb_init_module);
  165. /**
  166. * igb_exit_module - Driver Exit Cleanup Routine
  167. *
  168. * igb_exit_module is called just before the driver is removed
  169. * from memory.
  170. **/
  171. static void __exit igb_exit_module(void)
  172. {
  173. pci_unregister_driver(&igb_driver);
  174. }
  175. module_exit(igb_exit_module);
  176. /**
  177. * igb_alloc_queues - Allocate memory for all rings
  178. * @adapter: board private structure to initialize
  179. *
  180. * We allocate one ring per queue at run-time since we don't know the
  181. * number of queues at compile-time.
  182. **/
  183. static int igb_alloc_queues(struct igb_adapter *adapter)
  184. {
  185. int i;
  186. adapter->tx_ring = kcalloc(adapter->num_tx_queues,
  187. sizeof(struct igb_ring), GFP_KERNEL);
  188. if (!adapter->tx_ring)
  189. return -ENOMEM;
  190. adapter->rx_ring = kcalloc(adapter->num_rx_queues,
  191. sizeof(struct igb_ring), GFP_KERNEL);
  192. if (!adapter->rx_ring) {
  193. kfree(adapter->tx_ring);
  194. return -ENOMEM;
  195. }
  196. for (i = 0; i < adapter->num_rx_queues; i++) {
  197. struct igb_ring *ring = &(adapter->rx_ring[i]);
  198. ring->adapter = adapter;
  199. ring->itr_register = E1000_ITR;
  200. if (!ring->napi.poll)
  201. netif_napi_add(adapter->netdev, &ring->napi, igb_clean,
  202. adapter->napi.weight /
  203. adapter->num_rx_queues);
  204. }
  205. return 0;
  206. }
  207. #define IGB_N0_QUEUE -1
  208. static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue,
  209. int tx_queue, int msix_vector)
  210. {
  211. u32 msixbm = 0;
  212. struct e1000_hw *hw = &adapter->hw;
  213. /* The 82575 assigns vectors using a bitmask, which matches the
  214. bitmask for the EICR/EIMS/EIMC registers. To assign one
  215. or more queues to a vector, we write the appropriate bits
  216. into the MSIXBM register for that vector. */
  217. if (rx_queue > IGB_N0_QUEUE) {
  218. msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
  219. adapter->rx_ring[rx_queue].eims_value = msixbm;
  220. }
  221. if (tx_queue > IGB_N0_QUEUE) {
  222. msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
  223. adapter->tx_ring[tx_queue].eims_value =
  224. E1000_EICR_TX_QUEUE0 << tx_queue;
  225. }
  226. array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
  227. }
  228. /**
  229. * igb_configure_msix - Configure MSI-X hardware
  230. *
  231. * igb_configure_msix sets up the hardware to properly
  232. * generate MSI-X interrupts.
  233. **/
  234. static void igb_configure_msix(struct igb_adapter *adapter)
  235. {
  236. u32 tmp;
  237. int i, vector = 0;
  238. struct e1000_hw *hw = &adapter->hw;
  239. adapter->eims_enable_mask = 0;
  240. for (i = 0; i < adapter->num_tx_queues; i++) {
  241. struct igb_ring *tx_ring = &adapter->tx_ring[i];
  242. igb_assign_vector(adapter, IGB_N0_QUEUE, i, vector++);
  243. adapter->eims_enable_mask |= tx_ring->eims_value;
  244. if (tx_ring->itr_val)
  245. writel(1000000000 / (tx_ring->itr_val * 256),
  246. hw->hw_addr + tx_ring->itr_register);
  247. else
  248. writel(1, hw->hw_addr + tx_ring->itr_register);
  249. }
  250. for (i = 0; i < adapter->num_rx_queues; i++) {
  251. struct igb_ring *rx_ring = &adapter->rx_ring[i];
  252. igb_assign_vector(adapter, i, IGB_N0_QUEUE, vector++);
  253. adapter->eims_enable_mask |= rx_ring->eims_value;
  254. if (rx_ring->itr_val)
  255. writel(1000000000 / (rx_ring->itr_val * 256),
  256. hw->hw_addr + rx_ring->itr_register);
  257. else
  258. writel(1, hw->hw_addr + rx_ring->itr_register);
  259. }
  260. /* set vector for other causes, i.e. link changes */
  261. array_wr32(E1000_MSIXBM(0), vector++,
  262. E1000_EIMS_OTHER);
  263. /* disable IAM for ICR interrupt bits */
  264. wr32(E1000_IAM, 0);
  265. tmp = rd32(E1000_CTRL_EXT);
  266. /* enable MSI-X PBA support*/
  267. tmp |= E1000_CTRL_EXT_PBA_CLR;
  268. /* Auto-Mask interrupts upon ICR read. */
  269. tmp |= E1000_CTRL_EXT_EIAME;
  270. tmp |= E1000_CTRL_EXT_IRCA;
  271. wr32(E1000_CTRL_EXT, tmp);
  272. adapter->eims_enable_mask |= E1000_EIMS_OTHER;
  273. wrfl();
  274. }
  275. /**
  276. * igb_request_msix - Initialize MSI-X interrupts
  277. *
  278. * igb_request_msix allocates MSI-X vectors and requests interrupts from the
  279. * kernel.
  280. **/
  281. static int igb_request_msix(struct igb_adapter *adapter)
  282. {
  283. struct net_device *netdev = adapter->netdev;
  284. int i, err = 0, vector = 0;
  285. vector = 0;
  286. for (i = 0; i < adapter->num_tx_queues; i++) {
  287. struct igb_ring *ring = &(adapter->tx_ring[i]);
  288. sprintf(ring->name, "%s-tx%d", netdev->name, i);
  289. err = request_irq(adapter->msix_entries[vector].vector,
  290. &igb_msix_tx, 0, ring->name,
  291. &(adapter->tx_ring[i]));
  292. if (err)
  293. goto out;
  294. ring->itr_register = E1000_EITR(0) + (vector << 2);
  295. ring->itr_val = adapter->itr;
  296. vector++;
  297. }
  298. for (i = 0; i < adapter->num_rx_queues; i++) {
  299. struct igb_ring *ring = &(adapter->rx_ring[i]);
  300. if (strlen(netdev->name) < (IFNAMSIZ - 5))
  301. sprintf(ring->name, "%s-rx%d", netdev->name, i);
  302. else
  303. memcpy(ring->name, netdev->name, IFNAMSIZ);
  304. err = request_irq(adapter->msix_entries[vector].vector,
  305. &igb_msix_rx, 0, ring->name,
  306. &(adapter->rx_ring[i]));
  307. if (err)
  308. goto out;
  309. ring->itr_register = E1000_EITR(0) + (vector << 2);
  310. ring->itr_val = adapter->itr;
  311. vector++;
  312. }
  313. err = request_irq(adapter->msix_entries[vector].vector,
  314. &igb_msix_other, 0, netdev->name, netdev);
  315. if (err)
  316. goto out;
  317. adapter->napi.poll = igb_clean_rx_ring_msix;
  318. for (i = 0; i < adapter->num_rx_queues; i++)
  319. adapter->rx_ring[i].napi.poll = adapter->napi.poll;
  320. igb_configure_msix(adapter);
  321. return 0;
  322. out:
  323. return err;
  324. }
  325. static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
  326. {
  327. if (adapter->msix_entries) {
  328. pci_disable_msix(adapter->pdev);
  329. kfree(adapter->msix_entries);
  330. adapter->msix_entries = NULL;
  331. } else if (adapter->msi_enabled)
  332. pci_disable_msi(adapter->pdev);
  333. return;
  334. }
  335. /**
  336. * igb_set_interrupt_capability - set MSI or MSI-X if supported
  337. *
  338. * Attempt to configure interrupts using the best available
  339. * capabilities of the hardware and kernel.
  340. **/
  341. static void igb_set_interrupt_capability(struct igb_adapter *adapter)
  342. {
  343. int err;
  344. int numvecs, i;
  345. numvecs = adapter->num_tx_queues + adapter->num_rx_queues + 1;
  346. adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
  347. GFP_KERNEL);
  348. if (!adapter->msix_entries)
  349. goto msi_only;
  350. for (i = 0; i < numvecs; i++)
  351. adapter->msix_entries[i].entry = i;
  352. err = pci_enable_msix(adapter->pdev,
  353. adapter->msix_entries,
  354. numvecs);
  355. if (err == 0)
  356. return;
  357. igb_reset_interrupt_capability(adapter);
  358. /* If we can't do MSI-X, try MSI */
  359. msi_only:
  360. adapter->num_rx_queues = 1;
  361. if (!pci_enable_msi(adapter->pdev))
  362. adapter->msi_enabled = 1;
  363. return;
  364. }
  365. /**
  366. * igb_request_irq - initialize interrupts
  367. *
  368. * Attempts to configure interrupts using the best available
  369. * capabilities of the hardware and kernel.
  370. **/
  371. static int igb_request_irq(struct igb_adapter *adapter)
  372. {
  373. struct net_device *netdev = adapter->netdev;
  374. struct e1000_hw *hw = &adapter->hw;
  375. int err = 0;
  376. if (adapter->msix_entries) {
  377. err = igb_request_msix(adapter);
  378. if (!err) {
  379. /* enable IAM, auto-mask,
  380. * DO NOT USE EIAM or IAM in legacy mode */
  381. wr32(E1000_IAM, IMS_ENABLE_MASK);
  382. goto request_done;
  383. }
  384. /* fall back to MSI */
  385. igb_reset_interrupt_capability(adapter);
  386. if (!pci_enable_msi(adapter->pdev))
  387. adapter->msi_enabled = 1;
  388. igb_free_all_tx_resources(adapter);
  389. igb_free_all_rx_resources(adapter);
  390. adapter->num_rx_queues = 1;
  391. igb_alloc_queues(adapter);
  392. }
  393. if (adapter->msi_enabled) {
  394. err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
  395. netdev->name, netdev);
  396. if (!err)
  397. goto request_done;
  398. /* fall back to legacy interrupts */
  399. igb_reset_interrupt_capability(adapter);
  400. adapter->msi_enabled = 0;
  401. }
  402. err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
  403. netdev->name, netdev);
  404. if (err)
  405. dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
  406. err);
  407. request_done:
  408. return err;
  409. }
  410. static void igb_free_irq(struct igb_adapter *adapter)
  411. {
  412. struct net_device *netdev = adapter->netdev;
  413. if (adapter->msix_entries) {
  414. int vector = 0, i;
  415. for (i = 0; i < adapter->num_tx_queues; i++)
  416. free_irq(adapter->msix_entries[vector++].vector,
  417. &(adapter->tx_ring[i]));
  418. for (i = 0; i < adapter->num_rx_queues; i++)
  419. free_irq(adapter->msix_entries[vector++].vector,
  420. &(adapter->rx_ring[i]));
  421. free_irq(adapter->msix_entries[vector++].vector, netdev);
  422. return;
  423. }
  424. free_irq(adapter->pdev->irq, netdev);
  425. }
  426. /**
  427. * igb_irq_disable - Mask off interrupt generation on the NIC
  428. * @adapter: board private structure
  429. **/
  430. static void igb_irq_disable(struct igb_adapter *adapter)
  431. {
  432. struct e1000_hw *hw = &adapter->hw;
  433. if (adapter->msix_entries) {
  434. wr32(E1000_EIMC, ~0);
  435. wr32(E1000_EIAC, 0);
  436. }
  437. wr32(E1000_IMC, ~0);
  438. wrfl();
  439. synchronize_irq(adapter->pdev->irq);
  440. }
  441. /**
  442. * igb_irq_enable - Enable default interrupt generation settings
  443. * @adapter: board private structure
  444. **/
  445. static void igb_irq_enable(struct igb_adapter *adapter)
  446. {
  447. struct e1000_hw *hw = &adapter->hw;
  448. if (adapter->msix_entries) {
  449. wr32(E1000_EIMS,
  450. adapter->eims_enable_mask);
  451. wr32(E1000_EIAC,
  452. adapter->eims_enable_mask);
  453. wr32(E1000_IMS, E1000_IMS_LSC);
  454. } else
  455. wr32(E1000_IMS, IMS_ENABLE_MASK);
  456. }
  457. static void igb_update_mng_vlan(struct igb_adapter *adapter)
  458. {
  459. struct net_device *netdev = adapter->netdev;
  460. u16 vid = adapter->hw.mng_cookie.vlan_id;
  461. u16 old_vid = adapter->mng_vlan_id;
  462. if (adapter->vlgrp) {
  463. if (!vlan_group_get_device(adapter->vlgrp, vid)) {
  464. if (adapter->hw.mng_cookie.status &
  465. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
  466. igb_vlan_rx_add_vid(netdev, vid);
  467. adapter->mng_vlan_id = vid;
  468. } else
  469. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  470. if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
  471. (vid != old_vid) &&
  472. !vlan_group_get_device(adapter->vlgrp, old_vid))
  473. igb_vlan_rx_kill_vid(netdev, old_vid);
  474. } else
  475. adapter->mng_vlan_id = vid;
  476. }
  477. }
  478. /**
  479. * igb_release_hw_control - release control of the h/w to f/w
  480. * @adapter: address of board private structure
  481. *
  482. * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
  483. * For ASF and Pass Through versions of f/w this means that the
  484. * driver is no longer loaded.
  485. *
  486. **/
  487. static void igb_release_hw_control(struct igb_adapter *adapter)
  488. {
  489. struct e1000_hw *hw = &adapter->hw;
  490. u32 ctrl_ext;
  491. /* Let firmware take over control of h/w */
  492. ctrl_ext = rd32(E1000_CTRL_EXT);
  493. wr32(E1000_CTRL_EXT,
  494. ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
  495. }
  496. /**
  497. * igb_get_hw_control - get control of the h/w from f/w
  498. * @adapter: address of board private structure
  499. *
  500. * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
  501. * For ASF and Pass Through versions of f/w this means that
  502. * the driver is loaded.
  503. *
  504. **/
  505. static void igb_get_hw_control(struct igb_adapter *adapter)
  506. {
  507. struct e1000_hw *hw = &adapter->hw;
  508. u32 ctrl_ext;
  509. /* Let firmware know the driver has taken over */
  510. ctrl_ext = rd32(E1000_CTRL_EXT);
  511. wr32(E1000_CTRL_EXT,
  512. ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
  513. }
  514. static void igb_init_manageability(struct igb_adapter *adapter)
  515. {
  516. struct e1000_hw *hw = &adapter->hw;
  517. if (adapter->en_mng_pt) {
  518. u32 manc2h = rd32(E1000_MANC2H);
  519. u32 manc = rd32(E1000_MANC);
  520. /* enable receiving management packets to the host */
  521. /* this will probably generate destination unreachable messages
  522. * from the host OS, but the packets will be handled on SMBUS */
  523. manc |= E1000_MANC_EN_MNG2HOST;
  524. #define E1000_MNG2HOST_PORT_623 (1 << 5)
  525. #define E1000_MNG2HOST_PORT_664 (1 << 6)
  526. manc2h |= E1000_MNG2HOST_PORT_623;
  527. manc2h |= E1000_MNG2HOST_PORT_664;
  528. wr32(E1000_MANC2H, manc2h);
  529. wr32(E1000_MANC, manc);
  530. }
  531. }
  532. /**
  533. * igb_configure - configure the hardware for RX and TX
  534. * @adapter: private board structure
  535. **/
  536. static void igb_configure(struct igb_adapter *adapter)
  537. {
  538. struct net_device *netdev = adapter->netdev;
  539. int i;
  540. igb_get_hw_control(adapter);
  541. igb_set_multi(netdev);
  542. igb_restore_vlan(adapter);
  543. igb_init_manageability(adapter);
  544. igb_configure_tx(adapter);
  545. igb_setup_rctl(adapter);
  546. igb_configure_rx(adapter);
  547. /* call IGB_DESC_UNUSED which always leaves
  548. * at least 1 descriptor unused to make sure
  549. * next_to_use != next_to_clean */
  550. for (i = 0; i < adapter->num_rx_queues; i++) {
  551. struct igb_ring *ring = &adapter->rx_ring[i];
  552. igb_alloc_rx_buffers_adv(adapter, ring, IGB_DESC_UNUSED(ring));
  553. }
  554. adapter->tx_queue_len = netdev->tx_queue_len;
  555. }
  556. /**
  557. * igb_up - Open the interface and prepare it to handle traffic
  558. * @adapter: board private structure
  559. **/
  560. int igb_up(struct igb_adapter *adapter)
  561. {
  562. struct e1000_hw *hw = &adapter->hw;
  563. int i;
  564. /* hardware has been reset, we need to reload some things */
  565. igb_configure(adapter);
  566. clear_bit(__IGB_DOWN, &adapter->state);
  567. napi_enable(&adapter->napi);
  568. if (adapter->msix_entries) {
  569. for (i = 0; i < adapter->num_rx_queues; i++)
  570. napi_enable(&adapter->rx_ring[i].napi);
  571. igb_configure_msix(adapter);
  572. }
  573. /* Clear any pending interrupts. */
  574. rd32(E1000_ICR);
  575. igb_irq_enable(adapter);
  576. /* Fire a link change interrupt to start the watchdog. */
  577. wr32(E1000_ICS, E1000_ICS_LSC);
  578. return 0;
  579. }
  580. void igb_down(struct igb_adapter *adapter)
  581. {
  582. struct e1000_hw *hw = &adapter->hw;
  583. struct net_device *netdev = adapter->netdev;
  584. u32 tctl, rctl;
  585. int i;
  586. /* signal that we're down so the interrupt handler does not
  587. * reschedule our watchdog timer */
  588. set_bit(__IGB_DOWN, &adapter->state);
  589. /* disable receives in the hardware */
  590. rctl = rd32(E1000_RCTL);
  591. wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
  592. /* flush and sleep below */
  593. netif_stop_queue(netdev);
  594. /* disable transmits in the hardware */
  595. tctl = rd32(E1000_TCTL);
  596. tctl &= ~E1000_TCTL_EN;
  597. wr32(E1000_TCTL, tctl);
  598. /* flush both disables and wait for them to finish */
  599. wrfl();
  600. msleep(10);
  601. napi_disable(&adapter->napi);
  602. if (adapter->msix_entries)
  603. for (i = 0; i < adapter->num_rx_queues; i++)
  604. napi_disable(&adapter->rx_ring[i].napi);
  605. igb_irq_disable(adapter);
  606. del_timer_sync(&adapter->watchdog_timer);
  607. del_timer_sync(&adapter->phy_info_timer);
  608. netdev->tx_queue_len = adapter->tx_queue_len;
  609. netif_carrier_off(netdev);
  610. adapter->link_speed = 0;
  611. adapter->link_duplex = 0;
  612. igb_reset(adapter);
  613. igb_clean_all_tx_rings(adapter);
  614. igb_clean_all_rx_rings(adapter);
  615. }
  616. void igb_reinit_locked(struct igb_adapter *adapter)
  617. {
  618. WARN_ON(in_interrupt());
  619. while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
  620. msleep(1);
  621. igb_down(adapter);
  622. igb_up(adapter);
  623. clear_bit(__IGB_RESETTING, &adapter->state);
  624. }
  625. void igb_reset(struct igb_adapter *adapter)
  626. {
  627. struct e1000_hw *hw = &adapter->hw;
  628. struct e1000_fc_info *fc = &adapter->hw.fc;
  629. u32 pba = 0, tx_space, min_tx_space, min_rx_space;
  630. u16 hwm;
  631. /* Repartition Pba for greater than 9k mtu
  632. * To take effect CTRL.RST is required.
  633. */
  634. pba = E1000_PBA_34K;
  635. if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
  636. /* adjust PBA for jumbo frames */
  637. wr32(E1000_PBA, pba);
  638. /* To maintain wire speed transmits, the Tx FIFO should be
  639. * large enough to accommodate two full transmit packets,
  640. * rounded up to the next 1KB and expressed in KB. Likewise,
  641. * the Rx FIFO should be large enough to accommodate at least
  642. * one full receive packet and is similarly rounded up and
  643. * expressed in KB. */
  644. pba = rd32(E1000_PBA);
  645. /* upper 16 bits has Tx packet buffer allocation size in KB */
  646. tx_space = pba >> 16;
  647. /* lower 16 bits has Rx packet buffer allocation size in KB */
  648. pba &= 0xffff;
  649. /* the tx fifo also stores 16 bytes of information about the tx
  650. * but don't include ethernet FCS because hardware appends it */
  651. min_tx_space = (adapter->max_frame_size +
  652. sizeof(struct e1000_tx_desc) -
  653. ETH_FCS_LEN) * 2;
  654. min_tx_space = ALIGN(min_tx_space, 1024);
  655. min_tx_space >>= 10;
  656. /* software strips receive CRC, so leave room for it */
  657. min_rx_space = adapter->max_frame_size;
  658. min_rx_space = ALIGN(min_rx_space, 1024);
  659. min_rx_space >>= 10;
  660. /* If current Tx allocation is less than the min Tx FIFO size,
  661. * and the min Tx FIFO size is less than the current Rx FIFO
  662. * allocation, take space away from current Rx allocation */
  663. if (tx_space < min_tx_space &&
  664. ((min_tx_space - tx_space) < pba)) {
  665. pba = pba - (min_tx_space - tx_space);
  666. /* if short on rx space, rx wins and must trump tx
  667. * adjustment */
  668. if (pba < min_rx_space)
  669. pba = min_rx_space;
  670. }
  671. }
  672. wr32(E1000_PBA, pba);
  673. /* flow control settings */
  674. /* The high water mark must be low enough to fit one full frame
  675. * (or the size used for early receive) above it in the Rx FIFO.
  676. * Set it to the lower of:
  677. * - 90% of the Rx FIFO size, or
  678. * - the full Rx FIFO size minus one full frame */
  679. hwm = min(((pba << 10) * 9 / 10),
  680. ((pba << 10) - adapter->max_frame_size));
  681. fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
  682. fc->low_water = fc->high_water - 8;
  683. fc->pause_time = 0xFFFF;
  684. fc->send_xon = 1;
  685. fc->type = fc->original_type;
  686. /* Allow time for pending master requests to run */
  687. adapter->hw.mac.ops.reset_hw(&adapter->hw);
  688. wr32(E1000_WUC, 0);
  689. if (adapter->hw.mac.ops.init_hw(&adapter->hw))
  690. dev_err(&adapter->pdev->dev, "Hardware Error\n");
  691. igb_update_mng_vlan(adapter);
  692. /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
  693. wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
  694. igb_reset_adaptive(&adapter->hw);
  695. if (adapter->hw.phy.ops.get_phy_info)
  696. adapter->hw.phy.ops.get_phy_info(&adapter->hw);
  697. }
  698. /**
  699. * igb_probe - Device Initialization Routine
  700. * @pdev: PCI device information struct
  701. * @ent: entry in igb_pci_tbl
  702. *
  703. * Returns 0 on success, negative on failure
  704. *
  705. * igb_probe initializes an adapter identified by a pci_dev structure.
  706. * The OS initialization, configuring of the adapter private structure,
  707. * and a hardware reset occur.
  708. **/
  709. static int __devinit igb_probe(struct pci_dev *pdev,
  710. const struct pci_device_id *ent)
  711. {
  712. struct net_device *netdev;
  713. struct igb_adapter *adapter;
  714. struct e1000_hw *hw;
  715. const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
  716. unsigned long mmio_start, mmio_len;
  717. static int cards_found;
  718. int i, err, pci_using_dac;
  719. u16 eeprom_data = 0;
  720. u16 eeprom_apme_mask = IGB_EEPROM_APME;
  721. u32 part_num;
  722. err = pci_enable_device(pdev);
  723. if (err)
  724. return err;
  725. pci_using_dac = 0;
  726. err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
  727. if (!err) {
  728. err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
  729. if (!err)
  730. pci_using_dac = 1;
  731. } else {
  732. err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
  733. if (err) {
  734. err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
  735. if (err) {
  736. dev_err(&pdev->dev, "No usable DMA "
  737. "configuration, aborting\n");
  738. goto err_dma;
  739. }
  740. }
  741. }
  742. err = pci_request_regions(pdev, igb_driver_name);
  743. if (err)
  744. goto err_pci_reg;
  745. pci_set_master(pdev);
  746. pci_save_state(pdev);
  747. err = -ENOMEM;
  748. netdev = alloc_etherdev(sizeof(struct igb_adapter));
  749. if (!netdev)
  750. goto err_alloc_etherdev;
  751. SET_NETDEV_DEV(netdev, &pdev->dev);
  752. pci_set_drvdata(pdev, netdev);
  753. adapter = netdev_priv(netdev);
  754. adapter->netdev = netdev;
  755. adapter->pdev = pdev;
  756. hw = &adapter->hw;
  757. hw->back = adapter;
  758. adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
  759. mmio_start = pci_resource_start(pdev, 0);
  760. mmio_len = pci_resource_len(pdev, 0);
  761. err = -EIO;
  762. adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
  763. if (!adapter->hw.hw_addr)
  764. goto err_ioremap;
  765. netdev->open = &igb_open;
  766. netdev->stop = &igb_close;
  767. netdev->get_stats = &igb_get_stats;
  768. netdev->set_multicast_list = &igb_set_multi;
  769. netdev->set_mac_address = &igb_set_mac;
  770. netdev->change_mtu = &igb_change_mtu;
  771. netdev->do_ioctl = &igb_ioctl;
  772. igb_set_ethtool_ops(netdev);
  773. netdev->tx_timeout = &igb_tx_timeout;
  774. netdev->watchdog_timeo = 5 * HZ;
  775. netif_napi_add(netdev, &adapter->napi, igb_clean, 64);
  776. netdev->vlan_rx_register = igb_vlan_rx_register;
  777. netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
  778. netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
  779. #ifdef CONFIG_NET_POLL_CONTROLLER
  780. netdev->poll_controller = igb_netpoll;
  781. #endif
  782. netdev->hard_start_xmit = &igb_xmit_frame_adv;
  783. strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
  784. netdev->mem_start = mmio_start;
  785. netdev->mem_end = mmio_start + mmio_len;
  786. adapter->bd_number = cards_found;
  787. /* PCI config space info */
  788. hw->vendor_id = pdev->vendor;
  789. hw->device_id = pdev->device;
  790. hw->revision_id = pdev->revision;
  791. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  792. hw->subsystem_device_id = pdev->subsystem_device;
  793. /* setup the private structure */
  794. hw->back = adapter;
  795. /* Copy the default MAC, PHY and NVM function pointers */
  796. memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
  797. memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
  798. memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
  799. /* Initialize skew-specific constants */
  800. err = ei->get_invariants(hw);
  801. if (err)
  802. goto err_hw_init;
  803. err = igb_sw_init(adapter);
  804. if (err)
  805. goto err_sw_init;
  806. igb_get_bus_info_pcie(hw);
  807. hw->phy.autoneg_wait_to_complete = false;
  808. hw->mac.adaptive_ifs = true;
  809. /* Copper options */
  810. if (hw->phy.media_type == e1000_media_type_copper) {
  811. hw->phy.mdix = AUTO_ALL_MODES;
  812. hw->phy.disable_polarity_correction = false;
  813. hw->phy.ms_type = e1000_ms_hw_default;
  814. }
  815. if (igb_check_reset_block(hw))
  816. dev_info(&pdev->dev,
  817. "PHY reset is blocked due to SOL/IDER session.\n");
  818. netdev->features = NETIF_F_SG |
  819. NETIF_F_HW_CSUM |
  820. NETIF_F_HW_VLAN_TX |
  821. NETIF_F_HW_VLAN_RX |
  822. NETIF_F_HW_VLAN_FILTER;
  823. netdev->features |= NETIF_F_TSO;
  824. netdev->features |= NETIF_F_TSO6;
  825. if (pci_using_dac)
  826. netdev->features |= NETIF_F_HIGHDMA;
  827. netdev->features |= NETIF_F_LLTX;
  828. adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
  829. /* before reading the NVM, reset the controller to put the device in a
  830. * known good starting state */
  831. hw->mac.ops.reset_hw(hw);
  832. /* make sure the NVM is good */
  833. if (igb_validate_nvm_checksum(hw) < 0) {
  834. dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
  835. err = -EIO;
  836. goto err_eeprom;
  837. }
  838. /* copy the MAC address out of the NVM */
  839. if (hw->mac.ops.read_mac_addr(hw))
  840. dev_err(&pdev->dev, "NVM Read Error\n");
  841. memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
  842. memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
  843. if (!is_valid_ether_addr(netdev->perm_addr)) {
  844. dev_err(&pdev->dev, "Invalid MAC Address\n");
  845. err = -EIO;
  846. goto err_eeprom;
  847. }
  848. init_timer(&adapter->watchdog_timer);
  849. adapter->watchdog_timer.function = &igb_watchdog;
  850. adapter->watchdog_timer.data = (unsigned long) adapter;
  851. init_timer(&adapter->phy_info_timer);
  852. adapter->phy_info_timer.function = &igb_update_phy_info;
  853. adapter->phy_info_timer.data = (unsigned long) adapter;
  854. INIT_WORK(&adapter->reset_task, igb_reset_task);
  855. INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
  856. /* Initialize link & ring properties that are user-changeable */
  857. adapter->tx_ring->count = 256;
  858. for (i = 0; i < adapter->num_tx_queues; i++)
  859. adapter->tx_ring[i].count = adapter->tx_ring->count;
  860. adapter->rx_ring->count = 256;
  861. for (i = 0; i < adapter->num_rx_queues; i++)
  862. adapter->rx_ring[i].count = adapter->rx_ring->count;
  863. adapter->fc_autoneg = true;
  864. hw->mac.autoneg = true;
  865. hw->phy.autoneg_advertised = 0x2f;
  866. hw->fc.original_type = e1000_fc_default;
  867. hw->fc.type = e1000_fc_default;
  868. adapter->itr_setting = 3;
  869. adapter->itr = IGB_START_ITR;
  870. igb_validate_mdi_setting(hw);
  871. adapter->rx_csum = 1;
  872. /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
  873. * enable the ACPI Magic Packet filter
  874. */
  875. if (hw->bus.func == 0 ||
  876. hw->device_id == E1000_DEV_ID_82575EB_COPPER)
  877. hw->nvm.ops.read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1,
  878. &eeprom_data);
  879. if (eeprom_data & eeprom_apme_mask)
  880. adapter->eeprom_wol |= E1000_WUFC_MAG;
  881. /* now that we have the eeprom settings, apply the special cases where
  882. * the eeprom may be wrong or the board simply won't support wake on
  883. * lan on a particular port */
  884. switch (pdev->device) {
  885. case E1000_DEV_ID_82575GB_QUAD_COPPER:
  886. adapter->eeprom_wol = 0;
  887. break;
  888. case E1000_DEV_ID_82575EB_FIBER_SERDES:
  889. /* Wake events only supported on port A for dual fiber
  890. * regardless of eeprom setting */
  891. if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
  892. adapter->eeprom_wol = 0;
  893. break;
  894. }
  895. /* initialize the wol settings based on the eeprom settings */
  896. adapter->wol = adapter->eeprom_wol;
  897. /* reset the hardware with the new settings */
  898. igb_reset(adapter);
  899. /* let the f/w know that the h/w is now under the control of the
  900. * driver. */
  901. igb_get_hw_control(adapter);
  902. /* tell the stack to leave us alone until igb_open() is called */
  903. netif_carrier_off(netdev);
  904. netif_stop_queue(netdev);
  905. strcpy(netdev->name, "eth%d");
  906. err = register_netdev(netdev);
  907. if (err)
  908. goto err_register;
  909. dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
  910. /* print bus type/speed/width info */
  911. dev_info(&pdev->dev,
  912. "%s: (PCIe:%s:%s) %02x:%02x:%02x:%02x:%02x:%02x\n",
  913. netdev->name,
  914. ((hw->bus.speed == e1000_bus_speed_2500)
  915. ? "2.5Gb/s" : "unknown"),
  916. ((hw->bus.width == e1000_bus_width_pcie_x4)
  917. ? "Width x4" : (hw->bus.width == e1000_bus_width_pcie_x1)
  918. ? "Width x1" : "unknown"),
  919. netdev->dev_addr[0], netdev->dev_addr[1], netdev->dev_addr[2],
  920. netdev->dev_addr[3], netdev->dev_addr[4], netdev->dev_addr[5]);
  921. igb_read_part_num(hw, &part_num);
  922. dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
  923. (part_num >> 8), (part_num & 0xff));
  924. dev_info(&pdev->dev,
  925. "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
  926. adapter->msix_entries ? "MSI-X" :
  927. adapter->msi_enabled ? "MSI" : "legacy",
  928. adapter->num_rx_queues, adapter->num_tx_queues);
  929. cards_found++;
  930. return 0;
  931. err_register:
  932. igb_release_hw_control(adapter);
  933. err_eeprom:
  934. if (!igb_check_reset_block(hw))
  935. hw->phy.ops.reset_phy(hw);
  936. if (hw->flash_address)
  937. iounmap(hw->flash_address);
  938. igb_remove_device(hw);
  939. kfree(adapter->tx_ring);
  940. kfree(adapter->rx_ring);
  941. err_sw_init:
  942. err_hw_init:
  943. iounmap(hw->hw_addr);
  944. err_ioremap:
  945. free_netdev(netdev);
  946. err_alloc_etherdev:
  947. pci_release_regions(pdev);
  948. err_pci_reg:
  949. err_dma:
  950. pci_disable_device(pdev);
  951. return err;
  952. }
  953. /**
  954. * igb_remove - Device Removal Routine
  955. * @pdev: PCI device information struct
  956. *
  957. * igb_remove is called by the PCI subsystem to alert the driver
  958. * that it should release a PCI device. The could be caused by a
  959. * Hot-Plug event, or because the driver is going to be removed from
  960. * memory.
  961. **/
  962. static void __devexit igb_remove(struct pci_dev *pdev)
  963. {
  964. struct net_device *netdev = pci_get_drvdata(pdev);
  965. struct igb_adapter *adapter = netdev_priv(netdev);
  966. /* flush_scheduled work may reschedule our watchdog task, so
  967. * explicitly disable watchdog tasks from being rescheduled */
  968. set_bit(__IGB_DOWN, &adapter->state);
  969. del_timer_sync(&adapter->watchdog_timer);
  970. del_timer_sync(&adapter->phy_info_timer);
  971. flush_scheduled_work();
  972. /* Release control of h/w to f/w. If f/w is AMT enabled, this
  973. * would have already happened in close and is redundant. */
  974. igb_release_hw_control(adapter);
  975. unregister_netdev(netdev);
  976. if (!igb_check_reset_block(&adapter->hw))
  977. adapter->hw.phy.ops.reset_phy(&adapter->hw);
  978. igb_remove_device(&adapter->hw);
  979. igb_reset_interrupt_capability(adapter);
  980. kfree(adapter->tx_ring);
  981. kfree(adapter->rx_ring);
  982. iounmap(adapter->hw.hw_addr);
  983. if (adapter->hw.flash_address)
  984. iounmap(adapter->hw.flash_address);
  985. pci_release_regions(pdev);
  986. free_netdev(netdev);
  987. pci_disable_device(pdev);
  988. }
  989. /**
  990. * igb_sw_init - Initialize general software structures (struct igb_adapter)
  991. * @adapter: board private structure to initialize
  992. *
  993. * igb_sw_init initializes the Adapter private data structure.
  994. * Fields are initialized based on PCI device information and
  995. * OS network device settings (MTU size).
  996. **/
  997. static int __devinit igb_sw_init(struct igb_adapter *adapter)
  998. {
  999. struct e1000_hw *hw = &adapter->hw;
  1000. struct net_device *netdev = adapter->netdev;
  1001. struct pci_dev *pdev = adapter->pdev;
  1002. pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
  1003. adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
  1004. adapter->rx_ps_hdr_size = 0; /* disable packet split */
  1005. adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
  1006. adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
  1007. /* Number of supported queues. */
  1008. /* Having more queues than CPUs doesn't make sense. */
  1009. adapter->num_tx_queues = 1;
  1010. adapter->num_rx_queues = min(IGB_MAX_RX_QUEUES, num_online_cpus());
  1011. igb_set_interrupt_capability(adapter);
  1012. if (igb_alloc_queues(adapter)) {
  1013. dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
  1014. return -ENOMEM;
  1015. }
  1016. /* Explicitly disable IRQ since the NIC can be in any state. */
  1017. igb_irq_disable(adapter);
  1018. set_bit(__IGB_DOWN, &adapter->state);
  1019. return 0;
  1020. }
  1021. /**
  1022. * igb_open - Called when a network interface is made active
  1023. * @netdev: network interface device structure
  1024. *
  1025. * Returns 0 on success, negative value on failure
  1026. *
  1027. * The open entry point is called when a network interface is made
  1028. * active by the system (IFF_UP). At this point all resources needed
  1029. * for transmit and receive operations are allocated, the interrupt
  1030. * handler is registered with the OS, the watchdog timer is started,
  1031. * and the stack is notified that the interface is ready.
  1032. **/
  1033. static int igb_open(struct net_device *netdev)
  1034. {
  1035. struct igb_adapter *adapter = netdev_priv(netdev);
  1036. struct e1000_hw *hw = &adapter->hw;
  1037. int err;
  1038. int i;
  1039. /* disallow open during test */
  1040. if (test_bit(__IGB_TESTING, &adapter->state))
  1041. return -EBUSY;
  1042. /* allocate transmit descriptors */
  1043. err = igb_setup_all_tx_resources(adapter);
  1044. if (err)
  1045. goto err_setup_tx;
  1046. /* allocate receive descriptors */
  1047. err = igb_setup_all_rx_resources(adapter);
  1048. if (err)
  1049. goto err_setup_rx;
  1050. /* e1000_power_up_phy(adapter); */
  1051. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  1052. if ((adapter->hw.mng_cookie.status &
  1053. E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
  1054. igb_update_mng_vlan(adapter);
  1055. /* before we allocate an interrupt, we must be ready to handle it.
  1056. * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
  1057. * as soon as we call pci_request_irq, so we have to setup our
  1058. * clean_rx handler before we do so. */
  1059. igb_configure(adapter);
  1060. err = igb_request_irq(adapter);
  1061. if (err)
  1062. goto err_req_irq;
  1063. /* From here on the code is the same as igb_up() */
  1064. clear_bit(__IGB_DOWN, &adapter->state);
  1065. napi_enable(&adapter->napi);
  1066. if (adapter->msix_entries)
  1067. for (i = 0; i < adapter->num_rx_queues; i++)
  1068. napi_enable(&adapter->rx_ring[i].napi);
  1069. igb_irq_enable(adapter);
  1070. /* Clear any pending interrupts. */
  1071. rd32(E1000_ICR);
  1072. /* Fire a link status change interrupt to start the watchdog. */
  1073. wr32(E1000_ICS, E1000_ICS_LSC);
  1074. return 0;
  1075. err_req_irq:
  1076. igb_release_hw_control(adapter);
  1077. /* e1000_power_down_phy(adapter); */
  1078. igb_free_all_rx_resources(adapter);
  1079. err_setup_rx:
  1080. igb_free_all_tx_resources(adapter);
  1081. err_setup_tx:
  1082. igb_reset(adapter);
  1083. return err;
  1084. }
  1085. /**
  1086. * igb_close - Disables a network interface
  1087. * @netdev: network interface device structure
  1088. *
  1089. * Returns 0, this is not allowed to fail
  1090. *
  1091. * The close entry point is called when an interface is de-activated
  1092. * by the OS. The hardware is still under the driver's control, but
  1093. * needs to be disabled. A global MAC reset is issued to stop the
  1094. * hardware, and all transmit and receive resources are freed.
  1095. **/
  1096. static int igb_close(struct net_device *netdev)
  1097. {
  1098. struct igb_adapter *adapter = netdev_priv(netdev);
  1099. WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
  1100. igb_down(adapter);
  1101. igb_free_irq(adapter);
  1102. igb_free_all_tx_resources(adapter);
  1103. igb_free_all_rx_resources(adapter);
  1104. /* kill manageability vlan ID if supported, but not if a vlan with
  1105. * the same ID is registered on the host OS (let 8021q kill it) */
  1106. if ((adapter->hw.mng_cookie.status &
  1107. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
  1108. !(adapter->vlgrp &&
  1109. vlan_group_get_device(adapter->vlgrp, adapter->mng_vlan_id)))
  1110. igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
  1111. return 0;
  1112. }
  1113. /**
  1114. * igb_setup_tx_resources - allocate Tx resources (Descriptors)
  1115. * @adapter: board private structure
  1116. * @tx_ring: tx descriptor ring (for a specific queue) to setup
  1117. *
  1118. * Return 0 on success, negative on failure
  1119. **/
  1120. int igb_setup_tx_resources(struct igb_adapter *adapter,
  1121. struct igb_ring *tx_ring)
  1122. {
  1123. struct pci_dev *pdev = adapter->pdev;
  1124. int size;
  1125. size = sizeof(struct igb_buffer) * tx_ring->count;
  1126. tx_ring->buffer_info = vmalloc(size);
  1127. if (!tx_ring->buffer_info)
  1128. goto err;
  1129. memset(tx_ring->buffer_info, 0, size);
  1130. /* round up to nearest 4K */
  1131. tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc)
  1132. + sizeof(u32);
  1133. tx_ring->size = ALIGN(tx_ring->size, 4096);
  1134. tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
  1135. &tx_ring->dma);
  1136. if (!tx_ring->desc)
  1137. goto err;
  1138. tx_ring->adapter = adapter;
  1139. tx_ring->next_to_use = 0;
  1140. tx_ring->next_to_clean = 0;
  1141. spin_lock_init(&tx_ring->tx_clean_lock);
  1142. spin_lock_init(&tx_ring->tx_lock);
  1143. return 0;
  1144. err:
  1145. vfree(tx_ring->buffer_info);
  1146. dev_err(&adapter->pdev->dev,
  1147. "Unable to allocate memory for the transmit descriptor ring\n");
  1148. return -ENOMEM;
  1149. }
  1150. /**
  1151. * igb_setup_all_tx_resources - wrapper to allocate Tx resources
  1152. * (Descriptors) for all queues
  1153. * @adapter: board private structure
  1154. *
  1155. * Return 0 on success, negative on failure
  1156. **/
  1157. static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
  1158. {
  1159. int i, err = 0;
  1160. for (i = 0; i < adapter->num_tx_queues; i++) {
  1161. err = igb_setup_tx_resources(adapter, &adapter->tx_ring[i]);
  1162. if (err) {
  1163. dev_err(&adapter->pdev->dev,
  1164. "Allocation for Tx Queue %u failed\n", i);
  1165. for (i--; i >= 0; i--)
  1166. igb_free_tx_resources(adapter,
  1167. &adapter->tx_ring[i]);
  1168. break;
  1169. }
  1170. }
  1171. return err;
  1172. }
  1173. /**
  1174. * igb_configure_tx - Configure transmit Unit after Reset
  1175. * @adapter: board private structure
  1176. *
  1177. * Configure the Tx unit of the MAC after a reset.
  1178. **/
  1179. static void igb_configure_tx(struct igb_adapter *adapter)
  1180. {
  1181. u64 tdba, tdwba;
  1182. struct e1000_hw *hw = &adapter->hw;
  1183. u32 tctl;
  1184. u32 txdctl, txctrl;
  1185. int i;
  1186. for (i = 0; i < adapter->num_tx_queues; i++) {
  1187. struct igb_ring *ring = &(adapter->tx_ring[i]);
  1188. wr32(E1000_TDLEN(i),
  1189. ring->count * sizeof(struct e1000_tx_desc));
  1190. tdba = ring->dma;
  1191. wr32(E1000_TDBAL(i),
  1192. tdba & 0x00000000ffffffffULL);
  1193. wr32(E1000_TDBAH(i), tdba >> 32);
  1194. tdwba = ring->dma + ring->count * sizeof(struct e1000_tx_desc);
  1195. tdwba |= 1; /* enable head wb */
  1196. wr32(E1000_TDWBAL(i),
  1197. tdwba & 0x00000000ffffffffULL);
  1198. wr32(E1000_TDWBAH(i), tdwba >> 32);
  1199. ring->head = E1000_TDH(i);
  1200. ring->tail = E1000_TDT(i);
  1201. writel(0, hw->hw_addr + ring->tail);
  1202. writel(0, hw->hw_addr + ring->head);
  1203. txdctl = rd32(E1000_TXDCTL(i));
  1204. txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
  1205. wr32(E1000_TXDCTL(i), txdctl);
  1206. /* Turn off Relaxed Ordering on head write-backs. The
  1207. * writebacks MUST be delivered in order or it will
  1208. * completely screw up our bookeeping.
  1209. */
  1210. txctrl = rd32(E1000_DCA_TXCTRL(i));
  1211. txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
  1212. wr32(E1000_DCA_TXCTRL(i), txctrl);
  1213. }
  1214. /* Use the default values for the Tx Inter Packet Gap (IPG) timer */
  1215. /* Program the Transmit Control Register */
  1216. tctl = rd32(E1000_TCTL);
  1217. tctl &= ~E1000_TCTL_CT;
  1218. tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
  1219. (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
  1220. igb_config_collision_dist(hw);
  1221. /* Setup Transmit Descriptor Settings for eop descriptor */
  1222. adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS;
  1223. /* Enable transmits */
  1224. tctl |= E1000_TCTL_EN;
  1225. wr32(E1000_TCTL, tctl);
  1226. }
  1227. /**
  1228. * igb_setup_rx_resources - allocate Rx resources (Descriptors)
  1229. * @adapter: board private structure
  1230. * @rx_ring: rx descriptor ring (for a specific queue) to setup
  1231. *
  1232. * Returns 0 on success, negative on failure
  1233. **/
  1234. int igb_setup_rx_resources(struct igb_adapter *adapter,
  1235. struct igb_ring *rx_ring)
  1236. {
  1237. struct pci_dev *pdev = adapter->pdev;
  1238. int size, desc_len;
  1239. size = sizeof(struct igb_buffer) * rx_ring->count;
  1240. rx_ring->buffer_info = vmalloc(size);
  1241. if (!rx_ring->buffer_info)
  1242. goto err;
  1243. memset(rx_ring->buffer_info, 0, size);
  1244. desc_len = sizeof(union e1000_adv_rx_desc);
  1245. /* Round up to nearest 4K */
  1246. rx_ring->size = rx_ring->count * desc_len;
  1247. rx_ring->size = ALIGN(rx_ring->size, 4096);
  1248. rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
  1249. &rx_ring->dma);
  1250. if (!rx_ring->desc)
  1251. goto err;
  1252. rx_ring->next_to_clean = 0;
  1253. rx_ring->next_to_use = 0;
  1254. rx_ring->pending_skb = NULL;
  1255. rx_ring->adapter = adapter;
  1256. /* FIXME: do we want to setup ring->napi->poll here? */
  1257. rx_ring->napi.poll = adapter->napi.poll;
  1258. return 0;
  1259. err:
  1260. vfree(rx_ring->buffer_info);
  1261. dev_err(&adapter->pdev->dev, "Unable to allocate memory for "
  1262. "the receive descriptor ring\n");
  1263. return -ENOMEM;
  1264. }
  1265. /**
  1266. * igb_setup_all_rx_resources - wrapper to allocate Rx resources
  1267. * (Descriptors) for all queues
  1268. * @adapter: board private structure
  1269. *
  1270. * Return 0 on success, negative on failure
  1271. **/
  1272. static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
  1273. {
  1274. int i, err = 0;
  1275. for (i = 0; i < adapter->num_rx_queues; i++) {
  1276. err = igb_setup_rx_resources(adapter, &adapter->rx_ring[i]);
  1277. if (err) {
  1278. dev_err(&adapter->pdev->dev,
  1279. "Allocation for Rx Queue %u failed\n", i);
  1280. for (i--; i >= 0; i--)
  1281. igb_free_rx_resources(adapter,
  1282. &adapter->rx_ring[i]);
  1283. break;
  1284. }
  1285. }
  1286. return err;
  1287. }
  1288. /**
  1289. * igb_setup_rctl - configure the receive control registers
  1290. * @adapter: Board private structure
  1291. **/
  1292. static void igb_setup_rctl(struct igb_adapter *adapter)
  1293. {
  1294. struct e1000_hw *hw = &adapter->hw;
  1295. u32 rctl;
  1296. u32 srrctl = 0;
  1297. int i;
  1298. rctl = rd32(E1000_RCTL);
  1299. rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
  1300. rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
  1301. E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
  1302. (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
  1303. /* disable the stripping of CRC because it breaks
  1304. * BMC firmware connected over SMBUS
  1305. rctl |= E1000_RCTL_SECRC;
  1306. */
  1307. rctl &= ~E1000_RCTL_SBP;
  1308. if (adapter->netdev->mtu <= ETH_DATA_LEN)
  1309. rctl &= ~E1000_RCTL_LPE;
  1310. else
  1311. rctl |= E1000_RCTL_LPE;
  1312. if (adapter->rx_buffer_len <= IGB_RXBUFFER_2048) {
  1313. /* Setup buffer sizes */
  1314. rctl &= ~E1000_RCTL_SZ_4096;
  1315. rctl |= E1000_RCTL_BSEX;
  1316. switch (adapter->rx_buffer_len) {
  1317. case IGB_RXBUFFER_256:
  1318. rctl |= E1000_RCTL_SZ_256;
  1319. rctl &= ~E1000_RCTL_BSEX;
  1320. break;
  1321. case IGB_RXBUFFER_512:
  1322. rctl |= E1000_RCTL_SZ_512;
  1323. rctl &= ~E1000_RCTL_BSEX;
  1324. break;
  1325. case IGB_RXBUFFER_1024:
  1326. rctl |= E1000_RCTL_SZ_1024;
  1327. rctl &= ~E1000_RCTL_BSEX;
  1328. break;
  1329. case IGB_RXBUFFER_2048:
  1330. default:
  1331. rctl |= E1000_RCTL_SZ_2048;
  1332. rctl &= ~E1000_RCTL_BSEX;
  1333. break;
  1334. case IGB_RXBUFFER_4096:
  1335. rctl |= E1000_RCTL_SZ_4096;
  1336. break;
  1337. case IGB_RXBUFFER_8192:
  1338. rctl |= E1000_RCTL_SZ_8192;
  1339. break;
  1340. case IGB_RXBUFFER_16384:
  1341. rctl |= E1000_RCTL_SZ_16384;
  1342. break;
  1343. }
  1344. } else {
  1345. rctl &= ~E1000_RCTL_BSEX;
  1346. srrctl = adapter->rx_buffer_len >> E1000_SRRCTL_BSIZEPKT_SHIFT;
  1347. }
  1348. /* 82575 and greater support packet-split where the protocol
  1349. * header is placed in skb->data and the packet data is
  1350. * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
  1351. * In the case of a non-split, skb->data is linearly filled,
  1352. * followed by the page buffers. Therefore, skb->data is
  1353. * sized to hold the largest protocol header.
  1354. */
  1355. /* allocations using alloc_page take too long for regular MTU
  1356. * so only enable packet split for jumbo frames */
  1357. if (rctl & E1000_RCTL_LPE) {
  1358. adapter->rx_ps_hdr_size = IGB_RXBUFFER_128;
  1359. srrctl = adapter->rx_ps_hdr_size <<
  1360. E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
  1361. /* buffer size is ALWAYS one page */
  1362. srrctl |= PAGE_SIZE >> E1000_SRRCTL_BSIZEPKT_SHIFT;
  1363. srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
  1364. } else {
  1365. adapter->rx_ps_hdr_size = 0;
  1366. srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
  1367. }
  1368. for (i = 0; i < adapter->num_rx_queues; i++)
  1369. wr32(E1000_SRRCTL(i), srrctl);
  1370. wr32(E1000_RCTL, rctl);
  1371. }
  1372. /**
  1373. * igb_configure_rx - Configure receive Unit after Reset
  1374. * @adapter: board private structure
  1375. *
  1376. * Configure the Rx unit of the MAC after a reset.
  1377. **/
  1378. static void igb_configure_rx(struct igb_adapter *adapter)
  1379. {
  1380. u64 rdba;
  1381. struct e1000_hw *hw = &adapter->hw;
  1382. u32 rctl, rxcsum;
  1383. u32 rxdctl;
  1384. int i;
  1385. /* disable receives while setting up the descriptors */
  1386. rctl = rd32(E1000_RCTL);
  1387. wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
  1388. wrfl();
  1389. mdelay(10);
  1390. if (adapter->itr_setting > 3)
  1391. wr32(E1000_ITR,
  1392. 1000000000 / (adapter->itr * 256));
  1393. /* Setup the HW Rx Head and Tail Descriptor Pointers and
  1394. * the Base and Length of the Rx Descriptor Ring */
  1395. for (i = 0; i < adapter->num_rx_queues; i++) {
  1396. struct igb_ring *ring = &(adapter->rx_ring[i]);
  1397. rdba = ring->dma;
  1398. wr32(E1000_RDBAL(i),
  1399. rdba & 0x00000000ffffffffULL);
  1400. wr32(E1000_RDBAH(i), rdba >> 32);
  1401. wr32(E1000_RDLEN(i),
  1402. ring->count * sizeof(union e1000_adv_rx_desc));
  1403. ring->head = E1000_RDH(i);
  1404. ring->tail = E1000_RDT(i);
  1405. writel(0, hw->hw_addr + ring->tail);
  1406. writel(0, hw->hw_addr + ring->head);
  1407. rxdctl = rd32(E1000_RXDCTL(i));
  1408. rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
  1409. rxdctl &= 0xFFF00000;
  1410. rxdctl |= IGB_RX_PTHRESH;
  1411. rxdctl |= IGB_RX_HTHRESH << 8;
  1412. rxdctl |= IGB_RX_WTHRESH << 16;
  1413. wr32(E1000_RXDCTL(i), rxdctl);
  1414. }
  1415. if (adapter->num_rx_queues > 1) {
  1416. u32 random[10];
  1417. u32 mrqc;
  1418. u32 j, shift;
  1419. union e1000_reta {
  1420. u32 dword;
  1421. u8 bytes[4];
  1422. } reta;
  1423. get_random_bytes(&random[0], 40);
  1424. shift = 6;
  1425. for (j = 0; j < (32 * 4); j++) {
  1426. reta.bytes[j & 3] =
  1427. (j % adapter->num_rx_queues) << shift;
  1428. if ((j & 3) == 3)
  1429. writel(reta.dword,
  1430. hw->hw_addr + E1000_RETA(0) + (j & ~3));
  1431. }
  1432. mrqc = E1000_MRQC_ENABLE_RSS_4Q;
  1433. /* Fill out hash function seeds */
  1434. for (j = 0; j < 10; j++)
  1435. array_wr32(E1000_RSSRK(0), j, random[j]);
  1436. mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
  1437. E1000_MRQC_RSS_FIELD_IPV4_TCP);
  1438. mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
  1439. E1000_MRQC_RSS_FIELD_IPV6_TCP);
  1440. mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
  1441. E1000_MRQC_RSS_FIELD_IPV6_UDP);
  1442. mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
  1443. E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
  1444. wr32(E1000_MRQC, mrqc);
  1445. /* Multiqueue and raw packet checksumming are mutually
  1446. * exclusive. Note that this not the same as TCP/IP
  1447. * checksumming, which works fine. */
  1448. rxcsum = rd32(E1000_RXCSUM);
  1449. rxcsum |= E1000_RXCSUM_PCSD;
  1450. wr32(E1000_RXCSUM, rxcsum);
  1451. } else {
  1452. /* Enable Receive Checksum Offload for TCP and UDP */
  1453. rxcsum = rd32(E1000_RXCSUM);
  1454. if (adapter->rx_csum) {
  1455. rxcsum |= E1000_RXCSUM_TUOFL;
  1456. /* Enable IPv4 payload checksum for UDP fragments
  1457. * Must be used in conjunction with packet-split. */
  1458. if (adapter->rx_ps_hdr_size)
  1459. rxcsum |= E1000_RXCSUM_IPPCSE;
  1460. } else {
  1461. rxcsum &= ~E1000_RXCSUM_TUOFL;
  1462. /* don't need to clear IPPCSE as it defaults to 0 */
  1463. }
  1464. wr32(E1000_RXCSUM, rxcsum);
  1465. }
  1466. if (adapter->vlgrp)
  1467. wr32(E1000_RLPML,
  1468. adapter->max_frame_size + VLAN_TAG_SIZE);
  1469. else
  1470. wr32(E1000_RLPML, adapter->max_frame_size);
  1471. /* Enable Receives */
  1472. wr32(E1000_RCTL, rctl);
  1473. }
  1474. /**
  1475. * igb_free_tx_resources - Free Tx Resources per Queue
  1476. * @adapter: board private structure
  1477. * @tx_ring: Tx descriptor ring for a specific queue
  1478. *
  1479. * Free all transmit software resources
  1480. **/
  1481. static void igb_free_tx_resources(struct igb_adapter *adapter,
  1482. struct igb_ring *tx_ring)
  1483. {
  1484. struct pci_dev *pdev = adapter->pdev;
  1485. igb_clean_tx_ring(adapter, tx_ring);
  1486. vfree(tx_ring->buffer_info);
  1487. tx_ring->buffer_info = NULL;
  1488. pci_free_consistent(pdev, tx_ring->size, tx_ring->desc, tx_ring->dma);
  1489. tx_ring->desc = NULL;
  1490. }
  1491. /**
  1492. * igb_free_all_tx_resources - Free Tx Resources for All Queues
  1493. * @adapter: board private structure
  1494. *
  1495. * Free all transmit software resources
  1496. **/
  1497. static void igb_free_all_tx_resources(struct igb_adapter *adapter)
  1498. {
  1499. int i;
  1500. for (i = 0; i < adapter->num_tx_queues; i++)
  1501. igb_free_tx_resources(adapter, &adapter->tx_ring[i]);
  1502. }
  1503. static void igb_unmap_and_free_tx_resource(struct igb_adapter *adapter,
  1504. struct igb_buffer *buffer_info)
  1505. {
  1506. if (buffer_info->dma) {
  1507. pci_unmap_page(adapter->pdev,
  1508. buffer_info->dma,
  1509. buffer_info->length,
  1510. PCI_DMA_TODEVICE);
  1511. buffer_info->dma = 0;
  1512. }
  1513. if (buffer_info->skb) {
  1514. dev_kfree_skb_any(buffer_info->skb);
  1515. buffer_info->skb = NULL;
  1516. }
  1517. buffer_info->time_stamp = 0;
  1518. /* buffer_info must be completely set up in the transmit path */
  1519. }
  1520. /**
  1521. * igb_clean_tx_ring - Free Tx Buffers
  1522. * @adapter: board private structure
  1523. * @tx_ring: ring to be cleaned
  1524. **/
  1525. static void igb_clean_tx_ring(struct igb_adapter *adapter,
  1526. struct igb_ring *tx_ring)
  1527. {
  1528. struct igb_buffer *buffer_info;
  1529. unsigned long size;
  1530. unsigned int i;
  1531. if (!tx_ring->buffer_info)
  1532. return;
  1533. /* Free all the Tx ring sk_buffs */
  1534. for (i = 0; i < tx_ring->count; i++) {
  1535. buffer_info = &tx_ring->buffer_info[i];
  1536. igb_unmap_and_free_tx_resource(adapter, buffer_info);
  1537. }
  1538. size = sizeof(struct igb_buffer) * tx_ring->count;
  1539. memset(tx_ring->buffer_info, 0, size);
  1540. /* Zero out the descriptor ring */
  1541. memset(tx_ring->desc, 0, tx_ring->size);
  1542. tx_ring->next_to_use = 0;
  1543. tx_ring->next_to_clean = 0;
  1544. writel(0, adapter->hw.hw_addr + tx_ring->head);
  1545. writel(0, adapter->hw.hw_addr + tx_ring->tail);
  1546. }
  1547. /**
  1548. * igb_clean_all_tx_rings - Free Tx Buffers for all queues
  1549. * @adapter: board private structure
  1550. **/
  1551. static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
  1552. {
  1553. int i;
  1554. for (i = 0; i < adapter->num_tx_queues; i++)
  1555. igb_clean_tx_ring(adapter, &adapter->tx_ring[i]);
  1556. }
  1557. /**
  1558. * igb_free_rx_resources - Free Rx Resources
  1559. * @adapter: board private structure
  1560. * @rx_ring: ring to clean the resources from
  1561. *
  1562. * Free all receive software resources
  1563. **/
  1564. static void igb_free_rx_resources(struct igb_adapter *adapter,
  1565. struct igb_ring *rx_ring)
  1566. {
  1567. struct pci_dev *pdev = adapter->pdev;
  1568. igb_clean_rx_ring(adapter, rx_ring);
  1569. vfree(rx_ring->buffer_info);
  1570. rx_ring->buffer_info = NULL;
  1571. pci_free_consistent(pdev, rx_ring->size, rx_ring->desc, rx_ring->dma);
  1572. rx_ring->desc = NULL;
  1573. }
  1574. /**
  1575. * igb_free_all_rx_resources - Free Rx Resources for All Queues
  1576. * @adapter: board private structure
  1577. *
  1578. * Free all receive software resources
  1579. **/
  1580. static void igb_free_all_rx_resources(struct igb_adapter *adapter)
  1581. {
  1582. int i;
  1583. for (i = 0; i < adapter->num_rx_queues; i++)
  1584. igb_free_rx_resources(adapter, &adapter->rx_ring[i]);
  1585. }
  1586. /**
  1587. * igb_clean_rx_ring - Free Rx Buffers per Queue
  1588. * @adapter: board private structure
  1589. * @rx_ring: ring to free buffers from
  1590. **/
  1591. static void igb_clean_rx_ring(struct igb_adapter *adapter,
  1592. struct igb_ring *rx_ring)
  1593. {
  1594. struct igb_buffer *buffer_info;
  1595. struct pci_dev *pdev = adapter->pdev;
  1596. unsigned long size;
  1597. unsigned int i;
  1598. if (!rx_ring->buffer_info)
  1599. return;
  1600. /* Free all the Rx ring sk_buffs */
  1601. for (i = 0; i < rx_ring->count; i++) {
  1602. buffer_info = &rx_ring->buffer_info[i];
  1603. if (buffer_info->dma) {
  1604. if (adapter->rx_ps_hdr_size)
  1605. pci_unmap_single(pdev, buffer_info->dma,
  1606. adapter->rx_ps_hdr_size,
  1607. PCI_DMA_FROMDEVICE);
  1608. else
  1609. pci_unmap_single(pdev, buffer_info->dma,
  1610. adapter->rx_buffer_len,
  1611. PCI_DMA_FROMDEVICE);
  1612. buffer_info->dma = 0;
  1613. }
  1614. if (buffer_info->skb) {
  1615. dev_kfree_skb(buffer_info->skb);
  1616. buffer_info->skb = NULL;
  1617. }
  1618. if (buffer_info->page) {
  1619. pci_unmap_page(pdev, buffer_info->page_dma,
  1620. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  1621. put_page(buffer_info->page);
  1622. buffer_info->page = NULL;
  1623. buffer_info->page_dma = 0;
  1624. }
  1625. }
  1626. /* there also may be some cached data from a chained receive */
  1627. if (rx_ring->pending_skb) {
  1628. dev_kfree_skb(rx_ring->pending_skb);
  1629. rx_ring->pending_skb = NULL;
  1630. }
  1631. size = sizeof(struct igb_buffer) * rx_ring->count;
  1632. memset(rx_ring->buffer_info, 0, size);
  1633. /* Zero out the descriptor ring */
  1634. memset(rx_ring->desc, 0, rx_ring->size);
  1635. rx_ring->next_to_clean = 0;
  1636. rx_ring->next_to_use = 0;
  1637. writel(0, adapter->hw.hw_addr + rx_ring->head);
  1638. writel(0, adapter->hw.hw_addr + rx_ring->tail);
  1639. }
  1640. /**
  1641. * igb_clean_all_rx_rings - Free Rx Buffers for all queues
  1642. * @adapter: board private structure
  1643. **/
  1644. static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
  1645. {
  1646. int i;
  1647. for (i = 0; i < adapter->num_rx_queues; i++)
  1648. igb_clean_rx_ring(adapter, &adapter->rx_ring[i]);
  1649. }
  1650. /**
  1651. * igb_set_mac - Change the Ethernet Address of the NIC
  1652. * @netdev: network interface device structure
  1653. * @p: pointer to an address structure
  1654. *
  1655. * Returns 0 on success, negative on failure
  1656. **/
  1657. static int igb_set_mac(struct net_device *netdev, void *p)
  1658. {
  1659. struct igb_adapter *adapter = netdev_priv(netdev);
  1660. struct sockaddr *addr = p;
  1661. if (!is_valid_ether_addr(addr->sa_data))
  1662. return -EADDRNOTAVAIL;
  1663. memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
  1664. memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
  1665. adapter->hw.mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
  1666. return 0;
  1667. }
  1668. /**
  1669. * igb_set_multi - Multicast and Promiscuous mode set
  1670. * @netdev: network interface device structure
  1671. *
  1672. * The set_multi entry point is called whenever the multicast address
  1673. * list or the network interface flags are updated. This routine is
  1674. * responsible for configuring the hardware for proper multicast,
  1675. * promiscuous mode, and all-multi behavior.
  1676. **/
  1677. static void igb_set_multi(struct net_device *netdev)
  1678. {
  1679. struct igb_adapter *adapter = netdev_priv(netdev);
  1680. struct e1000_hw *hw = &adapter->hw;
  1681. struct e1000_mac_info *mac = &hw->mac;
  1682. struct dev_mc_list *mc_ptr;
  1683. u8 *mta_list;
  1684. u32 rctl;
  1685. int i;
  1686. /* Check for Promiscuous and All Multicast modes */
  1687. rctl = rd32(E1000_RCTL);
  1688. if (netdev->flags & IFF_PROMISC)
  1689. rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
  1690. else if (netdev->flags & IFF_ALLMULTI) {
  1691. rctl |= E1000_RCTL_MPE;
  1692. rctl &= ~E1000_RCTL_UPE;
  1693. } else
  1694. rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
  1695. wr32(E1000_RCTL, rctl);
  1696. if (!netdev->mc_count) {
  1697. /* nothing to program, so clear mc list */
  1698. igb_update_mc_addr_list(hw, NULL, 0, 1,
  1699. mac->rar_entry_count);
  1700. return;
  1701. }
  1702. mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
  1703. if (!mta_list)
  1704. return;
  1705. /* The shared function expects a packed array of only addresses. */
  1706. mc_ptr = netdev->mc_list;
  1707. for (i = 0; i < netdev->mc_count; i++) {
  1708. if (!mc_ptr)
  1709. break;
  1710. memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
  1711. mc_ptr = mc_ptr->next;
  1712. }
  1713. igb_update_mc_addr_list(hw, mta_list, i, 1, mac->rar_entry_count);
  1714. kfree(mta_list);
  1715. }
  1716. /* Need to wait a few seconds after link up to get diagnostic information from
  1717. * the phy */
  1718. static void igb_update_phy_info(unsigned long data)
  1719. {
  1720. struct igb_adapter *adapter = (struct igb_adapter *) data;
  1721. if (adapter->hw.phy.ops.get_phy_info)
  1722. adapter->hw.phy.ops.get_phy_info(&adapter->hw);
  1723. }
  1724. /**
  1725. * igb_watchdog - Timer Call-back
  1726. * @data: pointer to adapter cast into an unsigned long
  1727. **/
  1728. static void igb_watchdog(unsigned long data)
  1729. {
  1730. struct igb_adapter *adapter = (struct igb_adapter *)data;
  1731. /* Do the rest outside of interrupt context */
  1732. schedule_work(&adapter->watchdog_task);
  1733. }
  1734. static void igb_watchdog_task(struct work_struct *work)
  1735. {
  1736. struct igb_adapter *adapter = container_of(work,
  1737. struct igb_adapter, watchdog_task);
  1738. struct e1000_hw *hw = &adapter->hw;
  1739. struct net_device *netdev = adapter->netdev;
  1740. struct igb_ring *tx_ring = adapter->tx_ring;
  1741. struct e1000_mac_info *mac = &adapter->hw.mac;
  1742. u32 link;
  1743. s32 ret_val;
  1744. if ((netif_carrier_ok(netdev)) &&
  1745. (rd32(E1000_STATUS) & E1000_STATUS_LU))
  1746. goto link_up;
  1747. ret_val = hw->mac.ops.check_for_link(&adapter->hw);
  1748. if ((ret_val == E1000_ERR_PHY) &&
  1749. (hw->phy.type == e1000_phy_igp_3) &&
  1750. (rd32(E1000_CTRL) &
  1751. E1000_PHY_CTRL_GBE_DISABLE))
  1752. dev_info(&adapter->pdev->dev,
  1753. "Gigabit has been disabled, downgrading speed\n");
  1754. if ((hw->phy.media_type == e1000_media_type_internal_serdes) &&
  1755. !(rd32(E1000_TXCW) & E1000_TXCW_ANE))
  1756. link = mac->serdes_has_link;
  1757. else
  1758. link = rd32(E1000_STATUS) &
  1759. E1000_STATUS_LU;
  1760. if (link) {
  1761. if (!netif_carrier_ok(netdev)) {
  1762. u32 ctrl;
  1763. hw->mac.ops.get_speed_and_duplex(&adapter->hw,
  1764. &adapter->link_speed,
  1765. &adapter->link_duplex);
  1766. ctrl = rd32(E1000_CTRL);
  1767. dev_info(&adapter->pdev->dev,
  1768. "NIC Link is Up %d Mbps %s, "
  1769. "Flow Control: %s\n",
  1770. adapter->link_speed,
  1771. adapter->link_duplex == FULL_DUPLEX ?
  1772. "Full Duplex" : "Half Duplex",
  1773. ((ctrl & E1000_CTRL_TFCE) && (ctrl &
  1774. E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
  1775. E1000_CTRL_RFCE) ? "RX" : ((ctrl &
  1776. E1000_CTRL_TFCE) ? "TX" : "None")));
  1777. /* tweak tx_queue_len according to speed/duplex and
  1778. * adjust the timeout factor */
  1779. netdev->tx_queue_len = adapter->tx_queue_len;
  1780. adapter->tx_timeout_factor = 1;
  1781. switch (adapter->link_speed) {
  1782. case SPEED_10:
  1783. netdev->tx_queue_len = 10;
  1784. adapter->tx_timeout_factor = 14;
  1785. break;
  1786. case SPEED_100:
  1787. netdev->tx_queue_len = 100;
  1788. /* maybe add some timeout factor ? */
  1789. break;
  1790. }
  1791. netif_carrier_on(netdev);
  1792. netif_wake_queue(netdev);
  1793. if (!test_bit(__IGB_DOWN, &adapter->state))
  1794. mod_timer(&adapter->phy_info_timer,
  1795. round_jiffies(jiffies + 2 * HZ));
  1796. }
  1797. } else {
  1798. if (netif_carrier_ok(netdev)) {
  1799. adapter->link_speed = 0;
  1800. adapter->link_duplex = 0;
  1801. dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
  1802. netif_carrier_off(netdev);
  1803. netif_stop_queue(netdev);
  1804. if (!test_bit(__IGB_DOWN, &adapter->state))
  1805. mod_timer(&adapter->phy_info_timer,
  1806. round_jiffies(jiffies + 2 * HZ));
  1807. }
  1808. }
  1809. link_up:
  1810. igb_update_stats(adapter);
  1811. mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
  1812. adapter->tpt_old = adapter->stats.tpt;
  1813. mac->collision_delta = adapter->stats.colc - adapter->colc_old;
  1814. adapter->colc_old = adapter->stats.colc;
  1815. adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
  1816. adapter->gorc_old = adapter->stats.gorc;
  1817. adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
  1818. adapter->gotc_old = adapter->stats.gotc;
  1819. igb_update_adaptive(&adapter->hw);
  1820. if (!netif_carrier_ok(netdev)) {
  1821. if (IGB_DESC_UNUSED(tx_ring) + 1 < tx_ring->count) {
  1822. /* We've lost link, so the controller stops DMA,
  1823. * but we've got queued Tx work that's never going
  1824. * to get done, so reset controller to flush Tx.
  1825. * (Do the reset outside of interrupt context). */
  1826. adapter->tx_timeout_count++;
  1827. schedule_work(&adapter->reset_task);
  1828. }
  1829. }
  1830. /* Cause software interrupt to ensure rx ring is cleaned */
  1831. wr32(E1000_ICS, E1000_ICS_RXDMT0);
  1832. /* Force detection of hung controller every watchdog period */
  1833. tx_ring->detect_tx_hung = true;
  1834. /* Reset the timer */
  1835. if (!test_bit(__IGB_DOWN, &adapter->state))
  1836. mod_timer(&adapter->watchdog_timer,
  1837. round_jiffies(jiffies + 2 * HZ));
  1838. }
  1839. enum latency_range {
  1840. lowest_latency = 0,
  1841. low_latency = 1,
  1842. bulk_latency = 2,
  1843. latency_invalid = 255
  1844. };
  1845. static void igb_lower_rx_eitr(struct igb_adapter *adapter,
  1846. struct igb_ring *rx_ring)
  1847. {
  1848. struct e1000_hw *hw = &adapter->hw;
  1849. int new_val;
  1850. new_val = rx_ring->itr_val / 2;
  1851. if (new_val < IGB_MIN_DYN_ITR)
  1852. new_val = IGB_MIN_DYN_ITR;
  1853. if (new_val != rx_ring->itr_val) {
  1854. rx_ring->itr_val = new_val;
  1855. wr32(rx_ring->itr_register,
  1856. 1000000000 / (new_val * 256));
  1857. }
  1858. }
  1859. static void igb_raise_rx_eitr(struct igb_adapter *adapter,
  1860. struct igb_ring *rx_ring)
  1861. {
  1862. struct e1000_hw *hw = &adapter->hw;
  1863. int new_val;
  1864. new_val = rx_ring->itr_val * 2;
  1865. if (new_val > IGB_MAX_DYN_ITR)
  1866. new_val = IGB_MAX_DYN_ITR;
  1867. if (new_val != rx_ring->itr_val) {
  1868. rx_ring->itr_val = new_val;
  1869. wr32(rx_ring->itr_register,
  1870. 1000000000 / (new_val * 256));
  1871. }
  1872. }
  1873. /**
  1874. * igb_update_itr - update the dynamic ITR value based on statistics
  1875. * Stores a new ITR value based on packets and byte
  1876. * counts during the last interrupt. The advantage of per interrupt
  1877. * computation is faster updates and more accurate ITR for the current
  1878. * traffic pattern. Constants in this function were computed
  1879. * based on theoretical maximum wire speed and thresholds were set based
  1880. * on testing data as well as attempting to minimize response time
  1881. * while increasing bulk throughput.
  1882. * this functionality is controlled by the InterruptThrottleRate module
  1883. * parameter (see igb_param.c)
  1884. * NOTE: These calculations are only valid when operating in a single-
  1885. * queue environment.
  1886. * @adapter: pointer to adapter
  1887. * @itr_setting: current adapter->itr
  1888. * @packets: the number of packets during this measurement interval
  1889. * @bytes: the number of bytes during this measurement interval
  1890. **/
  1891. static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
  1892. int packets, int bytes)
  1893. {
  1894. unsigned int retval = itr_setting;
  1895. if (packets == 0)
  1896. goto update_itr_done;
  1897. switch (itr_setting) {
  1898. case lowest_latency:
  1899. /* handle TSO and jumbo frames */
  1900. if (bytes/packets > 8000)
  1901. retval = bulk_latency;
  1902. else if ((packets < 5) && (bytes > 512))
  1903. retval = low_latency;
  1904. break;
  1905. case low_latency: /* 50 usec aka 20000 ints/s */
  1906. if (bytes > 10000) {
  1907. /* this if handles the TSO accounting */
  1908. if (bytes/packets > 8000) {
  1909. retval = bulk_latency;
  1910. } else if ((packets < 10) || ((bytes/packets) > 1200)) {
  1911. retval = bulk_latency;
  1912. } else if ((packets > 35)) {
  1913. retval = lowest_latency;
  1914. }
  1915. } else if (bytes/packets > 2000) {
  1916. retval = bulk_latency;
  1917. } else if (packets <= 2 && bytes < 512) {
  1918. retval = lowest_latency;
  1919. }
  1920. break;
  1921. case bulk_latency: /* 250 usec aka 4000 ints/s */
  1922. if (bytes > 25000) {
  1923. if (packets > 35)
  1924. retval = low_latency;
  1925. } else if (bytes < 6000) {
  1926. retval = low_latency;
  1927. }
  1928. break;
  1929. }
  1930. update_itr_done:
  1931. return retval;
  1932. }
  1933. static void igb_set_itr(struct igb_adapter *adapter, u16 itr_register,
  1934. int rx_only)
  1935. {
  1936. u16 current_itr;
  1937. u32 new_itr = adapter->itr;
  1938. /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
  1939. if (adapter->link_speed != SPEED_1000) {
  1940. current_itr = 0;
  1941. new_itr = 4000;
  1942. goto set_itr_now;
  1943. }
  1944. adapter->rx_itr = igb_update_itr(adapter,
  1945. adapter->rx_itr,
  1946. adapter->rx_ring->total_packets,
  1947. adapter->rx_ring->total_bytes);
  1948. /* conservative mode (itr 3) eliminates the lowest_latency setting */
  1949. if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
  1950. adapter->rx_itr = low_latency;
  1951. if (!rx_only) {
  1952. adapter->tx_itr = igb_update_itr(adapter,
  1953. adapter->tx_itr,
  1954. adapter->tx_ring->total_packets,
  1955. adapter->tx_ring->total_bytes);
  1956. /* conservative mode (itr 3) eliminates the
  1957. * lowest_latency setting */
  1958. if (adapter->itr_setting == 3 &&
  1959. adapter->tx_itr == lowest_latency)
  1960. adapter->tx_itr = low_latency;
  1961. current_itr = max(adapter->rx_itr, adapter->tx_itr);
  1962. } else {
  1963. current_itr = adapter->rx_itr;
  1964. }
  1965. switch (current_itr) {
  1966. /* counts and packets in update_itr are dependent on these numbers */
  1967. case lowest_latency:
  1968. new_itr = 70000;
  1969. break;
  1970. case low_latency:
  1971. new_itr = 20000; /* aka hwitr = ~200 */
  1972. break;
  1973. case bulk_latency:
  1974. new_itr = 4000;
  1975. break;
  1976. default:
  1977. break;
  1978. }
  1979. set_itr_now:
  1980. if (new_itr != adapter->itr) {
  1981. /* this attempts to bias the interrupt rate towards Bulk
  1982. * by adding intermediate steps when interrupt rate is
  1983. * increasing */
  1984. new_itr = new_itr > adapter->itr ?
  1985. min(adapter->itr + (new_itr >> 2), new_itr) :
  1986. new_itr;
  1987. /* Don't write the value here; it resets the adapter's
  1988. * internal timer, and causes us to delay far longer than
  1989. * we should between interrupts. Instead, we write the ITR
  1990. * value at the beginning of the next interrupt so the timing
  1991. * ends up being correct.
  1992. */
  1993. adapter->itr = new_itr;
  1994. adapter->set_itr = 1;
  1995. }
  1996. return;
  1997. }
  1998. #define IGB_TX_FLAGS_CSUM 0x00000001
  1999. #define IGB_TX_FLAGS_VLAN 0x00000002
  2000. #define IGB_TX_FLAGS_TSO 0x00000004
  2001. #define IGB_TX_FLAGS_IPV4 0x00000008
  2002. #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
  2003. #define IGB_TX_FLAGS_VLAN_SHIFT 16
  2004. static inline int igb_tso_adv(struct igb_adapter *adapter,
  2005. struct igb_ring *tx_ring,
  2006. struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
  2007. {
  2008. struct e1000_adv_tx_context_desc *context_desc;
  2009. unsigned int i;
  2010. int err;
  2011. struct igb_buffer *buffer_info;
  2012. u32 info = 0, tu_cmd = 0;
  2013. u32 mss_l4len_idx, l4len;
  2014. *hdr_len = 0;
  2015. if (skb_header_cloned(skb)) {
  2016. err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  2017. if (err)
  2018. return err;
  2019. }
  2020. l4len = tcp_hdrlen(skb);
  2021. *hdr_len += l4len;
  2022. if (skb->protocol == htons(ETH_P_IP)) {
  2023. struct iphdr *iph = ip_hdr(skb);
  2024. iph->tot_len = 0;
  2025. iph->check = 0;
  2026. tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
  2027. iph->daddr, 0,
  2028. IPPROTO_TCP,
  2029. 0);
  2030. } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
  2031. ipv6_hdr(skb)->payload_len = 0;
  2032. tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
  2033. &ipv6_hdr(skb)->daddr,
  2034. 0, IPPROTO_TCP, 0);
  2035. }
  2036. i = tx_ring->next_to_use;
  2037. buffer_info = &tx_ring->buffer_info[i];
  2038. context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
  2039. /* VLAN MACLEN IPLEN */
  2040. if (tx_flags & IGB_TX_FLAGS_VLAN)
  2041. info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
  2042. info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
  2043. *hdr_len += skb_network_offset(skb);
  2044. info |= skb_network_header_len(skb);
  2045. *hdr_len += skb_network_header_len(skb);
  2046. context_desc->vlan_macip_lens = cpu_to_le32(info);
  2047. /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
  2048. tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
  2049. if (skb->protocol == htons(ETH_P_IP))
  2050. tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
  2051. tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
  2052. context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
  2053. /* MSS L4LEN IDX */
  2054. mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
  2055. mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
  2056. /* Context index must be unique per ring. Luckily, so is the interrupt
  2057. * mask value. */
  2058. mss_l4len_idx |= tx_ring->eims_value >> 4;
  2059. context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
  2060. context_desc->seqnum_seed = 0;
  2061. buffer_info->time_stamp = jiffies;
  2062. buffer_info->dma = 0;
  2063. i++;
  2064. if (i == tx_ring->count)
  2065. i = 0;
  2066. tx_ring->next_to_use = i;
  2067. return true;
  2068. }
  2069. static inline bool igb_tx_csum_adv(struct igb_adapter *adapter,
  2070. struct igb_ring *tx_ring,
  2071. struct sk_buff *skb, u32 tx_flags)
  2072. {
  2073. struct e1000_adv_tx_context_desc *context_desc;
  2074. unsigned int i;
  2075. struct igb_buffer *buffer_info;
  2076. u32 info = 0, tu_cmd = 0;
  2077. if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
  2078. (tx_flags & IGB_TX_FLAGS_VLAN)) {
  2079. i = tx_ring->next_to_use;
  2080. buffer_info = &tx_ring->buffer_info[i];
  2081. context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
  2082. if (tx_flags & IGB_TX_FLAGS_VLAN)
  2083. info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
  2084. info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
  2085. if (skb->ip_summed == CHECKSUM_PARTIAL)
  2086. info |= skb_network_header_len(skb);
  2087. context_desc->vlan_macip_lens = cpu_to_le32(info);
  2088. tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
  2089. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  2090. switch (skb->protocol) {
  2091. case __constant_htons(ETH_P_IP):
  2092. tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
  2093. if (ip_hdr(skb)->protocol == IPPROTO_TCP)
  2094. tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
  2095. break;
  2096. case __constant_htons(ETH_P_IPV6):
  2097. /* XXX what about other V6 headers?? */
  2098. if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
  2099. tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
  2100. break;
  2101. default:
  2102. if (unlikely(net_ratelimit()))
  2103. dev_warn(&adapter->pdev->dev,
  2104. "partial checksum but proto=%x!\n",
  2105. skb->protocol);
  2106. break;
  2107. }
  2108. }
  2109. context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
  2110. context_desc->seqnum_seed = 0;
  2111. context_desc->mss_l4len_idx =
  2112. cpu_to_le32(tx_ring->eims_value >> 4);
  2113. buffer_info->time_stamp = jiffies;
  2114. buffer_info->dma = 0;
  2115. i++;
  2116. if (i == tx_ring->count)
  2117. i = 0;
  2118. tx_ring->next_to_use = i;
  2119. return true;
  2120. }
  2121. return false;
  2122. }
  2123. #define IGB_MAX_TXD_PWR 16
  2124. #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
  2125. static inline int igb_tx_map_adv(struct igb_adapter *adapter,
  2126. struct igb_ring *tx_ring,
  2127. struct sk_buff *skb)
  2128. {
  2129. struct igb_buffer *buffer_info;
  2130. unsigned int len = skb_headlen(skb);
  2131. unsigned int count = 0, i;
  2132. unsigned int f;
  2133. i = tx_ring->next_to_use;
  2134. buffer_info = &tx_ring->buffer_info[i];
  2135. BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
  2136. buffer_info->length = len;
  2137. /* set time_stamp *before* dma to help avoid a possible race */
  2138. buffer_info->time_stamp = jiffies;
  2139. buffer_info->dma = pci_map_single(adapter->pdev, skb->data, len,
  2140. PCI_DMA_TODEVICE);
  2141. count++;
  2142. i++;
  2143. if (i == tx_ring->count)
  2144. i = 0;
  2145. for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
  2146. struct skb_frag_struct *frag;
  2147. frag = &skb_shinfo(skb)->frags[f];
  2148. len = frag->size;
  2149. buffer_info = &tx_ring->buffer_info[i];
  2150. BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
  2151. buffer_info->length = len;
  2152. buffer_info->time_stamp = jiffies;
  2153. buffer_info->dma = pci_map_page(adapter->pdev,
  2154. frag->page,
  2155. frag->page_offset,
  2156. len,
  2157. PCI_DMA_TODEVICE);
  2158. count++;
  2159. i++;
  2160. if (i == tx_ring->count)
  2161. i = 0;
  2162. }
  2163. i = (i == 0) ? tx_ring->count - 1 : i - 1;
  2164. tx_ring->buffer_info[i].skb = skb;
  2165. return count;
  2166. }
  2167. static inline void igb_tx_queue_adv(struct igb_adapter *adapter,
  2168. struct igb_ring *tx_ring,
  2169. int tx_flags, int count, u32 paylen,
  2170. u8 hdr_len)
  2171. {
  2172. union e1000_adv_tx_desc *tx_desc = NULL;
  2173. struct igb_buffer *buffer_info;
  2174. u32 olinfo_status = 0, cmd_type_len;
  2175. unsigned int i;
  2176. cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
  2177. E1000_ADVTXD_DCMD_DEXT);
  2178. if (tx_flags & IGB_TX_FLAGS_VLAN)
  2179. cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
  2180. if (tx_flags & IGB_TX_FLAGS_TSO) {
  2181. cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
  2182. /* insert tcp checksum */
  2183. olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
  2184. /* insert ip checksum */
  2185. if (tx_flags & IGB_TX_FLAGS_IPV4)
  2186. olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
  2187. } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
  2188. olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
  2189. }
  2190. if (tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_TSO |
  2191. IGB_TX_FLAGS_VLAN))
  2192. olinfo_status |= tx_ring->eims_value >> 4;
  2193. olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
  2194. i = tx_ring->next_to_use;
  2195. while (count--) {
  2196. buffer_info = &tx_ring->buffer_info[i];
  2197. tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
  2198. tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
  2199. tx_desc->read.cmd_type_len =
  2200. cpu_to_le32(cmd_type_len | buffer_info->length);
  2201. tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
  2202. i++;
  2203. if (i == tx_ring->count)
  2204. i = 0;
  2205. }
  2206. tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
  2207. /* Force memory writes to complete before letting h/w
  2208. * know there are new descriptors to fetch. (Only
  2209. * applicable for weak-ordered memory model archs,
  2210. * such as IA-64). */
  2211. wmb();
  2212. tx_ring->next_to_use = i;
  2213. writel(i, adapter->hw.hw_addr + tx_ring->tail);
  2214. /* we need this if more than one processor can write to our tail
  2215. * at a time, it syncronizes IO on IA64/Altix systems */
  2216. mmiowb();
  2217. }
  2218. static int __igb_maybe_stop_tx(struct net_device *netdev,
  2219. struct igb_ring *tx_ring, int size)
  2220. {
  2221. struct igb_adapter *adapter = netdev_priv(netdev);
  2222. netif_stop_queue(netdev);
  2223. /* Herbert's original patch had:
  2224. * smp_mb__after_netif_stop_queue();
  2225. * but since that doesn't exist yet, just open code it. */
  2226. smp_mb();
  2227. /* We need to check again in a case another CPU has just
  2228. * made room available. */
  2229. if (IGB_DESC_UNUSED(tx_ring) < size)
  2230. return -EBUSY;
  2231. /* A reprieve! */
  2232. netif_start_queue(netdev);
  2233. ++adapter->restart_queue;
  2234. return 0;
  2235. }
  2236. static int igb_maybe_stop_tx(struct net_device *netdev,
  2237. struct igb_ring *tx_ring, int size)
  2238. {
  2239. if (IGB_DESC_UNUSED(tx_ring) >= size)
  2240. return 0;
  2241. return __igb_maybe_stop_tx(netdev, tx_ring, size);
  2242. }
  2243. #define TXD_USE_COUNT(S) (((S) >> (IGB_MAX_TXD_PWR)) + 1)
  2244. static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
  2245. struct net_device *netdev,
  2246. struct igb_ring *tx_ring)
  2247. {
  2248. struct igb_adapter *adapter = netdev_priv(netdev);
  2249. unsigned int tx_flags = 0;
  2250. unsigned int len;
  2251. unsigned long irq_flags;
  2252. u8 hdr_len = 0;
  2253. int tso = 0;
  2254. len = skb_headlen(skb);
  2255. if (test_bit(__IGB_DOWN, &adapter->state)) {
  2256. dev_kfree_skb_any(skb);
  2257. return NETDEV_TX_OK;
  2258. }
  2259. if (skb->len <= 0) {
  2260. dev_kfree_skb_any(skb);
  2261. return NETDEV_TX_OK;
  2262. }
  2263. if (!spin_trylock_irqsave(&tx_ring->tx_lock, irq_flags))
  2264. /* Collision - tell upper layer to requeue */
  2265. return NETDEV_TX_LOCKED;
  2266. /* need: 1 descriptor per page,
  2267. * + 2 desc gap to keep tail from touching head,
  2268. * + 1 desc for skb->data,
  2269. * + 1 desc for context descriptor,
  2270. * otherwise try next time */
  2271. if (igb_maybe_stop_tx(netdev, tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
  2272. /* this is a hard error */
  2273. spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
  2274. return NETDEV_TX_BUSY;
  2275. }
  2276. if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
  2277. tx_flags |= IGB_TX_FLAGS_VLAN;
  2278. tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
  2279. }
  2280. tso = skb_is_gso(skb) ? igb_tso_adv(adapter, tx_ring, skb, tx_flags,
  2281. &hdr_len) : 0;
  2282. if (tso < 0) {
  2283. dev_kfree_skb_any(skb);
  2284. spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
  2285. return NETDEV_TX_OK;
  2286. }
  2287. if (tso)
  2288. tx_flags |= IGB_TX_FLAGS_TSO;
  2289. else if (igb_tx_csum_adv(adapter, tx_ring, skb, tx_flags))
  2290. if (skb->ip_summed == CHECKSUM_PARTIAL)
  2291. tx_flags |= IGB_TX_FLAGS_CSUM;
  2292. if (skb->protocol == htons(ETH_P_IP))
  2293. tx_flags |= IGB_TX_FLAGS_IPV4;
  2294. igb_tx_queue_adv(adapter, tx_ring, tx_flags,
  2295. igb_tx_map_adv(adapter, tx_ring, skb),
  2296. skb->len, hdr_len);
  2297. netdev->trans_start = jiffies;
  2298. /* Make sure there is space in the ring for the next send. */
  2299. igb_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 4);
  2300. spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
  2301. return NETDEV_TX_OK;
  2302. }
  2303. static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *netdev)
  2304. {
  2305. struct igb_adapter *adapter = netdev_priv(netdev);
  2306. struct igb_ring *tx_ring = &adapter->tx_ring[0];
  2307. /* This goes back to the question of how to logically map a tx queue
  2308. * to a flow. Right now, performance is impacted slightly negatively
  2309. * if using multiple tx queues. If the stack breaks away from a
  2310. * single qdisc implementation, we can look at this again. */
  2311. return (igb_xmit_frame_ring_adv(skb, netdev, tx_ring));
  2312. }
  2313. /**
  2314. * igb_tx_timeout - Respond to a Tx Hang
  2315. * @netdev: network interface device structure
  2316. **/
  2317. static void igb_tx_timeout(struct net_device *netdev)
  2318. {
  2319. struct igb_adapter *adapter = netdev_priv(netdev);
  2320. struct e1000_hw *hw = &adapter->hw;
  2321. /* Do the reset outside of interrupt context */
  2322. adapter->tx_timeout_count++;
  2323. schedule_work(&adapter->reset_task);
  2324. wr32(E1000_EICS, adapter->eims_enable_mask &
  2325. ~(E1000_EIMS_TCP_TIMER | E1000_EIMS_OTHER));
  2326. }
  2327. static void igb_reset_task(struct work_struct *work)
  2328. {
  2329. struct igb_adapter *adapter;
  2330. adapter = container_of(work, struct igb_adapter, reset_task);
  2331. igb_reinit_locked(adapter);
  2332. }
  2333. /**
  2334. * igb_get_stats - Get System Network Statistics
  2335. * @netdev: network interface device structure
  2336. *
  2337. * Returns the address of the device statistics structure.
  2338. * The statistics are actually updated from the timer callback.
  2339. **/
  2340. static struct net_device_stats *
  2341. igb_get_stats(struct net_device *netdev)
  2342. {
  2343. struct igb_adapter *adapter = netdev_priv(netdev);
  2344. /* only return the current stats */
  2345. return &adapter->net_stats;
  2346. }
  2347. /**
  2348. * igb_change_mtu - Change the Maximum Transfer Unit
  2349. * @netdev: network interface device structure
  2350. * @new_mtu: new value for maximum frame size
  2351. *
  2352. * Returns 0 on success, negative on failure
  2353. **/
  2354. static int igb_change_mtu(struct net_device *netdev, int new_mtu)
  2355. {
  2356. struct igb_adapter *adapter = netdev_priv(netdev);
  2357. int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
  2358. if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
  2359. (max_frame > MAX_JUMBO_FRAME_SIZE)) {
  2360. dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
  2361. return -EINVAL;
  2362. }
  2363. #define MAX_STD_JUMBO_FRAME_SIZE 9234
  2364. if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
  2365. dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
  2366. return -EINVAL;
  2367. }
  2368. while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
  2369. msleep(1);
  2370. /* igb_down has a dependency on max_frame_size */
  2371. adapter->max_frame_size = max_frame;
  2372. if (netif_running(netdev))
  2373. igb_down(adapter);
  2374. /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
  2375. * means we reserve 2 more, this pushes us to allocate from the next
  2376. * larger slab size.
  2377. * i.e. RXBUFFER_2048 --> size-4096 slab
  2378. */
  2379. if (max_frame <= IGB_RXBUFFER_256)
  2380. adapter->rx_buffer_len = IGB_RXBUFFER_256;
  2381. else if (max_frame <= IGB_RXBUFFER_512)
  2382. adapter->rx_buffer_len = IGB_RXBUFFER_512;
  2383. else if (max_frame <= IGB_RXBUFFER_1024)
  2384. adapter->rx_buffer_len = IGB_RXBUFFER_1024;
  2385. else if (max_frame <= IGB_RXBUFFER_2048)
  2386. adapter->rx_buffer_len = IGB_RXBUFFER_2048;
  2387. else
  2388. adapter->rx_buffer_len = IGB_RXBUFFER_4096;
  2389. /* adjust allocation if LPE protects us, and we aren't using SBP */
  2390. if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
  2391. (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE))
  2392. adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
  2393. dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
  2394. netdev->mtu, new_mtu);
  2395. netdev->mtu = new_mtu;
  2396. if (netif_running(netdev))
  2397. igb_up(adapter);
  2398. else
  2399. igb_reset(adapter);
  2400. clear_bit(__IGB_RESETTING, &adapter->state);
  2401. return 0;
  2402. }
  2403. /**
  2404. * igb_update_stats - Update the board statistics counters
  2405. * @adapter: board private structure
  2406. **/
  2407. void igb_update_stats(struct igb_adapter *adapter)
  2408. {
  2409. struct e1000_hw *hw = &adapter->hw;
  2410. struct pci_dev *pdev = adapter->pdev;
  2411. u16 phy_tmp;
  2412. #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
  2413. /*
  2414. * Prevent stats update while adapter is being reset, or if the pci
  2415. * connection is down.
  2416. */
  2417. if (adapter->link_speed == 0)
  2418. return;
  2419. if (pci_channel_offline(pdev))
  2420. return;
  2421. adapter->stats.crcerrs += rd32(E1000_CRCERRS);
  2422. adapter->stats.gprc += rd32(E1000_GPRC);
  2423. adapter->stats.gorc += rd32(E1000_GORCL);
  2424. rd32(E1000_GORCH); /* clear GORCL */
  2425. adapter->stats.bprc += rd32(E1000_BPRC);
  2426. adapter->stats.mprc += rd32(E1000_MPRC);
  2427. adapter->stats.roc += rd32(E1000_ROC);
  2428. adapter->stats.prc64 += rd32(E1000_PRC64);
  2429. adapter->stats.prc127 += rd32(E1000_PRC127);
  2430. adapter->stats.prc255 += rd32(E1000_PRC255);
  2431. adapter->stats.prc511 += rd32(E1000_PRC511);
  2432. adapter->stats.prc1023 += rd32(E1000_PRC1023);
  2433. adapter->stats.prc1522 += rd32(E1000_PRC1522);
  2434. adapter->stats.symerrs += rd32(E1000_SYMERRS);
  2435. adapter->stats.sec += rd32(E1000_SEC);
  2436. adapter->stats.mpc += rd32(E1000_MPC);
  2437. adapter->stats.scc += rd32(E1000_SCC);
  2438. adapter->stats.ecol += rd32(E1000_ECOL);
  2439. adapter->stats.mcc += rd32(E1000_MCC);
  2440. adapter->stats.latecol += rd32(E1000_LATECOL);
  2441. adapter->stats.dc += rd32(E1000_DC);
  2442. adapter->stats.rlec += rd32(E1000_RLEC);
  2443. adapter->stats.xonrxc += rd32(E1000_XONRXC);
  2444. adapter->stats.xontxc += rd32(E1000_XONTXC);
  2445. adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
  2446. adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
  2447. adapter->stats.fcruc += rd32(E1000_FCRUC);
  2448. adapter->stats.gptc += rd32(E1000_GPTC);
  2449. adapter->stats.gotc += rd32(E1000_GOTCL);
  2450. rd32(E1000_GOTCH); /* clear GOTCL */
  2451. adapter->stats.rnbc += rd32(E1000_RNBC);
  2452. adapter->stats.ruc += rd32(E1000_RUC);
  2453. adapter->stats.rfc += rd32(E1000_RFC);
  2454. adapter->stats.rjc += rd32(E1000_RJC);
  2455. adapter->stats.tor += rd32(E1000_TORH);
  2456. adapter->stats.tot += rd32(E1000_TOTH);
  2457. adapter->stats.tpr += rd32(E1000_TPR);
  2458. adapter->stats.ptc64 += rd32(E1000_PTC64);
  2459. adapter->stats.ptc127 += rd32(E1000_PTC127);
  2460. adapter->stats.ptc255 += rd32(E1000_PTC255);
  2461. adapter->stats.ptc511 += rd32(E1000_PTC511);
  2462. adapter->stats.ptc1023 += rd32(E1000_PTC1023);
  2463. adapter->stats.ptc1522 += rd32(E1000_PTC1522);
  2464. adapter->stats.mptc += rd32(E1000_MPTC);
  2465. adapter->stats.bptc += rd32(E1000_BPTC);
  2466. /* used for adaptive IFS */
  2467. hw->mac.tx_packet_delta = rd32(E1000_TPT);
  2468. adapter->stats.tpt += hw->mac.tx_packet_delta;
  2469. hw->mac.collision_delta = rd32(E1000_COLC);
  2470. adapter->stats.colc += hw->mac.collision_delta;
  2471. adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
  2472. adapter->stats.rxerrc += rd32(E1000_RXERRC);
  2473. adapter->stats.tncrs += rd32(E1000_TNCRS);
  2474. adapter->stats.tsctc += rd32(E1000_TSCTC);
  2475. adapter->stats.tsctfc += rd32(E1000_TSCTFC);
  2476. adapter->stats.iac += rd32(E1000_IAC);
  2477. adapter->stats.icrxoc += rd32(E1000_ICRXOC);
  2478. adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
  2479. adapter->stats.icrxatc += rd32(E1000_ICRXATC);
  2480. adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
  2481. adapter->stats.ictxatc += rd32(E1000_ICTXATC);
  2482. adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
  2483. adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
  2484. adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
  2485. /* Fill out the OS statistics structure */
  2486. adapter->net_stats.multicast = adapter->stats.mprc;
  2487. adapter->net_stats.collisions = adapter->stats.colc;
  2488. /* Rx Errors */
  2489. /* RLEC on some newer hardware can be incorrect so build
  2490. * our own version based on RUC and ROC */
  2491. adapter->net_stats.rx_errors = adapter->stats.rxerrc +
  2492. adapter->stats.crcerrs + adapter->stats.algnerrc +
  2493. adapter->stats.ruc + adapter->stats.roc +
  2494. adapter->stats.cexterr;
  2495. adapter->net_stats.rx_length_errors = adapter->stats.ruc +
  2496. adapter->stats.roc;
  2497. adapter->net_stats.rx_crc_errors = adapter->stats.crcerrs;
  2498. adapter->net_stats.rx_frame_errors = adapter->stats.algnerrc;
  2499. adapter->net_stats.rx_missed_errors = adapter->stats.mpc;
  2500. /* Tx Errors */
  2501. adapter->net_stats.tx_errors = adapter->stats.ecol +
  2502. adapter->stats.latecol;
  2503. adapter->net_stats.tx_aborted_errors = adapter->stats.ecol;
  2504. adapter->net_stats.tx_window_errors = adapter->stats.latecol;
  2505. adapter->net_stats.tx_carrier_errors = adapter->stats.tncrs;
  2506. /* Tx Dropped needs to be maintained elsewhere */
  2507. /* Phy Stats */
  2508. if (hw->phy.media_type == e1000_media_type_copper) {
  2509. if ((adapter->link_speed == SPEED_1000) &&
  2510. (!hw->phy.ops.read_phy_reg(hw, PHY_1000T_STATUS,
  2511. &phy_tmp))) {
  2512. phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
  2513. adapter->phy_stats.idle_errors += phy_tmp;
  2514. }
  2515. }
  2516. /* Management Stats */
  2517. adapter->stats.mgptc += rd32(E1000_MGTPTC);
  2518. adapter->stats.mgprc += rd32(E1000_MGTPRC);
  2519. adapter->stats.mgpdc += rd32(E1000_MGTPDC);
  2520. }
  2521. static irqreturn_t igb_msix_other(int irq, void *data)
  2522. {
  2523. struct net_device *netdev = data;
  2524. struct igb_adapter *adapter = netdev_priv(netdev);
  2525. struct e1000_hw *hw = &adapter->hw;
  2526. u32 eicr;
  2527. /* disable interrupts from the "other" bit, avoid re-entry */
  2528. wr32(E1000_EIMC, E1000_EIMS_OTHER);
  2529. eicr = rd32(E1000_EICR);
  2530. if (eicr & E1000_EIMS_OTHER) {
  2531. u32 icr = rd32(E1000_ICR);
  2532. /* reading ICR causes bit 31 of EICR to be cleared */
  2533. if (!(icr & E1000_ICR_LSC))
  2534. goto no_link_interrupt;
  2535. hw->mac.get_link_status = 1;
  2536. /* guard against interrupt when we're going down */
  2537. if (!test_bit(__IGB_DOWN, &adapter->state))
  2538. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  2539. }
  2540. no_link_interrupt:
  2541. wr32(E1000_IMS, E1000_IMS_LSC);
  2542. wr32(E1000_EIMS, E1000_EIMS_OTHER);
  2543. return IRQ_HANDLED;
  2544. }
  2545. static irqreturn_t igb_msix_tx(int irq, void *data)
  2546. {
  2547. struct igb_ring *tx_ring = data;
  2548. struct igb_adapter *adapter = tx_ring->adapter;
  2549. struct e1000_hw *hw = &adapter->hw;
  2550. if (!tx_ring->itr_val)
  2551. wr32(E1000_EIMC, tx_ring->eims_value);
  2552. tx_ring->total_bytes = 0;
  2553. tx_ring->total_packets = 0;
  2554. if (!igb_clean_tx_irq(adapter, tx_ring))
  2555. /* Ring was not completely cleaned, so fire another interrupt */
  2556. wr32(E1000_EICS, tx_ring->eims_value);
  2557. if (!tx_ring->itr_val)
  2558. wr32(E1000_EIMS, tx_ring->eims_value);
  2559. return IRQ_HANDLED;
  2560. }
  2561. static irqreturn_t igb_msix_rx(int irq, void *data)
  2562. {
  2563. struct igb_ring *rx_ring = data;
  2564. struct igb_adapter *adapter = rx_ring->adapter;
  2565. struct e1000_hw *hw = &adapter->hw;
  2566. if (!rx_ring->itr_val)
  2567. wr32(E1000_EIMC, rx_ring->eims_value);
  2568. if (netif_rx_schedule_prep(adapter->netdev, &rx_ring->napi)) {
  2569. rx_ring->total_bytes = 0;
  2570. rx_ring->total_packets = 0;
  2571. rx_ring->no_itr_adjust = 0;
  2572. __netif_rx_schedule(adapter->netdev, &rx_ring->napi);
  2573. } else {
  2574. if (!rx_ring->no_itr_adjust) {
  2575. igb_lower_rx_eitr(adapter, rx_ring);
  2576. rx_ring->no_itr_adjust = 1;
  2577. }
  2578. }
  2579. return IRQ_HANDLED;
  2580. }
  2581. /**
  2582. * igb_intr_msi - Interrupt Handler
  2583. * @irq: interrupt number
  2584. * @data: pointer to a network interface device structure
  2585. **/
  2586. static irqreturn_t igb_intr_msi(int irq, void *data)
  2587. {
  2588. struct net_device *netdev = data;
  2589. struct igb_adapter *adapter = netdev_priv(netdev);
  2590. struct napi_struct *napi = &adapter->napi;
  2591. struct e1000_hw *hw = &adapter->hw;
  2592. /* read ICR disables interrupts using IAM */
  2593. u32 icr = rd32(E1000_ICR);
  2594. /* Write the ITR value calculated at the end of the
  2595. * previous interrupt.
  2596. */
  2597. if (adapter->set_itr) {
  2598. wr32(E1000_ITR,
  2599. 1000000000 / (adapter->itr * 256));
  2600. adapter->set_itr = 0;
  2601. }
  2602. /* read ICR disables interrupts using IAM */
  2603. if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
  2604. hw->mac.get_link_status = 1;
  2605. if (!test_bit(__IGB_DOWN, &adapter->state))
  2606. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  2607. }
  2608. if (netif_rx_schedule_prep(netdev, napi)) {
  2609. adapter->tx_ring->total_bytes = 0;
  2610. adapter->tx_ring->total_packets = 0;
  2611. adapter->rx_ring->total_bytes = 0;
  2612. adapter->rx_ring->total_packets = 0;
  2613. __netif_rx_schedule(netdev, napi);
  2614. }
  2615. return IRQ_HANDLED;
  2616. }
  2617. /**
  2618. * igb_intr - Interrupt Handler
  2619. * @irq: interrupt number
  2620. * @data: pointer to a network interface device structure
  2621. **/
  2622. static irqreturn_t igb_intr(int irq, void *data)
  2623. {
  2624. struct net_device *netdev = data;
  2625. struct igb_adapter *adapter = netdev_priv(netdev);
  2626. struct napi_struct *napi = &adapter->napi;
  2627. struct e1000_hw *hw = &adapter->hw;
  2628. /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
  2629. * need for the IMC write */
  2630. u32 icr = rd32(E1000_ICR);
  2631. u32 eicr = 0;
  2632. if (!icr)
  2633. return IRQ_NONE; /* Not our interrupt */
  2634. /* Write the ITR value calculated at the end of the
  2635. * previous interrupt.
  2636. */
  2637. if (adapter->set_itr) {
  2638. wr32(E1000_ITR,
  2639. 1000000000 / (adapter->itr * 256));
  2640. adapter->set_itr = 0;
  2641. }
  2642. /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
  2643. * not set, then the adapter didn't send an interrupt */
  2644. if (!(icr & E1000_ICR_INT_ASSERTED))
  2645. return IRQ_NONE;
  2646. eicr = rd32(E1000_EICR);
  2647. if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
  2648. hw->mac.get_link_status = 1;
  2649. /* guard against interrupt when we're going down */
  2650. if (!test_bit(__IGB_DOWN, &adapter->state))
  2651. mod_timer(&adapter->watchdog_timer, jiffies + 1);
  2652. }
  2653. if (netif_rx_schedule_prep(netdev, napi)) {
  2654. adapter->tx_ring->total_bytes = 0;
  2655. adapter->rx_ring->total_bytes = 0;
  2656. adapter->tx_ring->total_packets = 0;
  2657. adapter->rx_ring->total_packets = 0;
  2658. __netif_rx_schedule(netdev, napi);
  2659. }
  2660. return IRQ_HANDLED;
  2661. }
  2662. /**
  2663. * igb_clean - NAPI Rx polling callback
  2664. * @adapter: board private structure
  2665. **/
  2666. static int igb_clean(struct napi_struct *napi, int budget)
  2667. {
  2668. struct igb_adapter *adapter = container_of(napi, struct igb_adapter,
  2669. napi);
  2670. struct net_device *netdev = adapter->netdev;
  2671. int tx_clean_complete = 1, work_done = 0;
  2672. int i;
  2673. /* Must NOT use netdev_priv macro here. */
  2674. adapter = netdev->priv;
  2675. /* Keep link state information with original netdev */
  2676. if (!netif_carrier_ok(netdev))
  2677. goto quit_polling;
  2678. /* igb_clean is called per-cpu. This lock protects tx_ring[i] from
  2679. * being cleaned by multiple cpus simultaneously. A failure obtaining
  2680. * the lock means tx_ring[i] is currently being cleaned anyway. */
  2681. for (i = 0; i < adapter->num_tx_queues; i++) {
  2682. if (spin_trylock(&adapter->tx_ring[i].tx_clean_lock)) {
  2683. tx_clean_complete &= igb_clean_tx_irq(adapter,
  2684. &adapter->tx_ring[i]);
  2685. spin_unlock(&adapter->tx_ring[i].tx_clean_lock);
  2686. }
  2687. }
  2688. for (i = 0; i < adapter->num_rx_queues; i++)
  2689. igb_clean_rx_irq_adv(adapter, &adapter->rx_ring[i], &work_done,
  2690. adapter->rx_ring[i].napi.weight);
  2691. /* If no Tx and not enough Rx work done, exit the polling mode */
  2692. if ((tx_clean_complete && (work_done < budget)) ||
  2693. !netif_running(netdev)) {
  2694. quit_polling:
  2695. if (adapter->itr_setting & 3)
  2696. igb_set_itr(adapter, E1000_ITR, false);
  2697. netif_rx_complete(netdev, napi);
  2698. if (!test_bit(__IGB_DOWN, &adapter->state))
  2699. igb_irq_enable(adapter);
  2700. return 0;
  2701. }
  2702. return 1;
  2703. }
  2704. static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
  2705. {
  2706. struct igb_ring *rx_ring = container_of(napi, struct igb_ring, napi);
  2707. struct igb_adapter *adapter = rx_ring->adapter;
  2708. struct e1000_hw *hw = &adapter->hw;
  2709. struct net_device *netdev = adapter->netdev;
  2710. int work_done = 0;
  2711. /* Keep link state information with original netdev */
  2712. if (!netif_carrier_ok(netdev))
  2713. goto quit_polling;
  2714. igb_clean_rx_irq_adv(adapter, rx_ring, &work_done, budget);
  2715. /* If not enough Rx work done, exit the polling mode */
  2716. if ((work_done == 0) || !netif_running(netdev)) {
  2717. quit_polling:
  2718. netif_rx_complete(netdev, napi);
  2719. wr32(E1000_EIMS, rx_ring->eims_value);
  2720. if ((adapter->itr_setting & 3) && !rx_ring->no_itr_adjust &&
  2721. (rx_ring->total_packets > IGB_DYN_ITR_PACKET_THRESHOLD)) {
  2722. int mean_size = rx_ring->total_bytes /
  2723. rx_ring->total_packets;
  2724. if (mean_size < IGB_DYN_ITR_LENGTH_LOW)
  2725. igb_raise_rx_eitr(adapter, rx_ring);
  2726. else if (mean_size > IGB_DYN_ITR_LENGTH_HIGH)
  2727. igb_lower_rx_eitr(adapter, rx_ring);
  2728. }
  2729. return 0;
  2730. }
  2731. return 1;
  2732. }
  2733. static inline u32 get_head(struct igb_ring *tx_ring)
  2734. {
  2735. void *end = (struct e1000_tx_desc *)tx_ring->desc + tx_ring->count;
  2736. return le32_to_cpu(*(volatile __le32 *)end);
  2737. }
  2738. /**
  2739. * igb_clean_tx_irq - Reclaim resources after transmit completes
  2740. * @adapter: board private structure
  2741. * returns true if ring is completely cleaned
  2742. **/
  2743. static bool igb_clean_tx_irq(struct igb_adapter *adapter,
  2744. struct igb_ring *tx_ring)
  2745. {
  2746. struct net_device *netdev = adapter->netdev;
  2747. struct e1000_hw *hw = &adapter->hw;
  2748. struct e1000_tx_desc *tx_desc;
  2749. struct igb_buffer *buffer_info;
  2750. struct sk_buff *skb;
  2751. unsigned int i;
  2752. u32 head, oldhead;
  2753. unsigned int count = 0;
  2754. bool cleaned = false;
  2755. bool retval = true;
  2756. unsigned int total_bytes = 0, total_packets = 0;
  2757. rmb();
  2758. head = get_head(tx_ring);
  2759. i = tx_ring->next_to_clean;
  2760. while (1) {
  2761. while (i != head) {
  2762. cleaned = true;
  2763. tx_desc = E1000_TX_DESC(*tx_ring, i);
  2764. buffer_info = &tx_ring->buffer_info[i];
  2765. skb = buffer_info->skb;
  2766. if (skb) {
  2767. unsigned int segs, bytecount;
  2768. /* gso_segs is currently only valid for tcp */
  2769. segs = skb_shinfo(skb)->gso_segs ?: 1;
  2770. /* multiply data chunks by size of headers */
  2771. bytecount = ((segs - 1) * skb_headlen(skb)) +
  2772. skb->len;
  2773. total_packets += segs;
  2774. total_bytes += bytecount;
  2775. }
  2776. igb_unmap_and_free_tx_resource(adapter, buffer_info);
  2777. tx_desc->upper.data = 0;
  2778. i++;
  2779. if (i == tx_ring->count)
  2780. i = 0;
  2781. count++;
  2782. if (count == IGB_MAX_TX_CLEAN) {
  2783. retval = false;
  2784. goto done_cleaning;
  2785. }
  2786. }
  2787. oldhead = head;
  2788. rmb();
  2789. head = get_head(tx_ring);
  2790. if (head == oldhead)
  2791. goto done_cleaning;
  2792. } /* while (1) */
  2793. done_cleaning:
  2794. tx_ring->next_to_clean = i;
  2795. if (unlikely(cleaned &&
  2796. netif_carrier_ok(netdev) &&
  2797. IGB_DESC_UNUSED(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
  2798. /* Make sure that anybody stopping the queue after this
  2799. * sees the new next_to_clean.
  2800. */
  2801. smp_mb();
  2802. if (netif_queue_stopped(netdev) &&
  2803. !(test_bit(__IGB_DOWN, &adapter->state))) {
  2804. netif_wake_queue(netdev);
  2805. ++adapter->restart_queue;
  2806. }
  2807. }
  2808. if (tx_ring->detect_tx_hung) {
  2809. /* Detect a transmit hang in hardware, this serializes the
  2810. * check with the clearing of time_stamp and movement of i */
  2811. tx_ring->detect_tx_hung = false;
  2812. if (tx_ring->buffer_info[i].time_stamp &&
  2813. time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
  2814. (adapter->tx_timeout_factor * HZ))
  2815. && !(rd32(E1000_STATUS) &
  2816. E1000_STATUS_TXOFF)) {
  2817. tx_desc = E1000_TX_DESC(*tx_ring, i);
  2818. /* detected Tx unit hang */
  2819. dev_err(&adapter->pdev->dev,
  2820. "Detected Tx Unit Hang\n"
  2821. " Tx Queue <%lu>\n"
  2822. " TDH <%x>\n"
  2823. " TDT <%x>\n"
  2824. " next_to_use <%x>\n"
  2825. " next_to_clean <%x>\n"
  2826. " head (WB) <%x>\n"
  2827. "buffer_info[next_to_clean]\n"
  2828. " time_stamp <%lx>\n"
  2829. " jiffies <%lx>\n"
  2830. " desc.status <%x>\n",
  2831. (unsigned long)((tx_ring - adapter->tx_ring) /
  2832. sizeof(struct igb_ring)),
  2833. readl(adapter->hw.hw_addr + tx_ring->head),
  2834. readl(adapter->hw.hw_addr + tx_ring->tail),
  2835. tx_ring->next_to_use,
  2836. tx_ring->next_to_clean,
  2837. head,
  2838. tx_ring->buffer_info[i].time_stamp,
  2839. jiffies,
  2840. tx_desc->upper.fields.status);
  2841. netif_stop_queue(netdev);
  2842. }
  2843. }
  2844. tx_ring->total_bytes += total_bytes;
  2845. tx_ring->total_packets += total_packets;
  2846. adapter->net_stats.tx_bytes += total_bytes;
  2847. adapter->net_stats.tx_packets += total_packets;
  2848. return retval;
  2849. }
  2850. /**
  2851. * igb_receive_skb - helper function to handle rx indications
  2852. * @adapter: board private structure
  2853. * @status: descriptor status field as written by hardware
  2854. * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
  2855. * @skb: pointer to sk_buff to be indicated to stack
  2856. **/
  2857. static void igb_receive_skb(struct igb_adapter *adapter, u8 status, __le16 vlan,
  2858. struct sk_buff *skb)
  2859. {
  2860. if (adapter->vlgrp && (status & E1000_RXD_STAT_VP))
  2861. vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
  2862. le16_to_cpu(vlan) &
  2863. E1000_RXD_SPC_VLAN_MASK);
  2864. else
  2865. netif_receive_skb(skb);
  2866. }
  2867. static inline void igb_rx_checksum_adv(struct igb_adapter *adapter,
  2868. u32 status_err, struct sk_buff *skb)
  2869. {
  2870. skb->ip_summed = CHECKSUM_NONE;
  2871. /* Ignore Checksum bit is set or checksum is disabled through ethtool */
  2872. if ((status_err & E1000_RXD_STAT_IXSM) || !adapter->rx_csum)
  2873. return;
  2874. /* TCP/UDP checksum error bit is set */
  2875. if (status_err &
  2876. (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
  2877. /* let the stack verify checksum errors */
  2878. adapter->hw_csum_err++;
  2879. return;
  2880. }
  2881. /* It must be a TCP or UDP packet with a valid checksum */
  2882. if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
  2883. skb->ip_summed = CHECKSUM_UNNECESSARY;
  2884. adapter->hw_csum_good++;
  2885. }
  2886. static bool igb_clean_rx_irq_adv(struct igb_adapter *adapter,
  2887. struct igb_ring *rx_ring,
  2888. int *work_done, int budget)
  2889. {
  2890. struct net_device *netdev = adapter->netdev;
  2891. struct pci_dev *pdev = adapter->pdev;
  2892. union e1000_adv_rx_desc *rx_desc , *next_rxd;
  2893. struct igb_buffer *buffer_info , *next_buffer;
  2894. struct sk_buff *skb;
  2895. unsigned int i, j;
  2896. u32 length, hlen, staterr;
  2897. bool cleaned = false;
  2898. int cleaned_count = 0;
  2899. unsigned int total_bytes = 0, total_packets = 0;
  2900. i = rx_ring->next_to_clean;
  2901. rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
  2902. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  2903. while (staterr & E1000_RXD_STAT_DD) {
  2904. if (*work_done >= budget)
  2905. break;
  2906. (*work_done)++;
  2907. buffer_info = &rx_ring->buffer_info[i];
  2908. /* HW will not DMA in data larger than the given buffer, even
  2909. * if it parses the (NFS, of course) header to be larger. In
  2910. * that case, it fills the header buffer and spills the rest
  2911. * into the page.
  2912. */
  2913. hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
  2914. E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
  2915. if (hlen > adapter->rx_ps_hdr_size)
  2916. hlen = adapter->rx_ps_hdr_size;
  2917. length = le16_to_cpu(rx_desc->wb.upper.length);
  2918. cleaned = true;
  2919. cleaned_count++;
  2920. if (rx_ring->pending_skb != NULL) {
  2921. skb = rx_ring->pending_skb;
  2922. rx_ring->pending_skb = NULL;
  2923. j = rx_ring->pending_skb_page;
  2924. } else {
  2925. skb = buffer_info->skb;
  2926. prefetch(skb->data - NET_IP_ALIGN);
  2927. buffer_info->skb = NULL;
  2928. if (hlen) {
  2929. pci_unmap_single(pdev, buffer_info->dma,
  2930. adapter->rx_ps_hdr_size +
  2931. NET_IP_ALIGN,
  2932. PCI_DMA_FROMDEVICE);
  2933. skb_put(skb, hlen);
  2934. } else {
  2935. pci_unmap_single(pdev, buffer_info->dma,
  2936. adapter->rx_buffer_len +
  2937. NET_IP_ALIGN,
  2938. PCI_DMA_FROMDEVICE);
  2939. skb_put(skb, length);
  2940. goto send_up;
  2941. }
  2942. j = 0;
  2943. }
  2944. while (length) {
  2945. pci_unmap_page(pdev, buffer_info->page_dma,
  2946. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  2947. buffer_info->page_dma = 0;
  2948. skb_fill_page_desc(skb, j, buffer_info->page,
  2949. 0, length);
  2950. buffer_info->page = NULL;
  2951. skb->len += length;
  2952. skb->data_len += length;
  2953. skb->truesize += length;
  2954. rx_desc->wb.upper.status_error = 0;
  2955. if (staterr & E1000_RXD_STAT_EOP)
  2956. break;
  2957. j++;
  2958. cleaned_count++;
  2959. i++;
  2960. if (i == rx_ring->count)
  2961. i = 0;
  2962. buffer_info = &rx_ring->buffer_info[i];
  2963. rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
  2964. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  2965. length = le16_to_cpu(rx_desc->wb.upper.length);
  2966. if (!(staterr & E1000_RXD_STAT_DD)) {
  2967. rx_ring->pending_skb = skb;
  2968. rx_ring->pending_skb_page = j;
  2969. goto out;
  2970. }
  2971. }
  2972. send_up:
  2973. pskb_trim(skb, skb->len - 4);
  2974. i++;
  2975. if (i == rx_ring->count)
  2976. i = 0;
  2977. next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
  2978. prefetch(next_rxd);
  2979. next_buffer = &rx_ring->buffer_info[i];
  2980. if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
  2981. dev_kfree_skb_irq(skb);
  2982. goto next_desc;
  2983. }
  2984. rx_ring->no_itr_adjust |= (staterr & E1000_RXD_STAT_DYNINT);
  2985. total_bytes += skb->len;
  2986. total_packets++;
  2987. igb_rx_checksum_adv(adapter, staterr, skb);
  2988. skb->protocol = eth_type_trans(skb, netdev);
  2989. igb_receive_skb(adapter, staterr, rx_desc->wb.upper.vlan, skb);
  2990. netdev->last_rx = jiffies;
  2991. next_desc:
  2992. rx_desc->wb.upper.status_error = 0;
  2993. /* return some buffers to hardware, one at a time is too slow */
  2994. if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
  2995. igb_alloc_rx_buffers_adv(adapter, rx_ring,
  2996. cleaned_count);
  2997. cleaned_count = 0;
  2998. }
  2999. /* use prefetched values */
  3000. rx_desc = next_rxd;
  3001. buffer_info = next_buffer;
  3002. staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
  3003. }
  3004. out:
  3005. rx_ring->next_to_clean = i;
  3006. cleaned_count = IGB_DESC_UNUSED(rx_ring);
  3007. if (cleaned_count)
  3008. igb_alloc_rx_buffers_adv(adapter, rx_ring, cleaned_count);
  3009. rx_ring->total_packets += total_packets;
  3010. rx_ring->total_bytes += total_bytes;
  3011. rx_ring->rx_stats.packets += total_packets;
  3012. rx_ring->rx_stats.bytes += total_bytes;
  3013. adapter->net_stats.rx_bytes += total_bytes;
  3014. adapter->net_stats.rx_packets += total_packets;
  3015. return cleaned;
  3016. }
  3017. /**
  3018. * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
  3019. * @adapter: address of board private structure
  3020. **/
  3021. static void igb_alloc_rx_buffers_adv(struct igb_adapter *adapter,
  3022. struct igb_ring *rx_ring,
  3023. int cleaned_count)
  3024. {
  3025. struct net_device *netdev = adapter->netdev;
  3026. struct pci_dev *pdev = adapter->pdev;
  3027. union e1000_adv_rx_desc *rx_desc;
  3028. struct igb_buffer *buffer_info;
  3029. struct sk_buff *skb;
  3030. unsigned int i;
  3031. i = rx_ring->next_to_use;
  3032. buffer_info = &rx_ring->buffer_info[i];
  3033. while (cleaned_count--) {
  3034. rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
  3035. if (adapter->rx_ps_hdr_size && !buffer_info->page) {
  3036. buffer_info->page = alloc_page(GFP_ATOMIC);
  3037. if (!buffer_info->page) {
  3038. adapter->alloc_rx_buff_failed++;
  3039. goto no_buffers;
  3040. }
  3041. buffer_info->page_dma =
  3042. pci_map_page(pdev,
  3043. buffer_info->page,
  3044. 0, PAGE_SIZE,
  3045. PCI_DMA_FROMDEVICE);
  3046. }
  3047. if (!buffer_info->skb) {
  3048. int bufsz;
  3049. if (adapter->rx_ps_hdr_size)
  3050. bufsz = adapter->rx_ps_hdr_size;
  3051. else
  3052. bufsz = adapter->rx_buffer_len;
  3053. bufsz += NET_IP_ALIGN;
  3054. skb = netdev_alloc_skb(netdev, bufsz);
  3055. if (!skb) {
  3056. adapter->alloc_rx_buff_failed++;
  3057. goto no_buffers;
  3058. }
  3059. /* Make buffer alignment 2 beyond a 16 byte boundary
  3060. * this will result in a 16 byte aligned IP header after
  3061. * the 14 byte MAC header is removed
  3062. */
  3063. skb_reserve(skb, NET_IP_ALIGN);
  3064. buffer_info->skb = skb;
  3065. buffer_info->dma = pci_map_single(pdev, skb->data,
  3066. bufsz,
  3067. PCI_DMA_FROMDEVICE);
  3068. }
  3069. /* Refresh the desc even if buffer_addrs didn't change because
  3070. * each write-back erases this info. */
  3071. if (adapter->rx_ps_hdr_size) {
  3072. rx_desc->read.pkt_addr =
  3073. cpu_to_le64(buffer_info->page_dma);
  3074. rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
  3075. } else {
  3076. rx_desc->read.pkt_addr =
  3077. cpu_to_le64(buffer_info->dma);
  3078. rx_desc->read.hdr_addr = 0;
  3079. }
  3080. i++;
  3081. if (i == rx_ring->count)
  3082. i = 0;
  3083. buffer_info = &rx_ring->buffer_info[i];
  3084. }
  3085. no_buffers:
  3086. if (rx_ring->next_to_use != i) {
  3087. rx_ring->next_to_use = i;
  3088. if (i == 0)
  3089. i = (rx_ring->count - 1);
  3090. else
  3091. i--;
  3092. /* Force memory writes to complete before letting h/w
  3093. * know there are new descriptors to fetch. (Only
  3094. * applicable for weak-ordered memory model archs,
  3095. * such as IA-64). */
  3096. wmb();
  3097. writel(i, adapter->hw.hw_addr + rx_ring->tail);
  3098. }
  3099. }
  3100. /**
  3101. * igb_mii_ioctl -
  3102. * @netdev:
  3103. * @ifreq:
  3104. * @cmd:
  3105. **/
  3106. static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  3107. {
  3108. struct igb_adapter *adapter = netdev_priv(netdev);
  3109. struct mii_ioctl_data *data = if_mii(ifr);
  3110. if (adapter->hw.phy.media_type != e1000_media_type_copper)
  3111. return -EOPNOTSUPP;
  3112. switch (cmd) {
  3113. case SIOCGMIIPHY:
  3114. data->phy_id = adapter->hw.phy.addr;
  3115. break;
  3116. case SIOCGMIIREG:
  3117. if (!capable(CAP_NET_ADMIN))
  3118. return -EPERM;
  3119. if (adapter->hw.phy.ops.read_phy_reg(&adapter->hw,
  3120. data->reg_num
  3121. & 0x1F, &data->val_out))
  3122. return -EIO;
  3123. break;
  3124. case SIOCSMIIREG:
  3125. default:
  3126. return -EOPNOTSUPP;
  3127. }
  3128. return 0;
  3129. }
  3130. /**
  3131. * igb_ioctl -
  3132. * @netdev:
  3133. * @ifreq:
  3134. * @cmd:
  3135. **/
  3136. static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  3137. {
  3138. switch (cmd) {
  3139. case SIOCGMIIPHY:
  3140. case SIOCGMIIREG:
  3141. case SIOCSMIIREG:
  3142. return igb_mii_ioctl(netdev, ifr, cmd);
  3143. default:
  3144. return -EOPNOTSUPP;
  3145. }
  3146. }
  3147. static void igb_vlan_rx_register(struct net_device *netdev,
  3148. struct vlan_group *grp)
  3149. {
  3150. struct igb_adapter *adapter = netdev_priv(netdev);
  3151. struct e1000_hw *hw = &adapter->hw;
  3152. u32 ctrl, rctl;
  3153. igb_irq_disable(adapter);
  3154. adapter->vlgrp = grp;
  3155. if (grp) {
  3156. /* enable VLAN tag insert/strip */
  3157. ctrl = rd32(E1000_CTRL);
  3158. ctrl |= E1000_CTRL_VME;
  3159. wr32(E1000_CTRL, ctrl);
  3160. /* enable VLAN receive filtering */
  3161. rctl = rd32(E1000_RCTL);
  3162. rctl |= E1000_RCTL_VFE;
  3163. rctl &= ~E1000_RCTL_CFIEN;
  3164. wr32(E1000_RCTL, rctl);
  3165. igb_update_mng_vlan(adapter);
  3166. wr32(E1000_RLPML,
  3167. adapter->max_frame_size + VLAN_TAG_SIZE);
  3168. } else {
  3169. /* disable VLAN tag insert/strip */
  3170. ctrl = rd32(E1000_CTRL);
  3171. ctrl &= ~E1000_CTRL_VME;
  3172. wr32(E1000_CTRL, ctrl);
  3173. /* disable VLAN filtering */
  3174. rctl = rd32(E1000_RCTL);
  3175. rctl &= ~E1000_RCTL_VFE;
  3176. wr32(E1000_RCTL, rctl);
  3177. if (adapter->mng_vlan_id != (u16)IGB_MNG_VLAN_NONE) {
  3178. igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
  3179. adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
  3180. }
  3181. wr32(E1000_RLPML,
  3182. adapter->max_frame_size);
  3183. }
  3184. if (!test_bit(__IGB_DOWN, &adapter->state))
  3185. igb_irq_enable(adapter);
  3186. }
  3187. static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
  3188. {
  3189. struct igb_adapter *adapter = netdev_priv(netdev);
  3190. struct e1000_hw *hw = &adapter->hw;
  3191. u32 vfta, index;
  3192. if ((adapter->hw.mng_cookie.status &
  3193. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
  3194. (vid == adapter->mng_vlan_id))
  3195. return;
  3196. /* add VID to filter table */
  3197. index = (vid >> 5) & 0x7F;
  3198. vfta = array_rd32(E1000_VFTA, index);
  3199. vfta |= (1 << (vid & 0x1F));
  3200. igb_write_vfta(&adapter->hw, index, vfta);
  3201. }
  3202. static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
  3203. {
  3204. struct igb_adapter *adapter = netdev_priv(netdev);
  3205. struct e1000_hw *hw = &adapter->hw;
  3206. u32 vfta, index;
  3207. igb_irq_disable(adapter);
  3208. vlan_group_set_device(adapter->vlgrp, vid, NULL);
  3209. if (!test_bit(__IGB_DOWN, &adapter->state))
  3210. igb_irq_enable(adapter);
  3211. if ((adapter->hw.mng_cookie.status &
  3212. E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
  3213. (vid == adapter->mng_vlan_id)) {
  3214. /* release control to f/w */
  3215. igb_release_hw_control(adapter);
  3216. return;
  3217. }
  3218. /* remove VID from filter table */
  3219. index = (vid >> 5) & 0x7F;
  3220. vfta = array_rd32(E1000_VFTA, index);
  3221. vfta &= ~(1 << (vid & 0x1F));
  3222. igb_write_vfta(&adapter->hw, index, vfta);
  3223. }
  3224. static void igb_restore_vlan(struct igb_adapter *adapter)
  3225. {
  3226. igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
  3227. if (adapter->vlgrp) {
  3228. u16 vid;
  3229. for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
  3230. if (!vlan_group_get_device(adapter->vlgrp, vid))
  3231. continue;
  3232. igb_vlan_rx_add_vid(adapter->netdev, vid);
  3233. }
  3234. }
  3235. }
  3236. int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
  3237. {
  3238. struct e1000_mac_info *mac = &adapter->hw.mac;
  3239. mac->autoneg = 0;
  3240. /* Fiber NICs only allow 1000 gbps Full duplex */
  3241. if ((adapter->hw.phy.media_type == e1000_media_type_fiber) &&
  3242. spddplx != (SPEED_1000 + DUPLEX_FULL)) {
  3243. dev_err(&adapter->pdev->dev,
  3244. "Unsupported Speed/Duplex configuration\n");
  3245. return -EINVAL;
  3246. }
  3247. switch (spddplx) {
  3248. case SPEED_10 + DUPLEX_HALF:
  3249. mac->forced_speed_duplex = ADVERTISE_10_HALF;
  3250. break;
  3251. case SPEED_10 + DUPLEX_FULL:
  3252. mac->forced_speed_duplex = ADVERTISE_10_FULL;
  3253. break;
  3254. case SPEED_100 + DUPLEX_HALF:
  3255. mac->forced_speed_duplex = ADVERTISE_100_HALF;
  3256. break;
  3257. case SPEED_100 + DUPLEX_FULL:
  3258. mac->forced_speed_duplex = ADVERTISE_100_FULL;
  3259. break;
  3260. case SPEED_1000 + DUPLEX_FULL:
  3261. mac->autoneg = 1;
  3262. adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
  3263. break;
  3264. case SPEED_1000 + DUPLEX_HALF: /* not supported */
  3265. default:
  3266. dev_err(&adapter->pdev->dev,
  3267. "Unsupported Speed/Duplex configuration\n");
  3268. return -EINVAL;
  3269. }
  3270. return 0;
  3271. }
  3272. static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
  3273. {
  3274. struct net_device *netdev = pci_get_drvdata(pdev);
  3275. struct igb_adapter *adapter = netdev_priv(netdev);
  3276. struct e1000_hw *hw = &adapter->hw;
  3277. u32 ctrl, ctrl_ext, rctl, status;
  3278. u32 wufc = adapter->wol;
  3279. #ifdef CONFIG_PM
  3280. int retval = 0;
  3281. #endif
  3282. netif_device_detach(netdev);
  3283. if (netif_running(netdev)) {
  3284. WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
  3285. igb_down(adapter);
  3286. igb_free_irq(adapter);
  3287. }
  3288. #ifdef CONFIG_PM
  3289. retval = pci_save_state(pdev);
  3290. if (retval)
  3291. return retval;
  3292. #endif
  3293. status = rd32(E1000_STATUS);
  3294. if (status & E1000_STATUS_LU)
  3295. wufc &= ~E1000_WUFC_LNKC;
  3296. if (wufc) {
  3297. igb_setup_rctl(adapter);
  3298. igb_set_multi(netdev);
  3299. /* turn on all-multi mode if wake on multicast is enabled */
  3300. if (wufc & E1000_WUFC_MC) {
  3301. rctl = rd32(E1000_RCTL);
  3302. rctl |= E1000_RCTL_MPE;
  3303. wr32(E1000_RCTL, rctl);
  3304. }
  3305. ctrl = rd32(E1000_CTRL);
  3306. /* advertise wake from D3Cold */
  3307. #define E1000_CTRL_ADVD3WUC 0x00100000
  3308. /* phy power management enable */
  3309. #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
  3310. ctrl |= E1000_CTRL_ADVD3WUC;
  3311. wr32(E1000_CTRL, ctrl);
  3312. if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
  3313. adapter->hw.phy.media_type ==
  3314. e1000_media_type_internal_serdes) {
  3315. /* keep the laser running in D3 */
  3316. ctrl_ext = rd32(E1000_CTRL_EXT);
  3317. ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
  3318. wr32(E1000_CTRL_EXT, ctrl_ext);
  3319. }
  3320. /* Allow time for pending master requests to run */
  3321. igb_disable_pcie_master(&adapter->hw);
  3322. wr32(E1000_WUC, E1000_WUC_PME_EN);
  3323. wr32(E1000_WUFC, wufc);
  3324. pci_enable_wake(pdev, PCI_D3hot, 1);
  3325. pci_enable_wake(pdev, PCI_D3cold, 1);
  3326. } else {
  3327. wr32(E1000_WUC, 0);
  3328. wr32(E1000_WUFC, 0);
  3329. pci_enable_wake(pdev, PCI_D3hot, 0);
  3330. pci_enable_wake(pdev, PCI_D3cold, 0);
  3331. }
  3332. /* make sure adapter isn't asleep if manageability is enabled */
  3333. if (adapter->en_mng_pt) {
  3334. pci_enable_wake(pdev, PCI_D3hot, 1);
  3335. pci_enable_wake(pdev, PCI_D3cold, 1);
  3336. }
  3337. /* Release control of h/w to f/w. If f/w is AMT enabled, this
  3338. * would have already happened in close and is redundant. */
  3339. igb_release_hw_control(adapter);
  3340. pci_disable_device(pdev);
  3341. pci_set_power_state(pdev, pci_choose_state(pdev, state));
  3342. return 0;
  3343. }
  3344. #ifdef CONFIG_PM
  3345. static int igb_resume(struct pci_dev *pdev)
  3346. {
  3347. struct net_device *netdev = pci_get_drvdata(pdev);
  3348. struct igb_adapter *adapter = netdev_priv(netdev);
  3349. struct e1000_hw *hw = &adapter->hw;
  3350. u32 err;
  3351. pci_set_power_state(pdev, PCI_D0);
  3352. pci_restore_state(pdev);
  3353. err = pci_enable_device(pdev);
  3354. if (err) {
  3355. dev_err(&pdev->dev,
  3356. "igb: Cannot enable PCI device from suspend\n");
  3357. return err;
  3358. }
  3359. pci_set_master(pdev);
  3360. pci_enable_wake(pdev, PCI_D3hot, 0);
  3361. pci_enable_wake(pdev, PCI_D3cold, 0);
  3362. if (netif_running(netdev)) {
  3363. err = igb_request_irq(adapter);
  3364. if (err)
  3365. return err;
  3366. }
  3367. /* e1000_power_up_phy(adapter); */
  3368. igb_reset(adapter);
  3369. wr32(E1000_WUS, ~0);
  3370. igb_init_manageability(adapter);
  3371. if (netif_running(netdev))
  3372. igb_up(adapter);
  3373. netif_device_attach(netdev);
  3374. /* let the f/w know that the h/w is now under the control of the
  3375. * driver. */
  3376. igb_get_hw_control(adapter);
  3377. return 0;
  3378. }
  3379. #endif
  3380. static void igb_shutdown(struct pci_dev *pdev)
  3381. {
  3382. igb_suspend(pdev, PMSG_SUSPEND);
  3383. }
  3384. #ifdef CONFIG_NET_POLL_CONTROLLER
  3385. /*
  3386. * Polling 'interrupt' - used by things like netconsole to send skbs
  3387. * without having to re-enable interrupts. It's not called while
  3388. * the interrupt routine is executing.
  3389. */
  3390. static void igb_netpoll(struct net_device *netdev)
  3391. {
  3392. struct igb_adapter *adapter = netdev_priv(netdev);
  3393. int i;
  3394. int work_done = 0;
  3395. igb_irq_disable(adapter);
  3396. for (i = 0; i < adapter->num_tx_queues; i++)
  3397. igb_clean_tx_irq(adapter, &adapter->tx_ring[i]);
  3398. for (i = 0; i < adapter->num_rx_queues; i++)
  3399. igb_clean_rx_irq_adv(adapter, &adapter->rx_ring[i],
  3400. &work_done,
  3401. adapter->rx_ring[i].napi.weight);
  3402. igb_irq_enable(adapter);
  3403. }
  3404. #endif /* CONFIG_NET_POLL_CONTROLLER */
  3405. /**
  3406. * igb_io_error_detected - called when PCI error is detected
  3407. * @pdev: Pointer to PCI device
  3408. * @state: The current pci connection state
  3409. *
  3410. * This function is called after a PCI bus error affecting
  3411. * this device has been detected.
  3412. */
  3413. static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
  3414. pci_channel_state_t state)
  3415. {
  3416. struct net_device *netdev = pci_get_drvdata(pdev);
  3417. struct igb_adapter *adapter = netdev_priv(netdev);
  3418. netif_device_detach(netdev);
  3419. if (netif_running(netdev))
  3420. igb_down(adapter);
  3421. pci_disable_device(pdev);
  3422. /* Request a slot slot reset. */
  3423. return PCI_ERS_RESULT_NEED_RESET;
  3424. }
  3425. /**
  3426. * igb_io_slot_reset - called after the pci bus has been reset.
  3427. * @pdev: Pointer to PCI device
  3428. *
  3429. * Restart the card from scratch, as if from a cold-boot. Implementation
  3430. * resembles the first-half of the igb_resume routine.
  3431. */
  3432. static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
  3433. {
  3434. struct net_device *netdev = pci_get_drvdata(pdev);
  3435. struct igb_adapter *adapter = netdev_priv(netdev);
  3436. struct e1000_hw *hw = &adapter->hw;
  3437. if (pci_enable_device(pdev)) {
  3438. dev_err(&pdev->dev,
  3439. "Cannot re-enable PCI device after reset.\n");
  3440. return PCI_ERS_RESULT_DISCONNECT;
  3441. }
  3442. pci_set_master(pdev);
  3443. pci_restore_state(pdev);
  3444. pci_enable_wake(pdev, PCI_D3hot, 0);
  3445. pci_enable_wake(pdev, PCI_D3cold, 0);
  3446. igb_reset(adapter);
  3447. wr32(E1000_WUS, ~0);
  3448. return PCI_ERS_RESULT_RECOVERED;
  3449. }
  3450. /**
  3451. * igb_io_resume - called when traffic can start flowing again.
  3452. * @pdev: Pointer to PCI device
  3453. *
  3454. * This callback is called when the error recovery driver tells us that
  3455. * its OK to resume normal operation. Implementation resembles the
  3456. * second-half of the igb_resume routine.
  3457. */
  3458. static void igb_io_resume(struct pci_dev *pdev)
  3459. {
  3460. struct net_device *netdev = pci_get_drvdata(pdev);
  3461. struct igb_adapter *adapter = netdev_priv(netdev);
  3462. igb_init_manageability(adapter);
  3463. if (netif_running(netdev)) {
  3464. if (igb_up(adapter)) {
  3465. dev_err(&pdev->dev, "igb_up failed after reset\n");
  3466. return;
  3467. }
  3468. }
  3469. netif_device_attach(netdev);
  3470. /* let the f/w know that the h/w is now under the control of the
  3471. * driver. */
  3472. igb_get_hw_control(adapter);
  3473. }
  3474. /* igb_main.c */