myri10ge.c 111 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071
  1. /*************************************************************************
  2. * myri10ge.c: Myricom Myri-10G Ethernet driver.
  3. *
  4. * Copyright (C) 2005 - 2009 Myricom, Inc.
  5. * All rights reserved.
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions
  9. * are met:
  10. * 1. Redistributions of source code must retain the above copyright
  11. * notice, this list of conditions and the following disclaimer.
  12. * 2. Redistributions in binary form must reproduce the above copyright
  13. * notice, this list of conditions and the following disclaimer in the
  14. * documentation and/or other materials provided with the distribution.
  15. * 3. Neither the name of Myricom, Inc. nor the names of its contributors
  16. * may be used to endorse or promote products derived from this software
  17. * without specific prior written permission.
  18. *
  19. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  23. * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29. * POSSIBILITY OF SUCH DAMAGE.
  30. *
  31. *
  32. * If the eeprom on your board is not recent enough, you will need to get a
  33. * newer firmware image at:
  34. * http://www.myri.com/scs/download-Myri10GE.html
  35. *
  36. * Contact Information:
  37. * <help@myri.com>
  38. * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006
  39. *************************************************************************/
  40. #include <linux/tcp.h>
  41. #include <linux/netdevice.h>
  42. #include <linux/skbuff.h>
  43. #include <linux/string.h>
  44. #include <linux/module.h>
  45. #include <linux/pci.h>
  46. #include <linux/dma-mapping.h>
  47. #include <linux/etherdevice.h>
  48. #include <linux/if_ether.h>
  49. #include <linux/if_vlan.h>
  50. #include <linux/inet_lro.h>
  51. #include <linux/dca.h>
  52. #include <linux/ip.h>
  53. #include <linux/inet.h>
  54. #include <linux/in.h>
  55. #include <linux/ethtool.h>
  56. #include <linux/firmware.h>
  57. #include <linux/delay.h>
  58. #include <linux/timer.h>
  59. #include <linux/vmalloc.h>
  60. #include <linux/crc32.h>
  61. #include <linux/moduleparam.h>
  62. #include <linux/io.h>
  63. #include <linux/log2.h>
  64. #include <net/checksum.h>
  65. #include <net/ip.h>
  66. #include <net/tcp.h>
  67. #include <asm/byteorder.h>
  68. #include <asm/io.h>
  69. #include <asm/processor.h>
  70. #ifdef CONFIG_MTRR
  71. #include <asm/mtrr.h>
  72. #endif
  73. #include "myri10ge_mcp.h"
  74. #include "myri10ge_mcp_gen_header.h"
  75. #define MYRI10GE_VERSION_STR "1.4.4-1.401"
  76. MODULE_DESCRIPTION("Myricom 10G driver (10GbE)");
  77. MODULE_AUTHOR("Maintainer: help@myri.com");
  78. MODULE_VERSION(MYRI10GE_VERSION_STR);
  79. MODULE_LICENSE("Dual BSD/GPL");
  80. #define MYRI10GE_MAX_ETHER_MTU 9014
  81. #define MYRI10GE_ETH_STOPPED 0
  82. #define MYRI10GE_ETH_STOPPING 1
  83. #define MYRI10GE_ETH_STARTING 2
  84. #define MYRI10GE_ETH_RUNNING 3
  85. #define MYRI10GE_ETH_OPEN_FAILED 4
  86. #define MYRI10GE_EEPROM_STRINGS_SIZE 256
  87. #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2)
  88. #define MYRI10GE_MAX_LRO_DESCRIPTORS 8
  89. #define MYRI10GE_LRO_MAX_PKTS 64
  90. #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff)
  91. #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff
  92. #define MYRI10GE_ALLOC_ORDER 0
  93. #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE)
  94. #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1)
  95. #define MYRI10GE_MAX_SLICES 32
  96. struct myri10ge_rx_buffer_state {
  97. struct page *page;
  98. int page_offset;
  99. DECLARE_PCI_UNMAP_ADDR(bus)
  100. DECLARE_PCI_UNMAP_LEN(len)
  101. };
  102. struct myri10ge_tx_buffer_state {
  103. struct sk_buff *skb;
  104. int last;
  105. DECLARE_PCI_UNMAP_ADDR(bus)
  106. DECLARE_PCI_UNMAP_LEN(len)
  107. };
  108. struct myri10ge_cmd {
  109. u32 data0;
  110. u32 data1;
  111. u32 data2;
  112. };
  113. struct myri10ge_rx_buf {
  114. struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */
  115. struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */
  116. struct myri10ge_rx_buffer_state *info;
  117. struct page *page;
  118. dma_addr_t bus;
  119. int page_offset;
  120. int cnt;
  121. int fill_cnt;
  122. int alloc_fail;
  123. int mask; /* number of rx slots -1 */
  124. int watchdog_needed;
  125. };
  126. struct myri10ge_tx_buf {
  127. struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */
  128. __be32 __iomem *send_go; /* "go" doorbell ptr */
  129. __be32 __iomem *send_stop; /* "stop" doorbell ptr */
  130. struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */
  131. char *req_bytes;
  132. struct myri10ge_tx_buffer_state *info;
  133. int mask; /* number of transmit slots -1 */
  134. int req ____cacheline_aligned; /* transmit slots submitted */
  135. int pkt_start; /* packets started */
  136. int stop_queue;
  137. int linearized;
  138. int done ____cacheline_aligned; /* transmit slots completed */
  139. int pkt_done; /* packets completed */
  140. int wake_queue;
  141. int queue_active;
  142. };
  143. struct myri10ge_rx_done {
  144. struct mcp_slot *entry;
  145. dma_addr_t bus;
  146. int cnt;
  147. int idx;
  148. struct net_lro_mgr lro_mgr;
  149. struct net_lro_desc lro_desc[MYRI10GE_MAX_LRO_DESCRIPTORS];
  150. };
  151. struct myri10ge_slice_netstats {
  152. unsigned long rx_packets;
  153. unsigned long tx_packets;
  154. unsigned long rx_bytes;
  155. unsigned long tx_bytes;
  156. unsigned long rx_dropped;
  157. unsigned long tx_dropped;
  158. };
  159. struct myri10ge_slice_state {
  160. struct myri10ge_tx_buf tx; /* transmit ring */
  161. struct myri10ge_rx_buf rx_small;
  162. struct myri10ge_rx_buf rx_big;
  163. struct myri10ge_rx_done rx_done;
  164. struct net_device *dev;
  165. struct napi_struct napi;
  166. struct myri10ge_priv *mgp;
  167. struct myri10ge_slice_netstats stats;
  168. __be32 __iomem *irq_claim;
  169. struct mcp_irq_data *fw_stats;
  170. dma_addr_t fw_stats_bus;
  171. int watchdog_tx_done;
  172. int watchdog_tx_req;
  173. #ifdef CONFIG_MYRI10GE_DCA
  174. int cached_dca_tag;
  175. int cpu;
  176. __be32 __iomem *dca_tag;
  177. #endif
  178. char irq_desc[32];
  179. };
  180. struct myri10ge_priv {
  181. struct myri10ge_slice_state *ss;
  182. int tx_boundary; /* boundary transmits cannot cross */
  183. int num_slices;
  184. int running; /* running? */
  185. int csum_flag; /* rx_csums? */
  186. int small_bytes;
  187. int big_bytes;
  188. int max_intr_slots;
  189. struct net_device *dev;
  190. struct net_device_stats stats;
  191. spinlock_t stats_lock;
  192. u8 __iomem *sram;
  193. int sram_size;
  194. unsigned long board_span;
  195. unsigned long iomem_base;
  196. __be32 __iomem *irq_deassert;
  197. char *mac_addr_string;
  198. struct mcp_cmd_response *cmd;
  199. dma_addr_t cmd_bus;
  200. struct pci_dev *pdev;
  201. int msi_enabled;
  202. int msix_enabled;
  203. struct msix_entry *msix_vectors;
  204. #ifdef CONFIG_MYRI10GE_DCA
  205. int dca_enabled;
  206. #endif
  207. u32 link_state;
  208. unsigned int rdma_tags_available;
  209. int intr_coal_delay;
  210. __be32 __iomem *intr_coal_delay_ptr;
  211. int mtrr;
  212. int wc_enabled;
  213. int down_cnt;
  214. wait_queue_head_t down_wq;
  215. struct work_struct watchdog_work;
  216. struct timer_list watchdog_timer;
  217. int watchdog_resets;
  218. int watchdog_pause;
  219. int pause;
  220. char *fw_name;
  221. char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE];
  222. char *product_code_string;
  223. char fw_version[128];
  224. int fw_ver_major;
  225. int fw_ver_minor;
  226. int fw_ver_tiny;
  227. int adopted_rx_filter_bug;
  228. u8 mac_addr[6]; /* eeprom mac address */
  229. unsigned long serial_number;
  230. int vendor_specific_offset;
  231. int fw_multicast_support;
  232. unsigned long features;
  233. u32 max_tso6;
  234. u32 read_dma;
  235. u32 write_dma;
  236. u32 read_write_dma;
  237. u32 link_changes;
  238. u32 msg_enable;
  239. };
  240. static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat";
  241. static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat";
  242. static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat";
  243. static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat";
  244. static char *myri10ge_fw_name = NULL;
  245. module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR);
  246. MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name");
  247. static int myri10ge_ecrc_enable = 1;
  248. module_param(myri10ge_ecrc_enable, int, S_IRUGO);
  249. MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E");
  250. static int myri10ge_small_bytes = -1; /* -1 == auto */
  251. module_param(myri10ge_small_bytes, int, S_IRUGO | S_IWUSR);
  252. MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets");
  253. static int myri10ge_msi = 1; /* enable msi by default */
  254. module_param(myri10ge_msi, int, S_IRUGO | S_IWUSR);
  255. MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts");
  256. static int myri10ge_intr_coal_delay = 75;
  257. module_param(myri10ge_intr_coal_delay, int, S_IRUGO);
  258. MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay");
  259. static int myri10ge_flow_control = 1;
  260. module_param(myri10ge_flow_control, int, S_IRUGO);
  261. MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter");
  262. static int myri10ge_deassert_wait = 1;
  263. module_param(myri10ge_deassert_wait, int, S_IRUGO | S_IWUSR);
  264. MODULE_PARM_DESC(myri10ge_deassert_wait,
  265. "Wait when deasserting legacy interrupts");
  266. static int myri10ge_force_firmware = 0;
  267. module_param(myri10ge_force_firmware, int, S_IRUGO);
  268. MODULE_PARM_DESC(myri10ge_force_firmware,
  269. "Force firmware to assume aligned completions");
  270. static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN;
  271. module_param(myri10ge_initial_mtu, int, S_IRUGO);
  272. MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU");
  273. static int myri10ge_napi_weight = 64;
  274. module_param(myri10ge_napi_weight, int, S_IRUGO);
  275. MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight");
  276. static int myri10ge_watchdog_timeout = 1;
  277. module_param(myri10ge_watchdog_timeout, int, S_IRUGO);
  278. MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout");
  279. static int myri10ge_max_irq_loops = 1048576;
  280. module_param(myri10ge_max_irq_loops, int, S_IRUGO);
  281. MODULE_PARM_DESC(myri10ge_max_irq_loops,
  282. "Set stuck legacy IRQ detection threshold");
  283. #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK
  284. static int myri10ge_debug = -1; /* defaults above */
  285. module_param(myri10ge_debug, int, 0);
  286. MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)");
  287. static int myri10ge_lro = 1;
  288. module_param(myri10ge_lro, int, S_IRUGO);
  289. MODULE_PARM_DESC(myri10ge_lro, "Enable large receive offload");
  290. static int myri10ge_lro_max_pkts = MYRI10GE_LRO_MAX_PKTS;
  291. module_param(myri10ge_lro_max_pkts, int, S_IRUGO);
  292. MODULE_PARM_DESC(myri10ge_lro_max_pkts,
  293. "Number of LRO packets to be aggregated");
  294. static int myri10ge_fill_thresh = 256;
  295. module_param(myri10ge_fill_thresh, int, S_IRUGO | S_IWUSR);
  296. MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed");
  297. static int myri10ge_reset_recover = 1;
  298. static int myri10ge_max_slices = 1;
  299. module_param(myri10ge_max_slices, int, S_IRUGO);
  300. MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues");
  301. static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT;
  302. module_param(myri10ge_rss_hash, int, S_IRUGO);
  303. MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do");
  304. static int myri10ge_dca = 1;
  305. module_param(myri10ge_dca, int, S_IRUGO);
  306. MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible");
  307. #define MYRI10GE_FW_OFFSET 1024*1024
  308. #define MYRI10GE_HIGHPART_TO_U32(X) \
  309. (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0)
  310. #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X))
  311. #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8)
  312. static void myri10ge_set_multicast_list(struct net_device *dev);
  313. static int myri10ge_sw_tso(struct sk_buff *skb, struct net_device *dev);
  314. static inline void put_be32(__be32 val, __be32 __iomem * p)
  315. {
  316. __raw_writel((__force __u32) val, (__force void __iomem *)p);
  317. }
  318. static int
  319. myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd,
  320. struct myri10ge_cmd *data, int atomic)
  321. {
  322. struct mcp_cmd *buf;
  323. char buf_bytes[sizeof(*buf) + 8];
  324. struct mcp_cmd_response *response = mgp->cmd;
  325. char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD;
  326. u32 dma_low, dma_high, result, value;
  327. int sleep_total = 0;
  328. /* ensure buf is aligned to 8 bytes */
  329. buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8);
  330. buf->data0 = htonl(data->data0);
  331. buf->data1 = htonl(data->data1);
  332. buf->data2 = htonl(data->data2);
  333. buf->cmd = htonl(cmd);
  334. dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus);
  335. dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus);
  336. buf->response_addr.low = htonl(dma_low);
  337. buf->response_addr.high = htonl(dma_high);
  338. response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT);
  339. mb();
  340. myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf));
  341. /* wait up to 15ms. Longest command is the DMA benchmark,
  342. * which is capped at 5ms, but runs from a timeout handler
  343. * that runs every 7.8ms. So a 15ms timeout leaves us with
  344. * a 2.2ms margin
  345. */
  346. if (atomic) {
  347. /* if atomic is set, do not sleep,
  348. * and try to get the completion quickly
  349. * (1ms will be enough for those commands) */
  350. for (sleep_total = 0;
  351. sleep_total < 1000
  352. && response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT);
  353. sleep_total += 10) {
  354. udelay(10);
  355. mb();
  356. }
  357. } else {
  358. /* use msleep for most command */
  359. for (sleep_total = 0;
  360. sleep_total < 15
  361. && response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT);
  362. sleep_total++)
  363. msleep(1);
  364. }
  365. result = ntohl(response->result);
  366. value = ntohl(response->data);
  367. if (result != MYRI10GE_NO_RESPONSE_RESULT) {
  368. if (result == 0) {
  369. data->data0 = value;
  370. return 0;
  371. } else if (result == MXGEFW_CMD_UNKNOWN) {
  372. return -ENOSYS;
  373. } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) {
  374. return -E2BIG;
  375. } else if (result == MXGEFW_CMD_ERROR_RANGE &&
  376. cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES &&
  377. (data->
  378. data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) !=
  379. 0) {
  380. return -ERANGE;
  381. } else {
  382. dev_err(&mgp->pdev->dev,
  383. "command %d failed, result = %d\n",
  384. cmd, result);
  385. return -ENXIO;
  386. }
  387. }
  388. dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n",
  389. cmd, result);
  390. return -EAGAIN;
  391. }
  392. /*
  393. * The eeprom strings on the lanaiX have the format
  394. * SN=x\0
  395. * MAC=x:x:x:x:x:x\0
  396. * PT:ddd mmm xx xx:xx:xx xx\0
  397. * PV:ddd mmm xx xx:xx:xx xx\0
  398. */
  399. static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp)
  400. {
  401. char *ptr, *limit;
  402. int i;
  403. ptr = mgp->eeprom_strings;
  404. limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE;
  405. while (*ptr != '\0' && ptr < limit) {
  406. if (memcmp(ptr, "MAC=", 4) == 0) {
  407. ptr += 4;
  408. mgp->mac_addr_string = ptr;
  409. for (i = 0; i < 6; i++) {
  410. if ((ptr + 2) > limit)
  411. goto abort;
  412. mgp->mac_addr[i] =
  413. simple_strtoul(ptr, &ptr, 16);
  414. ptr += 1;
  415. }
  416. }
  417. if (memcmp(ptr, "PC=", 3) == 0) {
  418. ptr += 3;
  419. mgp->product_code_string = ptr;
  420. }
  421. if (memcmp((const void *)ptr, "SN=", 3) == 0) {
  422. ptr += 3;
  423. mgp->serial_number = simple_strtoul(ptr, &ptr, 10);
  424. }
  425. while (ptr < limit && *ptr++) ;
  426. }
  427. return 0;
  428. abort:
  429. dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n");
  430. return -ENXIO;
  431. }
  432. /*
  433. * Enable or disable periodic RDMAs from the host to make certain
  434. * chipsets resend dropped PCIe messages
  435. */
  436. static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable)
  437. {
  438. char __iomem *submit;
  439. __be32 buf[16] __attribute__ ((__aligned__(8)));
  440. u32 dma_low, dma_high;
  441. int i;
  442. /* clear confirmation addr */
  443. mgp->cmd->data = 0;
  444. mb();
  445. /* send a rdma command to the PCIe engine, and wait for the
  446. * response in the confirmation address. The firmware should
  447. * write a -1 there to indicate it is alive and well
  448. */
  449. dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus);
  450. dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus);
  451. buf[0] = htonl(dma_high); /* confirm addr MSW */
  452. buf[1] = htonl(dma_low); /* confirm addr LSW */
  453. buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */
  454. buf[3] = htonl(dma_high); /* dummy addr MSW */
  455. buf[4] = htonl(dma_low); /* dummy addr LSW */
  456. buf[5] = htonl(enable); /* enable? */
  457. submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA;
  458. myri10ge_pio_copy(submit, &buf, sizeof(buf));
  459. for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++)
  460. msleep(1);
  461. if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA)
  462. dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n",
  463. (enable ? "enable" : "disable"));
  464. }
  465. static int
  466. myri10ge_validate_firmware(struct myri10ge_priv *mgp,
  467. struct mcp_gen_header *hdr)
  468. {
  469. struct device *dev = &mgp->pdev->dev;
  470. /* check firmware type */
  471. if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) {
  472. dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type));
  473. return -EINVAL;
  474. }
  475. /* save firmware version for ethtool */
  476. strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version));
  477. sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major,
  478. &mgp->fw_ver_minor, &mgp->fw_ver_tiny);
  479. if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR
  480. && mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) {
  481. dev_err(dev, "Found firmware version %s\n", mgp->fw_version);
  482. dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR,
  483. MXGEFW_VERSION_MINOR);
  484. return -EINVAL;
  485. }
  486. return 0;
  487. }
  488. static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size)
  489. {
  490. unsigned crc, reread_crc;
  491. const struct firmware *fw;
  492. struct device *dev = &mgp->pdev->dev;
  493. unsigned char *fw_readback;
  494. struct mcp_gen_header *hdr;
  495. size_t hdr_offset;
  496. int status;
  497. unsigned i;
  498. if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) {
  499. dev_err(dev, "Unable to load %s firmware image via hotplug\n",
  500. mgp->fw_name);
  501. status = -EINVAL;
  502. goto abort_with_nothing;
  503. }
  504. /* check size */
  505. if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET ||
  506. fw->size < MCP_HEADER_PTR_OFFSET + 4) {
  507. dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size);
  508. status = -EINVAL;
  509. goto abort_with_fw;
  510. }
  511. /* check id */
  512. hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET));
  513. if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) {
  514. dev_err(dev, "Bad firmware file\n");
  515. status = -EINVAL;
  516. goto abort_with_fw;
  517. }
  518. hdr = (void *)(fw->data + hdr_offset);
  519. status = myri10ge_validate_firmware(mgp, hdr);
  520. if (status != 0)
  521. goto abort_with_fw;
  522. crc = crc32(~0, fw->data, fw->size);
  523. for (i = 0; i < fw->size; i += 256) {
  524. myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i,
  525. fw->data + i,
  526. min(256U, (unsigned)(fw->size - i)));
  527. mb();
  528. readb(mgp->sram);
  529. }
  530. fw_readback = vmalloc(fw->size);
  531. if (!fw_readback) {
  532. status = -ENOMEM;
  533. goto abort_with_fw;
  534. }
  535. /* corruption checking is good for parity recovery and buggy chipset */
  536. memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size);
  537. reread_crc = crc32(~0, fw_readback, fw->size);
  538. vfree(fw_readback);
  539. if (crc != reread_crc) {
  540. dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n",
  541. (unsigned)fw->size, reread_crc, crc);
  542. status = -EIO;
  543. goto abort_with_fw;
  544. }
  545. *size = (u32) fw->size;
  546. abort_with_fw:
  547. release_firmware(fw);
  548. abort_with_nothing:
  549. return status;
  550. }
  551. static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp)
  552. {
  553. struct mcp_gen_header *hdr;
  554. struct device *dev = &mgp->pdev->dev;
  555. const size_t bytes = sizeof(struct mcp_gen_header);
  556. size_t hdr_offset;
  557. int status;
  558. /* find running firmware header */
  559. hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET));
  560. if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) {
  561. dev_err(dev, "Running firmware has bad header offset (%d)\n",
  562. (int)hdr_offset);
  563. return -EIO;
  564. }
  565. /* copy header of running firmware from SRAM to host memory to
  566. * validate firmware */
  567. hdr = kmalloc(bytes, GFP_KERNEL);
  568. if (hdr == NULL) {
  569. dev_err(dev, "could not malloc firmware hdr\n");
  570. return -ENOMEM;
  571. }
  572. memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes);
  573. status = myri10ge_validate_firmware(mgp, hdr);
  574. kfree(hdr);
  575. /* check to see if adopted firmware has bug where adopting
  576. * it will cause broadcasts to be filtered unless the NIC
  577. * is kept in ALLMULTI mode */
  578. if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 &&
  579. mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) {
  580. mgp->adopted_rx_filter_bug = 1;
  581. dev_warn(dev, "Adopting fw %d.%d.%d: "
  582. "working around rx filter bug\n",
  583. mgp->fw_ver_major, mgp->fw_ver_minor,
  584. mgp->fw_ver_tiny);
  585. }
  586. return status;
  587. }
  588. static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp)
  589. {
  590. struct myri10ge_cmd cmd;
  591. int status;
  592. /* probe for IPv6 TSO support */
  593. mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO;
  594. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE,
  595. &cmd, 0);
  596. if (status == 0) {
  597. mgp->max_tso6 = cmd.data0;
  598. mgp->features |= NETIF_F_TSO6;
  599. }
  600. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0);
  601. if (status != 0) {
  602. dev_err(&mgp->pdev->dev,
  603. "failed MXGEFW_CMD_GET_RX_RING_SIZE\n");
  604. return -ENXIO;
  605. }
  606. mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr));
  607. return 0;
  608. }
  609. static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt)
  610. {
  611. char __iomem *submit;
  612. __be32 buf[16] __attribute__ ((__aligned__(8)));
  613. u32 dma_low, dma_high, size;
  614. int status, i;
  615. size = 0;
  616. status = myri10ge_load_hotplug_firmware(mgp, &size);
  617. if (status) {
  618. if (!adopt)
  619. return status;
  620. dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n");
  621. /* Do not attempt to adopt firmware if there
  622. * was a bad crc */
  623. if (status == -EIO)
  624. return status;
  625. status = myri10ge_adopt_running_firmware(mgp);
  626. if (status != 0) {
  627. dev_err(&mgp->pdev->dev,
  628. "failed to adopt running firmware\n");
  629. return status;
  630. }
  631. dev_info(&mgp->pdev->dev,
  632. "Successfully adopted running firmware\n");
  633. if (mgp->tx_boundary == 4096) {
  634. dev_warn(&mgp->pdev->dev,
  635. "Using firmware currently running on NIC"
  636. ". For optimal\n");
  637. dev_warn(&mgp->pdev->dev,
  638. "performance consider loading optimized "
  639. "firmware\n");
  640. dev_warn(&mgp->pdev->dev, "via hotplug\n");
  641. }
  642. mgp->fw_name = "adopted";
  643. mgp->tx_boundary = 2048;
  644. myri10ge_dummy_rdma(mgp, 1);
  645. status = myri10ge_get_firmware_capabilities(mgp);
  646. return status;
  647. }
  648. /* clear confirmation addr */
  649. mgp->cmd->data = 0;
  650. mb();
  651. /* send a reload command to the bootstrap MCP, and wait for the
  652. * response in the confirmation address. The firmware should
  653. * write a -1 there to indicate it is alive and well
  654. */
  655. dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus);
  656. dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus);
  657. buf[0] = htonl(dma_high); /* confirm addr MSW */
  658. buf[1] = htonl(dma_low); /* confirm addr LSW */
  659. buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */
  660. /* FIX: All newest firmware should un-protect the bottom of
  661. * the sram before handoff. However, the very first interfaces
  662. * do not. Therefore the handoff copy must skip the first 8 bytes
  663. */
  664. buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */
  665. buf[4] = htonl(size - 8); /* length of code */
  666. buf[5] = htonl(8); /* where to copy to */
  667. buf[6] = htonl(0); /* where to jump to */
  668. submit = mgp->sram + MXGEFW_BOOT_HANDOFF;
  669. myri10ge_pio_copy(submit, &buf, sizeof(buf));
  670. mb();
  671. msleep(1);
  672. mb();
  673. i = 0;
  674. while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) {
  675. msleep(1 << i);
  676. i++;
  677. }
  678. if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) {
  679. dev_err(&mgp->pdev->dev, "handoff failed\n");
  680. return -ENXIO;
  681. }
  682. myri10ge_dummy_rdma(mgp, 1);
  683. status = myri10ge_get_firmware_capabilities(mgp);
  684. return status;
  685. }
  686. static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr)
  687. {
  688. struct myri10ge_cmd cmd;
  689. int status;
  690. cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
  691. | (addr[2] << 8) | addr[3]);
  692. cmd.data1 = ((addr[4] << 8) | (addr[5]));
  693. status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0);
  694. return status;
  695. }
  696. static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause)
  697. {
  698. struct myri10ge_cmd cmd;
  699. int status, ctl;
  700. ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL;
  701. status = myri10ge_send_cmd(mgp, ctl, &cmd, 0);
  702. if (status) {
  703. printk(KERN_ERR
  704. "myri10ge: %s: Failed to set flow control mode\n",
  705. mgp->dev->name);
  706. return status;
  707. }
  708. mgp->pause = pause;
  709. return 0;
  710. }
  711. static void
  712. myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic)
  713. {
  714. struct myri10ge_cmd cmd;
  715. int status, ctl;
  716. ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC;
  717. status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic);
  718. if (status)
  719. printk(KERN_ERR "myri10ge: %s: Failed to set promisc mode\n",
  720. mgp->dev->name);
  721. }
  722. static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type)
  723. {
  724. struct myri10ge_cmd cmd;
  725. int status;
  726. u32 len;
  727. struct page *dmatest_page;
  728. dma_addr_t dmatest_bus;
  729. char *test = " ";
  730. dmatest_page = alloc_page(GFP_KERNEL);
  731. if (!dmatest_page)
  732. return -ENOMEM;
  733. dmatest_bus = pci_map_page(mgp->pdev, dmatest_page, 0, PAGE_SIZE,
  734. DMA_BIDIRECTIONAL);
  735. /* Run a small DMA test.
  736. * The magic multipliers to the length tell the firmware
  737. * to do DMA read, write, or read+write tests. The
  738. * results are returned in cmd.data0. The upper 16
  739. * bits or the return is the number of transfers completed.
  740. * The lower 16 bits is the time in 0.5us ticks that the
  741. * transfers took to complete.
  742. */
  743. len = mgp->tx_boundary;
  744. cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus);
  745. cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus);
  746. cmd.data2 = len * 0x10000;
  747. status = myri10ge_send_cmd(mgp, test_type, &cmd, 0);
  748. if (status != 0) {
  749. test = "read";
  750. goto abort;
  751. }
  752. mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff);
  753. cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus);
  754. cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus);
  755. cmd.data2 = len * 0x1;
  756. status = myri10ge_send_cmd(mgp, test_type, &cmd, 0);
  757. if (status != 0) {
  758. test = "write";
  759. goto abort;
  760. }
  761. mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff);
  762. cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus);
  763. cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus);
  764. cmd.data2 = len * 0x10001;
  765. status = myri10ge_send_cmd(mgp, test_type, &cmd, 0);
  766. if (status != 0) {
  767. test = "read/write";
  768. goto abort;
  769. }
  770. mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) /
  771. (cmd.data0 & 0xffff);
  772. abort:
  773. pci_unmap_page(mgp->pdev, dmatest_bus, PAGE_SIZE, DMA_BIDIRECTIONAL);
  774. put_page(dmatest_page);
  775. if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
  776. dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n",
  777. test, status);
  778. return status;
  779. }
  780. static int myri10ge_reset(struct myri10ge_priv *mgp)
  781. {
  782. struct myri10ge_cmd cmd;
  783. struct myri10ge_slice_state *ss;
  784. int i, status;
  785. size_t bytes;
  786. #ifdef CONFIG_MYRI10GE_DCA
  787. unsigned long dca_tag_off;
  788. #endif
  789. /* try to send a reset command to the card to see if it
  790. * is alive */
  791. memset(&cmd, 0, sizeof(cmd));
  792. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0);
  793. if (status != 0) {
  794. dev_err(&mgp->pdev->dev, "failed reset\n");
  795. return -ENXIO;
  796. }
  797. (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST);
  798. /*
  799. * Use non-ndis mcp_slot (eg, 4 bytes total,
  800. * no toeplitz hash value returned. Older firmware will
  801. * not understand this command, but will use the correct
  802. * sized mcp_slot, so we ignore error returns
  803. */
  804. cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN;
  805. (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0);
  806. /* Now exchange information about interrupts */
  807. bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry);
  808. cmd.data0 = (u32) bytes;
  809. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0);
  810. /*
  811. * Even though we already know how many slices are supported
  812. * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES
  813. * has magic side effects, and must be called after a reset.
  814. * It must be called prior to calling any RSS related cmds,
  815. * including assigning an interrupt queue for anything but
  816. * slice 0. It must also be called *after*
  817. * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
  818. * the firmware to compute offsets.
  819. */
  820. if (mgp->num_slices > 1) {
  821. /* ask the maximum number of slices it supports */
  822. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
  823. &cmd, 0);
  824. if (status != 0) {
  825. dev_err(&mgp->pdev->dev,
  826. "failed to get number of slices\n");
  827. }
  828. /*
  829. * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
  830. * to setting up the interrupt queue DMA
  831. */
  832. cmd.data0 = mgp->num_slices;
  833. cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
  834. if (mgp->dev->real_num_tx_queues > 1)
  835. cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
  836. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
  837. &cmd, 0);
  838. /* Firmware older than 1.4.32 only supports multiple
  839. * RX queues, so if we get an error, first retry using a
  840. * single TX queue before giving up */
  841. if (status != 0 && mgp->dev->real_num_tx_queues > 1) {
  842. mgp->dev->real_num_tx_queues = 1;
  843. cmd.data0 = mgp->num_slices;
  844. cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
  845. status = myri10ge_send_cmd(mgp,
  846. MXGEFW_CMD_ENABLE_RSS_QUEUES,
  847. &cmd, 0);
  848. }
  849. if (status != 0) {
  850. dev_err(&mgp->pdev->dev,
  851. "failed to set number of slices\n");
  852. return status;
  853. }
  854. }
  855. for (i = 0; i < mgp->num_slices; i++) {
  856. ss = &mgp->ss[i];
  857. cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus);
  858. cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus);
  859. cmd.data2 = i;
  860. status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA,
  861. &cmd, 0);
  862. };
  863. status |=
  864. myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0);
  865. for (i = 0; i < mgp->num_slices; i++) {
  866. ss = &mgp->ss[i];
  867. ss->irq_claim =
  868. (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i);
  869. }
  870. status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
  871. &cmd, 0);
  872. mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0);
  873. status |= myri10ge_send_cmd
  874. (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0);
  875. mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0);
  876. if (status != 0) {
  877. dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n");
  878. return status;
  879. }
  880. put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr);
  881. #ifdef CONFIG_MYRI10GE_DCA
  882. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0);
  883. dca_tag_off = cmd.data0;
  884. for (i = 0; i < mgp->num_slices; i++) {
  885. ss = &mgp->ss[i];
  886. if (status == 0) {
  887. ss->dca_tag = (__iomem __be32 *)
  888. (mgp->sram + dca_tag_off + 4 * i);
  889. } else {
  890. ss->dca_tag = NULL;
  891. }
  892. }
  893. #endif /* CONFIG_MYRI10GE_DCA */
  894. /* reset mcp/driver shared state back to 0 */
  895. mgp->link_changes = 0;
  896. for (i = 0; i < mgp->num_slices; i++) {
  897. ss = &mgp->ss[i];
  898. memset(ss->rx_done.entry, 0, bytes);
  899. ss->tx.req = 0;
  900. ss->tx.done = 0;
  901. ss->tx.pkt_start = 0;
  902. ss->tx.pkt_done = 0;
  903. ss->rx_big.cnt = 0;
  904. ss->rx_small.cnt = 0;
  905. ss->rx_done.idx = 0;
  906. ss->rx_done.cnt = 0;
  907. ss->tx.wake_queue = 0;
  908. ss->tx.stop_queue = 0;
  909. }
  910. status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr);
  911. myri10ge_change_pause(mgp, mgp->pause);
  912. myri10ge_set_multicast_list(mgp->dev);
  913. return status;
  914. }
  915. #ifdef CONFIG_MYRI10GE_DCA
  916. static void
  917. myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag)
  918. {
  919. ss->cpu = cpu;
  920. ss->cached_dca_tag = tag;
  921. put_be32(htonl(tag), ss->dca_tag);
  922. }
  923. static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss)
  924. {
  925. int cpu = get_cpu();
  926. int tag;
  927. if (cpu != ss->cpu) {
  928. tag = dca_get_tag(cpu);
  929. if (ss->cached_dca_tag != tag)
  930. myri10ge_write_dca(ss, cpu, tag);
  931. }
  932. put_cpu();
  933. }
  934. static void myri10ge_setup_dca(struct myri10ge_priv *mgp)
  935. {
  936. int err, i;
  937. struct pci_dev *pdev = mgp->pdev;
  938. if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled)
  939. return;
  940. if (!myri10ge_dca) {
  941. dev_err(&pdev->dev, "dca disabled by administrator\n");
  942. return;
  943. }
  944. err = dca_add_requester(&pdev->dev);
  945. if (err) {
  946. if (err != -ENODEV)
  947. dev_err(&pdev->dev,
  948. "dca_add_requester() failed, err=%d\n", err);
  949. return;
  950. }
  951. mgp->dca_enabled = 1;
  952. for (i = 0; i < mgp->num_slices; i++)
  953. myri10ge_write_dca(&mgp->ss[i], -1, 0);
  954. }
  955. static void myri10ge_teardown_dca(struct myri10ge_priv *mgp)
  956. {
  957. struct pci_dev *pdev = mgp->pdev;
  958. int err;
  959. if (!mgp->dca_enabled)
  960. return;
  961. mgp->dca_enabled = 0;
  962. err = dca_remove_requester(&pdev->dev);
  963. }
  964. static int myri10ge_notify_dca_device(struct device *dev, void *data)
  965. {
  966. struct myri10ge_priv *mgp;
  967. unsigned long event;
  968. mgp = dev_get_drvdata(dev);
  969. event = *(unsigned long *)data;
  970. if (event == DCA_PROVIDER_ADD)
  971. myri10ge_setup_dca(mgp);
  972. else if (event == DCA_PROVIDER_REMOVE)
  973. myri10ge_teardown_dca(mgp);
  974. return 0;
  975. }
  976. #endif /* CONFIG_MYRI10GE_DCA */
  977. static inline void
  978. myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst,
  979. struct mcp_kreq_ether_recv *src)
  980. {
  981. __be32 low;
  982. low = src->addr_low;
  983. src->addr_low = htonl(DMA_BIT_MASK(32));
  984. myri10ge_pio_copy(dst, src, 4 * sizeof(*src));
  985. mb();
  986. myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src));
  987. mb();
  988. src->addr_low = low;
  989. put_be32(low, &dst->addr_low);
  990. mb();
  991. }
  992. static inline void myri10ge_vlan_ip_csum(struct sk_buff *skb, __wsum hw_csum)
  993. {
  994. struct vlan_hdr *vh = (struct vlan_hdr *)(skb->data);
  995. if ((skb->protocol == htons(ETH_P_8021Q)) &&
  996. (vh->h_vlan_encapsulated_proto == htons(ETH_P_IP) ||
  997. vh->h_vlan_encapsulated_proto == htons(ETH_P_IPV6))) {
  998. skb->csum = hw_csum;
  999. skb->ip_summed = CHECKSUM_COMPLETE;
  1000. }
  1001. }
  1002. static inline void
  1003. myri10ge_rx_skb_build(struct sk_buff *skb, u8 * va,
  1004. struct skb_frag_struct *rx_frags, int len, int hlen)
  1005. {
  1006. struct skb_frag_struct *skb_frags;
  1007. skb->len = skb->data_len = len;
  1008. skb->truesize = len + sizeof(struct sk_buff);
  1009. /* attach the page(s) */
  1010. skb_frags = skb_shinfo(skb)->frags;
  1011. while (len > 0) {
  1012. memcpy(skb_frags, rx_frags, sizeof(*skb_frags));
  1013. len -= rx_frags->size;
  1014. skb_frags++;
  1015. rx_frags++;
  1016. skb_shinfo(skb)->nr_frags++;
  1017. }
  1018. /* pskb_may_pull is not available in irq context, but
  1019. * skb_pull() (for ether_pad and eth_type_trans()) requires
  1020. * the beginning of the packet in skb_headlen(), move it
  1021. * manually */
  1022. skb_copy_to_linear_data(skb, va, hlen);
  1023. skb_shinfo(skb)->frags[0].page_offset += hlen;
  1024. skb_shinfo(skb)->frags[0].size -= hlen;
  1025. skb->data_len -= hlen;
  1026. skb->tail += hlen;
  1027. skb_pull(skb, MXGEFW_PAD);
  1028. }
  1029. static void
  1030. myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx,
  1031. int bytes, int watchdog)
  1032. {
  1033. struct page *page;
  1034. int idx;
  1035. if (unlikely(rx->watchdog_needed && !watchdog))
  1036. return;
  1037. /* try to refill entire ring */
  1038. while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) {
  1039. idx = rx->fill_cnt & rx->mask;
  1040. if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) {
  1041. /* we can use part of previous page */
  1042. get_page(rx->page);
  1043. } else {
  1044. /* we need a new page */
  1045. page =
  1046. alloc_pages(GFP_ATOMIC | __GFP_COMP,
  1047. MYRI10GE_ALLOC_ORDER);
  1048. if (unlikely(page == NULL)) {
  1049. if (rx->fill_cnt - rx->cnt < 16)
  1050. rx->watchdog_needed = 1;
  1051. return;
  1052. }
  1053. rx->page = page;
  1054. rx->page_offset = 0;
  1055. rx->bus = pci_map_page(mgp->pdev, page, 0,
  1056. MYRI10GE_ALLOC_SIZE,
  1057. PCI_DMA_FROMDEVICE);
  1058. }
  1059. rx->info[idx].page = rx->page;
  1060. rx->info[idx].page_offset = rx->page_offset;
  1061. /* note that this is the address of the start of the
  1062. * page */
  1063. pci_unmap_addr_set(&rx->info[idx], bus, rx->bus);
  1064. rx->shadow[idx].addr_low =
  1065. htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset);
  1066. rx->shadow[idx].addr_high =
  1067. htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus));
  1068. /* start next packet on a cacheline boundary */
  1069. rx->page_offset += SKB_DATA_ALIGN(bytes);
  1070. #if MYRI10GE_ALLOC_SIZE > 4096
  1071. /* don't cross a 4KB boundary */
  1072. if ((rx->page_offset >> 12) !=
  1073. ((rx->page_offset + bytes - 1) >> 12))
  1074. rx->page_offset = (rx->page_offset + 4096) & ~4095;
  1075. #endif
  1076. rx->fill_cnt++;
  1077. /* copy 8 descriptors to the firmware at a time */
  1078. if ((idx & 7) == 7) {
  1079. myri10ge_submit_8rx(&rx->lanai[idx - 7],
  1080. &rx->shadow[idx - 7]);
  1081. }
  1082. }
  1083. }
  1084. static inline void
  1085. myri10ge_unmap_rx_page(struct pci_dev *pdev,
  1086. struct myri10ge_rx_buffer_state *info, int bytes)
  1087. {
  1088. /* unmap the recvd page if we're the only or last user of it */
  1089. if (bytes >= MYRI10GE_ALLOC_SIZE / 2 ||
  1090. (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) {
  1091. pci_unmap_page(pdev, (pci_unmap_addr(info, bus)
  1092. & ~(MYRI10GE_ALLOC_SIZE - 1)),
  1093. MYRI10GE_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
  1094. }
  1095. }
  1096. #define MYRI10GE_HLEN 64 /* The number of bytes to copy from a
  1097. * page into an skb */
  1098. static inline int
  1099. myri10ge_rx_done(struct myri10ge_slice_state *ss, struct myri10ge_rx_buf *rx,
  1100. int bytes, int len, __wsum csum)
  1101. {
  1102. struct myri10ge_priv *mgp = ss->mgp;
  1103. struct sk_buff *skb;
  1104. struct skb_frag_struct rx_frags[MYRI10GE_MAX_FRAGS_PER_FRAME];
  1105. int i, idx, hlen, remainder;
  1106. struct pci_dev *pdev = mgp->pdev;
  1107. struct net_device *dev = mgp->dev;
  1108. u8 *va;
  1109. len += MXGEFW_PAD;
  1110. idx = rx->cnt & rx->mask;
  1111. va = page_address(rx->info[idx].page) + rx->info[idx].page_offset;
  1112. prefetch(va);
  1113. /* Fill skb_frag_struct(s) with data from our receive */
  1114. for (i = 0, remainder = len; remainder > 0; i++) {
  1115. myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes);
  1116. rx_frags[i].page = rx->info[idx].page;
  1117. rx_frags[i].page_offset = rx->info[idx].page_offset;
  1118. if (remainder < MYRI10GE_ALLOC_SIZE)
  1119. rx_frags[i].size = remainder;
  1120. else
  1121. rx_frags[i].size = MYRI10GE_ALLOC_SIZE;
  1122. rx->cnt++;
  1123. idx = rx->cnt & rx->mask;
  1124. remainder -= MYRI10GE_ALLOC_SIZE;
  1125. }
  1126. if (mgp->csum_flag && myri10ge_lro) {
  1127. rx_frags[0].page_offset += MXGEFW_PAD;
  1128. rx_frags[0].size -= MXGEFW_PAD;
  1129. len -= MXGEFW_PAD;
  1130. lro_receive_frags(&ss->rx_done.lro_mgr, rx_frags,
  1131. /* opaque, will come back in get_frag_header */
  1132. len, len,
  1133. (void *)(__force unsigned long)csum, csum);
  1134. return 1;
  1135. }
  1136. hlen = MYRI10GE_HLEN > len ? len : MYRI10GE_HLEN;
  1137. /* allocate an skb to attach the page(s) to. This is done
  1138. * after trying LRO, so as to avoid skb allocation overheads */
  1139. skb = netdev_alloc_skb(dev, MYRI10GE_HLEN + 16);
  1140. if (unlikely(skb == NULL)) {
  1141. ss->stats.rx_dropped++;
  1142. do {
  1143. i--;
  1144. put_page(rx_frags[i].page);
  1145. } while (i != 0);
  1146. return 0;
  1147. }
  1148. /* Attach the pages to the skb, and trim off any padding */
  1149. myri10ge_rx_skb_build(skb, va, rx_frags, len, hlen);
  1150. if (skb_shinfo(skb)->frags[0].size <= 0) {
  1151. put_page(skb_shinfo(skb)->frags[0].page);
  1152. skb_shinfo(skb)->nr_frags = 0;
  1153. }
  1154. skb->protocol = eth_type_trans(skb, dev);
  1155. skb_record_rx_queue(skb, ss - &mgp->ss[0]);
  1156. if (mgp->csum_flag) {
  1157. if ((skb->protocol == htons(ETH_P_IP)) ||
  1158. (skb->protocol == htons(ETH_P_IPV6))) {
  1159. skb->csum = csum;
  1160. skb->ip_summed = CHECKSUM_COMPLETE;
  1161. } else
  1162. myri10ge_vlan_ip_csum(skb, csum);
  1163. }
  1164. netif_receive_skb(skb);
  1165. return 1;
  1166. }
  1167. static inline void
  1168. myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index)
  1169. {
  1170. struct pci_dev *pdev = ss->mgp->pdev;
  1171. struct myri10ge_tx_buf *tx = &ss->tx;
  1172. struct netdev_queue *dev_queue;
  1173. struct sk_buff *skb;
  1174. int idx, len;
  1175. while (tx->pkt_done != mcp_index) {
  1176. idx = tx->done & tx->mask;
  1177. skb = tx->info[idx].skb;
  1178. /* Mark as free */
  1179. tx->info[idx].skb = NULL;
  1180. if (tx->info[idx].last) {
  1181. tx->pkt_done++;
  1182. tx->info[idx].last = 0;
  1183. }
  1184. tx->done++;
  1185. len = pci_unmap_len(&tx->info[idx], len);
  1186. pci_unmap_len_set(&tx->info[idx], len, 0);
  1187. if (skb) {
  1188. ss->stats.tx_bytes += skb->len;
  1189. ss->stats.tx_packets++;
  1190. dev_kfree_skb_irq(skb);
  1191. if (len)
  1192. pci_unmap_single(pdev,
  1193. pci_unmap_addr(&tx->info[idx],
  1194. bus), len,
  1195. PCI_DMA_TODEVICE);
  1196. } else {
  1197. if (len)
  1198. pci_unmap_page(pdev,
  1199. pci_unmap_addr(&tx->info[idx],
  1200. bus), len,
  1201. PCI_DMA_TODEVICE);
  1202. }
  1203. }
  1204. dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss);
  1205. /*
  1206. * Make a minimal effort to prevent the NIC from polling an
  1207. * idle tx queue. If we can't get the lock we leave the queue
  1208. * active. In this case, either a thread was about to start
  1209. * using the queue anyway, or we lost a race and the NIC will
  1210. * waste some of its resources polling an inactive queue for a
  1211. * while.
  1212. */
  1213. if ((ss->mgp->dev->real_num_tx_queues > 1) &&
  1214. __netif_tx_trylock(dev_queue)) {
  1215. if (tx->req == tx->done) {
  1216. tx->queue_active = 0;
  1217. put_be32(htonl(1), tx->send_stop);
  1218. mb();
  1219. mmiowb();
  1220. }
  1221. __netif_tx_unlock(dev_queue);
  1222. }
  1223. /* start the queue if we've stopped it */
  1224. if (netif_tx_queue_stopped(dev_queue)
  1225. && tx->req - tx->done < (tx->mask >> 1)) {
  1226. tx->wake_queue++;
  1227. netif_tx_wake_queue(dev_queue);
  1228. }
  1229. }
  1230. static inline int
  1231. myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget)
  1232. {
  1233. struct myri10ge_rx_done *rx_done = &ss->rx_done;
  1234. struct myri10ge_priv *mgp = ss->mgp;
  1235. unsigned long rx_bytes = 0;
  1236. unsigned long rx_packets = 0;
  1237. unsigned long rx_ok;
  1238. int idx = rx_done->idx;
  1239. int cnt = rx_done->cnt;
  1240. int work_done = 0;
  1241. u16 length;
  1242. __wsum checksum;
  1243. while (rx_done->entry[idx].length != 0 && work_done < budget) {
  1244. length = ntohs(rx_done->entry[idx].length);
  1245. rx_done->entry[idx].length = 0;
  1246. checksum = csum_unfold(rx_done->entry[idx].checksum);
  1247. if (length <= mgp->small_bytes)
  1248. rx_ok = myri10ge_rx_done(ss, &ss->rx_small,
  1249. mgp->small_bytes,
  1250. length, checksum);
  1251. else
  1252. rx_ok = myri10ge_rx_done(ss, &ss->rx_big,
  1253. mgp->big_bytes,
  1254. length, checksum);
  1255. rx_packets += rx_ok;
  1256. rx_bytes += rx_ok * (unsigned long)length;
  1257. cnt++;
  1258. idx = cnt & (mgp->max_intr_slots - 1);
  1259. work_done++;
  1260. }
  1261. rx_done->idx = idx;
  1262. rx_done->cnt = cnt;
  1263. ss->stats.rx_packets += rx_packets;
  1264. ss->stats.rx_bytes += rx_bytes;
  1265. if (myri10ge_lro)
  1266. lro_flush_all(&rx_done->lro_mgr);
  1267. /* restock receive rings if needed */
  1268. if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh)
  1269. myri10ge_alloc_rx_pages(mgp, &ss->rx_small,
  1270. mgp->small_bytes + MXGEFW_PAD, 0);
  1271. if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh)
  1272. myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0);
  1273. return work_done;
  1274. }
  1275. static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp)
  1276. {
  1277. struct mcp_irq_data *stats = mgp->ss[0].fw_stats;
  1278. if (unlikely(stats->stats_updated)) {
  1279. unsigned link_up = ntohl(stats->link_up);
  1280. if (mgp->link_state != link_up) {
  1281. mgp->link_state = link_up;
  1282. if (mgp->link_state == MXGEFW_LINK_UP) {
  1283. if (netif_msg_link(mgp))
  1284. printk(KERN_INFO
  1285. "myri10ge: %s: link up\n",
  1286. mgp->dev->name);
  1287. netif_carrier_on(mgp->dev);
  1288. mgp->link_changes++;
  1289. } else {
  1290. if (netif_msg_link(mgp))
  1291. printk(KERN_INFO
  1292. "myri10ge: %s: link %s\n",
  1293. mgp->dev->name,
  1294. (link_up == MXGEFW_LINK_MYRINET ?
  1295. "mismatch (Myrinet detected)" :
  1296. "down"));
  1297. netif_carrier_off(mgp->dev);
  1298. mgp->link_changes++;
  1299. }
  1300. }
  1301. if (mgp->rdma_tags_available !=
  1302. ntohl(stats->rdma_tags_available)) {
  1303. mgp->rdma_tags_available =
  1304. ntohl(stats->rdma_tags_available);
  1305. printk(KERN_WARNING "myri10ge: %s: RDMA timed out! "
  1306. "%d tags left\n", mgp->dev->name,
  1307. mgp->rdma_tags_available);
  1308. }
  1309. mgp->down_cnt += stats->link_down;
  1310. if (stats->link_down)
  1311. wake_up(&mgp->down_wq);
  1312. }
  1313. }
  1314. static int myri10ge_poll(struct napi_struct *napi, int budget)
  1315. {
  1316. struct myri10ge_slice_state *ss =
  1317. container_of(napi, struct myri10ge_slice_state, napi);
  1318. int work_done;
  1319. #ifdef CONFIG_MYRI10GE_DCA
  1320. if (ss->mgp->dca_enabled)
  1321. myri10ge_update_dca(ss);
  1322. #endif
  1323. /* process as many rx events as NAPI will allow */
  1324. work_done = myri10ge_clean_rx_done(ss, budget);
  1325. if (work_done < budget) {
  1326. napi_complete(napi);
  1327. put_be32(htonl(3), ss->irq_claim);
  1328. }
  1329. return work_done;
  1330. }
  1331. static irqreturn_t myri10ge_intr(int irq, void *arg)
  1332. {
  1333. struct myri10ge_slice_state *ss = arg;
  1334. struct myri10ge_priv *mgp = ss->mgp;
  1335. struct mcp_irq_data *stats = ss->fw_stats;
  1336. struct myri10ge_tx_buf *tx = &ss->tx;
  1337. u32 send_done_count;
  1338. int i;
  1339. /* an interrupt on a non-zero receive-only slice is implicitly
  1340. * valid since MSI-X irqs are not shared */
  1341. if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) {
  1342. napi_schedule(&ss->napi);
  1343. return (IRQ_HANDLED);
  1344. }
  1345. /* make sure it is our IRQ, and that the DMA has finished */
  1346. if (unlikely(!stats->valid))
  1347. return (IRQ_NONE);
  1348. /* low bit indicates receives are present, so schedule
  1349. * napi poll handler */
  1350. if (stats->valid & 1)
  1351. napi_schedule(&ss->napi);
  1352. if (!mgp->msi_enabled && !mgp->msix_enabled) {
  1353. put_be32(0, mgp->irq_deassert);
  1354. if (!myri10ge_deassert_wait)
  1355. stats->valid = 0;
  1356. mb();
  1357. } else
  1358. stats->valid = 0;
  1359. /* Wait for IRQ line to go low, if using INTx */
  1360. i = 0;
  1361. while (1) {
  1362. i++;
  1363. /* check for transmit completes and receives */
  1364. send_done_count = ntohl(stats->send_done_count);
  1365. if (send_done_count != tx->pkt_done)
  1366. myri10ge_tx_done(ss, (int)send_done_count);
  1367. if (unlikely(i > myri10ge_max_irq_loops)) {
  1368. printk(KERN_WARNING "myri10ge: %s: irq stuck?\n",
  1369. mgp->dev->name);
  1370. stats->valid = 0;
  1371. schedule_work(&mgp->watchdog_work);
  1372. }
  1373. if (likely(stats->valid == 0))
  1374. break;
  1375. cpu_relax();
  1376. barrier();
  1377. }
  1378. /* Only slice 0 updates stats */
  1379. if (ss == mgp->ss)
  1380. myri10ge_check_statblock(mgp);
  1381. put_be32(htonl(3), ss->irq_claim + 1);
  1382. return (IRQ_HANDLED);
  1383. }
  1384. static int
  1385. myri10ge_get_settings(struct net_device *netdev, struct ethtool_cmd *cmd)
  1386. {
  1387. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1388. char *ptr;
  1389. int i;
  1390. cmd->autoneg = AUTONEG_DISABLE;
  1391. cmd->speed = SPEED_10000;
  1392. cmd->duplex = DUPLEX_FULL;
  1393. /*
  1394. * parse the product code to deterimine the interface type
  1395. * (CX4, XFP, Quad Ribbon Fiber) by looking at the character
  1396. * after the 3rd dash in the driver's cached copy of the
  1397. * EEPROM's product code string.
  1398. */
  1399. ptr = mgp->product_code_string;
  1400. if (ptr == NULL) {
  1401. printk(KERN_ERR "myri10ge: %s: Missing product code\n",
  1402. netdev->name);
  1403. return 0;
  1404. }
  1405. for (i = 0; i < 3; i++, ptr++) {
  1406. ptr = strchr(ptr, '-');
  1407. if (ptr == NULL) {
  1408. printk(KERN_ERR "myri10ge: %s: Invalid product "
  1409. "code %s\n", netdev->name,
  1410. mgp->product_code_string);
  1411. return 0;
  1412. }
  1413. }
  1414. if (*ptr == 'R' || *ptr == 'Q') {
  1415. /* We've found either an XFP or quad ribbon fiber */
  1416. cmd->port = PORT_FIBRE;
  1417. }
  1418. return 0;
  1419. }
  1420. static void
  1421. myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info)
  1422. {
  1423. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1424. strlcpy(info->driver, "myri10ge", sizeof(info->driver));
  1425. strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version));
  1426. strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version));
  1427. strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info));
  1428. }
  1429. static int
  1430. myri10ge_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal)
  1431. {
  1432. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1433. coal->rx_coalesce_usecs = mgp->intr_coal_delay;
  1434. return 0;
  1435. }
  1436. static int
  1437. myri10ge_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal)
  1438. {
  1439. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1440. mgp->intr_coal_delay = coal->rx_coalesce_usecs;
  1441. put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr);
  1442. return 0;
  1443. }
  1444. static void
  1445. myri10ge_get_pauseparam(struct net_device *netdev,
  1446. struct ethtool_pauseparam *pause)
  1447. {
  1448. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1449. pause->autoneg = 0;
  1450. pause->rx_pause = mgp->pause;
  1451. pause->tx_pause = mgp->pause;
  1452. }
  1453. static int
  1454. myri10ge_set_pauseparam(struct net_device *netdev,
  1455. struct ethtool_pauseparam *pause)
  1456. {
  1457. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1458. if (pause->tx_pause != mgp->pause)
  1459. return myri10ge_change_pause(mgp, pause->tx_pause);
  1460. if (pause->rx_pause != mgp->pause)
  1461. return myri10ge_change_pause(mgp, pause->tx_pause);
  1462. if (pause->autoneg != 0)
  1463. return -EINVAL;
  1464. return 0;
  1465. }
  1466. static void
  1467. myri10ge_get_ringparam(struct net_device *netdev,
  1468. struct ethtool_ringparam *ring)
  1469. {
  1470. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1471. ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1;
  1472. ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1;
  1473. ring->rx_jumbo_max_pending = 0;
  1474. ring->tx_max_pending = mgp->ss[0].rx_small.mask + 1;
  1475. ring->rx_mini_pending = ring->rx_mini_max_pending;
  1476. ring->rx_pending = ring->rx_max_pending;
  1477. ring->rx_jumbo_pending = ring->rx_jumbo_max_pending;
  1478. ring->tx_pending = ring->tx_max_pending;
  1479. }
  1480. static u32 myri10ge_get_rx_csum(struct net_device *netdev)
  1481. {
  1482. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1483. if (mgp->csum_flag)
  1484. return 1;
  1485. else
  1486. return 0;
  1487. }
  1488. static int myri10ge_set_rx_csum(struct net_device *netdev, u32 csum_enabled)
  1489. {
  1490. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1491. if (csum_enabled)
  1492. mgp->csum_flag = MXGEFW_FLAGS_CKSUM;
  1493. else
  1494. mgp->csum_flag = 0;
  1495. return 0;
  1496. }
  1497. static int myri10ge_set_tso(struct net_device *netdev, u32 tso_enabled)
  1498. {
  1499. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1500. unsigned long flags = mgp->features & (NETIF_F_TSO6 | NETIF_F_TSO);
  1501. if (tso_enabled)
  1502. netdev->features |= flags;
  1503. else
  1504. netdev->features &= ~flags;
  1505. return 0;
  1506. }
  1507. static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = {
  1508. "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors",
  1509. "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions",
  1510. "rx_length_errors", "rx_over_errors", "rx_crc_errors",
  1511. "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors",
  1512. "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors",
  1513. "tx_heartbeat_errors", "tx_window_errors",
  1514. /* device-specific stats */
  1515. "tx_boundary", "WC", "irq", "MSI", "MSIX",
  1516. "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs",
  1517. "serial_number", "watchdog_resets",
  1518. #ifdef CONFIG_MYRI10GE_DCA
  1519. "dca_capable_firmware", "dca_device_present",
  1520. #endif
  1521. "link_changes", "link_up", "dropped_link_overflow",
  1522. "dropped_link_error_or_filtered",
  1523. "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32",
  1524. "dropped_unicast_filtered", "dropped_multicast_filtered",
  1525. "dropped_runt", "dropped_overrun", "dropped_no_small_buffer",
  1526. "dropped_no_big_buffer"
  1527. };
  1528. static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = {
  1529. "----------- slice ---------",
  1530. "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done",
  1531. "rx_small_cnt", "rx_big_cnt",
  1532. "wake_queue", "stop_queue", "tx_linearized", "LRO aggregated",
  1533. "LRO flushed",
  1534. "LRO avg aggr", "LRO no_desc"
  1535. };
  1536. #define MYRI10GE_NET_STATS_LEN 21
  1537. #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats)
  1538. #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats)
  1539. static void
  1540. myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data)
  1541. {
  1542. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1543. int i;
  1544. switch (stringset) {
  1545. case ETH_SS_STATS:
  1546. memcpy(data, *myri10ge_gstrings_main_stats,
  1547. sizeof(myri10ge_gstrings_main_stats));
  1548. data += sizeof(myri10ge_gstrings_main_stats);
  1549. for (i = 0; i < mgp->num_slices; i++) {
  1550. memcpy(data, *myri10ge_gstrings_slice_stats,
  1551. sizeof(myri10ge_gstrings_slice_stats));
  1552. data += sizeof(myri10ge_gstrings_slice_stats);
  1553. }
  1554. break;
  1555. }
  1556. }
  1557. static int myri10ge_get_sset_count(struct net_device *netdev, int sset)
  1558. {
  1559. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1560. switch (sset) {
  1561. case ETH_SS_STATS:
  1562. return MYRI10GE_MAIN_STATS_LEN +
  1563. mgp->num_slices * MYRI10GE_SLICE_STATS_LEN;
  1564. default:
  1565. return -EOPNOTSUPP;
  1566. }
  1567. }
  1568. static void
  1569. myri10ge_get_ethtool_stats(struct net_device *netdev,
  1570. struct ethtool_stats *stats, u64 * data)
  1571. {
  1572. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1573. struct myri10ge_slice_state *ss;
  1574. int slice;
  1575. int i;
  1576. for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++)
  1577. data[i] = ((unsigned long *)&mgp->stats)[i];
  1578. data[i++] = (unsigned int)mgp->tx_boundary;
  1579. data[i++] = (unsigned int)mgp->wc_enabled;
  1580. data[i++] = (unsigned int)mgp->pdev->irq;
  1581. data[i++] = (unsigned int)mgp->msi_enabled;
  1582. data[i++] = (unsigned int)mgp->msix_enabled;
  1583. data[i++] = (unsigned int)mgp->read_dma;
  1584. data[i++] = (unsigned int)mgp->write_dma;
  1585. data[i++] = (unsigned int)mgp->read_write_dma;
  1586. data[i++] = (unsigned int)mgp->serial_number;
  1587. data[i++] = (unsigned int)mgp->watchdog_resets;
  1588. #ifdef CONFIG_MYRI10GE_DCA
  1589. data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL);
  1590. data[i++] = (unsigned int)(mgp->dca_enabled);
  1591. #endif
  1592. data[i++] = (unsigned int)mgp->link_changes;
  1593. /* firmware stats are useful only in the first slice */
  1594. ss = &mgp->ss[0];
  1595. data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up);
  1596. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow);
  1597. data[i++] =
  1598. (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered);
  1599. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause);
  1600. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy);
  1601. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32);
  1602. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered);
  1603. data[i++] =
  1604. (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered);
  1605. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt);
  1606. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun);
  1607. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer);
  1608. data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer);
  1609. for (slice = 0; slice < mgp->num_slices; slice++) {
  1610. ss = &mgp->ss[slice];
  1611. data[i++] = slice;
  1612. data[i++] = (unsigned int)ss->tx.pkt_start;
  1613. data[i++] = (unsigned int)ss->tx.pkt_done;
  1614. data[i++] = (unsigned int)ss->tx.req;
  1615. data[i++] = (unsigned int)ss->tx.done;
  1616. data[i++] = (unsigned int)ss->rx_small.cnt;
  1617. data[i++] = (unsigned int)ss->rx_big.cnt;
  1618. data[i++] = (unsigned int)ss->tx.wake_queue;
  1619. data[i++] = (unsigned int)ss->tx.stop_queue;
  1620. data[i++] = (unsigned int)ss->tx.linearized;
  1621. data[i++] = ss->rx_done.lro_mgr.stats.aggregated;
  1622. data[i++] = ss->rx_done.lro_mgr.stats.flushed;
  1623. if (ss->rx_done.lro_mgr.stats.flushed)
  1624. data[i++] = ss->rx_done.lro_mgr.stats.aggregated /
  1625. ss->rx_done.lro_mgr.stats.flushed;
  1626. else
  1627. data[i++] = 0;
  1628. data[i++] = ss->rx_done.lro_mgr.stats.no_desc;
  1629. }
  1630. }
  1631. static void myri10ge_set_msglevel(struct net_device *netdev, u32 value)
  1632. {
  1633. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1634. mgp->msg_enable = value;
  1635. }
  1636. static u32 myri10ge_get_msglevel(struct net_device *netdev)
  1637. {
  1638. struct myri10ge_priv *mgp = netdev_priv(netdev);
  1639. return mgp->msg_enable;
  1640. }
  1641. static const struct ethtool_ops myri10ge_ethtool_ops = {
  1642. .get_settings = myri10ge_get_settings,
  1643. .get_drvinfo = myri10ge_get_drvinfo,
  1644. .get_coalesce = myri10ge_get_coalesce,
  1645. .set_coalesce = myri10ge_set_coalesce,
  1646. .get_pauseparam = myri10ge_get_pauseparam,
  1647. .set_pauseparam = myri10ge_set_pauseparam,
  1648. .get_ringparam = myri10ge_get_ringparam,
  1649. .get_rx_csum = myri10ge_get_rx_csum,
  1650. .set_rx_csum = myri10ge_set_rx_csum,
  1651. .set_tx_csum = ethtool_op_set_tx_hw_csum,
  1652. .set_sg = ethtool_op_set_sg,
  1653. .set_tso = myri10ge_set_tso,
  1654. .get_link = ethtool_op_get_link,
  1655. .get_strings = myri10ge_get_strings,
  1656. .get_sset_count = myri10ge_get_sset_count,
  1657. .get_ethtool_stats = myri10ge_get_ethtool_stats,
  1658. .set_msglevel = myri10ge_set_msglevel,
  1659. .get_msglevel = myri10ge_get_msglevel
  1660. };
  1661. static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss)
  1662. {
  1663. struct myri10ge_priv *mgp = ss->mgp;
  1664. struct myri10ge_cmd cmd;
  1665. struct net_device *dev = mgp->dev;
  1666. int tx_ring_size, rx_ring_size;
  1667. int tx_ring_entries, rx_ring_entries;
  1668. int i, slice, status;
  1669. size_t bytes;
  1670. /* get ring sizes */
  1671. slice = ss - mgp->ss;
  1672. cmd.data0 = slice;
  1673. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0);
  1674. tx_ring_size = cmd.data0;
  1675. cmd.data0 = slice;
  1676. status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0);
  1677. if (status != 0)
  1678. return status;
  1679. rx_ring_size = cmd.data0;
  1680. tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send);
  1681. rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr);
  1682. ss->tx.mask = tx_ring_entries - 1;
  1683. ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
  1684. status = -ENOMEM;
  1685. /* allocate the host shadow rings */
  1686. bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4)
  1687. * sizeof(*ss->tx.req_list);
  1688. ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL);
  1689. if (ss->tx.req_bytes == NULL)
  1690. goto abort_with_nothing;
  1691. /* ensure req_list entries are aligned to 8 bytes */
  1692. ss->tx.req_list = (struct mcp_kreq_ether_send *)
  1693. ALIGN((unsigned long)ss->tx.req_bytes, 8);
  1694. ss->tx.queue_active = 0;
  1695. bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow);
  1696. ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL);
  1697. if (ss->rx_small.shadow == NULL)
  1698. goto abort_with_tx_req_bytes;
  1699. bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow);
  1700. ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL);
  1701. if (ss->rx_big.shadow == NULL)
  1702. goto abort_with_rx_small_shadow;
  1703. /* allocate the host info rings */
  1704. bytes = tx_ring_entries * sizeof(*ss->tx.info);
  1705. ss->tx.info = kzalloc(bytes, GFP_KERNEL);
  1706. if (ss->tx.info == NULL)
  1707. goto abort_with_rx_big_shadow;
  1708. bytes = rx_ring_entries * sizeof(*ss->rx_small.info);
  1709. ss->rx_small.info = kzalloc(bytes, GFP_KERNEL);
  1710. if (ss->rx_small.info == NULL)
  1711. goto abort_with_tx_info;
  1712. bytes = rx_ring_entries * sizeof(*ss->rx_big.info);
  1713. ss->rx_big.info = kzalloc(bytes, GFP_KERNEL);
  1714. if (ss->rx_big.info == NULL)
  1715. goto abort_with_rx_small_info;
  1716. /* Fill the receive rings */
  1717. ss->rx_big.cnt = 0;
  1718. ss->rx_small.cnt = 0;
  1719. ss->rx_big.fill_cnt = 0;
  1720. ss->rx_small.fill_cnt = 0;
  1721. ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE;
  1722. ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE;
  1723. ss->rx_small.watchdog_needed = 0;
  1724. ss->rx_big.watchdog_needed = 0;
  1725. myri10ge_alloc_rx_pages(mgp, &ss->rx_small,
  1726. mgp->small_bytes + MXGEFW_PAD, 0);
  1727. if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) {
  1728. printk(KERN_ERR
  1729. "myri10ge: %s:slice-%d: alloced only %d small bufs\n",
  1730. dev->name, slice, ss->rx_small.fill_cnt);
  1731. goto abort_with_rx_small_ring;
  1732. }
  1733. myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0);
  1734. if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) {
  1735. printk(KERN_ERR
  1736. "myri10ge: %s:slice-%d: alloced only %d big bufs\n",
  1737. dev->name, slice, ss->rx_big.fill_cnt);
  1738. goto abort_with_rx_big_ring;
  1739. }
  1740. return 0;
  1741. abort_with_rx_big_ring:
  1742. for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) {
  1743. int idx = i & ss->rx_big.mask;
  1744. myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx],
  1745. mgp->big_bytes);
  1746. put_page(ss->rx_big.info[idx].page);
  1747. }
  1748. abort_with_rx_small_ring:
  1749. for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) {
  1750. int idx = i & ss->rx_small.mask;
  1751. myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx],
  1752. mgp->small_bytes + MXGEFW_PAD);
  1753. put_page(ss->rx_small.info[idx].page);
  1754. }
  1755. kfree(ss->rx_big.info);
  1756. abort_with_rx_small_info:
  1757. kfree(ss->rx_small.info);
  1758. abort_with_tx_info:
  1759. kfree(ss->tx.info);
  1760. abort_with_rx_big_shadow:
  1761. kfree(ss->rx_big.shadow);
  1762. abort_with_rx_small_shadow:
  1763. kfree(ss->rx_small.shadow);
  1764. abort_with_tx_req_bytes:
  1765. kfree(ss->tx.req_bytes);
  1766. ss->tx.req_bytes = NULL;
  1767. ss->tx.req_list = NULL;
  1768. abort_with_nothing:
  1769. return status;
  1770. }
  1771. static void myri10ge_free_rings(struct myri10ge_slice_state *ss)
  1772. {
  1773. struct myri10ge_priv *mgp = ss->mgp;
  1774. struct sk_buff *skb;
  1775. struct myri10ge_tx_buf *tx;
  1776. int i, len, idx;
  1777. /* If not allocated, skip it */
  1778. if (ss->tx.req_list == NULL)
  1779. return;
  1780. for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) {
  1781. idx = i & ss->rx_big.mask;
  1782. if (i == ss->rx_big.fill_cnt - 1)
  1783. ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE;
  1784. myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx],
  1785. mgp->big_bytes);
  1786. put_page(ss->rx_big.info[idx].page);
  1787. }
  1788. for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) {
  1789. idx = i & ss->rx_small.mask;
  1790. if (i == ss->rx_small.fill_cnt - 1)
  1791. ss->rx_small.info[idx].page_offset =
  1792. MYRI10GE_ALLOC_SIZE;
  1793. myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx],
  1794. mgp->small_bytes + MXGEFW_PAD);
  1795. put_page(ss->rx_small.info[idx].page);
  1796. }
  1797. tx = &ss->tx;
  1798. while (tx->done != tx->req) {
  1799. idx = tx->done & tx->mask;
  1800. skb = tx->info[idx].skb;
  1801. /* Mark as free */
  1802. tx->info[idx].skb = NULL;
  1803. tx->done++;
  1804. len = pci_unmap_len(&tx->info[idx], len);
  1805. pci_unmap_len_set(&tx->info[idx], len, 0);
  1806. if (skb) {
  1807. ss->stats.tx_dropped++;
  1808. dev_kfree_skb_any(skb);
  1809. if (len)
  1810. pci_unmap_single(mgp->pdev,
  1811. pci_unmap_addr(&tx->info[idx],
  1812. bus), len,
  1813. PCI_DMA_TODEVICE);
  1814. } else {
  1815. if (len)
  1816. pci_unmap_page(mgp->pdev,
  1817. pci_unmap_addr(&tx->info[idx],
  1818. bus), len,
  1819. PCI_DMA_TODEVICE);
  1820. }
  1821. }
  1822. kfree(ss->rx_big.info);
  1823. kfree(ss->rx_small.info);
  1824. kfree(ss->tx.info);
  1825. kfree(ss->rx_big.shadow);
  1826. kfree(ss->rx_small.shadow);
  1827. kfree(ss->tx.req_bytes);
  1828. ss->tx.req_bytes = NULL;
  1829. ss->tx.req_list = NULL;
  1830. }
  1831. static int myri10ge_request_irq(struct myri10ge_priv *mgp)
  1832. {
  1833. struct pci_dev *pdev = mgp->pdev;
  1834. struct myri10ge_slice_state *ss;
  1835. struct net_device *netdev = mgp->dev;
  1836. int i;
  1837. int status;
  1838. mgp->msi_enabled = 0;
  1839. mgp->msix_enabled = 0;
  1840. status = 0;
  1841. if (myri10ge_msi) {
  1842. if (mgp->num_slices > 1) {
  1843. status =
  1844. pci_enable_msix(pdev, mgp->msix_vectors,
  1845. mgp->num_slices);
  1846. if (status == 0) {
  1847. mgp->msix_enabled = 1;
  1848. } else {
  1849. dev_err(&pdev->dev,
  1850. "Error %d setting up MSI-X\n", status);
  1851. return status;
  1852. }
  1853. }
  1854. if (mgp->msix_enabled == 0) {
  1855. status = pci_enable_msi(pdev);
  1856. if (status != 0) {
  1857. dev_err(&pdev->dev,
  1858. "Error %d setting up MSI; falling back to xPIC\n",
  1859. status);
  1860. } else {
  1861. mgp->msi_enabled = 1;
  1862. }
  1863. }
  1864. }
  1865. if (mgp->msix_enabled) {
  1866. for (i = 0; i < mgp->num_slices; i++) {
  1867. ss = &mgp->ss[i];
  1868. snprintf(ss->irq_desc, sizeof(ss->irq_desc),
  1869. "%s:slice-%d", netdev->name, i);
  1870. status = request_irq(mgp->msix_vectors[i].vector,
  1871. myri10ge_intr, 0, ss->irq_desc,
  1872. ss);
  1873. if (status != 0) {
  1874. dev_err(&pdev->dev,
  1875. "slice %d failed to allocate IRQ\n", i);
  1876. i--;
  1877. while (i >= 0) {
  1878. free_irq(mgp->msix_vectors[i].vector,
  1879. &mgp->ss[i]);
  1880. i--;
  1881. }
  1882. pci_disable_msix(pdev);
  1883. return status;
  1884. }
  1885. }
  1886. } else {
  1887. status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED,
  1888. mgp->dev->name, &mgp->ss[0]);
  1889. if (status != 0) {
  1890. dev_err(&pdev->dev, "failed to allocate IRQ\n");
  1891. if (mgp->msi_enabled)
  1892. pci_disable_msi(pdev);
  1893. }
  1894. }
  1895. return status;
  1896. }
  1897. static void myri10ge_free_irq(struct myri10ge_priv *mgp)
  1898. {
  1899. struct pci_dev *pdev = mgp->pdev;
  1900. int i;
  1901. if (mgp->msix_enabled) {
  1902. for (i = 0; i < mgp->num_slices; i++)
  1903. free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]);
  1904. } else {
  1905. free_irq(pdev->irq, &mgp->ss[0]);
  1906. }
  1907. if (mgp->msi_enabled)
  1908. pci_disable_msi(pdev);
  1909. if (mgp->msix_enabled)
  1910. pci_disable_msix(pdev);
  1911. }
  1912. static int
  1913. myri10ge_get_frag_header(struct skb_frag_struct *frag, void **mac_hdr,
  1914. void **ip_hdr, void **tcpudp_hdr,
  1915. u64 * hdr_flags, void *priv)
  1916. {
  1917. struct ethhdr *eh;
  1918. struct vlan_ethhdr *veh;
  1919. struct iphdr *iph;
  1920. u8 *va = page_address(frag->page) + frag->page_offset;
  1921. unsigned long ll_hlen;
  1922. /* passed opaque through lro_receive_frags() */
  1923. __wsum csum = (__force __wsum) (unsigned long)priv;
  1924. /* find the mac header, aborting if not IPv4 */
  1925. eh = (struct ethhdr *)va;
  1926. *mac_hdr = eh;
  1927. ll_hlen = ETH_HLEN;
  1928. if (eh->h_proto != htons(ETH_P_IP)) {
  1929. if (eh->h_proto == htons(ETH_P_8021Q)) {
  1930. veh = (struct vlan_ethhdr *)va;
  1931. if (veh->h_vlan_encapsulated_proto != htons(ETH_P_IP))
  1932. return -1;
  1933. ll_hlen += VLAN_HLEN;
  1934. /*
  1935. * HW checksum starts ETH_HLEN bytes into
  1936. * frame, so we must subtract off the VLAN
  1937. * header's checksum before csum can be used
  1938. */
  1939. csum = csum_sub(csum, csum_partial(va + ETH_HLEN,
  1940. VLAN_HLEN, 0));
  1941. } else {
  1942. return -1;
  1943. }
  1944. }
  1945. *hdr_flags = LRO_IPV4;
  1946. iph = (struct iphdr *)(va + ll_hlen);
  1947. *ip_hdr = iph;
  1948. if (iph->protocol != IPPROTO_TCP)
  1949. return -1;
  1950. if (iph->frag_off & htons(IP_MF | IP_OFFSET))
  1951. return -1;
  1952. *hdr_flags |= LRO_TCP;
  1953. *tcpudp_hdr = (u8 *) (*ip_hdr) + (iph->ihl << 2);
  1954. /* verify the IP checksum */
  1955. if (unlikely(ip_fast_csum((u8 *) iph, iph->ihl)))
  1956. return -1;
  1957. /* verify the checksum */
  1958. if (unlikely(csum_tcpudp_magic(iph->saddr, iph->daddr,
  1959. ntohs(iph->tot_len) - (iph->ihl << 2),
  1960. IPPROTO_TCP, csum)))
  1961. return -1;
  1962. return 0;
  1963. }
  1964. static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice)
  1965. {
  1966. struct myri10ge_cmd cmd;
  1967. struct myri10ge_slice_state *ss;
  1968. int status;
  1969. ss = &mgp->ss[slice];
  1970. status = 0;
  1971. if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) {
  1972. cmd.data0 = slice;
  1973. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET,
  1974. &cmd, 0);
  1975. ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *)
  1976. (mgp->sram + cmd.data0);
  1977. }
  1978. cmd.data0 = slice;
  1979. status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET,
  1980. &cmd, 0);
  1981. ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *)
  1982. (mgp->sram + cmd.data0);
  1983. cmd.data0 = slice;
  1984. status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0);
  1985. ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *)
  1986. (mgp->sram + cmd.data0);
  1987. ss->tx.send_go = (__iomem __be32 *)
  1988. (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice);
  1989. ss->tx.send_stop = (__iomem __be32 *)
  1990. (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice);
  1991. return status;
  1992. }
  1993. static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice)
  1994. {
  1995. struct myri10ge_cmd cmd;
  1996. struct myri10ge_slice_state *ss;
  1997. int status;
  1998. ss = &mgp->ss[slice];
  1999. cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus);
  2000. cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus);
  2001. cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16);
  2002. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0);
  2003. if (status == -ENOSYS) {
  2004. dma_addr_t bus = ss->fw_stats_bus;
  2005. if (slice != 0)
  2006. return -EINVAL;
  2007. bus += offsetof(struct mcp_irq_data, send_done_count);
  2008. cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus);
  2009. cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus);
  2010. status = myri10ge_send_cmd(mgp,
  2011. MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
  2012. &cmd, 0);
  2013. /* Firmware cannot support multicast without STATS_DMA_V2 */
  2014. mgp->fw_multicast_support = 0;
  2015. } else {
  2016. mgp->fw_multicast_support = 1;
  2017. }
  2018. return 0;
  2019. }
  2020. static int myri10ge_open(struct net_device *dev)
  2021. {
  2022. struct myri10ge_slice_state *ss;
  2023. struct myri10ge_priv *mgp = netdev_priv(dev);
  2024. struct myri10ge_cmd cmd;
  2025. int i, status, big_pow2, slice;
  2026. u8 *itable;
  2027. struct net_lro_mgr *lro_mgr;
  2028. if (mgp->running != MYRI10GE_ETH_STOPPED)
  2029. return -EBUSY;
  2030. mgp->running = MYRI10GE_ETH_STARTING;
  2031. status = myri10ge_reset(mgp);
  2032. if (status != 0) {
  2033. printk(KERN_ERR "myri10ge: %s: failed reset\n", dev->name);
  2034. goto abort_with_nothing;
  2035. }
  2036. if (mgp->num_slices > 1) {
  2037. cmd.data0 = mgp->num_slices;
  2038. cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
  2039. if (mgp->dev->real_num_tx_queues > 1)
  2040. cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
  2041. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
  2042. &cmd, 0);
  2043. if (status != 0) {
  2044. printk(KERN_ERR
  2045. "myri10ge: %s: failed to set number of slices\n",
  2046. dev->name);
  2047. goto abort_with_nothing;
  2048. }
  2049. /* setup the indirection table */
  2050. cmd.data0 = mgp->num_slices;
  2051. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
  2052. &cmd, 0);
  2053. status |= myri10ge_send_cmd(mgp,
  2054. MXGEFW_CMD_GET_RSS_TABLE_OFFSET,
  2055. &cmd, 0);
  2056. if (status != 0) {
  2057. printk(KERN_ERR
  2058. "myri10ge: %s: failed to setup rss tables\n",
  2059. dev->name);
  2060. goto abort_with_nothing;
  2061. }
  2062. /* just enable an identity mapping */
  2063. itable = mgp->sram + cmd.data0;
  2064. for (i = 0; i < mgp->num_slices; i++)
  2065. __raw_writeb(i, &itable[i]);
  2066. cmd.data0 = 1;
  2067. cmd.data1 = myri10ge_rss_hash;
  2068. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE,
  2069. &cmd, 0);
  2070. if (status != 0) {
  2071. printk(KERN_ERR
  2072. "myri10ge: %s: failed to enable slices\n",
  2073. dev->name);
  2074. goto abort_with_nothing;
  2075. }
  2076. }
  2077. status = myri10ge_request_irq(mgp);
  2078. if (status != 0)
  2079. goto abort_with_nothing;
  2080. /* decide what small buffer size to use. For good TCP rx
  2081. * performance, it is important to not receive 1514 byte
  2082. * frames into jumbo buffers, as it confuses the socket buffer
  2083. * accounting code, leading to drops and erratic performance.
  2084. */
  2085. if (dev->mtu <= ETH_DATA_LEN)
  2086. /* enough for a TCP header */
  2087. mgp->small_bytes = (128 > SMP_CACHE_BYTES)
  2088. ? (128 - MXGEFW_PAD)
  2089. : (SMP_CACHE_BYTES - MXGEFW_PAD);
  2090. else
  2091. /* enough for a vlan encapsulated ETH_DATA_LEN frame */
  2092. mgp->small_bytes = VLAN_ETH_FRAME_LEN;
  2093. /* Override the small buffer size? */
  2094. if (myri10ge_small_bytes > 0)
  2095. mgp->small_bytes = myri10ge_small_bytes;
  2096. /* Firmware needs the big buff size as a power of 2. Lie and
  2097. * tell him the buffer is larger, because we only use 1
  2098. * buffer/pkt, and the mtu will prevent overruns.
  2099. */
  2100. big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD;
  2101. if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) {
  2102. while (!is_power_of_2(big_pow2))
  2103. big_pow2++;
  2104. mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD;
  2105. } else {
  2106. big_pow2 = MYRI10GE_ALLOC_SIZE;
  2107. mgp->big_bytes = big_pow2;
  2108. }
  2109. /* setup the per-slice data structures */
  2110. for (slice = 0; slice < mgp->num_slices; slice++) {
  2111. ss = &mgp->ss[slice];
  2112. status = myri10ge_get_txrx(mgp, slice);
  2113. if (status != 0) {
  2114. printk(KERN_ERR
  2115. "myri10ge: %s: failed to get ring sizes or locations\n",
  2116. dev->name);
  2117. goto abort_with_rings;
  2118. }
  2119. status = myri10ge_allocate_rings(ss);
  2120. if (status != 0)
  2121. goto abort_with_rings;
  2122. /* only firmware which supports multiple TX queues
  2123. * supports setting up the tx stats on non-zero
  2124. * slices */
  2125. if (slice == 0 || mgp->dev->real_num_tx_queues > 1)
  2126. status = myri10ge_set_stats(mgp, slice);
  2127. if (status) {
  2128. printk(KERN_ERR
  2129. "myri10ge: %s: Couldn't set stats DMA\n",
  2130. dev->name);
  2131. goto abort_with_rings;
  2132. }
  2133. lro_mgr = &ss->rx_done.lro_mgr;
  2134. lro_mgr->dev = dev;
  2135. lro_mgr->features = LRO_F_NAPI;
  2136. lro_mgr->ip_summed = CHECKSUM_COMPLETE;
  2137. lro_mgr->ip_summed_aggr = CHECKSUM_UNNECESSARY;
  2138. lro_mgr->max_desc = MYRI10GE_MAX_LRO_DESCRIPTORS;
  2139. lro_mgr->lro_arr = ss->rx_done.lro_desc;
  2140. lro_mgr->get_frag_header = myri10ge_get_frag_header;
  2141. lro_mgr->max_aggr = myri10ge_lro_max_pkts;
  2142. if (lro_mgr->max_aggr > MAX_SKB_FRAGS)
  2143. lro_mgr->max_aggr = MAX_SKB_FRAGS;
  2144. /* must happen prior to any irq */
  2145. napi_enable(&(ss)->napi);
  2146. }
  2147. /* now give firmware buffers sizes, and MTU */
  2148. cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN;
  2149. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0);
  2150. cmd.data0 = mgp->small_bytes;
  2151. status |=
  2152. myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0);
  2153. cmd.data0 = big_pow2;
  2154. status |=
  2155. myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0);
  2156. if (status) {
  2157. printk(KERN_ERR "myri10ge: %s: Couldn't set buffer sizes\n",
  2158. dev->name);
  2159. goto abort_with_rings;
  2160. }
  2161. /*
  2162. * Set Linux style TSO mode; this is needed only on newer
  2163. * firmware versions. Older versions default to Linux
  2164. * style TSO
  2165. */
  2166. cmd.data0 = 0;
  2167. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0);
  2168. if (status && status != -ENOSYS) {
  2169. printk(KERN_ERR "myri10ge: %s: Couldn't set TSO mode\n",
  2170. dev->name);
  2171. goto abort_with_rings;
  2172. }
  2173. mgp->link_state = ~0U;
  2174. mgp->rdma_tags_available = 15;
  2175. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0);
  2176. if (status) {
  2177. printk(KERN_ERR "myri10ge: %s: Couldn't bring up link\n",
  2178. dev->name);
  2179. goto abort_with_rings;
  2180. }
  2181. mgp->running = MYRI10GE_ETH_RUNNING;
  2182. mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ;
  2183. add_timer(&mgp->watchdog_timer);
  2184. netif_tx_wake_all_queues(dev);
  2185. return 0;
  2186. abort_with_rings:
  2187. while (slice) {
  2188. slice--;
  2189. napi_disable(&mgp->ss[slice].napi);
  2190. }
  2191. for (i = 0; i < mgp->num_slices; i++)
  2192. myri10ge_free_rings(&mgp->ss[i]);
  2193. myri10ge_free_irq(mgp);
  2194. abort_with_nothing:
  2195. mgp->running = MYRI10GE_ETH_STOPPED;
  2196. return -ENOMEM;
  2197. }
  2198. static int myri10ge_close(struct net_device *dev)
  2199. {
  2200. struct myri10ge_priv *mgp = netdev_priv(dev);
  2201. struct myri10ge_cmd cmd;
  2202. int status, old_down_cnt;
  2203. int i;
  2204. if (mgp->running != MYRI10GE_ETH_RUNNING)
  2205. return 0;
  2206. if (mgp->ss[0].tx.req_bytes == NULL)
  2207. return 0;
  2208. del_timer_sync(&mgp->watchdog_timer);
  2209. mgp->running = MYRI10GE_ETH_STOPPING;
  2210. for (i = 0; i < mgp->num_slices; i++) {
  2211. napi_disable(&mgp->ss[i].napi);
  2212. }
  2213. netif_carrier_off(dev);
  2214. netif_tx_stop_all_queues(dev);
  2215. old_down_cnt = mgp->down_cnt;
  2216. mb();
  2217. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0);
  2218. if (status)
  2219. printk(KERN_ERR "myri10ge: %s: Couldn't bring down link\n",
  2220. dev->name);
  2221. wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, HZ);
  2222. if (old_down_cnt == mgp->down_cnt)
  2223. printk(KERN_ERR "myri10ge: %s never got down irq\n", dev->name);
  2224. netif_tx_disable(dev);
  2225. myri10ge_free_irq(mgp);
  2226. for (i = 0; i < mgp->num_slices; i++)
  2227. myri10ge_free_rings(&mgp->ss[i]);
  2228. mgp->running = MYRI10GE_ETH_STOPPED;
  2229. return 0;
  2230. }
  2231. /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy
  2232. * backwards one at a time and handle ring wraps */
  2233. static inline void
  2234. myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx,
  2235. struct mcp_kreq_ether_send *src, int cnt)
  2236. {
  2237. int idx, starting_slot;
  2238. starting_slot = tx->req;
  2239. while (cnt > 1) {
  2240. cnt--;
  2241. idx = (starting_slot + cnt) & tx->mask;
  2242. myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src));
  2243. mb();
  2244. }
  2245. }
  2246. /*
  2247. * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy
  2248. * at most 32 bytes at a time, so as to avoid involving the software
  2249. * pio handler in the nic. We re-write the first segment's flags
  2250. * to mark them valid only after writing the entire chain.
  2251. */
  2252. static inline void
  2253. myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src,
  2254. int cnt)
  2255. {
  2256. int idx, i;
  2257. struct mcp_kreq_ether_send __iomem *dstp, *dst;
  2258. struct mcp_kreq_ether_send *srcp;
  2259. u8 last_flags;
  2260. idx = tx->req & tx->mask;
  2261. last_flags = src->flags;
  2262. src->flags = 0;
  2263. mb();
  2264. dst = dstp = &tx->lanai[idx];
  2265. srcp = src;
  2266. if ((idx + cnt) < tx->mask) {
  2267. for (i = 0; i < (cnt - 1); i += 2) {
  2268. myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src));
  2269. mb(); /* force write every 32 bytes */
  2270. srcp += 2;
  2271. dstp += 2;
  2272. }
  2273. } else {
  2274. /* submit all but the first request, and ensure
  2275. * that it is submitted below */
  2276. myri10ge_submit_req_backwards(tx, src, cnt);
  2277. i = 0;
  2278. }
  2279. if (i < cnt) {
  2280. /* submit the first request */
  2281. myri10ge_pio_copy(dstp, srcp, sizeof(*src));
  2282. mb(); /* barrier before setting valid flag */
  2283. }
  2284. /* re-write the last 32-bits with the valid flags */
  2285. src->flags = last_flags;
  2286. put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3);
  2287. tx->req += cnt;
  2288. mb();
  2289. }
  2290. /*
  2291. * Transmit a packet. We need to split the packet so that a single
  2292. * segment does not cross myri10ge->tx_boundary, so this makes segment
  2293. * counting tricky. So rather than try to count segments up front, we
  2294. * just give up if there are too few segments to hold a reasonably
  2295. * fragmented packet currently available. If we run
  2296. * out of segments while preparing a packet for DMA, we just linearize
  2297. * it and try again.
  2298. */
  2299. static int myri10ge_xmit(struct sk_buff *skb, struct net_device *dev)
  2300. {
  2301. struct myri10ge_priv *mgp = netdev_priv(dev);
  2302. struct myri10ge_slice_state *ss;
  2303. struct mcp_kreq_ether_send *req;
  2304. struct myri10ge_tx_buf *tx;
  2305. struct skb_frag_struct *frag;
  2306. struct netdev_queue *netdev_queue;
  2307. dma_addr_t bus;
  2308. u32 low;
  2309. __be32 high_swapped;
  2310. unsigned int len;
  2311. int idx, last_idx, avail, frag_cnt, frag_idx, count, mss, max_segments;
  2312. u16 pseudo_hdr_offset, cksum_offset, queue;
  2313. int cum_len, seglen, boundary, rdma_count;
  2314. u8 flags, odd_flag;
  2315. queue = skb_get_queue_mapping(skb);
  2316. ss = &mgp->ss[queue];
  2317. netdev_queue = netdev_get_tx_queue(mgp->dev, queue);
  2318. tx = &ss->tx;
  2319. again:
  2320. req = tx->req_list;
  2321. avail = tx->mask - 1 - (tx->req - tx->done);
  2322. mss = 0;
  2323. max_segments = MXGEFW_MAX_SEND_DESC;
  2324. if (skb_is_gso(skb)) {
  2325. mss = skb_shinfo(skb)->gso_size;
  2326. max_segments = MYRI10GE_MAX_SEND_DESC_TSO;
  2327. }
  2328. if ((unlikely(avail < max_segments))) {
  2329. /* we are out of transmit resources */
  2330. tx->stop_queue++;
  2331. netif_tx_stop_queue(netdev_queue);
  2332. return 1;
  2333. }
  2334. /* Setup checksum offloading, if needed */
  2335. cksum_offset = 0;
  2336. pseudo_hdr_offset = 0;
  2337. odd_flag = 0;
  2338. flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST);
  2339. if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
  2340. cksum_offset = skb_transport_offset(skb);
  2341. pseudo_hdr_offset = cksum_offset + skb->csum_offset;
  2342. /* If the headers are excessively large, then we must
  2343. * fall back to a software checksum */
  2344. if (unlikely(!mss && (cksum_offset > 255 ||
  2345. pseudo_hdr_offset > 127))) {
  2346. if (skb_checksum_help(skb))
  2347. goto drop;
  2348. cksum_offset = 0;
  2349. pseudo_hdr_offset = 0;
  2350. } else {
  2351. odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
  2352. flags |= MXGEFW_FLAGS_CKSUM;
  2353. }
  2354. }
  2355. cum_len = 0;
  2356. if (mss) { /* TSO */
  2357. /* this removes any CKSUM flag from before */
  2358. flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST);
  2359. /* negative cum_len signifies to the
  2360. * send loop that we are still in the
  2361. * header portion of the TSO packet.
  2362. * TSO header can be at most 1KB long */
  2363. cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb));
  2364. /* for IPv6 TSO, the checksum offset stores the
  2365. * TCP header length, to save the firmware from
  2366. * the need to parse the headers */
  2367. if (skb_is_gso_v6(skb)) {
  2368. cksum_offset = tcp_hdrlen(skb);
  2369. /* Can only handle headers <= max_tso6 long */
  2370. if (unlikely(-cum_len > mgp->max_tso6))
  2371. return myri10ge_sw_tso(skb, dev);
  2372. }
  2373. /* for TSO, pseudo_hdr_offset holds mss.
  2374. * The firmware figures out where to put
  2375. * the checksum by parsing the header. */
  2376. pseudo_hdr_offset = mss;
  2377. } else
  2378. /* Mark small packets, and pad out tiny packets */
  2379. if (skb->len <= MXGEFW_SEND_SMALL_SIZE) {
  2380. flags |= MXGEFW_FLAGS_SMALL;
  2381. /* pad frames to at least ETH_ZLEN bytes */
  2382. if (unlikely(skb->len < ETH_ZLEN)) {
  2383. if (skb_padto(skb, ETH_ZLEN)) {
  2384. /* The packet is gone, so we must
  2385. * return 0 */
  2386. ss->stats.tx_dropped += 1;
  2387. return 0;
  2388. }
  2389. /* adjust the len to account for the zero pad
  2390. * so that the nic can know how long it is */
  2391. skb->len = ETH_ZLEN;
  2392. }
  2393. }
  2394. /* map the skb for DMA */
  2395. len = skb->len - skb->data_len;
  2396. idx = tx->req & tx->mask;
  2397. tx->info[idx].skb = skb;
  2398. bus = pci_map_single(mgp->pdev, skb->data, len, PCI_DMA_TODEVICE);
  2399. pci_unmap_addr_set(&tx->info[idx], bus, bus);
  2400. pci_unmap_len_set(&tx->info[idx], len, len);
  2401. frag_cnt = skb_shinfo(skb)->nr_frags;
  2402. frag_idx = 0;
  2403. count = 0;
  2404. rdma_count = 0;
  2405. /* "rdma_count" is the number of RDMAs belonging to the
  2406. * current packet BEFORE the current send request. For
  2407. * non-TSO packets, this is equal to "count".
  2408. * For TSO packets, rdma_count needs to be reset
  2409. * to 0 after a segment cut.
  2410. *
  2411. * The rdma_count field of the send request is
  2412. * the number of RDMAs of the packet starting at
  2413. * that request. For TSO send requests with one ore more cuts
  2414. * in the middle, this is the number of RDMAs starting
  2415. * after the last cut in the request. All previous
  2416. * segments before the last cut implicitly have 1 RDMA.
  2417. *
  2418. * Since the number of RDMAs is not known beforehand,
  2419. * it must be filled-in retroactively - after each
  2420. * segmentation cut or at the end of the entire packet.
  2421. */
  2422. while (1) {
  2423. /* Break the SKB or Fragment up into pieces which
  2424. * do not cross mgp->tx_boundary */
  2425. low = MYRI10GE_LOWPART_TO_U32(bus);
  2426. high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus));
  2427. while (len) {
  2428. u8 flags_next;
  2429. int cum_len_next;
  2430. if (unlikely(count == max_segments))
  2431. goto abort_linearize;
  2432. boundary =
  2433. (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1);
  2434. seglen = boundary - low;
  2435. if (seglen > len)
  2436. seglen = len;
  2437. flags_next = flags & ~MXGEFW_FLAGS_FIRST;
  2438. cum_len_next = cum_len + seglen;
  2439. if (mss) { /* TSO */
  2440. (req - rdma_count)->rdma_count = rdma_count + 1;
  2441. if (likely(cum_len >= 0)) { /* payload */
  2442. int next_is_first, chop;
  2443. chop = (cum_len_next > mss);
  2444. cum_len_next = cum_len_next % mss;
  2445. next_is_first = (cum_len_next == 0);
  2446. flags |= chop * MXGEFW_FLAGS_TSO_CHOP;
  2447. flags_next |= next_is_first *
  2448. MXGEFW_FLAGS_FIRST;
  2449. rdma_count |= -(chop | next_is_first);
  2450. rdma_count += chop & !next_is_first;
  2451. } else if (likely(cum_len_next >= 0)) { /* header ends */
  2452. int small;
  2453. rdma_count = -1;
  2454. cum_len_next = 0;
  2455. seglen = -cum_len;
  2456. small = (mss <= MXGEFW_SEND_SMALL_SIZE);
  2457. flags_next = MXGEFW_FLAGS_TSO_PLD |
  2458. MXGEFW_FLAGS_FIRST |
  2459. (small * MXGEFW_FLAGS_SMALL);
  2460. }
  2461. }
  2462. req->addr_high = high_swapped;
  2463. req->addr_low = htonl(low);
  2464. req->pseudo_hdr_offset = htons(pseudo_hdr_offset);
  2465. req->pad = 0; /* complete solid 16-byte block; does this matter? */
  2466. req->rdma_count = 1;
  2467. req->length = htons(seglen);
  2468. req->cksum_offset = cksum_offset;
  2469. req->flags = flags | ((cum_len & 1) * odd_flag);
  2470. low += seglen;
  2471. len -= seglen;
  2472. cum_len = cum_len_next;
  2473. flags = flags_next;
  2474. req++;
  2475. count++;
  2476. rdma_count++;
  2477. if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) {
  2478. if (unlikely(cksum_offset > seglen))
  2479. cksum_offset -= seglen;
  2480. else
  2481. cksum_offset = 0;
  2482. }
  2483. }
  2484. if (frag_idx == frag_cnt)
  2485. break;
  2486. /* map next fragment for DMA */
  2487. idx = (count + tx->req) & tx->mask;
  2488. frag = &skb_shinfo(skb)->frags[frag_idx];
  2489. frag_idx++;
  2490. len = frag->size;
  2491. bus = pci_map_page(mgp->pdev, frag->page, frag->page_offset,
  2492. len, PCI_DMA_TODEVICE);
  2493. pci_unmap_addr_set(&tx->info[idx], bus, bus);
  2494. pci_unmap_len_set(&tx->info[idx], len, len);
  2495. }
  2496. (req - rdma_count)->rdma_count = rdma_count;
  2497. if (mss)
  2498. do {
  2499. req--;
  2500. req->flags |= MXGEFW_FLAGS_TSO_LAST;
  2501. } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP |
  2502. MXGEFW_FLAGS_FIRST)));
  2503. idx = ((count - 1) + tx->req) & tx->mask;
  2504. tx->info[idx].last = 1;
  2505. myri10ge_submit_req(tx, tx->req_list, count);
  2506. /* if using multiple tx queues, make sure NIC polls the
  2507. * current slice */
  2508. if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) {
  2509. tx->queue_active = 1;
  2510. put_be32(htonl(1), tx->send_go);
  2511. mb();
  2512. mmiowb();
  2513. }
  2514. tx->pkt_start++;
  2515. if ((avail - count) < MXGEFW_MAX_SEND_DESC) {
  2516. tx->stop_queue++;
  2517. netif_tx_stop_queue(netdev_queue);
  2518. }
  2519. dev->trans_start = jiffies;
  2520. return 0;
  2521. abort_linearize:
  2522. /* Free any DMA resources we've alloced and clear out the skb
  2523. * slot so as to not trip up assertions, and to avoid a
  2524. * double-free if linearizing fails */
  2525. last_idx = (idx + 1) & tx->mask;
  2526. idx = tx->req & tx->mask;
  2527. tx->info[idx].skb = NULL;
  2528. do {
  2529. len = pci_unmap_len(&tx->info[idx], len);
  2530. if (len) {
  2531. if (tx->info[idx].skb != NULL)
  2532. pci_unmap_single(mgp->pdev,
  2533. pci_unmap_addr(&tx->info[idx],
  2534. bus), len,
  2535. PCI_DMA_TODEVICE);
  2536. else
  2537. pci_unmap_page(mgp->pdev,
  2538. pci_unmap_addr(&tx->info[idx],
  2539. bus), len,
  2540. PCI_DMA_TODEVICE);
  2541. pci_unmap_len_set(&tx->info[idx], len, 0);
  2542. tx->info[idx].skb = NULL;
  2543. }
  2544. idx = (idx + 1) & tx->mask;
  2545. } while (idx != last_idx);
  2546. if (skb_is_gso(skb)) {
  2547. printk(KERN_ERR
  2548. "myri10ge: %s: TSO but wanted to linearize?!?!?\n",
  2549. mgp->dev->name);
  2550. goto drop;
  2551. }
  2552. if (skb_linearize(skb))
  2553. goto drop;
  2554. tx->linearized++;
  2555. goto again;
  2556. drop:
  2557. dev_kfree_skb_any(skb);
  2558. ss->stats.tx_dropped += 1;
  2559. return 0;
  2560. }
  2561. static int myri10ge_sw_tso(struct sk_buff *skb, struct net_device *dev)
  2562. {
  2563. struct sk_buff *segs, *curr;
  2564. struct myri10ge_priv *mgp = netdev_priv(dev);
  2565. struct myri10ge_slice_state *ss;
  2566. int status;
  2567. segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6);
  2568. if (IS_ERR(segs))
  2569. goto drop;
  2570. while (segs) {
  2571. curr = segs;
  2572. segs = segs->next;
  2573. curr->next = NULL;
  2574. status = myri10ge_xmit(curr, dev);
  2575. if (status != 0) {
  2576. dev_kfree_skb_any(curr);
  2577. if (segs != NULL) {
  2578. curr = segs;
  2579. segs = segs->next;
  2580. curr->next = NULL;
  2581. dev_kfree_skb_any(segs);
  2582. }
  2583. goto drop;
  2584. }
  2585. }
  2586. dev_kfree_skb_any(skb);
  2587. return 0;
  2588. drop:
  2589. ss = &mgp->ss[skb_get_queue_mapping(skb)];
  2590. dev_kfree_skb_any(skb);
  2591. ss->stats.tx_dropped += 1;
  2592. return 0;
  2593. }
  2594. static struct net_device_stats *myri10ge_get_stats(struct net_device *dev)
  2595. {
  2596. struct myri10ge_priv *mgp = netdev_priv(dev);
  2597. struct myri10ge_slice_netstats *slice_stats;
  2598. struct net_device_stats *stats = &mgp->stats;
  2599. int i;
  2600. memset(stats, 0, sizeof(*stats));
  2601. for (i = 0; i < mgp->num_slices; i++) {
  2602. slice_stats = &mgp->ss[i].stats;
  2603. stats->rx_packets += slice_stats->rx_packets;
  2604. stats->tx_packets += slice_stats->tx_packets;
  2605. stats->rx_bytes += slice_stats->rx_bytes;
  2606. stats->tx_bytes += slice_stats->tx_bytes;
  2607. stats->rx_dropped += slice_stats->rx_dropped;
  2608. stats->tx_dropped += slice_stats->tx_dropped;
  2609. }
  2610. return stats;
  2611. }
  2612. static void myri10ge_set_multicast_list(struct net_device *dev)
  2613. {
  2614. struct myri10ge_priv *mgp = netdev_priv(dev);
  2615. struct myri10ge_cmd cmd;
  2616. struct dev_mc_list *mc_list;
  2617. __be32 data[2] = { 0, 0 };
  2618. int err;
  2619. /* can be called from atomic contexts,
  2620. * pass 1 to force atomicity in myri10ge_send_cmd() */
  2621. myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1);
  2622. /* This firmware is known to not support multicast */
  2623. if (!mgp->fw_multicast_support)
  2624. return;
  2625. /* Disable multicast filtering */
  2626. err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1);
  2627. if (err != 0) {
  2628. printk(KERN_ERR "myri10ge: %s: Failed MXGEFW_ENABLE_ALLMULTI,"
  2629. " error status: %d\n", dev->name, err);
  2630. goto abort;
  2631. }
  2632. if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) {
  2633. /* request to disable multicast filtering, so quit here */
  2634. return;
  2635. }
  2636. /* Flush the filters */
  2637. err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS,
  2638. &cmd, 1);
  2639. if (err != 0) {
  2640. printk(KERN_ERR
  2641. "myri10ge: %s: Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS"
  2642. ", error status: %d\n", dev->name, err);
  2643. goto abort;
  2644. }
  2645. /* Walk the multicast list, and add each address */
  2646. for (mc_list = dev->mc_list; mc_list != NULL; mc_list = mc_list->next) {
  2647. memcpy(data, &mc_list->dmi_addr, 6);
  2648. cmd.data0 = ntohl(data[0]);
  2649. cmd.data1 = ntohl(data[1]);
  2650. err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP,
  2651. &cmd, 1);
  2652. if (err != 0) {
  2653. printk(KERN_ERR "myri10ge: %s: Failed "
  2654. "MXGEFW_JOIN_MULTICAST_GROUP, error status:"
  2655. "%d\t", dev->name, err);
  2656. printk(KERN_ERR "MAC %pM\n", mc_list->dmi_addr);
  2657. goto abort;
  2658. }
  2659. }
  2660. /* Enable multicast filtering */
  2661. err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1);
  2662. if (err != 0) {
  2663. printk(KERN_ERR "myri10ge: %s: Failed MXGEFW_DISABLE_ALLMULTI,"
  2664. "error status: %d\n", dev->name, err);
  2665. goto abort;
  2666. }
  2667. return;
  2668. abort:
  2669. return;
  2670. }
  2671. static int myri10ge_set_mac_address(struct net_device *dev, void *addr)
  2672. {
  2673. struct sockaddr *sa = addr;
  2674. struct myri10ge_priv *mgp = netdev_priv(dev);
  2675. int status;
  2676. if (!is_valid_ether_addr(sa->sa_data))
  2677. return -EADDRNOTAVAIL;
  2678. status = myri10ge_update_mac_address(mgp, sa->sa_data);
  2679. if (status != 0) {
  2680. printk(KERN_ERR
  2681. "myri10ge: %s: changing mac address failed with %d\n",
  2682. dev->name, status);
  2683. return status;
  2684. }
  2685. /* change the dev structure */
  2686. memcpy(dev->dev_addr, sa->sa_data, 6);
  2687. return 0;
  2688. }
  2689. static int myri10ge_change_mtu(struct net_device *dev, int new_mtu)
  2690. {
  2691. struct myri10ge_priv *mgp = netdev_priv(dev);
  2692. int error = 0;
  2693. if ((new_mtu < 68) || (ETH_HLEN + new_mtu > MYRI10GE_MAX_ETHER_MTU)) {
  2694. printk(KERN_ERR "myri10ge: %s: new mtu (%d) is not valid\n",
  2695. dev->name, new_mtu);
  2696. return -EINVAL;
  2697. }
  2698. printk(KERN_INFO "%s: changing mtu from %d to %d\n",
  2699. dev->name, dev->mtu, new_mtu);
  2700. if (mgp->running) {
  2701. /* if we change the mtu on an active device, we must
  2702. * reset the device so the firmware sees the change */
  2703. myri10ge_close(dev);
  2704. dev->mtu = new_mtu;
  2705. myri10ge_open(dev);
  2706. } else
  2707. dev->mtu = new_mtu;
  2708. return error;
  2709. }
  2710. /*
  2711. * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary.
  2712. * Only do it if the bridge is a root port since we don't want to disturb
  2713. * any other device, except if forced with myri10ge_ecrc_enable > 1.
  2714. */
  2715. static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp)
  2716. {
  2717. struct pci_dev *bridge = mgp->pdev->bus->self;
  2718. struct device *dev = &mgp->pdev->dev;
  2719. unsigned cap;
  2720. unsigned err_cap;
  2721. u16 val;
  2722. u8 ext_type;
  2723. int ret;
  2724. if (!myri10ge_ecrc_enable || !bridge)
  2725. return;
  2726. /* check that the bridge is a root port */
  2727. cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
  2728. pci_read_config_word(bridge, cap + PCI_CAP_FLAGS, &val);
  2729. ext_type = (val & PCI_EXP_FLAGS_TYPE) >> 4;
  2730. if (ext_type != PCI_EXP_TYPE_ROOT_PORT) {
  2731. if (myri10ge_ecrc_enable > 1) {
  2732. struct pci_dev *prev_bridge, *old_bridge = bridge;
  2733. /* Walk the hierarchy up to the root port
  2734. * where ECRC has to be enabled */
  2735. do {
  2736. prev_bridge = bridge;
  2737. bridge = bridge->bus->self;
  2738. if (!bridge || prev_bridge == bridge) {
  2739. dev_err(dev,
  2740. "Failed to find root port"
  2741. " to force ECRC\n");
  2742. return;
  2743. }
  2744. cap =
  2745. pci_find_capability(bridge, PCI_CAP_ID_EXP);
  2746. pci_read_config_word(bridge,
  2747. cap + PCI_CAP_FLAGS, &val);
  2748. ext_type = (val & PCI_EXP_FLAGS_TYPE) >> 4;
  2749. } while (ext_type != PCI_EXP_TYPE_ROOT_PORT);
  2750. dev_info(dev,
  2751. "Forcing ECRC on non-root port %s"
  2752. " (enabling on root port %s)\n",
  2753. pci_name(old_bridge), pci_name(bridge));
  2754. } else {
  2755. dev_err(dev,
  2756. "Not enabling ECRC on non-root port %s\n",
  2757. pci_name(bridge));
  2758. return;
  2759. }
  2760. }
  2761. cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR);
  2762. if (!cap)
  2763. return;
  2764. ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap);
  2765. if (ret) {
  2766. dev_err(dev, "failed reading ext-conf-space of %s\n",
  2767. pci_name(bridge));
  2768. dev_err(dev, "\t pci=nommconf in use? "
  2769. "or buggy/incomplete/absent ACPI MCFG attr?\n");
  2770. return;
  2771. }
  2772. if (!(err_cap & PCI_ERR_CAP_ECRC_GENC))
  2773. return;
  2774. err_cap |= PCI_ERR_CAP_ECRC_GENE;
  2775. pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap);
  2776. dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge));
  2777. }
  2778. /*
  2779. * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput
  2780. * when the PCI-E Completion packets are aligned on an 8-byte
  2781. * boundary. Some PCI-E chip sets always align Completion packets; on
  2782. * the ones that do not, the alignment can be enforced by enabling
  2783. * ECRC generation (if supported).
  2784. *
  2785. * When PCI-E Completion packets are not aligned, it is actually more
  2786. * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
  2787. *
  2788. * If the driver can neither enable ECRC nor verify that it has
  2789. * already been enabled, then it must use a firmware image which works
  2790. * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it
  2791. * should also ensure that it never gives the device a Read-DMA which is
  2792. * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is
  2793. * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat)
  2794. * firmware image, and set tx_boundary to 4KB.
  2795. */
  2796. static void myri10ge_firmware_probe(struct myri10ge_priv *mgp)
  2797. {
  2798. struct pci_dev *pdev = mgp->pdev;
  2799. struct device *dev = &pdev->dev;
  2800. int status;
  2801. mgp->tx_boundary = 4096;
  2802. /*
  2803. * Verify the max read request size was set to 4KB
  2804. * before trying the test with 4KB.
  2805. */
  2806. status = pcie_get_readrq(pdev);
  2807. if (status < 0) {
  2808. dev_err(dev, "Couldn't read max read req size: %d\n", status);
  2809. goto abort;
  2810. }
  2811. if (status != 4096) {
  2812. dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status);
  2813. mgp->tx_boundary = 2048;
  2814. }
  2815. /*
  2816. * load the optimized firmware (which assumes aligned PCIe
  2817. * completions) in order to see if it works on this host.
  2818. */
  2819. mgp->fw_name = myri10ge_fw_aligned;
  2820. status = myri10ge_load_firmware(mgp, 1);
  2821. if (status != 0) {
  2822. goto abort;
  2823. }
  2824. /*
  2825. * Enable ECRC if possible
  2826. */
  2827. myri10ge_enable_ecrc(mgp);
  2828. /*
  2829. * Run a DMA test which watches for unaligned completions and
  2830. * aborts on the first one seen.
  2831. */
  2832. status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST);
  2833. if (status == 0)
  2834. return; /* keep the aligned firmware */
  2835. if (status != -E2BIG)
  2836. dev_warn(dev, "DMA test failed: %d\n", status);
  2837. if (status == -ENOSYS)
  2838. dev_warn(dev, "Falling back to ethp! "
  2839. "Please install up to date fw\n");
  2840. abort:
  2841. /* fall back to using the unaligned firmware */
  2842. mgp->tx_boundary = 2048;
  2843. mgp->fw_name = myri10ge_fw_unaligned;
  2844. }
  2845. static void myri10ge_select_firmware(struct myri10ge_priv *mgp)
  2846. {
  2847. if (myri10ge_force_firmware == 0) {
  2848. int link_width, exp_cap;
  2849. u16 lnk;
  2850. exp_cap = pci_find_capability(mgp->pdev, PCI_CAP_ID_EXP);
  2851. pci_read_config_word(mgp->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
  2852. link_width = (lnk >> 4) & 0x3f;
  2853. /* Check to see if Link is less than 8 or if the
  2854. * upstream bridge is known to provide aligned
  2855. * completions */
  2856. if (link_width < 8) {
  2857. dev_info(&mgp->pdev->dev, "PCIE x%d Link\n",
  2858. link_width);
  2859. mgp->tx_boundary = 4096;
  2860. mgp->fw_name = myri10ge_fw_aligned;
  2861. } else {
  2862. myri10ge_firmware_probe(mgp);
  2863. }
  2864. } else {
  2865. if (myri10ge_force_firmware == 1) {
  2866. dev_info(&mgp->pdev->dev,
  2867. "Assuming aligned completions (forced)\n");
  2868. mgp->tx_boundary = 4096;
  2869. mgp->fw_name = myri10ge_fw_aligned;
  2870. } else {
  2871. dev_info(&mgp->pdev->dev,
  2872. "Assuming unaligned completions (forced)\n");
  2873. mgp->tx_boundary = 2048;
  2874. mgp->fw_name = myri10ge_fw_unaligned;
  2875. }
  2876. }
  2877. if (myri10ge_fw_name != NULL) {
  2878. dev_info(&mgp->pdev->dev, "overriding firmware to %s\n",
  2879. myri10ge_fw_name);
  2880. mgp->fw_name = myri10ge_fw_name;
  2881. }
  2882. }
  2883. #ifdef CONFIG_PM
  2884. static int myri10ge_suspend(struct pci_dev *pdev, pm_message_t state)
  2885. {
  2886. struct myri10ge_priv *mgp;
  2887. struct net_device *netdev;
  2888. mgp = pci_get_drvdata(pdev);
  2889. if (mgp == NULL)
  2890. return -EINVAL;
  2891. netdev = mgp->dev;
  2892. netif_device_detach(netdev);
  2893. if (netif_running(netdev)) {
  2894. printk(KERN_INFO "myri10ge: closing %s\n", netdev->name);
  2895. rtnl_lock();
  2896. myri10ge_close(netdev);
  2897. rtnl_unlock();
  2898. }
  2899. myri10ge_dummy_rdma(mgp, 0);
  2900. pci_save_state(pdev);
  2901. pci_disable_device(pdev);
  2902. return pci_set_power_state(pdev, pci_choose_state(pdev, state));
  2903. }
  2904. static int myri10ge_resume(struct pci_dev *pdev)
  2905. {
  2906. struct myri10ge_priv *mgp;
  2907. struct net_device *netdev;
  2908. int status;
  2909. u16 vendor;
  2910. mgp = pci_get_drvdata(pdev);
  2911. if (mgp == NULL)
  2912. return -EINVAL;
  2913. netdev = mgp->dev;
  2914. pci_set_power_state(pdev, 0); /* zeros conf space as a side effect */
  2915. msleep(5); /* give card time to respond */
  2916. pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor);
  2917. if (vendor == 0xffff) {
  2918. printk(KERN_ERR "myri10ge: %s: device disappeared!\n",
  2919. mgp->dev->name);
  2920. return -EIO;
  2921. }
  2922. status = pci_restore_state(pdev);
  2923. if (status)
  2924. return status;
  2925. status = pci_enable_device(pdev);
  2926. if (status) {
  2927. dev_err(&pdev->dev, "failed to enable device\n");
  2928. return status;
  2929. }
  2930. pci_set_master(pdev);
  2931. myri10ge_reset(mgp);
  2932. myri10ge_dummy_rdma(mgp, 1);
  2933. /* Save configuration space to be restored if the
  2934. * nic resets due to a parity error */
  2935. pci_save_state(pdev);
  2936. if (netif_running(netdev)) {
  2937. rtnl_lock();
  2938. status = myri10ge_open(netdev);
  2939. rtnl_unlock();
  2940. if (status != 0)
  2941. goto abort_with_enabled;
  2942. }
  2943. netif_device_attach(netdev);
  2944. return 0;
  2945. abort_with_enabled:
  2946. pci_disable_device(pdev);
  2947. return -EIO;
  2948. }
  2949. #endif /* CONFIG_PM */
  2950. static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp)
  2951. {
  2952. struct pci_dev *pdev = mgp->pdev;
  2953. int vs = mgp->vendor_specific_offset;
  2954. u32 reboot;
  2955. /*enter read32 mode */
  2956. pci_write_config_byte(pdev, vs + 0x10, 0x3);
  2957. /*read REBOOT_STATUS (0xfffffff0) */
  2958. pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0);
  2959. pci_read_config_dword(pdev, vs + 0x14, &reboot);
  2960. return reboot;
  2961. }
  2962. /*
  2963. * This watchdog is used to check whether the board has suffered
  2964. * from a parity error and needs to be recovered.
  2965. */
  2966. static void myri10ge_watchdog(struct work_struct *work)
  2967. {
  2968. struct myri10ge_priv *mgp =
  2969. container_of(work, struct myri10ge_priv, watchdog_work);
  2970. struct myri10ge_tx_buf *tx;
  2971. u32 reboot;
  2972. int status;
  2973. int i;
  2974. u16 cmd, vendor;
  2975. mgp->watchdog_resets++;
  2976. pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd);
  2977. if ((cmd & PCI_COMMAND_MASTER) == 0) {
  2978. /* Bus master DMA disabled? Check to see
  2979. * if the card rebooted due to a parity error
  2980. * For now, just report it */
  2981. reboot = myri10ge_read_reboot(mgp);
  2982. printk(KERN_ERR
  2983. "myri10ge: %s: NIC rebooted (0x%x),%s resetting\n",
  2984. mgp->dev->name, reboot,
  2985. myri10ge_reset_recover ? " " : " not");
  2986. if (myri10ge_reset_recover == 0)
  2987. return;
  2988. myri10ge_reset_recover--;
  2989. /*
  2990. * A rebooted nic will come back with config space as
  2991. * it was after power was applied to PCIe bus.
  2992. * Attempt to restore config space which was saved
  2993. * when the driver was loaded, or the last time the
  2994. * nic was resumed from power saving mode.
  2995. */
  2996. pci_restore_state(mgp->pdev);
  2997. /* save state again for accounting reasons */
  2998. pci_save_state(mgp->pdev);
  2999. } else {
  3000. /* if we get back -1's from our slot, perhaps somebody
  3001. * powered off our card. Don't try to reset it in
  3002. * this case */
  3003. if (cmd == 0xffff) {
  3004. pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor);
  3005. if (vendor == 0xffff) {
  3006. printk(KERN_ERR
  3007. "myri10ge: %s: device disappeared!\n",
  3008. mgp->dev->name);
  3009. return;
  3010. }
  3011. }
  3012. /* Perhaps it is a software error. Try to reset */
  3013. printk(KERN_ERR "myri10ge: %s: device timeout, resetting\n",
  3014. mgp->dev->name);
  3015. for (i = 0; i < mgp->num_slices; i++) {
  3016. tx = &mgp->ss[i].tx;
  3017. printk(KERN_INFO
  3018. "myri10ge: %s: (%d): %d %d %d %d %d %d\n",
  3019. mgp->dev->name, i, tx->queue_active, tx->req,
  3020. tx->done, tx->pkt_start, tx->pkt_done,
  3021. (int)ntohl(mgp->ss[i].fw_stats->
  3022. send_done_count));
  3023. msleep(2000);
  3024. printk(KERN_INFO
  3025. "myri10ge: %s: (%d): %d %d %d %d %d %d\n",
  3026. mgp->dev->name, i, tx->queue_active, tx->req,
  3027. tx->done, tx->pkt_start, tx->pkt_done,
  3028. (int)ntohl(mgp->ss[i].fw_stats->
  3029. send_done_count));
  3030. }
  3031. }
  3032. rtnl_lock();
  3033. myri10ge_close(mgp->dev);
  3034. status = myri10ge_load_firmware(mgp, 1);
  3035. if (status != 0)
  3036. printk(KERN_ERR "myri10ge: %s: failed to load firmware\n",
  3037. mgp->dev->name);
  3038. else
  3039. myri10ge_open(mgp->dev);
  3040. rtnl_unlock();
  3041. }
  3042. /*
  3043. * We use our own timer routine rather than relying upon
  3044. * netdev->tx_timeout because we have a very large hardware transmit
  3045. * queue. Due to the large queue, the netdev->tx_timeout function
  3046. * cannot detect a NIC with a parity error in a timely fashion if the
  3047. * NIC is lightly loaded.
  3048. */
  3049. static void myri10ge_watchdog_timer(unsigned long arg)
  3050. {
  3051. struct myri10ge_priv *mgp;
  3052. struct myri10ge_slice_state *ss;
  3053. int i, reset_needed;
  3054. u32 rx_pause_cnt;
  3055. mgp = (struct myri10ge_priv *)arg;
  3056. rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause);
  3057. for (i = 0, reset_needed = 0;
  3058. i < mgp->num_slices && reset_needed == 0; ++i) {
  3059. ss = &mgp->ss[i];
  3060. if (ss->rx_small.watchdog_needed) {
  3061. myri10ge_alloc_rx_pages(mgp, &ss->rx_small,
  3062. mgp->small_bytes + MXGEFW_PAD,
  3063. 1);
  3064. if (ss->rx_small.fill_cnt - ss->rx_small.cnt >=
  3065. myri10ge_fill_thresh)
  3066. ss->rx_small.watchdog_needed = 0;
  3067. }
  3068. if (ss->rx_big.watchdog_needed) {
  3069. myri10ge_alloc_rx_pages(mgp, &ss->rx_big,
  3070. mgp->big_bytes, 1);
  3071. if (ss->rx_big.fill_cnt - ss->rx_big.cnt >=
  3072. myri10ge_fill_thresh)
  3073. ss->rx_big.watchdog_needed = 0;
  3074. }
  3075. if (ss->tx.req != ss->tx.done &&
  3076. ss->tx.done == ss->watchdog_tx_done &&
  3077. ss->watchdog_tx_req != ss->watchdog_tx_done) {
  3078. /* nic seems like it might be stuck.. */
  3079. if (rx_pause_cnt != mgp->watchdog_pause) {
  3080. if (net_ratelimit())
  3081. printk(KERN_WARNING
  3082. "myri10ge %s slice %d:"
  3083. "TX paused, check link partner\n",
  3084. mgp->dev->name, i);
  3085. } else {
  3086. printk(KERN_WARNING
  3087. "myri10ge %s slice %d stuck:",
  3088. mgp->dev->name, i);
  3089. reset_needed = 1;
  3090. }
  3091. }
  3092. ss->watchdog_tx_done = ss->tx.done;
  3093. ss->watchdog_tx_req = ss->tx.req;
  3094. }
  3095. mgp->watchdog_pause = rx_pause_cnt;
  3096. if (reset_needed) {
  3097. schedule_work(&mgp->watchdog_work);
  3098. } else {
  3099. /* rearm timer */
  3100. mod_timer(&mgp->watchdog_timer,
  3101. jiffies + myri10ge_watchdog_timeout * HZ);
  3102. }
  3103. }
  3104. static void myri10ge_free_slices(struct myri10ge_priv *mgp)
  3105. {
  3106. struct myri10ge_slice_state *ss;
  3107. struct pci_dev *pdev = mgp->pdev;
  3108. size_t bytes;
  3109. int i;
  3110. if (mgp->ss == NULL)
  3111. return;
  3112. for (i = 0; i < mgp->num_slices; i++) {
  3113. ss = &mgp->ss[i];
  3114. if (ss->rx_done.entry != NULL) {
  3115. bytes = mgp->max_intr_slots *
  3116. sizeof(*ss->rx_done.entry);
  3117. dma_free_coherent(&pdev->dev, bytes,
  3118. ss->rx_done.entry, ss->rx_done.bus);
  3119. ss->rx_done.entry = NULL;
  3120. }
  3121. if (ss->fw_stats != NULL) {
  3122. bytes = sizeof(*ss->fw_stats);
  3123. dma_free_coherent(&pdev->dev, bytes,
  3124. ss->fw_stats, ss->fw_stats_bus);
  3125. ss->fw_stats = NULL;
  3126. }
  3127. }
  3128. kfree(mgp->ss);
  3129. mgp->ss = NULL;
  3130. }
  3131. static int myri10ge_alloc_slices(struct myri10ge_priv *mgp)
  3132. {
  3133. struct myri10ge_slice_state *ss;
  3134. struct pci_dev *pdev = mgp->pdev;
  3135. size_t bytes;
  3136. int i;
  3137. bytes = sizeof(*mgp->ss) * mgp->num_slices;
  3138. mgp->ss = kzalloc(bytes, GFP_KERNEL);
  3139. if (mgp->ss == NULL) {
  3140. return -ENOMEM;
  3141. }
  3142. for (i = 0; i < mgp->num_slices; i++) {
  3143. ss = &mgp->ss[i];
  3144. bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry);
  3145. ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes,
  3146. &ss->rx_done.bus,
  3147. GFP_KERNEL);
  3148. if (ss->rx_done.entry == NULL)
  3149. goto abort;
  3150. memset(ss->rx_done.entry, 0, bytes);
  3151. bytes = sizeof(*ss->fw_stats);
  3152. ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes,
  3153. &ss->fw_stats_bus,
  3154. GFP_KERNEL);
  3155. if (ss->fw_stats == NULL)
  3156. goto abort;
  3157. ss->mgp = mgp;
  3158. ss->dev = mgp->dev;
  3159. netif_napi_add(ss->dev, &ss->napi, myri10ge_poll,
  3160. myri10ge_napi_weight);
  3161. }
  3162. return 0;
  3163. abort:
  3164. myri10ge_free_slices(mgp);
  3165. return -ENOMEM;
  3166. }
  3167. /*
  3168. * This function determines the number of slices supported.
  3169. * The number slices is the minumum of the number of CPUS,
  3170. * the number of MSI-X irqs supported, the number of slices
  3171. * supported by the firmware
  3172. */
  3173. static void myri10ge_probe_slices(struct myri10ge_priv *mgp)
  3174. {
  3175. struct myri10ge_cmd cmd;
  3176. struct pci_dev *pdev = mgp->pdev;
  3177. char *old_fw;
  3178. int i, status, ncpus, msix_cap;
  3179. mgp->num_slices = 1;
  3180. msix_cap = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
  3181. ncpus = num_online_cpus();
  3182. if (myri10ge_max_slices == 1 || msix_cap == 0 ||
  3183. (myri10ge_max_slices == -1 && ncpus < 2))
  3184. return;
  3185. /* try to load the slice aware rss firmware */
  3186. old_fw = mgp->fw_name;
  3187. if (myri10ge_fw_name != NULL) {
  3188. dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n",
  3189. myri10ge_fw_name);
  3190. mgp->fw_name = myri10ge_fw_name;
  3191. } else if (old_fw == myri10ge_fw_aligned)
  3192. mgp->fw_name = myri10ge_fw_rss_aligned;
  3193. else
  3194. mgp->fw_name = myri10ge_fw_rss_unaligned;
  3195. status = myri10ge_load_firmware(mgp, 0);
  3196. if (status != 0) {
  3197. dev_info(&pdev->dev, "Rss firmware not found\n");
  3198. return;
  3199. }
  3200. /* hit the board with a reset to ensure it is alive */
  3201. memset(&cmd, 0, sizeof(cmd));
  3202. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0);
  3203. if (status != 0) {
  3204. dev_err(&mgp->pdev->dev, "failed reset\n");
  3205. goto abort_with_fw;
  3206. return;
  3207. }
  3208. mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot);
  3209. /* tell it the size of the interrupt queues */
  3210. cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot);
  3211. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0);
  3212. if (status != 0) {
  3213. dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
  3214. goto abort_with_fw;
  3215. }
  3216. /* ask the maximum number of slices it supports */
  3217. status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0);
  3218. if (status != 0)
  3219. goto abort_with_fw;
  3220. else
  3221. mgp->num_slices = cmd.data0;
  3222. /* Only allow multiple slices if MSI-X is usable */
  3223. if (!myri10ge_msi) {
  3224. goto abort_with_fw;
  3225. }
  3226. /* if the admin did not specify a limit to how many
  3227. * slices we should use, cap it automatically to the
  3228. * number of CPUs currently online */
  3229. if (myri10ge_max_slices == -1)
  3230. myri10ge_max_slices = ncpus;
  3231. if (mgp->num_slices > myri10ge_max_slices)
  3232. mgp->num_slices = myri10ge_max_slices;
  3233. /* Now try to allocate as many MSI-X vectors as we have
  3234. * slices. We give up on MSI-X if we can only get a single
  3235. * vector. */
  3236. mgp->msix_vectors = kzalloc(mgp->num_slices *
  3237. sizeof(*mgp->msix_vectors), GFP_KERNEL);
  3238. if (mgp->msix_vectors == NULL)
  3239. goto disable_msix;
  3240. for (i = 0; i < mgp->num_slices; i++) {
  3241. mgp->msix_vectors[i].entry = i;
  3242. }
  3243. while (mgp->num_slices > 1) {
  3244. /* make sure it is a power of two */
  3245. while (!is_power_of_2(mgp->num_slices))
  3246. mgp->num_slices--;
  3247. if (mgp->num_slices == 1)
  3248. goto disable_msix;
  3249. status = pci_enable_msix(pdev, mgp->msix_vectors,
  3250. mgp->num_slices);
  3251. if (status == 0) {
  3252. pci_disable_msix(pdev);
  3253. return;
  3254. }
  3255. if (status > 0)
  3256. mgp->num_slices = status;
  3257. else
  3258. goto disable_msix;
  3259. }
  3260. disable_msix:
  3261. if (mgp->msix_vectors != NULL) {
  3262. kfree(mgp->msix_vectors);
  3263. mgp->msix_vectors = NULL;
  3264. }
  3265. abort_with_fw:
  3266. mgp->num_slices = 1;
  3267. mgp->fw_name = old_fw;
  3268. myri10ge_load_firmware(mgp, 0);
  3269. }
  3270. static const struct net_device_ops myri10ge_netdev_ops = {
  3271. .ndo_open = myri10ge_open,
  3272. .ndo_stop = myri10ge_close,
  3273. .ndo_start_xmit = myri10ge_xmit,
  3274. .ndo_get_stats = myri10ge_get_stats,
  3275. .ndo_validate_addr = eth_validate_addr,
  3276. .ndo_change_mtu = myri10ge_change_mtu,
  3277. .ndo_set_multicast_list = myri10ge_set_multicast_list,
  3278. .ndo_set_mac_address = myri10ge_set_mac_address,
  3279. };
  3280. static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  3281. {
  3282. struct net_device *netdev;
  3283. struct myri10ge_priv *mgp;
  3284. struct device *dev = &pdev->dev;
  3285. int i;
  3286. int status = -ENXIO;
  3287. int dac_enabled;
  3288. unsigned hdr_offset, ss_offset;
  3289. netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES);
  3290. if (netdev == NULL) {
  3291. dev_err(dev, "Could not allocate ethernet device\n");
  3292. return -ENOMEM;
  3293. }
  3294. SET_NETDEV_DEV(netdev, &pdev->dev);
  3295. mgp = netdev_priv(netdev);
  3296. mgp->dev = netdev;
  3297. mgp->pdev = pdev;
  3298. mgp->csum_flag = MXGEFW_FLAGS_CKSUM;
  3299. mgp->pause = myri10ge_flow_control;
  3300. mgp->intr_coal_delay = myri10ge_intr_coal_delay;
  3301. mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT);
  3302. init_waitqueue_head(&mgp->down_wq);
  3303. if (pci_enable_device(pdev)) {
  3304. dev_err(&pdev->dev, "pci_enable_device call failed\n");
  3305. status = -ENODEV;
  3306. goto abort_with_netdev;
  3307. }
  3308. /* Find the vendor-specific cap so we can check
  3309. * the reboot register later on */
  3310. mgp->vendor_specific_offset
  3311. = pci_find_capability(pdev, PCI_CAP_ID_VNDR);
  3312. /* Set our max read request to 4KB */
  3313. status = pcie_set_readrq(pdev, 4096);
  3314. if (status != 0) {
  3315. dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n",
  3316. status);
  3317. goto abort_with_enabled;
  3318. }
  3319. pci_set_master(pdev);
  3320. dac_enabled = 1;
  3321. status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
  3322. if (status != 0) {
  3323. dac_enabled = 0;
  3324. dev_err(&pdev->dev,
  3325. "64-bit pci address mask was refused, "
  3326. "trying 32-bit\n");
  3327. status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  3328. }
  3329. if (status != 0) {
  3330. dev_err(&pdev->dev, "Error %d setting DMA mask\n", status);
  3331. goto abort_with_enabled;
  3332. }
  3333. (void)pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  3334. mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd),
  3335. &mgp->cmd_bus, GFP_KERNEL);
  3336. if (mgp->cmd == NULL)
  3337. goto abort_with_enabled;
  3338. mgp->board_span = pci_resource_len(pdev, 0);
  3339. mgp->iomem_base = pci_resource_start(pdev, 0);
  3340. mgp->mtrr = -1;
  3341. mgp->wc_enabled = 0;
  3342. #ifdef CONFIG_MTRR
  3343. mgp->mtrr = mtrr_add(mgp->iomem_base, mgp->board_span,
  3344. MTRR_TYPE_WRCOMB, 1);
  3345. if (mgp->mtrr >= 0)
  3346. mgp->wc_enabled = 1;
  3347. #endif
  3348. mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span);
  3349. if (mgp->sram == NULL) {
  3350. dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n",
  3351. mgp->board_span, mgp->iomem_base);
  3352. status = -ENXIO;
  3353. goto abort_with_mtrr;
  3354. }
  3355. hdr_offset =
  3356. ntohl(__raw_readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc;
  3357. ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs);
  3358. mgp->sram_size = ntohl(__raw_readl(mgp->sram + ss_offset));
  3359. if (mgp->sram_size > mgp->board_span ||
  3360. mgp->sram_size <= MYRI10GE_FW_OFFSET) {
  3361. dev_err(&pdev->dev,
  3362. "invalid sram_size %dB or board span %ldB\n",
  3363. mgp->sram_size, mgp->board_span);
  3364. goto abort_with_ioremap;
  3365. }
  3366. memcpy_fromio(mgp->eeprom_strings,
  3367. mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE);
  3368. memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2);
  3369. status = myri10ge_read_mac_addr(mgp);
  3370. if (status)
  3371. goto abort_with_ioremap;
  3372. for (i = 0; i < ETH_ALEN; i++)
  3373. netdev->dev_addr[i] = mgp->mac_addr[i];
  3374. myri10ge_select_firmware(mgp);
  3375. status = myri10ge_load_firmware(mgp, 1);
  3376. if (status != 0) {
  3377. dev_err(&pdev->dev, "failed to load firmware\n");
  3378. goto abort_with_ioremap;
  3379. }
  3380. myri10ge_probe_slices(mgp);
  3381. status = myri10ge_alloc_slices(mgp);
  3382. if (status != 0) {
  3383. dev_err(&pdev->dev, "failed to alloc slice state\n");
  3384. goto abort_with_firmware;
  3385. }
  3386. netdev->real_num_tx_queues = mgp->num_slices;
  3387. status = myri10ge_reset(mgp);
  3388. if (status != 0) {
  3389. dev_err(&pdev->dev, "failed reset\n");
  3390. goto abort_with_slices;
  3391. }
  3392. #ifdef CONFIG_MYRI10GE_DCA
  3393. myri10ge_setup_dca(mgp);
  3394. #endif
  3395. pci_set_drvdata(pdev, mgp);
  3396. if ((myri10ge_initial_mtu + ETH_HLEN) > MYRI10GE_MAX_ETHER_MTU)
  3397. myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN;
  3398. if ((myri10ge_initial_mtu + ETH_HLEN) < 68)
  3399. myri10ge_initial_mtu = 68;
  3400. netdev->netdev_ops = &myri10ge_netdev_ops;
  3401. netdev->mtu = myri10ge_initial_mtu;
  3402. netdev->base_addr = mgp->iomem_base;
  3403. netdev->features = mgp->features;
  3404. if (dac_enabled)
  3405. netdev->features |= NETIF_F_HIGHDMA;
  3406. /* make sure we can get an irq, and that MSI can be
  3407. * setup (if available). Also ensure netdev->irq
  3408. * is set to correct value if MSI is enabled */
  3409. status = myri10ge_request_irq(mgp);
  3410. if (status != 0)
  3411. goto abort_with_firmware;
  3412. netdev->irq = pdev->irq;
  3413. myri10ge_free_irq(mgp);
  3414. /* Save configuration space to be restored if the
  3415. * nic resets due to a parity error */
  3416. pci_save_state(pdev);
  3417. /* Setup the watchdog timer */
  3418. setup_timer(&mgp->watchdog_timer, myri10ge_watchdog_timer,
  3419. (unsigned long)mgp);
  3420. SET_ETHTOOL_OPS(netdev, &myri10ge_ethtool_ops);
  3421. INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog);
  3422. status = register_netdev(netdev);
  3423. if (status != 0) {
  3424. dev_err(&pdev->dev, "register_netdev failed: %d\n", status);
  3425. goto abort_with_state;
  3426. }
  3427. if (mgp->msix_enabled)
  3428. dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, WC %s\n",
  3429. mgp->num_slices, mgp->tx_boundary, mgp->fw_name,
  3430. (mgp->wc_enabled ? "Enabled" : "Disabled"));
  3431. else
  3432. dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, WC %s\n",
  3433. mgp->msi_enabled ? "MSI" : "xPIC",
  3434. netdev->irq, mgp->tx_boundary, mgp->fw_name,
  3435. (mgp->wc_enabled ? "Enabled" : "Disabled"));
  3436. return 0;
  3437. abort_with_state:
  3438. pci_restore_state(pdev);
  3439. abort_with_slices:
  3440. myri10ge_free_slices(mgp);
  3441. abort_with_firmware:
  3442. myri10ge_dummy_rdma(mgp, 0);
  3443. abort_with_ioremap:
  3444. if (mgp->mac_addr_string != NULL)
  3445. dev_err(&pdev->dev,
  3446. "myri10ge_probe() failed: MAC=%s, SN=%ld\n",
  3447. mgp->mac_addr_string, mgp->serial_number);
  3448. iounmap(mgp->sram);
  3449. abort_with_mtrr:
  3450. #ifdef CONFIG_MTRR
  3451. if (mgp->mtrr >= 0)
  3452. mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span);
  3453. #endif
  3454. dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd),
  3455. mgp->cmd, mgp->cmd_bus);
  3456. abort_with_enabled:
  3457. pci_disable_device(pdev);
  3458. abort_with_netdev:
  3459. free_netdev(netdev);
  3460. return status;
  3461. }
  3462. /*
  3463. * myri10ge_remove
  3464. *
  3465. * Does what is necessary to shutdown one Myrinet device. Called
  3466. * once for each Myrinet card by the kernel when a module is
  3467. * unloaded.
  3468. */
  3469. static void myri10ge_remove(struct pci_dev *pdev)
  3470. {
  3471. struct myri10ge_priv *mgp;
  3472. struct net_device *netdev;
  3473. mgp = pci_get_drvdata(pdev);
  3474. if (mgp == NULL)
  3475. return;
  3476. flush_scheduled_work();
  3477. netdev = mgp->dev;
  3478. unregister_netdev(netdev);
  3479. #ifdef CONFIG_MYRI10GE_DCA
  3480. myri10ge_teardown_dca(mgp);
  3481. #endif
  3482. myri10ge_dummy_rdma(mgp, 0);
  3483. /* avoid a memory leak */
  3484. pci_restore_state(pdev);
  3485. iounmap(mgp->sram);
  3486. #ifdef CONFIG_MTRR
  3487. if (mgp->mtrr >= 0)
  3488. mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span);
  3489. #endif
  3490. myri10ge_free_slices(mgp);
  3491. if (mgp->msix_vectors != NULL)
  3492. kfree(mgp->msix_vectors);
  3493. dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd),
  3494. mgp->cmd, mgp->cmd_bus);
  3495. free_netdev(netdev);
  3496. pci_disable_device(pdev);
  3497. pci_set_drvdata(pdev, NULL);
  3498. }
  3499. #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008
  3500. #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009
  3501. static struct pci_device_id myri10ge_pci_tbl[] = {
  3502. {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)},
  3503. {PCI_DEVICE
  3504. (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)},
  3505. {0},
  3506. };
  3507. static struct pci_driver myri10ge_driver = {
  3508. .name = "myri10ge",
  3509. .probe = myri10ge_probe,
  3510. .remove = myri10ge_remove,
  3511. .id_table = myri10ge_pci_tbl,
  3512. #ifdef CONFIG_PM
  3513. .suspend = myri10ge_suspend,
  3514. .resume = myri10ge_resume,
  3515. #endif
  3516. };
  3517. #ifdef CONFIG_MYRI10GE_DCA
  3518. static int
  3519. myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p)
  3520. {
  3521. int err = driver_for_each_device(&myri10ge_driver.driver,
  3522. NULL, &event,
  3523. myri10ge_notify_dca_device);
  3524. if (err)
  3525. return NOTIFY_BAD;
  3526. return NOTIFY_DONE;
  3527. }
  3528. static struct notifier_block myri10ge_dca_notifier = {
  3529. .notifier_call = myri10ge_notify_dca,
  3530. .next = NULL,
  3531. .priority = 0,
  3532. };
  3533. #endif /* CONFIG_MYRI10GE_DCA */
  3534. static __init int myri10ge_init_module(void)
  3535. {
  3536. printk(KERN_INFO "%s: Version %s\n", myri10ge_driver.name,
  3537. MYRI10GE_VERSION_STR);
  3538. if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) {
  3539. printk(KERN_ERR
  3540. "%s: Illegal rssh hash type %d, defaulting to source port\n",
  3541. myri10ge_driver.name, myri10ge_rss_hash);
  3542. myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT;
  3543. }
  3544. #ifdef CONFIG_MYRI10GE_DCA
  3545. dca_register_notify(&myri10ge_dca_notifier);
  3546. #endif
  3547. if (myri10ge_max_slices > MYRI10GE_MAX_SLICES)
  3548. myri10ge_max_slices = MYRI10GE_MAX_SLICES;
  3549. return pci_register_driver(&myri10ge_driver);
  3550. }
  3551. module_init(myri10ge_init_module);
  3552. static __exit void myri10ge_cleanup_module(void)
  3553. {
  3554. #ifdef CONFIG_MYRI10GE_DCA
  3555. dca_unregister_notify(&myri10ge_dca_notifier);
  3556. #endif
  3557. pci_unregister_driver(&myri10ge_driver);
  3558. }
  3559. module_exit(myri10ge_cleanup_module);