efx.c 69 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672
  1. /****************************************************************************
  2. * Driver for Solarflare Solarstorm network controllers and boards
  3. * Copyright 2005-2006 Fen Systems Ltd.
  4. * Copyright 2005-2011 Solarflare Communications Inc.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation, incorporated herein by reference.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/pci.h>
  12. #include <linux/netdevice.h>
  13. #include <linux/etherdevice.h>
  14. #include <linux/delay.h>
  15. #include <linux/notifier.h>
  16. #include <linux/ip.h>
  17. #include <linux/tcp.h>
  18. #include <linux/in.h>
  19. #include <linux/crc32.h>
  20. #include <linux/ethtool.h>
  21. #include <linux/topology.h>
  22. #include <linux/gfp.h>
  23. #include <linux/cpu_rmap.h>
  24. #include "net_driver.h"
  25. #include "efx.h"
  26. #include "nic.h"
  27. #include "mcdi.h"
  28. #include "workarounds.h"
  29. /**************************************************************************
  30. *
  31. * Type name strings
  32. *
  33. **************************************************************************
  34. */
  35. /* Loopback mode names (see LOOPBACK_MODE()) */
  36. const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
  37. const char *const efx_loopback_mode_names[] = {
  38. [LOOPBACK_NONE] = "NONE",
  39. [LOOPBACK_DATA] = "DATAPATH",
  40. [LOOPBACK_GMAC] = "GMAC",
  41. [LOOPBACK_XGMII] = "XGMII",
  42. [LOOPBACK_XGXS] = "XGXS",
  43. [LOOPBACK_XAUI] = "XAUI",
  44. [LOOPBACK_GMII] = "GMII",
  45. [LOOPBACK_SGMII] = "SGMII",
  46. [LOOPBACK_XGBR] = "XGBR",
  47. [LOOPBACK_XFI] = "XFI",
  48. [LOOPBACK_XAUI_FAR] = "XAUI_FAR",
  49. [LOOPBACK_GMII_FAR] = "GMII_FAR",
  50. [LOOPBACK_SGMII_FAR] = "SGMII_FAR",
  51. [LOOPBACK_XFI_FAR] = "XFI_FAR",
  52. [LOOPBACK_GPHY] = "GPHY",
  53. [LOOPBACK_PHYXS] = "PHYXS",
  54. [LOOPBACK_PCS] = "PCS",
  55. [LOOPBACK_PMAPMD] = "PMA/PMD",
  56. [LOOPBACK_XPORT] = "XPORT",
  57. [LOOPBACK_XGMII_WS] = "XGMII_WS",
  58. [LOOPBACK_XAUI_WS] = "XAUI_WS",
  59. [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR",
  60. [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
  61. [LOOPBACK_GMII_WS] = "GMII_WS",
  62. [LOOPBACK_XFI_WS] = "XFI_WS",
  63. [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR",
  64. [LOOPBACK_PHYXS_WS] = "PHYXS_WS",
  65. };
  66. const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
  67. const char *const efx_reset_type_names[] = {
  68. [RESET_TYPE_INVISIBLE] = "INVISIBLE",
  69. [RESET_TYPE_ALL] = "ALL",
  70. [RESET_TYPE_WORLD] = "WORLD",
  71. [RESET_TYPE_DISABLE] = "DISABLE",
  72. [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG",
  73. [RESET_TYPE_INT_ERROR] = "INT_ERROR",
  74. [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY",
  75. [RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH",
  76. [RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH",
  77. [RESET_TYPE_TX_SKIP] = "TX_SKIP",
  78. [RESET_TYPE_MC_FAILURE] = "MC_FAILURE",
  79. };
  80. #define EFX_MAX_MTU (9 * 1024)
  81. /* Reset workqueue. If any NIC has a hardware failure then a reset will be
  82. * queued onto this work queue. This is not a per-nic work queue, because
  83. * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
  84. */
  85. static struct workqueue_struct *reset_workqueue;
  86. /**************************************************************************
  87. *
  88. * Configurable values
  89. *
  90. *************************************************************************/
  91. /*
  92. * Use separate channels for TX and RX events
  93. *
  94. * Set this to 1 to use separate channels for TX and RX. It allows us
  95. * to control interrupt affinity separately for TX and RX.
  96. *
  97. * This is only used in MSI-X interrupt mode
  98. */
  99. static unsigned int separate_tx_channels;
  100. module_param(separate_tx_channels, uint, 0444);
  101. MODULE_PARM_DESC(separate_tx_channels,
  102. "Use separate channels for TX and RX");
  103. /* This is the weight assigned to each of the (per-channel) virtual
  104. * NAPI devices.
  105. */
  106. static int napi_weight = 64;
  107. /* This is the time (in jiffies) between invocations of the hardware
  108. * monitor. On Falcon-based NICs, this will:
  109. * - Check the on-board hardware monitor;
  110. * - Poll the link state and reconfigure the hardware as necessary.
  111. */
  112. static unsigned int efx_monitor_interval = 1 * HZ;
  113. /* Initial interrupt moderation settings. They can be modified after
  114. * module load with ethtool.
  115. *
  116. * The default for RX should strike a balance between increasing the
  117. * round-trip latency and reducing overhead.
  118. */
  119. static unsigned int rx_irq_mod_usec = 60;
  120. /* Initial interrupt moderation settings. They can be modified after
  121. * module load with ethtool.
  122. *
  123. * This default is chosen to ensure that a 10G link does not go idle
  124. * while a TX queue is stopped after it has become full. A queue is
  125. * restarted when it drops below half full. The time this takes (assuming
  126. * worst case 3 descriptors per packet and 1024 descriptors) is
  127. * 512 / 3 * 1.2 = 205 usec.
  128. */
  129. static unsigned int tx_irq_mod_usec = 150;
  130. /* This is the first interrupt mode to try out of:
  131. * 0 => MSI-X
  132. * 1 => MSI
  133. * 2 => legacy
  134. */
  135. static unsigned int interrupt_mode;
  136. /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  137. * i.e. the number of CPUs among which we may distribute simultaneous
  138. * interrupt handling.
  139. *
  140. * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  141. * The default (0) means to assign an interrupt to each core.
  142. */
  143. static unsigned int rss_cpus;
  144. module_param(rss_cpus, uint, 0444);
  145. MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
  146. static int phy_flash_cfg;
  147. module_param(phy_flash_cfg, int, 0644);
  148. MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
  149. static unsigned irq_adapt_low_thresh = 10000;
  150. module_param(irq_adapt_low_thresh, uint, 0644);
  151. MODULE_PARM_DESC(irq_adapt_low_thresh,
  152. "Threshold score for reducing IRQ moderation");
  153. static unsigned irq_adapt_high_thresh = 20000;
  154. module_param(irq_adapt_high_thresh, uint, 0644);
  155. MODULE_PARM_DESC(irq_adapt_high_thresh,
  156. "Threshold score for increasing IRQ moderation");
  157. static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  158. NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
  159. NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
  160. NETIF_MSG_TX_ERR | NETIF_MSG_HW);
  161. module_param(debug, uint, 0);
  162. MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
  163. /**************************************************************************
  164. *
  165. * Utility functions and prototypes
  166. *
  167. *************************************************************************/
  168. static void efx_start_interrupts(struct efx_nic *efx);
  169. static void efx_stop_interrupts(struct efx_nic *efx);
  170. static void efx_remove_channels(struct efx_nic *efx);
  171. static void efx_remove_port(struct efx_nic *efx);
  172. static void efx_init_napi(struct efx_nic *efx);
  173. static void efx_fini_napi(struct efx_nic *efx);
  174. static void efx_fini_napi_channel(struct efx_channel *channel);
  175. static void efx_fini_struct(struct efx_nic *efx);
  176. static void efx_start_all(struct efx_nic *efx);
  177. static void efx_stop_all(struct efx_nic *efx);
  178. #define EFX_ASSERT_RESET_SERIALISED(efx) \
  179. do { \
  180. if ((efx->state == STATE_RUNNING) || \
  181. (efx->state == STATE_DISABLED)) \
  182. ASSERT_RTNL(); \
  183. } while (0)
  184. /**************************************************************************
  185. *
  186. * Event queue processing
  187. *
  188. *************************************************************************/
  189. /* Process channel's event queue
  190. *
  191. * This function is responsible for processing the event queue of a
  192. * single channel. The caller must guarantee that this function will
  193. * never be concurrently called more than once on the same channel,
  194. * though different channels may be being processed concurrently.
  195. */
  196. static int efx_process_channel(struct efx_channel *channel, int budget)
  197. {
  198. int spent;
  199. if (unlikely(!channel->enabled))
  200. return 0;
  201. spent = efx_nic_process_eventq(channel, budget);
  202. if (spent && efx_channel_has_rx_queue(channel)) {
  203. struct efx_rx_queue *rx_queue =
  204. efx_channel_get_rx_queue(channel);
  205. /* Deliver last RX packet. */
  206. if (channel->rx_pkt) {
  207. __efx_rx_packet(channel, channel->rx_pkt);
  208. channel->rx_pkt = NULL;
  209. }
  210. if (rx_queue->enabled) {
  211. efx_rx_strategy(channel);
  212. efx_fast_push_rx_descriptors(rx_queue);
  213. }
  214. }
  215. return spent;
  216. }
  217. /* Mark channel as finished processing
  218. *
  219. * Note that since we will not receive further interrupts for this
  220. * channel before we finish processing and call the eventq_read_ack()
  221. * method, there is no need to use the interrupt hold-off timers.
  222. */
  223. static inline void efx_channel_processed(struct efx_channel *channel)
  224. {
  225. /* The interrupt handler for this channel may set work_pending
  226. * as soon as we acknowledge the events we've seen. Make sure
  227. * it's cleared before then. */
  228. channel->work_pending = false;
  229. smp_wmb();
  230. efx_nic_eventq_read_ack(channel);
  231. }
  232. /* NAPI poll handler
  233. *
  234. * NAPI guarantees serialisation of polls of the same device, which
  235. * provides the guarantee required by efx_process_channel().
  236. */
  237. static int efx_poll(struct napi_struct *napi, int budget)
  238. {
  239. struct efx_channel *channel =
  240. container_of(napi, struct efx_channel, napi_str);
  241. struct efx_nic *efx = channel->efx;
  242. int spent;
  243. netif_vdbg(efx, intr, efx->net_dev,
  244. "channel %d NAPI poll executing on CPU %d\n",
  245. channel->channel, raw_smp_processor_id());
  246. spent = efx_process_channel(channel, budget);
  247. if (spent < budget) {
  248. if (efx_channel_has_rx_queue(channel) &&
  249. efx->irq_rx_adaptive &&
  250. unlikely(++channel->irq_count == 1000)) {
  251. if (unlikely(channel->irq_mod_score <
  252. irq_adapt_low_thresh)) {
  253. if (channel->irq_moderation > 1) {
  254. channel->irq_moderation -= 1;
  255. efx->type->push_irq_moderation(channel);
  256. }
  257. } else if (unlikely(channel->irq_mod_score >
  258. irq_adapt_high_thresh)) {
  259. if (channel->irq_moderation <
  260. efx->irq_rx_moderation) {
  261. channel->irq_moderation += 1;
  262. efx->type->push_irq_moderation(channel);
  263. }
  264. }
  265. channel->irq_count = 0;
  266. channel->irq_mod_score = 0;
  267. }
  268. efx_filter_rfs_expire(channel);
  269. /* There is no race here; although napi_disable() will
  270. * only wait for napi_complete(), this isn't a problem
  271. * since efx_channel_processed() will have no effect if
  272. * interrupts have already been disabled.
  273. */
  274. napi_complete(napi);
  275. efx_channel_processed(channel);
  276. }
  277. return spent;
  278. }
  279. /* Process the eventq of the specified channel immediately on this CPU
  280. *
  281. * Disable hardware generated interrupts, wait for any existing
  282. * processing to finish, then directly poll (and ack ) the eventq.
  283. * Finally reenable NAPI and interrupts.
  284. *
  285. * This is for use only during a loopback self-test. It must not
  286. * deliver any packets up the stack as this can result in deadlock.
  287. */
  288. void efx_process_channel_now(struct efx_channel *channel)
  289. {
  290. struct efx_nic *efx = channel->efx;
  291. BUG_ON(channel->channel >= efx->n_channels);
  292. BUG_ON(!channel->enabled);
  293. BUG_ON(!efx->loopback_selftest);
  294. /* Disable interrupts and wait for ISRs to complete */
  295. efx_nic_disable_interrupts(efx);
  296. if (efx->legacy_irq) {
  297. synchronize_irq(efx->legacy_irq);
  298. efx->legacy_irq_enabled = false;
  299. }
  300. if (channel->irq)
  301. synchronize_irq(channel->irq);
  302. /* Wait for any NAPI processing to complete */
  303. napi_disable(&channel->napi_str);
  304. /* Poll the channel */
  305. efx_process_channel(channel, channel->eventq_mask + 1);
  306. /* Ack the eventq. This may cause an interrupt to be generated
  307. * when they are reenabled */
  308. efx_channel_processed(channel);
  309. napi_enable(&channel->napi_str);
  310. if (efx->legacy_irq)
  311. efx->legacy_irq_enabled = true;
  312. efx_nic_enable_interrupts(efx);
  313. }
  314. /* Create event queue
  315. * Event queue memory allocations are done only once. If the channel
  316. * is reset, the memory buffer will be reused; this guards against
  317. * errors during channel reset and also simplifies interrupt handling.
  318. */
  319. static int efx_probe_eventq(struct efx_channel *channel)
  320. {
  321. struct efx_nic *efx = channel->efx;
  322. unsigned long entries;
  323. netif_dbg(efx, probe, efx->net_dev,
  324. "chan %d create event queue\n", channel->channel);
  325. /* Build an event queue with room for one event per tx and rx buffer,
  326. * plus some extra for link state events and MCDI completions. */
  327. entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
  328. EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
  329. channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
  330. return efx_nic_probe_eventq(channel);
  331. }
  332. /* Prepare channel's event queue */
  333. static void efx_init_eventq(struct efx_channel *channel)
  334. {
  335. netif_dbg(channel->efx, drv, channel->efx->net_dev,
  336. "chan %d init event queue\n", channel->channel);
  337. channel->eventq_read_ptr = 0;
  338. efx_nic_init_eventq(channel);
  339. }
  340. /* Enable event queue processing and NAPI */
  341. static void efx_start_eventq(struct efx_channel *channel)
  342. {
  343. netif_dbg(channel->efx, ifup, channel->efx->net_dev,
  344. "chan %d start event queue\n", channel->channel);
  345. /* The interrupt handler for this channel may set work_pending
  346. * as soon as we enable it. Make sure it's cleared before
  347. * then. Similarly, make sure it sees the enabled flag set.
  348. */
  349. channel->work_pending = false;
  350. channel->enabled = true;
  351. smp_wmb();
  352. napi_enable(&channel->napi_str);
  353. efx_nic_eventq_read_ack(channel);
  354. }
  355. /* Disable event queue processing and NAPI */
  356. static void efx_stop_eventq(struct efx_channel *channel)
  357. {
  358. if (!channel->enabled)
  359. return;
  360. napi_disable(&channel->napi_str);
  361. channel->enabled = false;
  362. }
  363. static void efx_fini_eventq(struct efx_channel *channel)
  364. {
  365. netif_dbg(channel->efx, drv, channel->efx->net_dev,
  366. "chan %d fini event queue\n", channel->channel);
  367. efx_nic_fini_eventq(channel);
  368. }
  369. static void efx_remove_eventq(struct efx_channel *channel)
  370. {
  371. netif_dbg(channel->efx, drv, channel->efx->net_dev,
  372. "chan %d remove event queue\n", channel->channel);
  373. efx_nic_remove_eventq(channel);
  374. }
  375. /**************************************************************************
  376. *
  377. * Channel handling
  378. *
  379. *************************************************************************/
  380. /* Allocate and initialise a channel structure, optionally copying
  381. * parameters (but not resources) from an old channel structure. */
  382. static struct efx_channel *
  383. efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
  384. {
  385. struct efx_channel *channel;
  386. struct efx_rx_queue *rx_queue;
  387. struct efx_tx_queue *tx_queue;
  388. int j;
  389. if (old_channel) {
  390. channel = kmalloc(sizeof(*channel), GFP_KERNEL);
  391. if (!channel)
  392. return NULL;
  393. *channel = *old_channel;
  394. channel->napi_dev = NULL;
  395. memset(&channel->eventq, 0, sizeof(channel->eventq));
  396. rx_queue = &channel->rx_queue;
  397. rx_queue->buffer = NULL;
  398. memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
  399. for (j = 0; j < EFX_TXQ_TYPES; j++) {
  400. tx_queue = &channel->tx_queue[j];
  401. if (tx_queue->channel)
  402. tx_queue->channel = channel;
  403. tx_queue->buffer = NULL;
  404. memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
  405. }
  406. } else {
  407. channel = kzalloc(sizeof(*channel), GFP_KERNEL);
  408. if (!channel)
  409. return NULL;
  410. channel->efx = efx;
  411. channel->channel = i;
  412. for (j = 0; j < EFX_TXQ_TYPES; j++) {
  413. tx_queue = &channel->tx_queue[j];
  414. tx_queue->efx = efx;
  415. tx_queue->queue = i * EFX_TXQ_TYPES + j;
  416. tx_queue->channel = channel;
  417. }
  418. }
  419. rx_queue = &channel->rx_queue;
  420. rx_queue->efx = efx;
  421. setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
  422. (unsigned long)rx_queue);
  423. return channel;
  424. }
  425. static int efx_probe_channel(struct efx_channel *channel)
  426. {
  427. struct efx_tx_queue *tx_queue;
  428. struct efx_rx_queue *rx_queue;
  429. int rc;
  430. netif_dbg(channel->efx, probe, channel->efx->net_dev,
  431. "creating channel %d\n", channel->channel);
  432. rc = efx_probe_eventq(channel);
  433. if (rc)
  434. goto fail1;
  435. efx_for_each_channel_tx_queue(tx_queue, channel) {
  436. rc = efx_probe_tx_queue(tx_queue);
  437. if (rc)
  438. goto fail2;
  439. }
  440. efx_for_each_channel_rx_queue(rx_queue, channel) {
  441. rc = efx_probe_rx_queue(rx_queue);
  442. if (rc)
  443. goto fail3;
  444. }
  445. channel->n_rx_frm_trunc = 0;
  446. return 0;
  447. fail3:
  448. efx_for_each_channel_rx_queue(rx_queue, channel)
  449. efx_remove_rx_queue(rx_queue);
  450. fail2:
  451. efx_for_each_channel_tx_queue(tx_queue, channel)
  452. efx_remove_tx_queue(tx_queue);
  453. fail1:
  454. return rc;
  455. }
  456. static void efx_set_channel_names(struct efx_nic *efx)
  457. {
  458. struct efx_channel *channel;
  459. const char *type = "";
  460. int number;
  461. efx_for_each_channel(channel, efx) {
  462. number = channel->channel;
  463. if (efx->n_channels > efx->n_rx_channels) {
  464. if (channel->channel < efx->n_rx_channels) {
  465. type = "-rx";
  466. } else {
  467. type = "-tx";
  468. number -= efx->n_rx_channels;
  469. }
  470. }
  471. snprintf(efx->channel_name[channel->channel],
  472. sizeof(efx->channel_name[0]),
  473. "%s%s-%d", efx->name, type, number);
  474. }
  475. }
  476. static int efx_probe_channels(struct efx_nic *efx)
  477. {
  478. struct efx_channel *channel;
  479. int rc;
  480. /* Restart special buffer allocation */
  481. efx->next_buffer_table = 0;
  482. efx_for_each_channel(channel, efx) {
  483. rc = efx_probe_channel(channel);
  484. if (rc) {
  485. netif_err(efx, probe, efx->net_dev,
  486. "failed to create channel %d\n",
  487. channel->channel);
  488. goto fail;
  489. }
  490. }
  491. efx_set_channel_names(efx);
  492. return 0;
  493. fail:
  494. efx_remove_channels(efx);
  495. return rc;
  496. }
  497. /* Channels are shutdown and reinitialised whilst the NIC is running
  498. * to propagate configuration changes (mtu, checksum offload), or
  499. * to clear hardware error conditions
  500. */
  501. static void efx_start_datapath(struct efx_nic *efx)
  502. {
  503. struct efx_tx_queue *tx_queue;
  504. struct efx_rx_queue *rx_queue;
  505. struct efx_channel *channel;
  506. /* Calculate the rx buffer allocation parameters required to
  507. * support the current MTU, including padding for header
  508. * alignment and overruns.
  509. */
  510. efx->rx_buffer_len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +
  511. EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
  512. efx->type->rx_buffer_hash_size +
  513. efx->type->rx_buffer_padding);
  514. efx->rx_buffer_order = get_order(efx->rx_buffer_len +
  515. sizeof(struct efx_rx_page_state));
  516. /* Initialise the channels */
  517. efx_for_each_channel(channel, efx) {
  518. efx_for_each_channel_tx_queue(tx_queue, channel)
  519. efx_init_tx_queue(tx_queue);
  520. /* The rx buffer allocation strategy is MTU dependent */
  521. efx_rx_strategy(channel);
  522. efx_for_each_channel_rx_queue(rx_queue, channel) {
  523. efx_init_rx_queue(rx_queue);
  524. efx_nic_generate_fill_event(rx_queue);
  525. }
  526. WARN_ON(channel->rx_pkt != NULL);
  527. efx_rx_strategy(channel);
  528. }
  529. if (netif_device_present(efx->net_dev))
  530. netif_tx_wake_all_queues(efx->net_dev);
  531. }
  532. static void efx_stop_datapath(struct efx_nic *efx)
  533. {
  534. struct efx_channel *channel;
  535. struct efx_tx_queue *tx_queue;
  536. struct efx_rx_queue *rx_queue;
  537. int rc;
  538. EFX_ASSERT_RESET_SERIALISED(efx);
  539. BUG_ON(efx->port_enabled);
  540. rc = efx_nic_flush_queues(efx);
  541. if (rc && EFX_WORKAROUND_7803(efx)) {
  542. /* Schedule a reset to recover from the flush failure. The
  543. * descriptor caches reference memory we're about to free,
  544. * but falcon_reconfigure_mac_wrapper() won't reconnect
  545. * the MACs because of the pending reset. */
  546. netif_err(efx, drv, efx->net_dev,
  547. "Resetting to recover from flush failure\n");
  548. efx_schedule_reset(efx, RESET_TYPE_ALL);
  549. } else if (rc) {
  550. netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
  551. } else {
  552. netif_dbg(efx, drv, efx->net_dev,
  553. "successfully flushed all queues\n");
  554. }
  555. efx_for_each_channel(channel, efx) {
  556. /* RX packet processing is pipelined, so wait for the
  557. * NAPI handler to complete. At least event queue 0
  558. * might be kept active by non-data events, so don't
  559. * use napi_synchronize() but actually disable NAPI
  560. * temporarily.
  561. */
  562. if (efx_channel_has_rx_queue(channel)) {
  563. efx_stop_eventq(channel);
  564. efx_start_eventq(channel);
  565. }
  566. efx_for_each_channel_rx_queue(rx_queue, channel)
  567. efx_fini_rx_queue(rx_queue);
  568. efx_for_each_possible_channel_tx_queue(tx_queue, channel)
  569. efx_fini_tx_queue(tx_queue);
  570. }
  571. }
  572. static void efx_remove_channel(struct efx_channel *channel)
  573. {
  574. struct efx_tx_queue *tx_queue;
  575. struct efx_rx_queue *rx_queue;
  576. netif_dbg(channel->efx, drv, channel->efx->net_dev,
  577. "destroy chan %d\n", channel->channel);
  578. efx_for_each_channel_rx_queue(rx_queue, channel)
  579. efx_remove_rx_queue(rx_queue);
  580. efx_for_each_possible_channel_tx_queue(tx_queue, channel)
  581. efx_remove_tx_queue(tx_queue);
  582. efx_remove_eventq(channel);
  583. }
  584. static void efx_remove_channels(struct efx_nic *efx)
  585. {
  586. struct efx_channel *channel;
  587. efx_for_each_channel(channel, efx)
  588. efx_remove_channel(channel);
  589. }
  590. int
  591. efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
  592. {
  593. struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
  594. u32 old_rxq_entries, old_txq_entries;
  595. unsigned i;
  596. int rc;
  597. efx_stop_all(efx);
  598. efx_stop_interrupts(efx);
  599. /* Clone channels */
  600. memset(other_channel, 0, sizeof(other_channel));
  601. for (i = 0; i < efx->n_channels; i++) {
  602. channel = efx_alloc_channel(efx, i, efx->channel[i]);
  603. if (!channel) {
  604. rc = -ENOMEM;
  605. goto out;
  606. }
  607. other_channel[i] = channel;
  608. }
  609. /* Swap entry counts and channel pointers */
  610. old_rxq_entries = efx->rxq_entries;
  611. old_txq_entries = efx->txq_entries;
  612. efx->rxq_entries = rxq_entries;
  613. efx->txq_entries = txq_entries;
  614. for (i = 0; i < efx->n_channels; i++) {
  615. channel = efx->channel[i];
  616. efx->channel[i] = other_channel[i];
  617. other_channel[i] = channel;
  618. }
  619. rc = efx_probe_channels(efx);
  620. if (rc)
  621. goto rollback;
  622. efx_init_napi(efx);
  623. /* Destroy old channels */
  624. for (i = 0; i < efx->n_channels; i++) {
  625. efx_fini_napi_channel(other_channel[i]);
  626. efx_remove_channel(other_channel[i]);
  627. }
  628. out:
  629. /* Free unused channel structures */
  630. for (i = 0; i < efx->n_channels; i++)
  631. kfree(other_channel[i]);
  632. efx_start_interrupts(efx);
  633. efx_start_all(efx);
  634. return rc;
  635. rollback:
  636. /* Swap back */
  637. efx->rxq_entries = old_rxq_entries;
  638. efx->txq_entries = old_txq_entries;
  639. for (i = 0; i < efx->n_channels; i++) {
  640. channel = efx->channel[i];
  641. efx->channel[i] = other_channel[i];
  642. other_channel[i] = channel;
  643. }
  644. goto out;
  645. }
  646. void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
  647. {
  648. mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
  649. }
  650. /**************************************************************************
  651. *
  652. * Port handling
  653. *
  654. **************************************************************************/
  655. /* This ensures that the kernel is kept informed (via
  656. * netif_carrier_on/off) of the link status, and also maintains the
  657. * link status's stop on the port's TX queue.
  658. */
  659. void efx_link_status_changed(struct efx_nic *efx)
  660. {
  661. struct efx_link_state *link_state = &efx->link_state;
  662. /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
  663. * that no events are triggered between unregister_netdev() and the
  664. * driver unloading. A more general condition is that NETDEV_CHANGE
  665. * can only be generated between NETDEV_UP and NETDEV_DOWN */
  666. if (!netif_running(efx->net_dev))
  667. return;
  668. if (link_state->up != netif_carrier_ok(efx->net_dev)) {
  669. efx->n_link_state_changes++;
  670. if (link_state->up)
  671. netif_carrier_on(efx->net_dev);
  672. else
  673. netif_carrier_off(efx->net_dev);
  674. }
  675. /* Status message for kernel log */
  676. if (link_state->up)
  677. netif_info(efx, link, efx->net_dev,
  678. "link up at %uMbps %s-duplex (MTU %d)%s\n",
  679. link_state->speed, link_state->fd ? "full" : "half",
  680. efx->net_dev->mtu,
  681. (efx->promiscuous ? " [PROMISC]" : ""));
  682. else
  683. netif_info(efx, link, efx->net_dev, "link down\n");
  684. }
  685. void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
  686. {
  687. efx->link_advertising = advertising;
  688. if (advertising) {
  689. if (advertising & ADVERTISED_Pause)
  690. efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
  691. else
  692. efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
  693. if (advertising & ADVERTISED_Asym_Pause)
  694. efx->wanted_fc ^= EFX_FC_TX;
  695. }
  696. }
  697. void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
  698. {
  699. efx->wanted_fc = wanted_fc;
  700. if (efx->link_advertising) {
  701. if (wanted_fc & EFX_FC_RX)
  702. efx->link_advertising |= (ADVERTISED_Pause |
  703. ADVERTISED_Asym_Pause);
  704. else
  705. efx->link_advertising &= ~(ADVERTISED_Pause |
  706. ADVERTISED_Asym_Pause);
  707. if (wanted_fc & EFX_FC_TX)
  708. efx->link_advertising ^= ADVERTISED_Asym_Pause;
  709. }
  710. }
  711. static void efx_fini_port(struct efx_nic *efx);
  712. /* Push loopback/power/transmit disable settings to the PHY, and reconfigure
  713. * the MAC appropriately. All other PHY configuration changes are pushed
  714. * through phy_op->set_settings(), and pushed asynchronously to the MAC
  715. * through efx_monitor().
  716. *
  717. * Callers must hold the mac_lock
  718. */
  719. int __efx_reconfigure_port(struct efx_nic *efx)
  720. {
  721. enum efx_phy_mode phy_mode;
  722. int rc;
  723. WARN_ON(!mutex_is_locked(&efx->mac_lock));
  724. /* Serialise the promiscuous flag with efx_set_rx_mode. */
  725. netif_addr_lock_bh(efx->net_dev);
  726. netif_addr_unlock_bh(efx->net_dev);
  727. /* Disable PHY transmit in mac level loopbacks */
  728. phy_mode = efx->phy_mode;
  729. if (LOOPBACK_INTERNAL(efx))
  730. efx->phy_mode |= PHY_MODE_TX_DISABLED;
  731. else
  732. efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
  733. rc = efx->type->reconfigure_port(efx);
  734. if (rc)
  735. efx->phy_mode = phy_mode;
  736. return rc;
  737. }
  738. /* Reinitialise the MAC to pick up new PHY settings, even if the port is
  739. * disabled. */
  740. int efx_reconfigure_port(struct efx_nic *efx)
  741. {
  742. int rc;
  743. EFX_ASSERT_RESET_SERIALISED(efx);
  744. mutex_lock(&efx->mac_lock);
  745. rc = __efx_reconfigure_port(efx);
  746. mutex_unlock(&efx->mac_lock);
  747. return rc;
  748. }
  749. /* Asynchronous work item for changing MAC promiscuity and multicast
  750. * hash. Avoid a drain/rx_ingress enable by reconfiguring the current
  751. * MAC directly. */
  752. static void efx_mac_work(struct work_struct *data)
  753. {
  754. struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
  755. mutex_lock(&efx->mac_lock);
  756. if (efx->port_enabled)
  757. efx->type->reconfigure_mac(efx);
  758. mutex_unlock(&efx->mac_lock);
  759. }
  760. static int efx_probe_port(struct efx_nic *efx)
  761. {
  762. int rc;
  763. netif_dbg(efx, probe, efx->net_dev, "create port\n");
  764. if (phy_flash_cfg)
  765. efx->phy_mode = PHY_MODE_SPECIAL;
  766. /* Connect up MAC/PHY operations table */
  767. rc = efx->type->probe_port(efx);
  768. if (rc)
  769. return rc;
  770. /* Initialise MAC address to permanent address */
  771. memcpy(efx->net_dev->dev_addr, efx->net_dev->perm_addr, ETH_ALEN);
  772. return 0;
  773. }
  774. static int efx_init_port(struct efx_nic *efx)
  775. {
  776. int rc;
  777. netif_dbg(efx, drv, efx->net_dev, "init port\n");
  778. mutex_lock(&efx->mac_lock);
  779. rc = efx->phy_op->init(efx);
  780. if (rc)
  781. goto fail1;
  782. efx->port_initialized = true;
  783. /* Reconfigure the MAC before creating dma queues (required for
  784. * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
  785. efx->type->reconfigure_mac(efx);
  786. /* Ensure the PHY advertises the correct flow control settings */
  787. rc = efx->phy_op->reconfigure(efx);
  788. if (rc)
  789. goto fail2;
  790. mutex_unlock(&efx->mac_lock);
  791. return 0;
  792. fail2:
  793. efx->phy_op->fini(efx);
  794. fail1:
  795. mutex_unlock(&efx->mac_lock);
  796. return rc;
  797. }
  798. static void efx_start_port(struct efx_nic *efx)
  799. {
  800. netif_dbg(efx, ifup, efx->net_dev, "start port\n");
  801. BUG_ON(efx->port_enabled);
  802. mutex_lock(&efx->mac_lock);
  803. efx->port_enabled = true;
  804. /* efx_mac_work() might have been scheduled after efx_stop_port(),
  805. * and then cancelled by efx_flush_all() */
  806. efx->type->reconfigure_mac(efx);
  807. mutex_unlock(&efx->mac_lock);
  808. }
  809. /* Prevent efx_mac_work() and efx_monitor() from working */
  810. static void efx_stop_port(struct efx_nic *efx)
  811. {
  812. netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
  813. mutex_lock(&efx->mac_lock);
  814. efx->port_enabled = false;
  815. mutex_unlock(&efx->mac_lock);
  816. /* Serialise against efx_set_multicast_list() */
  817. netif_addr_lock_bh(efx->net_dev);
  818. netif_addr_unlock_bh(efx->net_dev);
  819. }
  820. static void efx_fini_port(struct efx_nic *efx)
  821. {
  822. netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
  823. if (!efx->port_initialized)
  824. return;
  825. efx->phy_op->fini(efx);
  826. efx->port_initialized = false;
  827. efx->link_state.up = false;
  828. efx_link_status_changed(efx);
  829. }
  830. static void efx_remove_port(struct efx_nic *efx)
  831. {
  832. netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
  833. efx->type->remove_port(efx);
  834. }
  835. /**************************************************************************
  836. *
  837. * NIC handling
  838. *
  839. **************************************************************************/
  840. /* This configures the PCI device to enable I/O and DMA. */
  841. static int efx_init_io(struct efx_nic *efx)
  842. {
  843. struct pci_dev *pci_dev = efx->pci_dev;
  844. dma_addr_t dma_mask = efx->type->max_dma_mask;
  845. int rc;
  846. netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
  847. rc = pci_enable_device(pci_dev);
  848. if (rc) {
  849. netif_err(efx, probe, efx->net_dev,
  850. "failed to enable PCI device\n");
  851. goto fail1;
  852. }
  853. pci_set_master(pci_dev);
  854. /* Set the PCI DMA mask. Try all possibilities from our
  855. * genuine mask down to 32 bits, because some architectures
  856. * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
  857. * masks event though they reject 46 bit masks.
  858. */
  859. while (dma_mask > 0x7fffffffUL) {
  860. if (pci_dma_supported(pci_dev, dma_mask)) {
  861. rc = pci_set_dma_mask(pci_dev, dma_mask);
  862. if (rc == 0)
  863. break;
  864. }
  865. dma_mask >>= 1;
  866. }
  867. if (rc) {
  868. netif_err(efx, probe, efx->net_dev,
  869. "could not find a suitable DMA mask\n");
  870. goto fail2;
  871. }
  872. netif_dbg(efx, probe, efx->net_dev,
  873. "using DMA mask %llx\n", (unsigned long long) dma_mask);
  874. rc = pci_set_consistent_dma_mask(pci_dev, dma_mask);
  875. if (rc) {
  876. /* pci_set_consistent_dma_mask() is not *allowed* to
  877. * fail with a mask that pci_set_dma_mask() accepted,
  878. * but just in case...
  879. */
  880. netif_err(efx, probe, efx->net_dev,
  881. "failed to set consistent DMA mask\n");
  882. goto fail2;
  883. }
  884. efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR);
  885. rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc");
  886. if (rc) {
  887. netif_err(efx, probe, efx->net_dev,
  888. "request for memory BAR failed\n");
  889. rc = -EIO;
  890. goto fail3;
  891. }
  892. efx->membase = ioremap_nocache(efx->membase_phys,
  893. efx->type->mem_map_size);
  894. if (!efx->membase) {
  895. netif_err(efx, probe, efx->net_dev,
  896. "could not map memory BAR at %llx+%x\n",
  897. (unsigned long long)efx->membase_phys,
  898. efx->type->mem_map_size);
  899. rc = -ENOMEM;
  900. goto fail4;
  901. }
  902. netif_dbg(efx, probe, efx->net_dev,
  903. "memory BAR at %llx+%x (virtual %p)\n",
  904. (unsigned long long)efx->membase_phys,
  905. efx->type->mem_map_size, efx->membase);
  906. return 0;
  907. fail4:
  908. pci_release_region(efx->pci_dev, EFX_MEM_BAR);
  909. fail3:
  910. efx->membase_phys = 0;
  911. fail2:
  912. pci_disable_device(efx->pci_dev);
  913. fail1:
  914. return rc;
  915. }
  916. static void efx_fini_io(struct efx_nic *efx)
  917. {
  918. netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
  919. if (efx->membase) {
  920. iounmap(efx->membase);
  921. efx->membase = NULL;
  922. }
  923. if (efx->membase_phys) {
  924. pci_release_region(efx->pci_dev, EFX_MEM_BAR);
  925. efx->membase_phys = 0;
  926. }
  927. pci_disable_device(efx->pci_dev);
  928. }
  929. static int efx_wanted_parallelism(void)
  930. {
  931. cpumask_var_t thread_mask;
  932. int count;
  933. int cpu;
  934. if (rss_cpus)
  935. return rss_cpus;
  936. if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
  937. printk(KERN_WARNING
  938. "sfc: RSS disabled due to allocation failure\n");
  939. return 1;
  940. }
  941. count = 0;
  942. for_each_online_cpu(cpu) {
  943. if (!cpumask_test_cpu(cpu, thread_mask)) {
  944. ++count;
  945. cpumask_or(thread_mask, thread_mask,
  946. topology_thread_cpumask(cpu));
  947. }
  948. }
  949. free_cpumask_var(thread_mask);
  950. return count;
  951. }
  952. static int
  953. efx_init_rx_cpu_rmap(struct efx_nic *efx, struct msix_entry *xentries)
  954. {
  955. #ifdef CONFIG_RFS_ACCEL
  956. int i, rc;
  957. efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels);
  958. if (!efx->net_dev->rx_cpu_rmap)
  959. return -ENOMEM;
  960. for (i = 0; i < efx->n_rx_channels; i++) {
  961. rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
  962. xentries[i].vector);
  963. if (rc) {
  964. free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
  965. efx->net_dev->rx_cpu_rmap = NULL;
  966. return rc;
  967. }
  968. }
  969. #endif
  970. return 0;
  971. }
  972. /* Probe the number and type of interrupts we are able to obtain, and
  973. * the resulting numbers of channels and RX queues.
  974. */
  975. static int efx_probe_interrupts(struct efx_nic *efx)
  976. {
  977. int max_channels =
  978. min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS);
  979. int rc, i;
  980. if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
  981. struct msix_entry xentries[EFX_MAX_CHANNELS];
  982. int n_channels;
  983. n_channels = efx_wanted_parallelism();
  984. if (separate_tx_channels)
  985. n_channels *= 2;
  986. n_channels = min(n_channels, max_channels);
  987. for (i = 0; i < n_channels; i++)
  988. xentries[i].entry = i;
  989. rc = pci_enable_msix(efx->pci_dev, xentries, n_channels);
  990. if (rc > 0) {
  991. netif_err(efx, drv, efx->net_dev,
  992. "WARNING: Insufficient MSI-X vectors"
  993. " available (%d < %d).\n", rc, n_channels);
  994. netif_err(efx, drv, efx->net_dev,
  995. "WARNING: Performance may be reduced.\n");
  996. EFX_BUG_ON_PARANOID(rc >= n_channels);
  997. n_channels = rc;
  998. rc = pci_enable_msix(efx->pci_dev, xentries,
  999. n_channels);
  1000. }
  1001. if (rc == 0) {
  1002. efx->n_channels = n_channels;
  1003. if (separate_tx_channels) {
  1004. efx->n_tx_channels =
  1005. max(efx->n_channels / 2, 1U);
  1006. efx->n_rx_channels =
  1007. max(efx->n_channels -
  1008. efx->n_tx_channels, 1U);
  1009. } else {
  1010. efx->n_tx_channels = efx->n_channels;
  1011. efx->n_rx_channels = efx->n_channels;
  1012. }
  1013. rc = efx_init_rx_cpu_rmap(efx, xentries);
  1014. if (rc) {
  1015. pci_disable_msix(efx->pci_dev);
  1016. return rc;
  1017. }
  1018. for (i = 0; i < n_channels; i++)
  1019. efx_get_channel(efx, i)->irq =
  1020. xentries[i].vector;
  1021. } else {
  1022. /* Fall back to single channel MSI */
  1023. efx->interrupt_mode = EFX_INT_MODE_MSI;
  1024. netif_err(efx, drv, efx->net_dev,
  1025. "could not enable MSI-X\n");
  1026. }
  1027. }
  1028. /* Try single interrupt MSI */
  1029. if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
  1030. efx->n_channels = 1;
  1031. efx->n_rx_channels = 1;
  1032. efx->n_tx_channels = 1;
  1033. rc = pci_enable_msi(efx->pci_dev);
  1034. if (rc == 0) {
  1035. efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
  1036. } else {
  1037. netif_err(efx, drv, efx->net_dev,
  1038. "could not enable MSI\n");
  1039. efx->interrupt_mode = EFX_INT_MODE_LEGACY;
  1040. }
  1041. }
  1042. /* Assume legacy interrupts */
  1043. if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
  1044. efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);
  1045. efx->n_rx_channels = 1;
  1046. efx->n_tx_channels = 1;
  1047. efx->legacy_irq = efx->pci_dev->irq;
  1048. }
  1049. return 0;
  1050. }
  1051. /* Enable interrupts, then probe and start the event queues */
  1052. static void efx_start_interrupts(struct efx_nic *efx)
  1053. {
  1054. struct efx_channel *channel;
  1055. if (efx->legacy_irq)
  1056. efx->legacy_irq_enabled = true;
  1057. efx_nic_enable_interrupts(efx);
  1058. efx_for_each_channel(channel, efx) {
  1059. efx_init_eventq(channel);
  1060. efx_start_eventq(channel);
  1061. }
  1062. efx_mcdi_mode_event(efx);
  1063. }
  1064. static void efx_stop_interrupts(struct efx_nic *efx)
  1065. {
  1066. struct efx_channel *channel;
  1067. efx_mcdi_mode_poll(efx);
  1068. efx_nic_disable_interrupts(efx);
  1069. if (efx->legacy_irq) {
  1070. synchronize_irq(efx->legacy_irq);
  1071. efx->legacy_irq_enabled = false;
  1072. }
  1073. efx_for_each_channel(channel, efx) {
  1074. if (channel->irq)
  1075. synchronize_irq(channel->irq);
  1076. efx_stop_eventq(channel);
  1077. efx_fini_eventq(channel);
  1078. }
  1079. }
  1080. static void efx_remove_interrupts(struct efx_nic *efx)
  1081. {
  1082. struct efx_channel *channel;
  1083. /* Remove MSI/MSI-X interrupts */
  1084. efx_for_each_channel(channel, efx)
  1085. channel->irq = 0;
  1086. pci_disable_msi(efx->pci_dev);
  1087. pci_disable_msix(efx->pci_dev);
  1088. /* Remove legacy interrupt */
  1089. efx->legacy_irq = 0;
  1090. }
  1091. static void efx_set_channels(struct efx_nic *efx)
  1092. {
  1093. struct efx_channel *channel;
  1094. struct efx_tx_queue *tx_queue;
  1095. efx->tx_channel_offset =
  1096. separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0;
  1097. /* We need to adjust the TX queue numbers if we have separate
  1098. * RX-only and TX-only channels.
  1099. */
  1100. efx_for_each_channel(channel, efx) {
  1101. efx_for_each_channel_tx_queue(tx_queue, channel)
  1102. tx_queue->queue -= (efx->tx_channel_offset *
  1103. EFX_TXQ_TYPES);
  1104. }
  1105. }
  1106. static int efx_probe_nic(struct efx_nic *efx)
  1107. {
  1108. size_t i;
  1109. int rc;
  1110. netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
  1111. /* Carry out hardware-type specific initialisation */
  1112. rc = efx->type->probe(efx);
  1113. if (rc)
  1114. return rc;
  1115. /* Determine the number of channels and queues by trying to hook
  1116. * in MSI-X interrupts. */
  1117. rc = efx_probe_interrupts(efx);
  1118. if (rc)
  1119. goto fail;
  1120. if (efx->n_channels > 1)
  1121. get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key));
  1122. for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)
  1123. efx->rx_indir_table[i] =
  1124. ethtool_rxfh_indir_default(i, efx->n_rx_channels);
  1125. efx_set_channels(efx);
  1126. netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
  1127. netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
  1128. /* Initialise the interrupt moderation settings */
  1129. efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
  1130. true);
  1131. return 0;
  1132. fail:
  1133. efx->type->remove(efx);
  1134. return rc;
  1135. }
  1136. static void efx_remove_nic(struct efx_nic *efx)
  1137. {
  1138. netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
  1139. efx_remove_interrupts(efx);
  1140. efx->type->remove(efx);
  1141. }
  1142. /**************************************************************************
  1143. *
  1144. * NIC startup/shutdown
  1145. *
  1146. *************************************************************************/
  1147. static int efx_probe_all(struct efx_nic *efx)
  1148. {
  1149. int rc;
  1150. rc = efx_probe_nic(efx);
  1151. if (rc) {
  1152. netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
  1153. goto fail1;
  1154. }
  1155. rc = efx_probe_port(efx);
  1156. if (rc) {
  1157. netif_err(efx, probe, efx->net_dev, "failed to create port\n");
  1158. goto fail2;
  1159. }
  1160. efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
  1161. rc = efx_probe_channels(efx);
  1162. if (rc)
  1163. goto fail3;
  1164. rc = efx_probe_filters(efx);
  1165. if (rc) {
  1166. netif_err(efx, probe, efx->net_dev,
  1167. "failed to create filter tables\n");
  1168. goto fail4;
  1169. }
  1170. return 0;
  1171. fail4:
  1172. efx_remove_channels(efx);
  1173. fail3:
  1174. efx_remove_port(efx);
  1175. fail2:
  1176. efx_remove_nic(efx);
  1177. fail1:
  1178. return rc;
  1179. }
  1180. /* Called after previous invocation(s) of efx_stop_all, restarts the port,
  1181. * kernel transmit queues and NAPI processing, and ensures that the port is
  1182. * scheduled to be reconfigured. This function is safe to call multiple
  1183. * times when the NIC is in any state.
  1184. */
  1185. static void efx_start_all(struct efx_nic *efx)
  1186. {
  1187. EFX_ASSERT_RESET_SERIALISED(efx);
  1188. /* Check that it is appropriate to restart the interface. All
  1189. * of these flags are safe to read under just the rtnl lock */
  1190. if (efx->port_enabled)
  1191. return;
  1192. if ((efx->state != STATE_RUNNING) && (efx->state != STATE_INIT))
  1193. return;
  1194. if (!netif_running(efx->net_dev))
  1195. return;
  1196. efx_start_port(efx);
  1197. efx_start_datapath(efx);
  1198. /* Start the hardware monitor if there is one. Otherwise (we're link
  1199. * event driven), we have to poll the PHY because after an event queue
  1200. * flush, we could have a missed a link state change */
  1201. if (efx->type->monitor != NULL) {
  1202. queue_delayed_work(efx->workqueue, &efx->monitor_work,
  1203. efx_monitor_interval);
  1204. } else {
  1205. mutex_lock(&efx->mac_lock);
  1206. if (efx->phy_op->poll(efx))
  1207. efx_link_status_changed(efx);
  1208. mutex_unlock(&efx->mac_lock);
  1209. }
  1210. efx->type->start_stats(efx);
  1211. }
  1212. /* Flush all delayed work. Should only be called when no more delayed work
  1213. * will be scheduled. This doesn't flush pending online resets (efx_reset),
  1214. * since we're holding the rtnl_lock at this point. */
  1215. static void efx_flush_all(struct efx_nic *efx)
  1216. {
  1217. /* Make sure the hardware monitor is stopped */
  1218. cancel_delayed_work_sync(&efx->monitor_work);
  1219. /* Stop scheduled port reconfigurations */
  1220. cancel_work_sync(&efx->mac_work);
  1221. }
  1222. /* Quiesce hardware and software without bringing the link down.
  1223. * Safe to call multiple times, when the nic and interface is in any
  1224. * state. The caller is guaranteed to subsequently be in a position
  1225. * to modify any hardware and software state they see fit without
  1226. * taking locks. */
  1227. static void efx_stop_all(struct efx_nic *efx)
  1228. {
  1229. EFX_ASSERT_RESET_SERIALISED(efx);
  1230. /* port_enabled can be read safely under the rtnl lock */
  1231. if (!efx->port_enabled)
  1232. return;
  1233. efx->type->stop_stats(efx);
  1234. efx_stop_port(efx);
  1235. /* Flush efx_mac_work(), refill_workqueue, monitor_work */
  1236. efx_flush_all(efx);
  1237. /* Stop the kernel transmit interface late, so the watchdog
  1238. * timer isn't ticking over the flush */
  1239. netif_tx_disable(efx->net_dev);
  1240. efx_stop_datapath(efx);
  1241. }
  1242. static void efx_remove_all(struct efx_nic *efx)
  1243. {
  1244. efx_remove_filters(efx);
  1245. efx_remove_channels(efx);
  1246. efx_remove_port(efx);
  1247. efx_remove_nic(efx);
  1248. }
  1249. /**************************************************************************
  1250. *
  1251. * Interrupt moderation
  1252. *
  1253. **************************************************************************/
  1254. static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns)
  1255. {
  1256. if (usecs == 0)
  1257. return 0;
  1258. if (usecs * 1000 < quantum_ns)
  1259. return 1; /* never round down to 0 */
  1260. return usecs * 1000 / quantum_ns;
  1261. }
  1262. /* Set interrupt moderation parameters */
  1263. int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
  1264. unsigned int rx_usecs, bool rx_adaptive,
  1265. bool rx_may_override_tx)
  1266. {
  1267. struct efx_channel *channel;
  1268. unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max *
  1269. efx->timer_quantum_ns,
  1270. 1000);
  1271. unsigned int tx_ticks;
  1272. unsigned int rx_ticks;
  1273. EFX_ASSERT_RESET_SERIALISED(efx);
  1274. if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max)
  1275. return -EINVAL;
  1276. tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns);
  1277. rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns);
  1278. if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 &&
  1279. !rx_may_override_tx) {
  1280. netif_err(efx, drv, efx->net_dev, "Channels are shared. "
  1281. "RX and TX IRQ moderation must be equal\n");
  1282. return -EINVAL;
  1283. }
  1284. efx->irq_rx_adaptive = rx_adaptive;
  1285. efx->irq_rx_moderation = rx_ticks;
  1286. efx_for_each_channel(channel, efx) {
  1287. if (efx_channel_has_rx_queue(channel))
  1288. channel->irq_moderation = rx_ticks;
  1289. else if (efx_channel_has_tx_queues(channel))
  1290. channel->irq_moderation = tx_ticks;
  1291. }
  1292. return 0;
  1293. }
  1294. void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
  1295. unsigned int *rx_usecs, bool *rx_adaptive)
  1296. {
  1297. /* We must round up when converting ticks to microseconds
  1298. * because we round down when converting the other way.
  1299. */
  1300. *rx_adaptive = efx->irq_rx_adaptive;
  1301. *rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation *
  1302. efx->timer_quantum_ns,
  1303. 1000);
  1304. /* If channels are shared between RX and TX, so is IRQ
  1305. * moderation. Otherwise, IRQ moderation is the same for all
  1306. * TX channels and is not adaptive.
  1307. */
  1308. if (efx->tx_channel_offset == 0)
  1309. *tx_usecs = *rx_usecs;
  1310. else
  1311. *tx_usecs = DIV_ROUND_UP(
  1312. efx->channel[efx->tx_channel_offset]->irq_moderation *
  1313. efx->timer_quantum_ns,
  1314. 1000);
  1315. }
  1316. /**************************************************************************
  1317. *
  1318. * Hardware monitor
  1319. *
  1320. **************************************************************************/
  1321. /* Run periodically off the general workqueue */
  1322. static void efx_monitor(struct work_struct *data)
  1323. {
  1324. struct efx_nic *efx = container_of(data, struct efx_nic,
  1325. monitor_work.work);
  1326. netif_vdbg(efx, timer, efx->net_dev,
  1327. "hardware monitor executing on CPU %d\n",
  1328. raw_smp_processor_id());
  1329. BUG_ON(efx->type->monitor == NULL);
  1330. /* If the mac_lock is already held then it is likely a port
  1331. * reconfiguration is already in place, which will likely do
  1332. * most of the work of monitor() anyway. */
  1333. if (mutex_trylock(&efx->mac_lock)) {
  1334. if (efx->port_enabled)
  1335. efx->type->monitor(efx);
  1336. mutex_unlock(&efx->mac_lock);
  1337. }
  1338. queue_delayed_work(efx->workqueue, &efx->monitor_work,
  1339. efx_monitor_interval);
  1340. }
  1341. /**************************************************************************
  1342. *
  1343. * ioctls
  1344. *
  1345. *************************************************************************/
  1346. /* Net device ioctl
  1347. * Context: process, rtnl_lock() held.
  1348. */
  1349. static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
  1350. {
  1351. struct efx_nic *efx = netdev_priv(net_dev);
  1352. struct mii_ioctl_data *data = if_mii(ifr);
  1353. EFX_ASSERT_RESET_SERIALISED(efx);
  1354. /* Convert phy_id from older PRTAD/DEVAD format */
  1355. if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
  1356. (data->phy_id & 0xfc00) == 0x0400)
  1357. data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
  1358. return mdio_mii_ioctl(&efx->mdio, data, cmd);
  1359. }
  1360. /**************************************************************************
  1361. *
  1362. * NAPI interface
  1363. *
  1364. **************************************************************************/
  1365. static void efx_init_napi(struct efx_nic *efx)
  1366. {
  1367. struct efx_channel *channel;
  1368. efx_for_each_channel(channel, efx) {
  1369. channel->napi_dev = efx->net_dev;
  1370. netif_napi_add(channel->napi_dev, &channel->napi_str,
  1371. efx_poll, napi_weight);
  1372. }
  1373. }
  1374. static void efx_fini_napi_channel(struct efx_channel *channel)
  1375. {
  1376. if (channel->napi_dev)
  1377. netif_napi_del(&channel->napi_str);
  1378. channel->napi_dev = NULL;
  1379. }
  1380. static void efx_fini_napi(struct efx_nic *efx)
  1381. {
  1382. struct efx_channel *channel;
  1383. efx_for_each_channel(channel, efx)
  1384. efx_fini_napi_channel(channel);
  1385. }
  1386. /**************************************************************************
  1387. *
  1388. * Kernel netpoll interface
  1389. *
  1390. *************************************************************************/
  1391. #ifdef CONFIG_NET_POLL_CONTROLLER
  1392. /* Although in the common case interrupts will be disabled, this is not
  1393. * guaranteed. However, all our work happens inside the NAPI callback,
  1394. * so no locking is required.
  1395. */
  1396. static void efx_netpoll(struct net_device *net_dev)
  1397. {
  1398. struct efx_nic *efx = netdev_priv(net_dev);
  1399. struct efx_channel *channel;
  1400. efx_for_each_channel(channel, efx)
  1401. efx_schedule_channel(channel);
  1402. }
  1403. #endif
  1404. /**************************************************************************
  1405. *
  1406. * Kernel net device interface
  1407. *
  1408. *************************************************************************/
  1409. /* Context: process, rtnl_lock() held. */
  1410. static int efx_net_open(struct net_device *net_dev)
  1411. {
  1412. struct efx_nic *efx = netdev_priv(net_dev);
  1413. EFX_ASSERT_RESET_SERIALISED(efx);
  1414. netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
  1415. raw_smp_processor_id());
  1416. if (efx->state == STATE_DISABLED)
  1417. return -EIO;
  1418. if (efx->phy_mode & PHY_MODE_SPECIAL)
  1419. return -EBUSY;
  1420. if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
  1421. return -EIO;
  1422. /* Notify the kernel of the link state polled during driver load,
  1423. * before the monitor starts running */
  1424. efx_link_status_changed(efx);
  1425. efx_start_all(efx);
  1426. return 0;
  1427. }
  1428. /* Context: process, rtnl_lock() held.
  1429. * Note that the kernel will ignore our return code; this method
  1430. * should really be a void.
  1431. */
  1432. static int efx_net_stop(struct net_device *net_dev)
  1433. {
  1434. struct efx_nic *efx = netdev_priv(net_dev);
  1435. netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
  1436. raw_smp_processor_id());
  1437. if (efx->state != STATE_DISABLED) {
  1438. /* Stop the device and flush all the channels */
  1439. efx_stop_all(efx);
  1440. }
  1441. return 0;
  1442. }
  1443. /* Context: process, dev_base_lock or RTNL held, non-blocking. */
  1444. static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev,
  1445. struct rtnl_link_stats64 *stats)
  1446. {
  1447. struct efx_nic *efx = netdev_priv(net_dev);
  1448. struct efx_mac_stats *mac_stats = &efx->mac_stats;
  1449. spin_lock_bh(&efx->stats_lock);
  1450. efx->type->update_stats(efx);
  1451. stats->rx_packets = mac_stats->rx_packets;
  1452. stats->tx_packets = mac_stats->tx_packets;
  1453. stats->rx_bytes = mac_stats->rx_bytes;
  1454. stats->tx_bytes = mac_stats->tx_bytes;
  1455. stats->rx_dropped = efx->n_rx_nodesc_drop_cnt;
  1456. stats->multicast = mac_stats->rx_multicast;
  1457. stats->collisions = mac_stats->tx_collision;
  1458. stats->rx_length_errors = (mac_stats->rx_gtjumbo +
  1459. mac_stats->rx_length_error);
  1460. stats->rx_crc_errors = mac_stats->rx_bad;
  1461. stats->rx_frame_errors = mac_stats->rx_align_error;
  1462. stats->rx_fifo_errors = mac_stats->rx_overflow;
  1463. stats->rx_missed_errors = mac_stats->rx_missed;
  1464. stats->tx_window_errors = mac_stats->tx_late_collision;
  1465. stats->rx_errors = (stats->rx_length_errors +
  1466. stats->rx_crc_errors +
  1467. stats->rx_frame_errors +
  1468. mac_stats->rx_symbol_error);
  1469. stats->tx_errors = (stats->tx_window_errors +
  1470. mac_stats->tx_bad);
  1471. spin_unlock_bh(&efx->stats_lock);
  1472. return stats;
  1473. }
  1474. /* Context: netif_tx_lock held, BHs disabled. */
  1475. static void efx_watchdog(struct net_device *net_dev)
  1476. {
  1477. struct efx_nic *efx = netdev_priv(net_dev);
  1478. netif_err(efx, tx_err, efx->net_dev,
  1479. "TX stuck with port_enabled=%d: resetting channels\n",
  1480. efx->port_enabled);
  1481. efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
  1482. }
  1483. /* Context: process, rtnl_lock() held. */
  1484. static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
  1485. {
  1486. struct efx_nic *efx = netdev_priv(net_dev);
  1487. EFX_ASSERT_RESET_SERIALISED(efx);
  1488. if (new_mtu > EFX_MAX_MTU)
  1489. return -EINVAL;
  1490. efx_stop_all(efx);
  1491. netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
  1492. mutex_lock(&efx->mac_lock);
  1493. /* Reconfigure the MAC before enabling the dma queues so that
  1494. * the RX buffers don't overflow */
  1495. net_dev->mtu = new_mtu;
  1496. efx->type->reconfigure_mac(efx);
  1497. mutex_unlock(&efx->mac_lock);
  1498. efx_start_all(efx);
  1499. return 0;
  1500. }
  1501. static int efx_set_mac_address(struct net_device *net_dev, void *data)
  1502. {
  1503. struct efx_nic *efx = netdev_priv(net_dev);
  1504. struct sockaddr *addr = data;
  1505. char *new_addr = addr->sa_data;
  1506. EFX_ASSERT_RESET_SERIALISED(efx);
  1507. if (!is_valid_ether_addr(new_addr)) {
  1508. netif_err(efx, drv, efx->net_dev,
  1509. "invalid ethernet MAC address requested: %pM\n",
  1510. new_addr);
  1511. return -EINVAL;
  1512. }
  1513. memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);
  1514. /* Reconfigure the MAC */
  1515. mutex_lock(&efx->mac_lock);
  1516. efx->type->reconfigure_mac(efx);
  1517. mutex_unlock(&efx->mac_lock);
  1518. return 0;
  1519. }
  1520. /* Context: netif_addr_lock held, BHs disabled. */
  1521. static void efx_set_rx_mode(struct net_device *net_dev)
  1522. {
  1523. struct efx_nic *efx = netdev_priv(net_dev);
  1524. struct netdev_hw_addr *ha;
  1525. union efx_multicast_hash *mc_hash = &efx->multicast_hash;
  1526. u32 crc;
  1527. int bit;
  1528. efx->promiscuous = !!(net_dev->flags & IFF_PROMISC);
  1529. /* Build multicast hash table */
  1530. if (efx->promiscuous || (net_dev->flags & IFF_ALLMULTI)) {
  1531. memset(mc_hash, 0xff, sizeof(*mc_hash));
  1532. } else {
  1533. memset(mc_hash, 0x00, sizeof(*mc_hash));
  1534. netdev_for_each_mc_addr(ha, net_dev) {
  1535. crc = ether_crc_le(ETH_ALEN, ha->addr);
  1536. bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
  1537. set_bit_le(bit, mc_hash->byte);
  1538. }
  1539. /* Broadcast packets go through the multicast hash filter.
  1540. * ether_crc_le() of the broadcast address is 0xbe2612ff
  1541. * so we always add bit 0xff to the mask.
  1542. */
  1543. set_bit_le(0xff, mc_hash->byte);
  1544. }
  1545. if (efx->port_enabled)
  1546. queue_work(efx->workqueue, &efx->mac_work);
  1547. /* Otherwise efx_start_port() will do this */
  1548. }
  1549. static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
  1550. {
  1551. struct efx_nic *efx = netdev_priv(net_dev);
  1552. /* If disabling RX n-tuple filtering, clear existing filters */
  1553. if (net_dev->features & ~data & NETIF_F_NTUPLE)
  1554. efx_filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
  1555. return 0;
  1556. }
  1557. static const struct net_device_ops efx_netdev_ops = {
  1558. .ndo_open = efx_net_open,
  1559. .ndo_stop = efx_net_stop,
  1560. .ndo_get_stats64 = efx_net_stats,
  1561. .ndo_tx_timeout = efx_watchdog,
  1562. .ndo_start_xmit = efx_hard_start_xmit,
  1563. .ndo_validate_addr = eth_validate_addr,
  1564. .ndo_do_ioctl = efx_ioctl,
  1565. .ndo_change_mtu = efx_change_mtu,
  1566. .ndo_set_mac_address = efx_set_mac_address,
  1567. .ndo_set_rx_mode = efx_set_rx_mode,
  1568. .ndo_set_features = efx_set_features,
  1569. #ifdef CONFIG_NET_POLL_CONTROLLER
  1570. .ndo_poll_controller = efx_netpoll,
  1571. #endif
  1572. .ndo_setup_tc = efx_setup_tc,
  1573. #ifdef CONFIG_RFS_ACCEL
  1574. .ndo_rx_flow_steer = efx_filter_rfs,
  1575. #endif
  1576. };
  1577. static void efx_update_name(struct efx_nic *efx)
  1578. {
  1579. strcpy(efx->name, efx->net_dev->name);
  1580. efx_mtd_rename(efx);
  1581. efx_set_channel_names(efx);
  1582. }
  1583. static int efx_netdev_event(struct notifier_block *this,
  1584. unsigned long event, void *ptr)
  1585. {
  1586. struct net_device *net_dev = ptr;
  1587. if (net_dev->netdev_ops == &efx_netdev_ops &&
  1588. event == NETDEV_CHANGENAME)
  1589. efx_update_name(netdev_priv(net_dev));
  1590. return NOTIFY_DONE;
  1591. }
  1592. static struct notifier_block efx_netdev_notifier = {
  1593. .notifier_call = efx_netdev_event,
  1594. };
  1595. static ssize_t
  1596. show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
  1597. {
  1598. struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
  1599. return sprintf(buf, "%d\n", efx->phy_type);
  1600. }
  1601. static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);
  1602. static int efx_register_netdev(struct efx_nic *efx)
  1603. {
  1604. struct net_device *net_dev = efx->net_dev;
  1605. struct efx_channel *channel;
  1606. int rc;
  1607. net_dev->watchdog_timeo = 5 * HZ;
  1608. net_dev->irq = efx->pci_dev->irq;
  1609. net_dev->netdev_ops = &efx_netdev_ops;
  1610. SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);
  1611. rtnl_lock();
  1612. rc = dev_alloc_name(net_dev, net_dev->name);
  1613. if (rc < 0)
  1614. goto fail_locked;
  1615. efx_update_name(efx);
  1616. rc = register_netdevice(net_dev);
  1617. if (rc)
  1618. goto fail_locked;
  1619. efx_for_each_channel(channel, efx) {
  1620. struct efx_tx_queue *tx_queue;
  1621. efx_for_each_channel_tx_queue(tx_queue, channel)
  1622. efx_init_tx_queue_core_txq(tx_queue);
  1623. }
  1624. /* Always start with carrier off; PHY events will detect the link */
  1625. netif_carrier_off(net_dev);
  1626. rtnl_unlock();
  1627. rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
  1628. if (rc) {
  1629. netif_err(efx, drv, efx->net_dev,
  1630. "failed to init net dev attributes\n");
  1631. goto fail_registered;
  1632. }
  1633. return 0;
  1634. fail_locked:
  1635. rtnl_unlock();
  1636. netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
  1637. return rc;
  1638. fail_registered:
  1639. unregister_netdev(net_dev);
  1640. return rc;
  1641. }
  1642. static void efx_unregister_netdev(struct efx_nic *efx)
  1643. {
  1644. struct efx_channel *channel;
  1645. struct efx_tx_queue *tx_queue;
  1646. if (!efx->net_dev)
  1647. return;
  1648. BUG_ON(netdev_priv(efx->net_dev) != efx);
  1649. /* Free up any skbs still remaining. This has to happen before
  1650. * we try to unregister the netdev as running their destructors
  1651. * may be needed to get the device ref. count to 0. */
  1652. efx_for_each_channel(channel, efx) {
  1653. efx_for_each_channel_tx_queue(tx_queue, channel)
  1654. efx_release_tx_buffers(tx_queue);
  1655. }
  1656. strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
  1657. device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
  1658. unregister_netdev(efx->net_dev);
  1659. }
  1660. /**************************************************************************
  1661. *
  1662. * Device reset and suspend
  1663. *
  1664. **************************************************************************/
  1665. /* Tears down the entire software state and most of the hardware state
  1666. * before reset. */
  1667. void efx_reset_down(struct efx_nic *efx, enum reset_type method)
  1668. {
  1669. EFX_ASSERT_RESET_SERIALISED(efx);
  1670. efx_stop_all(efx);
  1671. mutex_lock(&efx->mac_lock);
  1672. efx_stop_interrupts(efx);
  1673. if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
  1674. efx->phy_op->fini(efx);
  1675. efx->type->fini(efx);
  1676. }
  1677. /* This function will always ensure that the locks acquired in
  1678. * efx_reset_down() are released. A failure return code indicates
  1679. * that we were unable to reinitialise the hardware, and the
  1680. * driver should be disabled. If ok is false, then the rx and tx
  1681. * engines are not restarted, pending a RESET_DISABLE. */
  1682. int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
  1683. {
  1684. int rc;
  1685. EFX_ASSERT_RESET_SERIALISED(efx);
  1686. rc = efx->type->init(efx);
  1687. if (rc) {
  1688. netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
  1689. goto fail;
  1690. }
  1691. if (!ok)
  1692. goto fail;
  1693. if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {
  1694. rc = efx->phy_op->init(efx);
  1695. if (rc)
  1696. goto fail;
  1697. if (efx->phy_op->reconfigure(efx))
  1698. netif_err(efx, drv, efx->net_dev,
  1699. "could not restore PHY settings\n");
  1700. }
  1701. efx->type->reconfigure_mac(efx);
  1702. efx_start_interrupts(efx);
  1703. efx_restore_filters(efx);
  1704. mutex_unlock(&efx->mac_lock);
  1705. efx_start_all(efx);
  1706. return 0;
  1707. fail:
  1708. efx->port_initialized = false;
  1709. mutex_unlock(&efx->mac_lock);
  1710. return rc;
  1711. }
  1712. /* Reset the NIC using the specified method. Note that the reset may
  1713. * fail, in which case the card will be left in an unusable state.
  1714. *
  1715. * Caller must hold the rtnl_lock.
  1716. */
  1717. int efx_reset(struct efx_nic *efx, enum reset_type method)
  1718. {
  1719. int rc, rc2;
  1720. bool disabled;
  1721. netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
  1722. RESET_TYPE(method));
  1723. netif_device_detach(efx->net_dev);
  1724. efx_reset_down(efx, method);
  1725. rc = efx->type->reset(efx, method);
  1726. if (rc) {
  1727. netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
  1728. goto out;
  1729. }
  1730. /* Clear flags for the scopes we covered. We assume the NIC and
  1731. * driver are now quiescent so that there is no race here.
  1732. */
  1733. efx->reset_pending &= -(1 << (method + 1));
  1734. /* Reinitialise bus-mastering, which may have been turned off before
  1735. * the reset was scheduled. This is still appropriate, even in the
  1736. * RESET_TYPE_DISABLE since this driver generally assumes the hardware
  1737. * can respond to requests. */
  1738. pci_set_master(efx->pci_dev);
  1739. out:
  1740. /* Leave device stopped if necessary */
  1741. disabled = rc || method == RESET_TYPE_DISABLE;
  1742. rc2 = efx_reset_up(efx, method, !disabled);
  1743. if (rc2) {
  1744. disabled = true;
  1745. if (!rc)
  1746. rc = rc2;
  1747. }
  1748. if (disabled) {
  1749. dev_close(efx->net_dev);
  1750. netif_err(efx, drv, efx->net_dev, "has been disabled\n");
  1751. efx->state = STATE_DISABLED;
  1752. } else {
  1753. netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
  1754. netif_device_attach(efx->net_dev);
  1755. }
  1756. return rc;
  1757. }
  1758. /* The worker thread exists so that code that cannot sleep can
  1759. * schedule a reset for later.
  1760. */
  1761. static void efx_reset_work(struct work_struct *data)
  1762. {
  1763. struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
  1764. unsigned long pending = ACCESS_ONCE(efx->reset_pending);
  1765. if (!pending)
  1766. return;
  1767. /* If we're not RUNNING then don't reset. Leave the reset_pending
  1768. * flags set so that efx_pci_probe_main will be retried */
  1769. if (efx->state != STATE_RUNNING) {
  1770. netif_info(efx, drv, efx->net_dev,
  1771. "scheduled reset quenched. NIC not RUNNING\n");
  1772. return;
  1773. }
  1774. rtnl_lock();
  1775. (void)efx_reset(efx, fls(pending) - 1);
  1776. rtnl_unlock();
  1777. }
  1778. void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
  1779. {
  1780. enum reset_type method;
  1781. switch (type) {
  1782. case RESET_TYPE_INVISIBLE:
  1783. case RESET_TYPE_ALL:
  1784. case RESET_TYPE_WORLD:
  1785. case RESET_TYPE_DISABLE:
  1786. method = type;
  1787. netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
  1788. RESET_TYPE(method));
  1789. break;
  1790. default:
  1791. method = efx->type->map_reset_reason(type);
  1792. netif_dbg(efx, drv, efx->net_dev,
  1793. "scheduling %s reset for %s\n",
  1794. RESET_TYPE(method), RESET_TYPE(type));
  1795. break;
  1796. }
  1797. set_bit(method, &efx->reset_pending);
  1798. /* efx_process_channel() will no longer read events once a
  1799. * reset is scheduled. So switch back to poll'd MCDI completions. */
  1800. efx_mcdi_mode_poll(efx);
  1801. queue_work(reset_workqueue, &efx->reset_work);
  1802. }
  1803. /**************************************************************************
  1804. *
  1805. * List of NICs we support
  1806. *
  1807. **************************************************************************/
  1808. /* PCI device ID table */
  1809. static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = {
  1810. {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE,
  1811. PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0),
  1812. .driver_data = (unsigned long) &falcon_a1_nic_type},
  1813. {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE,
  1814. PCI_DEVICE_ID_SOLARFLARE_SFC4000B),
  1815. .driver_data = (unsigned long) &falcon_b0_nic_type},
  1816. {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */
  1817. .driver_data = (unsigned long) &siena_a0_nic_type},
  1818. {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */
  1819. .driver_data = (unsigned long) &siena_a0_nic_type},
  1820. {0} /* end of list */
  1821. };
  1822. /**************************************************************************
  1823. *
  1824. * Dummy PHY/MAC operations
  1825. *
  1826. * Can be used for some unimplemented operations
  1827. * Needed so all function pointers are valid and do not have to be tested
  1828. * before use
  1829. *
  1830. **************************************************************************/
  1831. int efx_port_dummy_op_int(struct efx_nic *efx)
  1832. {
  1833. return 0;
  1834. }
  1835. void efx_port_dummy_op_void(struct efx_nic *efx) {}
  1836. static bool efx_port_dummy_op_poll(struct efx_nic *efx)
  1837. {
  1838. return false;
  1839. }
  1840. static const struct efx_phy_operations efx_dummy_phy_operations = {
  1841. .init = efx_port_dummy_op_int,
  1842. .reconfigure = efx_port_dummy_op_int,
  1843. .poll = efx_port_dummy_op_poll,
  1844. .fini = efx_port_dummy_op_void,
  1845. };
  1846. /**************************************************************************
  1847. *
  1848. * Data housekeeping
  1849. *
  1850. **************************************************************************/
  1851. /* This zeroes out and then fills in the invariants in a struct
  1852. * efx_nic (including all sub-structures).
  1853. */
  1854. static int efx_init_struct(struct efx_nic *efx, const struct efx_nic_type *type,
  1855. struct pci_dev *pci_dev, struct net_device *net_dev)
  1856. {
  1857. int i;
  1858. /* Initialise common structures */
  1859. memset(efx, 0, sizeof(*efx));
  1860. spin_lock_init(&efx->biu_lock);
  1861. #ifdef CONFIG_SFC_MTD
  1862. INIT_LIST_HEAD(&efx->mtd_list);
  1863. #endif
  1864. INIT_WORK(&efx->reset_work, efx_reset_work);
  1865. INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
  1866. efx->pci_dev = pci_dev;
  1867. efx->msg_enable = debug;
  1868. efx->state = STATE_INIT;
  1869. strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
  1870. efx->net_dev = net_dev;
  1871. spin_lock_init(&efx->stats_lock);
  1872. mutex_init(&efx->mac_lock);
  1873. efx->phy_op = &efx_dummy_phy_operations;
  1874. efx->mdio.dev = net_dev;
  1875. INIT_WORK(&efx->mac_work, efx_mac_work);
  1876. init_waitqueue_head(&efx->flush_wq);
  1877. for (i = 0; i < EFX_MAX_CHANNELS; i++) {
  1878. efx->channel[i] = efx_alloc_channel(efx, i, NULL);
  1879. if (!efx->channel[i])
  1880. goto fail;
  1881. }
  1882. efx->type = type;
  1883. EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);
  1884. /* Higher numbered interrupt modes are less capable! */
  1885. efx->interrupt_mode = max(efx->type->max_interrupt_mode,
  1886. interrupt_mode);
  1887. /* Would be good to use the net_dev name, but we're too early */
  1888. snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
  1889. pci_name(pci_dev));
  1890. efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
  1891. if (!efx->workqueue)
  1892. goto fail;
  1893. return 0;
  1894. fail:
  1895. efx_fini_struct(efx);
  1896. return -ENOMEM;
  1897. }
  1898. static void efx_fini_struct(struct efx_nic *efx)
  1899. {
  1900. int i;
  1901. for (i = 0; i < EFX_MAX_CHANNELS; i++)
  1902. kfree(efx->channel[i]);
  1903. if (efx->workqueue) {
  1904. destroy_workqueue(efx->workqueue);
  1905. efx->workqueue = NULL;
  1906. }
  1907. }
  1908. /**************************************************************************
  1909. *
  1910. * PCI interface
  1911. *
  1912. **************************************************************************/
  1913. /* Main body of final NIC shutdown code
  1914. * This is called only at module unload (or hotplug removal).
  1915. */
  1916. static void efx_pci_remove_main(struct efx_nic *efx)
  1917. {
  1918. #ifdef CONFIG_RFS_ACCEL
  1919. free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
  1920. efx->net_dev->rx_cpu_rmap = NULL;
  1921. #endif
  1922. efx_stop_interrupts(efx);
  1923. efx_nic_fini_interrupt(efx);
  1924. efx_fini_port(efx);
  1925. efx->type->fini(efx);
  1926. efx_fini_napi(efx);
  1927. efx_remove_all(efx);
  1928. }
  1929. /* Final NIC shutdown
  1930. * This is called only at module unload (or hotplug removal).
  1931. */
  1932. static void efx_pci_remove(struct pci_dev *pci_dev)
  1933. {
  1934. struct efx_nic *efx;
  1935. efx = pci_get_drvdata(pci_dev);
  1936. if (!efx)
  1937. return;
  1938. /* Mark the NIC as fini, then stop the interface */
  1939. rtnl_lock();
  1940. efx->state = STATE_FINI;
  1941. dev_close(efx->net_dev);
  1942. /* Allow any queued efx_resets() to complete */
  1943. rtnl_unlock();
  1944. efx_stop_interrupts(efx);
  1945. efx_unregister_netdev(efx);
  1946. efx_mtd_remove(efx);
  1947. /* Wait for any scheduled resets to complete. No more will be
  1948. * scheduled from this point because efx_stop_all() has been
  1949. * called, we are no longer registered with driverlink, and
  1950. * the net_device's have been removed. */
  1951. cancel_work_sync(&efx->reset_work);
  1952. efx_pci_remove_main(efx);
  1953. efx_fini_io(efx);
  1954. netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
  1955. pci_set_drvdata(pci_dev, NULL);
  1956. efx_fini_struct(efx);
  1957. free_netdev(efx->net_dev);
  1958. };
  1959. /* Main body of NIC initialisation
  1960. * This is called at module load (or hotplug insertion, theoretically).
  1961. */
  1962. static int efx_pci_probe_main(struct efx_nic *efx)
  1963. {
  1964. int rc;
  1965. /* Do start-of-day initialisation */
  1966. rc = efx_probe_all(efx);
  1967. if (rc)
  1968. goto fail1;
  1969. efx_init_napi(efx);
  1970. rc = efx->type->init(efx);
  1971. if (rc) {
  1972. netif_err(efx, probe, efx->net_dev,
  1973. "failed to initialise NIC\n");
  1974. goto fail3;
  1975. }
  1976. rc = efx_init_port(efx);
  1977. if (rc) {
  1978. netif_err(efx, probe, efx->net_dev,
  1979. "failed to initialise port\n");
  1980. goto fail4;
  1981. }
  1982. rc = efx_nic_init_interrupt(efx);
  1983. if (rc)
  1984. goto fail5;
  1985. efx_start_interrupts(efx);
  1986. return 0;
  1987. fail5:
  1988. efx_fini_port(efx);
  1989. fail4:
  1990. efx->type->fini(efx);
  1991. fail3:
  1992. efx_fini_napi(efx);
  1993. efx_remove_all(efx);
  1994. fail1:
  1995. return rc;
  1996. }
  1997. /* NIC initialisation
  1998. *
  1999. * This is called at module load (or hotplug insertion,
  2000. * theoretically). It sets up PCI mappings, resets the NIC,
  2001. * sets up and registers the network devices with the kernel and hooks
  2002. * the interrupt service routine. It does not prepare the device for
  2003. * transmission; this is left to the first time one of the network
  2004. * interfaces is brought up (i.e. efx_net_open).
  2005. */
  2006. static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
  2007. const struct pci_device_id *entry)
  2008. {
  2009. const struct efx_nic_type *type = (const struct efx_nic_type *) entry->driver_data;
  2010. struct net_device *net_dev;
  2011. struct efx_nic *efx;
  2012. int rc;
  2013. /* Allocate and initialise a struct net_device and struct efx_nic */
  2014. net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
  2015. EFX_MAX_RX_QUEUES);
  2016. if (!net_dev)
  2017. return -ENOMEM;
  2018. net_dev->features |= (type->offload_features | NETIF_F_SG |
  2019. NETIF_F_HIGHDMA | NETIF_F_TSO |
  2020. NETIF_F_RXCSUM);
  2021. if (type->offload_features & NETIF_F_V6_CSUM)
  2022. net_dev->features |= NETIF_F_TSO6;
  2023. /* Mask for features that also apply to VLAN devices */
  2024. net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG |
  2025. NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
  2026. NETIF_F_RXCSUM);
  2027. /* All offloads can be toggled */
  2028. net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;
  2029. efx = netdev_priv(net_dev);
  2030. pci_set_drvdata(pci_dev, efx);
  2031. SET_NETDEV_DEV(net_dev, &pci_dev->dev);
  2032. rc = efx_init_struct(efx, type, pci_dev, net_dev);
  2033. if (rc)
  2034. goto fail1;
  2035. netif_info(efx, probe, efx->net_dev,
  2036. "Solarflare NIC detected\n");
  2037. /* Set up basic I/O (BAR mappings etc) */
  2038. rc = efx_init_io(efx);
  2039. if (rc)
  2040. goto fail2;
  2041. rc = efx_pci_probe_main(efx);
  2042. /* Serialise against efx_reset(). No more resets will be
  2043. * scheduled since efx_stop_all() has been called, and we have
  2044. * not and never have been registered.
  2045. */
  2046. cancel_work_sync(&efx->reset_work);
  2047. if (rc)
  2048. goto fail3;
  2049. /* If there was a scheduled reset during probe, the NIC is
  2050. * probably hosed anyway.
  2051. */
  2052. if (efx->reset_pending) {
  2053. rc = -EIO;
  2054. goto fail4;
  2055. }
  2056. /* Switch to the running state before we expose the device to the OS,
  2057. * so that dev_open()|efx_start_all() will actually start the device */
  2058. efx->state = STATE_RUNNING;
  2059. rc = efx_register_netdev(efx);
  2060. if (rc)
  2061. goto fail4;
  2062. netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
  2063. /* Try to create MTDs, but allow this to fail */
  2064. rtnl_lock();
  2065. rc = efx_mtd_probe(efx);
  2066. rtnl_unlock();
  2067. if (rc)
  2068. netif_warn(efx, probe, efx->net_dev,
  2069. "failed to create MTDs (%d)\n", rc);
  2070. return 0;
  2071. fail4:
  2072. efx_pci_remove_main(efx);
  2073. fail3:
  2074. efx_fini_io(efx);
  2075. fail2:
  2076. efx_fini_struct(efx);
  2077. fail1:
  2078. WARN_ON(rc > 0);
  2079. netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
  2080. free_netdev(net_dev);
  2081. return rc;
  2082. }
  2083. static int efx_pm_freeze(struct device *dev)
  2084. {
  2085. struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
  2086. efx->state = STATE_FINI;
  2087. netif_device_detach(efx->net_dev);
  2088. efx_stop_all(efx);
  2089. efx_stop_interrupts(efx);
  2090. return 0;
  2091. }
  2092. static int efx_pm_thaw(struct device *dev)
  2093. {
  2094. struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
  2095. efx->state = STATE_INIT;
  2096. efx_start_interrupts(efx);
  2097. mutex_lock(&efx->mac_lock);
  2098. efx->phy_op->reconfigure(efx);
  2099. mutex_unlock(&efx->mac_lock);
  2100. efx_start_all(efx);
  2101. netif_device_attach(efx->net_dev);
  2102. efx->state = STATE_RUNNING;
  2103. efx->type->resume_wol(efx);
  2104. /* Reschedule any quenched resets scheduled during efx_pm_freeze() */
  2105. queue_work(reset_workqueue, &efx->reset_work);
  2106. return 0;
  2107. }
  2108. static int efx_pm_poweroff(struct device *dev)
  2109. {
  2110. struct pci_dev *pci_dev = to_pci_dev(dev);
  2111. struct efx_nic *efx = pci_get_drvdata(pci_dev);
  2112. efx->type->fini(efx);
  2113. efx->reset_pending = 0;
  2114. pci_save_state(pci_dev);
  2115. return pci_set_power_state(pci_dev, PCI_D3hot);
  2116. }
  2117. /* Used for both resume and restore */
  2118. static int efx_pm_resume(struct device *dev)
  2119. {
  2120. struct pci_dev *pci_dev = to_pci_dev(dev);
  2121. struct efx_nic *efx = pci_get_drvdata(pci_dev);
  2122. int rc;
  2123. rc = pci_set_power_state(pci_dev, PCI_D0);
  2124. if (rc)
  2125. return rc;
  2126. pci_restore_state(pci_dev);
  2127. rc = pci_enable_device(pci_dev);
  2128. if (rc)
  2129. return rc;
  2130. pci_set_master(efx->pci_dev);
  2131. rc = efx->type->reset(efx, RESET_TYPE_ALL);
  2132. if (rc)
  2133. return rc;
  2134. rc = efx->type->init(efx);
  2135. if (rc)
  2136. return rc;
  2137. efx_pm_thaw(dev);
  2138. return 0;
  2139. }
  2140. static int efx_pm_suspend(struct device *dev)
  2141. {
  2142. int rc;
  2143. efx_pm_freeze(dev);
  2144. rc = efx_pm_poweroff(dev);
  2145. if (rc)
  2146. efx_pm_resume(dev);
  2147. return rc;
  2148. }
  2149. static const struct dev_pm_ops efx_pm_ops = {
  2150. .suspend = efx_pm_suspend,
  2151. .resume = efx_pm_resume,
  2152. .freeze = efx_pm_freeze,
  2153. .thaw = efx_pm_thaw,
  2154. .poweroff = efx_pm_poweroff,
  2155. .restore = efx_pm_resume,
  2156. };
  2157. static struct pci_driver efx_pci_driver = {
  2158. .name = KBUILD_MODNAME,
  2159. .id_table = efx_pci_table,
  2160. .probe = efx_pci_probe,
  2161. .remove = efx_pci_remove,
  2162. .driver.pm = &efx_pm_ops,
  2163. };
  2164. /**************************************************************************
  2165. *
  2166. * Kernel module interface
  2167. *
  2168. *************************************************************************/
  2169. module_param(interrupt_mode, uint, 0444);
  2170. MODULE_PARM_DESC(interrupt_mode,
  2171. "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
  2172. static int __init efx_init_module(void)
  2173. {
  2174. int rc;
  2175. printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
  2176. rc = register_netdevice_notifier(&efx_netdev_notifier);
  2177. if (rc)
  2178. goto err_notifier;
  2179. reset_workqueue = create_singlethread_workqueue("sfc_reset");
  2180. if (!reset_workqueue) {
  2181. rc = -ENOMEM;
  2182. goto err_reset;
  2183. }
  2184. rc = pci_register_driver(&efx_pci_driver);
  2185. if (rc < 0)
  2186. goto err_pci;
  2187. return 0;
  2188. err_pci:
  2189. destroy_workqueue(reset_workqueue);
  2190. err_reset:
  2191. unregister_netdevice_notifier(&efx_netdev_notifier);
  2192. err_notifier:
  2193. return rc;
  2194. }
  2195. static void __exit efx_exit_module(void)
  2196. {
  2197. printk(KERN_INFO "Solarflare NET driver unloading\n");
  2198. pci_unregister_driver(&efx_pci_driver);
  2199. destroy_workqueue(reset_workqueue);
  2200. unregister_netdevice_notifier(&efx_netdev_notifier);
  2201. }
  2202. module_init(efx_init_module);
  2203. module_exit(efx_exit_module);
  2204. MODULE_AUTHOR("Solarflare Communications and "
  2205. "Michael Brown <mbrown@fensystems.co.uk>");
  2206. MODULE_DESCRIPTION("Solarflare Communications network driver");
  2207. MODULE_LICENSE("GPL");
  2208. MODULE_DEVICE_TABLE(pci, efx_pci_table);