efx.c 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210
  1. /****************************************************************************
  2. * Driver for Solarflare Solarstorm network controllers and boards
  3. * Copyright 2005-2006 Fen Systems Ltd.
  4. * Copyright 2005-2008 Solarflare Communications Inc.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation, incorporated herein by reference.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/pci.h>
  12. #include <linux/netdevice.h>
  13. #include <linux/etherdevice.h>
  14. #include <linux/delay.h>
  15. #include <linux/notifier.h>
  16. #include <linux/ip.h>
  17. #include <linux/tcp.h>
  18. #include <linux/in.h>
  19. #include <linux/crc32.h>
  20. #include <linux/ethtool.h>
  21. #include "net_driver.h"
  22. #include "gmii.h"
  23. #include "ethtool.h"
  24. #include "tx.h"
  25. #include "rx.h"
  26. #include "efx.h"
  27. #include "mdio_10g.h"
  28. #include "falcon.h"
  29. #include "workarounds.h"
  30. #include "mac.h"
  31. #define EFX_MAX_MTU (9 * 1024)
  32. /* RX slow fill workqueue. If memory allocation fails in the fast path,
  33. * a work item is pushed onto this work queue to retry the allocation later,
  34. * to avoid the NIC being starved of RX buffers. Since this is a per cpu
  35. * workqueue, there is nothing to be gained in making it per NIC
  36. */
  37. static struct workqueue_struct *refill_workqueue;
  38. /**************************************************************************
  39. *
  40. * Configurable values
  41. *
  42. *************************************************************************/
  43. /*
  44. * Enable large receive offload (LRO) aka soft segment reassembly (SSR)
  45. *
  46. * This sets the default for new devices. It can be controlled later
  47. * using ethtool.
  48. */
  49. static int lro = 1;
  50. module_param(lro, int, 0644);
  51. MODULE_PARM_DESC(lro, "Large receive offload acceleration");
  52. /*
  53. * Use separate channels for TX and RX events
  54. *
  55. * Set this to 1 to use separate channels for TX and RX. It allows us to
  56. * apply a higher level of interrupt moderation to TX events.
  57. *
  58. * This is forced to 0 for MSI interrupt mode as the interrupt vector
  59. * is not written
  60. */
  61. static unsigned int separate_tx_and_rx_channels = 1;
  62. /* This is the weight assigned to each of the (per-channel) virtual
  63. * NAPI devices.
  64. */
  65. static int napi_weight = 64;
  66. /* This is the time (in jiffies) between invocations of the hardware
  67. * monitor, which checks for known hardware bugs and resets the
  68. * hardware and driver as necessary.
  69. */
  70. unsigned int efx_monitor_interval = 1 * HZ;
  71. /* This controls whether or not the hardware monitor will trigger a
  72. * reset when it detects an error condition.
  73. */
  74. static unsigned int monitor_reset = 1;
  75. /* This controls whether or not the driver will initialise devices
  76. * with invalid MAC addresses stored in the EEPROM or flash. If true,
  77. * such devices will be initialised with a random locally-generated
  78. * MAC address. This allows for loading the sfc_mtd driver to
  79. * reprogram the flash, even if the flash contents (including the MAC
  80. * address) have previously been erased.
  81. */
  82. static unsigned int allow_bad_hwaddr;
  83. /* Initial interrupt moderation settings. They can be modified after
  84. * module load with ethtool.
  85. *
  86. * The default for RX should strike a balance between increasing the
  87. * round-trip latency and reducing overhead.
  88. */
  89. static unsigned int rx_irq_mod_usec = 60;
  90. /* Initial interrupt moderation settings. They can be modified after
  91. * module load with ethtool.
  92. *
  93. * This default is chosen to ensure that a 10G link does not go idle
  94. * while a TX queue is stopped after it has become full. A queue is
  95. * restarted when it drops below half full. The time this takes (assuming
  96. * worst case 3 descriptors per packet and 1024 descriptors) is
  97. * 512 / 3 * 1.2 = 205 usec.
  98. */
  99. static unsigned int tx_irq_mod_usec = 150;
  100. /* This is the first interrupt mode to try out of:
  101. * 0 => MSI-X
  102. * 1 => MSI
  103. * 2 => legacy
  104. */
  105. static unsigned int interrupt_mode;
  106. /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  107. * i.e. the number of CPUs among which we may distribute simultaneous
  108. * interrupt handling.
  109. *
  110. * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  111. * The default (0) means to assign an interrupt to each package (level II cache)
  112. */
  113. static unsigned int rss_cpus;
  114. module_param(rss_cpus, uint, 0444);
  115. MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
  116. /**************************************************************************
  117. *
  118. * Utility functions and prototypes
  119. *
  120. *************************************************************************/
  121. static void efx_remove_channel(struct efx_channel *channel);
  122. static void efx_remove_port(struct efx_nic *efx);
  123. static void efx_fini_napi(struct efx_nic *efx);
  124. static void efx_fini_channels(struct efx_nic *efx);
  125. #define EFX_ASSERT_RESET_SERIALISED(efx) \
  126. do { \
  127. if ((efx->state == STATE_RUNNING) || \
  128. (efx->state == STATE_RESETTING)) \
  129. ASSERT_RTNL(); \
  130. } while (0)
  131. /**************************************************************************
  132. *
  133. * Event queue processing
  134. *
  135. *************************************************************************/
  136. /* Process channel's event queue
  137. *
  138. * This function is responsible for processing the event queue of a
  139. * single channel. The caller must guarantee that this function will
  140. * never be concurrently called more than once on the same channel,
  141. * though different channels may be being processed concurrently.
  142. */
  143. static inline int efx_process_channel(struct efx_channel *channel, int rx_quota)
  144. {
  145. int rxdmaqs;
  146. struct efx_rx_queue *rx_queue;
  147. if (unlikely(channel->efx->reset_pending != RESET_TYPE_NONE ||
  148. !channel->enabled))
  149. return rx_quota;
  150. rxdmaqs = falcon_process_eventq(channel, &rx_quota);
  151. /* Deliver last RX packet. */
  152. if (channel->rx_pkt) {
  153. __efx_rx_packet(channel, channel->rx_pkt,
  154. channel->rx_pkt_csummed);
  155. channel->rx_pkt = NULL;
  156. }
  157. efx_flush_lro(channel);
  158. efx_rx_strategy(channel);
  159. /* Refill descriptor rings as necessary */
  160. rx_queue = &channel->efx->rx_queue[0];
  161. while (rxdmaqs) {
  162. if (rxdmaqs & 0x01)
  163. efx_fast_push_rx_descriptors(rx_queue);
  164. rx_queue++;
  165. rxdmaqs >>= 1;
  166. }
  167. return rx_quota;
  168. }
  169. /* Mark channel as finished processing
  170. *
  171. * Note that since we will not receive further interrupts for this
  172. * channel before we finish processing and call the eventq_read_ack()
  173. * method, there is no need to use the interrupt hold-off timers.
  174. */
  175. static inline void efx_channel_processed(struct efx_channel *channel)
  176. {
  177. /* Write to EVQ_RPTR_REG. If a new event arrived in a race
  178. * with finishing processing, a new interrupt will be raised.
  179. */
  180. channel->work_pending = 0;
  181. smp_wmb(); /* Ensure channel updated before any new interrupt. */
  182. falcon_eventq_read_ack(channel);
  183. }
  184. /* NAPI poll handler
  185. *
  186. * NAPI guarantees serialisation of polls of the same device, which
  187. * provides the guarantee required by efx_process_channel().
  188. */
  189. static int efx_poll(struct napi_struct *napi, int budget)
  190. {
  191. struct efx_channel *channel =
  192. container_of(napi, struct efx_channel, napi_str);
  193. struct net_device *napi_dev = channel->napi_dev;
  194. int unused;
  195. int rx_packets;
  196. EFX_TRACE(channel->efx, "channel %d NAPI poll executing on CPU %d\n",
  197. channel->channel, raw_smp_processor_id());
  198. unused = efx_process_channel(channel, budget);
  199. rx_packets = (budget - unused);
  200. if (rx_packets < budget) {
  201. /* There is no race here; although napi_disable() will
  202. * only wait for netif_rx_complete(), this isn't a problem
  203. * since efx_channel_processed() will have no effect if
  204. * interrupts have already been disabled.
  205. */
  206. netif_rx_complete(napi_dev, napi);
  207. efx_channel_processed(channel);
  208. }
  209. return rx_packets;
  210. }
  211. /* Process the eventq of the specified channel immediately on this CPU
  212. *
  213. * Disable hardware generated interrupts, wait for any existing
  214. * processing to finish, then directly poll (and ack ) the eventq.
  215. * Finally reenable NAPI and interrupts.
  216. *
  217. * Since we are touching interrupts the caller should hold the suspend lock
  218. */
  219. void efx_process_channel_now(struct efx_channel *channel)
  220. {
  221. struct efx_nic *efx = channel->efx;
  222. BUG_ON(!channel->used_flags);
  223. BUG_ON(!channel->enabled);
  224. /* Disable interrupts and wait for ISRs to complete */
  225. falcon_disable_interrupts(efx);
  226. if (efx->legacy_irq)
  227. synchronize_irq(efx->legacy_irq);
  228. if (channel->has_interrupt && channel->irq)
  229. synchronize_irq(channel->irq);
  230. /* Wait for any NAPI processing to complete */
  231. napi_disable(&channel->napi_str);
  232. /* Poll the channel */
  233. (void) efx_process_channel(channel, efx->type->evq_size);
  234. /* Ack the eventq. This may cause an interrupt to be generated
  235. * when they are reenabled */
  236. efx_channel_processed(channel);
  237. napi_enable(&channel->napi_str);
  238. falcon_enable_interrupts(efx);
  239. }
  240. /* Create event queue
  241. * Event queue memory allocations are done only once. If the channel
  242. * is reset, the memory buffer will be reused; this guards against
  243. * errors during channel reset and also simplifies interrupt handling.
  244. */
  245. static int efx_probe_eventq(struct efx_channel *channel)
  246. {
  247. EFX_LOG(channel->efx, "chan %d create event queue\n", channel->channel);
  248. return falcon_probe_eventq(channel);
  249. }
  250. /* Prepare channel's event queue */
  251. static int efx_init_eventq(struct efx_channel *channel)
  252. {
  253. EFX_LOG(channel->efx, "chan %d init event queue\n", channel->channel);
  254. channel->eventq_read_ptr = 0;
  255. return falcon_init_eventq(channel);
  256. }
  257. static void efx_fini_eventq(struct efx_channel *channel)
  258. {
  259. EFX_LOG(channel->efx, "chan %d fini event queue\n", channel->channel);
  260. falcon_fini_eventq(channel);
  261. }
  262. static void efx_remove_eventq(struct efx_channel *channel)
  263. {
  264. EFX_LOG(channel->efx, "chan %d remove event queue\n", channel->channel);
  265. falcon_remove_eventq(channel);
  266. }
  267. /**************************************************************************
  268. *
  269. * Channel handling
  270. *
  271. *************************************************************************/
  272. /* Setup per-NIC RX buffer parameters.
  273. * Calculate the rx buffer allocation parameters required to support
  274. * the current MTU, including padding for header alignment and overruns.
  275. */
  276. static void efx_calc_rx_buffer_params(struct efx_nic *efx)
  277. {
  278. unsigned int order, len;
  279. len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +
  280. EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
  281. efx->type->rx_buffer_padding);
  282. /* Calculate page-order */
  283. for (order = 0; ((1u << order) * PAGE_SIZE) < len; ++order)
  284. ;
  285. efx->rx_buffer_len = len;
  286. efx->rx_buffer_order = order;
  287. }
  288. static int efx_probe_channel(struct efx_channel *channel)
  289. {
  290. struct efx_tx_queue *tx_queue;
  291. struct efx_rx_queue *rx_queue;
  292. int rc;
  293. EFX_LOG(channel->efx, "creating channel %d\n", channel->channel);
  294. rc = efx_probe_eventq(channel);
  295. if (rc)
  296. goto fail1;
  297. efx_for_each_channel_tx_queue(tx_queue, channel) {
  298. rc = efx_probe_tx_queue(tx_queue);
  299. if (rc)
  300. goto fail2;
  301. }
  302. efx_for_each_channel_rx_queue(rx_queue, channel) {
  303. rc = efx_probe_rx_queue(rx_queue);
  304. if (rc)
  305. goto fail3;
  306. }
  307. channel->n_rx_frm_trunc = 0;
  308. return 0;
  309. fail3:
  310. efx_for_each_channel_rx_queue(rx_queue, channel)
  311. efx_remove_rx_queue(rx_queue);
  312. fail2:
  313. efx_for_each_channel_tx_queue(tx_queue, channel)
  314. efx_remove_tx_queue(tx_queue);
  315. fail1:
  316. return rc;
  317. }
  318. /* Channels are shutdown and reinitialised whilst the NIC is running
  319. * to propagate configuration changes (mtu, checksum offload), or
  320. * to clear hardware error conditions
  321. */
  322. static int efx_init_channels(struct efx_nic *efx)
  323. {
  324. struct efx_tx_queue *tx_queue;
  325. struct efx_rx_queue *rx_queue;
  326. struct efx_channel *channel;
  327. int rc = 0;
  328. efx_calc_rx_buffer_params(efx);
  329. /* Initialise the channels */
  330. efx_for_each_channel(channel, efx) {
  331. EFX_LOG(channel->efx, "init chan %d\n", channel->channel);
  332. rc = efx_init_eventq(channel);
  333. if (rc)
  334. goto err;
  335. efx_for_each_channel_tx_queue(tx_queue, channel) {
  336. rc = efx_init_tx_queue(tx_queue);
  337. if (rc)
  338. goto err;
  339. }
  340. /* The rx buffer allocation strategy is MTU dependent */
  341. efx_rx_strategy(channel);
  342. efx_for_each_channel_rx_queue(rx_queue, channel) {
  343. rc = efx_init_rx_queue(rx_queue);
  344. if (rc)
  345. goto err;
  346. }
  347. WARN_ON(channel->rx_pkt != NULL);
  348. efx_rx_strategy(channel);
  349. }
  350. return 0;
  351. err:
  352. EFX_ERR(efx, "failed to initialise channel %d\n",
  353. channel ? channel->channel : -1);
  354. efx_fini_channels(efx);
  355. return rc;
  356. }
  357. /* This enables event queue processing and packet transmission.
  358. *
  359. * Note that this function is not allowed to fail, since that would
  360. * introduce too much complexity into the suspend/resume path.
  361. */
  362. static void efx_start_channel(struct efx_channel *channel)
  363. {
  364. struct efx_rx_queue *rx_queue;
  365. EFX_LOG(channel->efx, "starting chan %d\n", channel->channel);
  366. if (!(channel->efx->net_dev->flags & IFF_UP))
  367. netif_napi_add(channel->napi_dev, &channel->napi_str,
  368. efx_poll, napi_weight);
  369. channel->work_pending = 0;
  370. channel->enabled = 1;
  371. smp_wmb(); /* ensure channel updated before first interrupt */
  372. napi_enable(&channel->napi_str);
  373. /* Load up RX descriptors */
  374. efx_for_each_channel_rx_queue(rx_queue, channel)
  375. efx_fast_push_rx_descriptors(rx_queue);
  376. }
  377. /* This disables event queue processing and packet transmission.
  378. * This function does not guarantee that all queue processing
  379. * (e.g. RX refill) is complete.
  380. */
  381. static void efx_stop_channel(struct efx_channel *channel)
  382. {
  383. struct efx_rx_queue *rx_queue;
  384. if (!channel->enabled)
  385. return;
  386. EFX_LOG(channel->efx, "stop chan %d\n", channel->channel);
  387. channel->enabled = 0;
  388. napi_disable(&channel->napi_str);
  389. /* Ensure that any worker threads have exited or will be no-ops */
  390. efx_for_each_channel_rx_queue(rx_queue, channel) {
  391. spin_lock_bh(&rx_queue->add_lock);
  392. spin_unlock_bh(&rx_queue->add_lock);
  393. }
  394. }
  395. static void efx_fini_channels(struct efx_nic *efx)
  396. {
  397. struct efx_channel *channel;
  398. struct efx_tx_queue *tx_queue;
  399. struct efx_rx_queue *rx_queue;
  400. EFX_ASSERT_RESET_SERIALISED(efx);
  401. BUG_ON(efx->port_enabled);
  402. efx_for_each_channel(channel, efx) {
  403. EFX_LOG(channel->efx, "shut down chan %d\n", channel->channel);
  404. efx_for_each_channel_rx_queue(rx_queue, channel)
  405. efx_fini_rx_queue(rx_queue);
  406. efx_for_each_channel_tx_queue(tx_queue, channel)
  407. efx_fini_tx_queue(tx_queue);
  408. }
  409. /* Do the event queues last so that we can handle flush events
  410. * for all DMA queues. */
  411. efx_for_each_channel(channel, efx) {
  412. EFX_LOG(channel->efx, "shut down evq %d\n", channel->channel);
  413. efx_fini_eventq(channel);
  414. }
  415. }
  416. static void efx_remove_channel(struct efx_channel *channel)
  417. {
  418. struct efx_tx_queue *tx_queue;
  419. struct efx_rx_queue *rx_queue;
  420. EFX_LOG(channel->efx, "destroy chan %d\n", channel->channel);
  421. efx_for_each_channel_rx_queue(rx_queue, channel)
  422. efx_remove_rx_queue(rx_queue);
  423. efx_for_each_channel_tx_queue(tx_queue, channel)
  424. efx_remove_tx_queue(tx_queue);
  425. efx_remove_eventq(channel);
  426. channel->used_flags = 0;
  427. }
  428. void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue, int delay)
  429. {
  430. queue_delayed_work(refill_workqueue, &rx_queue->work, delay);
  431. }
  432. /**************************************************************************
  433. *
  434. * Port handling
  435. *
  436. **************************************************************************/
  437. /* This ensures that the kernel is kept informed (via
  438. * netif_carrier_on/off) of the link status, and also maintains the
  439. * link status's stop on the port's TX queue.
  440. */
  441. static void efx_link_status_changed(struct efx_nic *efx)
  442. {
  443. int carrier_ok;
  444. /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
  445. * that no events are triggered between unregister_netdev() and the
  446. * driver unloading. A more general condition is that NETDEV_CHANGE
  447. * can only be generated between NETDEV_UP and NETDEV_DOWN */
  448. if (!netif_running(efx->net_dev))
  449. return;
  450. carrier_ok = netif_carrier_ok(efx->net_dev) ? 1 : 0;
  451. if (efx->link_up != carrier_ok) {
  452. efx->n_link_state_changes++;
  453. if (efx->link_up)
  454. netif_carrier_on(efx->net_dev);
  455. else
  456. netif_carrier_off(efx->net_dev);
  457. }
  458. /* Status message for kernel log */
  459. if (efx->link_up) {
  460. struct mii_if_info *gmii = &efx->mii;
  461. unsigned adv, lpa;
  462. /* NONE here means direct XAUI from the controller, with no
  463. * MDIO-attached device we can query. */
  464. if (efx->phy_type != PHY_TYPE_NONE) {
  465. adv = gmii_advertised(gmii);
  466. lpa = gmii_lpa(gmii);
  467. } else {
  468. lpa = GM_LPA_10000 | LPA_DUPLEX;
  469. adv = lpa;
  470. }
  471. EFX_INFO(efx, "link up at %dMbps %s-duplex "
  472. "(adv %04x lpa %04x) (MTU %d)%s\n",
  473. (efx->link_options & GM_LPA_10000 ? 10000 :
  474. (efx->link_options & GM_LPA_1000 ? 1000 :
  475. (efx->link_options & GM_LPA_100 ? 100 :
  476. 10))),
  477. (efx->link_options & GM_LPA_DUPLEX ?
  478. "full" : "half"),
  479. adv, lpa,
  480. efx->net_dev->mtu,
  481. (efx->promiscuous ? " [PROMISC]" : ""));
  482. } else {
  483. EFX_INFO(efx, "link down\n");
  484. }
  485. }
  486. /* This call reinitialises the MAC to pick up new PHY settings. The
  487. * caller must hold the mac_lock */
  488. static void __efx_reconfigure_port(struct efx_nic *efx)
  489. {
  490. WARN_ON(!mutex_is_locked(&efx->mac_lock));
  491. EFX_LOG(efx, "reconfiguring MAC from PHY settings on CPU %d\n",
  492. raw_smp_processor_id());
  493. falcon_reconfigure_xmac(efx);
  494. /* Inform kernel of loss/gain of carrier */
  495. efx_link_status_changed(efx);
  496. }
  497. /* Reinitialise the MAC to pick up new PHY settings, even if the port is
  498. * disabled. */
  499. void efx_reconfigure_port(struct efx_nic *efx)
  500. {
  501. EFX_ASSERT_RESET_SERIALISED(efx);
  502. mutex_lock(&efx->mac_lock);
  503. __efx_reconfigure_port(efx);
  504. mutex_unlock(&efx->mac_lock);
  505. }
  506. /* Asynchronous efx_reconfigure_port work item. To speed up efx_flush_all()
  507. * we don't efx_reconfigure_port() if the port is disabled. Care is taken
  508. * in efx_stop_all() and efx_start_port() to prevent PHY events being lost */
  509. static void efx_reconfigure_work(struct work_struct *data)
  510. {
  511. struct efx_nic *efx = container_of(data, struct efx_nic,
  512. reconfigure_work);
  513. mutex_lock(&efx->mac_lock);
  514. if (efx->port_enabled)
  515. __efx_reconfigure_port(efx);
  516. mutex_unlock(&efx->mac_lock);
  517. }
  518. static int efx_probe_port(struct efx_nic *efx)
  519. {
  520. int rc;
  521. EFX_LOG(efx, "create port\n");
  522. /* Connect up MAC/PHY operations table and read MAC address */
  523. rc = falcon_probe_port(efx);
  524. if (rc)
  525. goto err;
  526. /* Sanity check MAC address */
  527. if (is_valid_ether_addr(efx->mac_address)) {
  528. memcpy(efx->net_dev->dev_addr, efx->mac_address, ETH_ALEN);
  529. } else {
  530. DECLARE_MAC_BUF(mac);
  531. EFX_ERR(efx, "invalid MAC address %s\n",
  532. print_mac(mac, efx->mac_address));
  533. if (!allow_bad_hwaddr) {
  534. rc = -EINVAL;
  535. goto err;
  536. }
  537. random_ether_addr(efx->net_dev->dev_addr);
  538. EFX_INFO(efx, "using locally-generated MAC %s\n",
  539. print_mac(mac, efx->net_dev->dev_addr));
  540. }
  541. return 0;
  542. err:
  543. efx_remove_port(efx);
  544. return rc;
  545. }
  546. static int efx_init_port(struct efx_nic *efx)
  547. {
  548. int rc;
  549. EFX_LOG(efx, "init port\n");
  550. /* Initialise the MAC and PHY */
  551. rc = falcon_init_xmac(efx);
  552. if (rc)
  553. return rc;
  554. efx->port_initialized = 1;
  555. /* Reconfigure port to program MAC registers */
  556. falcon_reconfigure_xmac(efx);
  557. return 0;
  558. }
  559. /* Allow efx_reconfigure_port() to be scheduled, and close the window
  560. * between efx_stop_port and efx_flush_all whereby a previously scheduled
  561. * efx_reconfigure_port() may have been cancelled */
  562. static void efx_start_port(struct efx_nic *efx)
  563. {
  564. EFX_LOG(efx, "start port\n");
  565. BUG_ON(efx->port_enabled);
  566. mutex_lock(&efx->mac_lock);
  567. efx->port_enabled = 1;
  568. __efx_reconfigure_port(efx);
  569. mutex_unlock(&efx->mac_lock);
  570. }
  571. /* Prevent efx_reconfigure_work and efx_monitor() from executing, and
  572. * efx_set_multicast_list() from scheduling efx_reconfigure_work.
  573. * efx_reconfigure_work can still be scheduled via NAPI processing
  574. * until efx_flush_all() is called */
  575. static void efx_stop_port(struct efx_nic *efx)
  576. {
  577. EFX_LOG(efx, "stop port\n");
  578. mutex_lock(&efx->mac_lock);
  579. efx->port_enabled = 0;
  580. mutex_unlock(&efx->mac_lock);
  581. /* Serialise against efx_set_multicast_list() */
  582. if (NET_DEV_REGISTERED(efx)) {
  583. netif_tx_lock_bh(efx->net_dev);
  584. netif_tx_unlock_bh(efx->net_dev);
  585. }
  586. }
  587. static void efx_fini_port(struct efx_nic *efx)
  588. {
  589. EFX_LOG(efx, "shut down port\n");
  590. if (!efx->port_initialized)
  591. return;
  592. falcon_fini_xmac(efx);
  593. efx->port_initialized = 0;
  594. efx->link_up = 0;
  595. efx_link_status_changed(efx);
  596. }
  597. static void efx_remove_port(struct efx_nic *efx)
  598. {
  599. EFX_LOG(efx, "destroying port\n");
  600. falcon_remove_port(efx);
  601. }
  602. /**************************************************************************
  603. *
  604. * NIC handling
  605. *
  606. **************************************************************************/
  607. /* This configures the PCI device to enable I/O and DMA. */
  608. static int efx_init_io(struct efx_nic *efx)
  609. {
  610. struct pci_dev *pci_dev = efx->pci_dev;
  611. dma_addr_t dma_mask = efx->type->max_dma_mask;
  612. int rc;
  613. EFX_LOG(efx, "initialising I/O\n");
  614. rc = pci_enable_device(pci_dev);
  615. if (rc) {
  616. EFX_ERR(efx, "failed to enable PCI device\n");
  617. goto fail1;
  618. }
  619. pci_set_master(pci_dev);
  620. /* Set the PCI DMA mask. Try all possibilities from our
  621. * genuine mask down to 32 bits, because some architectures
  622. * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
  623. * masks event though they reject 46 bit masks.
  624. */
  625. while (dma_mask > 0x7fffffffUL) {
  626. if (pci_dma_supported(pci_dev, dma_mask) &&
  627. ((rc = pci_set_dma_mask(pci_dev, dma_mask)) == 0))
  628. break;
  629. dma_mask >>= 1;
  630. }
  631. if (rc) {
  632. EFX_ERR(efx, "could not find a suitable DMA mask\n");
  633. goto fail2;
  634. }
  635. EFX_LOG(efx, "using DMA mask %llx\n", (unsigned long long) dma_mask);
  636. rc = pci_set_consistent_dma_mask(pci_dev, dma_mask);
  637. if (rc) {
  638. /* pci_set_consistent_dma_mask() is not *allowed* to
  639. * fail with a mask that pci_set_dma_mask() accepted,
  640. * but just in case...
  641. */
  642. EFX_ERR(efx, "failed to set consistent DMA mask\n");
  643. goto fail2;
  644. }
  645. efx->membase_phys = pci_resource_start(efx->pci_dev,
  646. efx->type->mem_bar);
  647. rc = pci_request_region(pci_dev, efx->type->mem_bar, "sfc");
  648. if (rc) {
  649. EFX_ERR(efx, "request for memory BAR failed\n");
  650. rc = -EIO;
  651. goto fail3;
  652. }
  653. efx->membase = ioremap_nocache(efx->membase_phys,
  654. efx->type->mem_map_size);
  655. if (!efx->membase) {
  656. EFX_ERR(efx, "could not map memory BAR %d at %lx+%x\n",
  657. efx->type->mem_bar, efx->membase_phys,
  658. efx->type->mem_map_size);
  659. rc = -ENOMEM;
  660. goto fail4;
  661. }
  662. EFX_LOG(efx, "memory BAR %u at %lx+%x (virtual %p)\n",
  663. efx->type->mem_bar, efx->membase_phys, efx->type->mem_map_size,
  664. efx->membase);
  665. return 0;
  666. fail4:
  667. release_mem_region(efx->membase_phys, efx->type->mem_map_size);
  668. fail3:
  669. efx->membase_phys = 0UL;
  670. fail2:
  671. pci_disable_device(efx->pci_dev);
  672. fail1:
  673. return rc;
  674. }
  675. static void efx_fini_io(struct efx_nic *efx)
  676. {
  677. EFX_LOG(efx, "shutting down I/O\n");
  678. if (efx->membase) {
  679. iounmap(efx->membase);
  680. efx->membase = NULL;
  681. }
  682. if (efx->membase_phys) {
  683. pci_release_region(efx->pci_dev, efx->type->mem_bar);
  684. efx->membase_phys = 0UL;
  685. }
  686. pci_disable_device(efx->pci_dev);
  687. }
  688. /* Probe the number and type of interrupts we are able to obtain. */
  689. static void efx_probe_interrupts(struct efx_nic *efx)
  690. {
  691. int max_channel = efx->type->phys_addr_channels - 1;
  692. struct msix_entry xentries[EFX_MAX_CHANNELS];
  693. int rc, i;
  694. if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
  695. BUG_ON(!pci_find_capability(efx->pci_dev, PCI_CAP_ID_MSIX));
  696. efx->rss_queues = rss_cpus ? rss_cpus : num_online_cpus();
  697. efx->rss_queues = min(efx->rss_queues, max_channel + 1);
  698. efx->rss_queues = min(efx->rss_queues, EFX_MAX_CHANNELS);
  699. /* Request maximum number of MSI interrupts, and fill out
  700. * the channel interrupt information the allowed allocation */
  701. for (i = 0; i < efx->rss_queues; i++)
  702. xentries[i].entry = i;
  703. rc = pci_enable_msix(efx->pci_dev, xentries, efx->rss_queues);
  704. if (rc > 0) {
  705. EFX_BUG_ON_PARANOID(rc >= efx->rss_queues);
  706. efx->rss_queues = rc;
  707. rc = pci_enable_msix(efx->pci_dev, xentries,
  708. efx->rss_queues);
  709. }
  710. if (rc == 0) {
  711. for (i = 0; i < efx->rss_queues; i++) {
  712. efx->channel[i].has_interrupt = 1;
  713. efx->channel[i].irq = xentries[i].vector;
  714. }
  715. } else {
  716. /* Fall back to single channel MSI */
  717. efx->interrupt_mode = EFX_INT_MODE_MSI;
  718. EFX_ERR(efx, "could not enable MSI-X\n");
  719. }
  720. }
  721. /* Try single interrupt MSI */
  722. if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
  723. efx->rss_queues = 1;
  724. rc = pci_enable_msi(efx->pci_dev);
  725. if (rc == 0) {
  726. efx->channel[0].irq = efx->pci_dev->irq;
  727. efx->channel[0].has_interrupt = 1;
  728. } else {
  729. EFX_ERR(efx, "could not enable MSI\n");
  730. efx->interrupt_mode = EFX_INT_MODE_LEGACY;
  731. }
  732. }
  733. /* Assume legacy interrupts */
  734. if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
  735. efx->rss_queues = 1;
  736. /* Every channel is interruptible */
  737. for (i = 0; i < EFX_MAX_CHANNELS; i++)
  738. efx->channel[i].has_interrupt = 1;
  739. efx->legacy_irq = efx->pci_dev->irq;
  740. }
  741. }
  742. static void efx_remove_interrupts(struct efx_nic *efx)
  743. {
  744. struct efx_channel *channel;
  745. /* Remove MSI/MSI-X interrupts */
  746. efx_for_each_channel_with_interrupt(channel, efx)
  747. channel->irq = 0;
  748. pci_disable_msi(efx->pci_dev);
  749. pci_disable_msix(efx->pci_dev);
  750. /* Remove legacy interrupt */
  751. efx->legacy_irq = 0;
  752. }
  753. /* Select number of used resources
  754. * Should be called after probe_interrupts()
  755. */
  756. static void efx_select_used(struct efx_nic *efx)
  757. {
  758. struct efx_tx_queue *tx_queue;
  759. struct efx_rx_queue *rx_queue;
  760. int i;
  761. /* TX queues. One per port per channel with TX capability
  762. * (more than one per port won't work on Linux, due to out
  763. * of order issues... but will be fine on Solaris)
  764. */
  765. tx_queue = &efx->tx_queue[0];
  766. /* Perform this for each channel with TX capabilities.
  767. * At the moment, we only support a single TX queue
  768. */
  769. tx_queue->used = 1;
  770. if ((!EFX_INT_MODE_USE_MSI(efx)) && separate_tx_and_rx_channels)
  771. tx_queue->channel = &efx->channel[1];
  772. else
  773. tx_queue->channel = &efx->channel[0];
  774. tx_queue->channel->used_flags |= EFX_USED_BY_TX;
  775. tx_queue++;
  776. /* RX queues. Each has a dedicated channel. */
  777. for (i = 0; i < EFX_MAX_RX_QUEUES; i++) {
  778. rx_queue = &efx->rx_queue[i];
  779. if (i < efx->rss_queues) {
  780. rx_queue->used = 1;
  781. /* If we allow multiple RX queues per channel
  782. * we need to decide that here
  783. */
  784. rx_queue->channel = &efx->channel[rx_queue->queue];
  785. rx_queue->channel->used_flags |= EFX_USED_BY_RX;
  786. rx_queue++;
  787. }
  788. }
  789. }
  790. static int efx_probe_nic(struct efx_nic *efx)
  791. {
  792. int rc;
  793. EFX_LOG(efx, "creating NIC\n");
  794. /* Carry out hardware-type specific initialisation */
  795. rc = falcon_probe_nic(efx);
  796. if (rc)
  797. return rc;
  798. /* Determine the number of channels and RX queues by trying to hook
  799. * in MSI-X interrupts. */
  800. efx_probe_interrupts(efx);
  801. /* Determine number of RX queues and TX queues */
  802. efx_select_used(efx);
  803. /* Initialise the interrupt moderation settings */
  804. efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec);
  805. return 0;
  806. }
  807. static void efx_remove_nic(struct efx_nic *efx)
  808. {
  809. EFX_LOG(efx, "destroying NIC\n");
  810. efx_remove_interrupts(efx);
  811. falcon_remove_nic(efx);
  812. }
  813. /**************************************************************************
  814. *
  815. * NIC startup/shutdown
  816. *
  817. *************************************************************************/
  818. static int efx_probe_all(struct efx_nic *efx)
  819. {
  820. struct efx_channel *channel;
  821. int rc;
  822. /* Create NIC */
  823. rc = efx_probe_nic(efx);
  824. if (rc) {
  825. EFX_ERR(efx, "failed to create NIC\n");
  826. goto fail1;
  827. }
  828. /* Create port */
  829. rc = efx_probe_port(efx);
  830. if (rc) {
  831. EFX_ERR(efx, "failed to create port\n");
  832. goto fail2;
  833. }
  834. /* Create channels */
  835. efx_for_each_channel(channel, efx) {
  836. rc = efx_probe_channel(channel);
  837. if (rc) {
  838. EFX_ERR(efx, "failed to create channel %d\n",
  839. channel->channel);
  840. goto fail3;
  841. }
  842. }
  843. return 0;
  844. fail3:
  845. efx_for_each_channel(channel, efx)
  846. efx_remove_channel(channel);
  847. efx_remove_port(efx);
  848. fail2:
  849. efx_remove_nic(efx);
  850. fail1:
  851. return rc;
  852. }
  853. /* Called after previous invocation(s) of efx_stop_all, restarts the
  854. * port, kernel transmit queue, NAPI processing and hardware interrupts,
  855. * and ensures that the port is scheduled to be reconfigured.
  856. * This function is safe to call multiple times when the NIC is in any
  857. * state. */
  858. static void efx_start_all(struct efx_nic *efx)
  859. {
  860. struct efx_channel *channel;
  861. EFX_ASSERT_RESET_SERIALISED(efx);
  862. /* Check that it is appropriate to restart the interface. All
  863. * of these flags are safe to read under just the rtnl lock */
  864. if (efx->port_enabled)
  865. return;
  866. if ((efx->state != STATE_RUNNING) && (efx->state != STATE_INIT))
  867. return;
  868. if (NET_DEV_REGISTERED(efx) && !netif_running(efx->net_dev))
  869. return;
  870. /* Mark the port as enabled so port reconfigurations can start, then
  871. * restart the transmit interface early so the watchdog timer stops */
  872. efx_start_port(efx);
  873. efx_wake_queue(efx);
  874. efx_for_each_channel(channel, efx)
  875. efx_start_channel(channel);
  876. falcon_enable_interrupts(efx);
  877. /* Start hardware monitor if we're in RUNNING */
  878. if (efx->state == STATE_RUNNING)
  879. queue_delayed_work(efx->workqueue, &efx->monitor_work,
  880. efx_monitor_interval);
  881. }
  882. /* Flush all delayed work. Should only be called when no more delayed work
  883. * will be scheduled. This doesn't flush pending online resets (efx_reset),
  884. * since we're holding the rtnl_lock at this point. */
  885. static void efx_flush_all(struct efx_nic *efx)
  886. {
  887. struct efx_rx_queue *rx_queue;
  888. /* Make sure the hardware monitor is stopped */
  889. cancel_delayed_work_sync(&efx->monitor_work);
  890. /* Ensure that all RX slow refills are complete. */
  891. efx_for_each_rx_queue(rx_queue, efx) {
  892. cancel_delayed_work_sync(&rx_queue->work);
  893. }
  894. /* Stop scheduled port reconfigurations */
  895. cancel_work_sync(&efx->reconfigure_work);
  896. }
  897. /* Quiesce hardware and software without bringing the link down.
  898. * Safe to call multiple times, when the nic and interface is in any
  899. * state. The caller is guaranteed to subsequently be in a position
  900. * to modify any hardware and software state they see fit without
  901. * taking locks. */
  902. static void efx_stop_all(struct efx_nic *efx)
  903. {
  904. struct efx_channel *channel;
  905. EFX_ASSERT_RESET_SERIALISED(efx);
  906. /* port_enabled can be read safely under the rtnl lock */
  907. if (!efx->port_enabled)
  908. return;
  909. /* Disable interrupts and wait for ISR to complete */
  910. falcon_disable_interrupts(efx);
  911. if (efx->legacy_irq)
  912. synchronize_irq(efx->legacy_irq);
  913. efx_for_each_channel_with_interrupt(channel, efx)
  914. if (channel->irq)
  915. synchronize_irq(channel->irq);
  916. /* Stop all NAPI processing and synchronous rx refills */
  917. efx_for_each_channel(channel, efx)
  918. efx_stop_channel(channel);
  919. /* Stop all asynchronous port reconfigurations. Since all
  920. * event processing has already been stopped, there is no
  921. * window to loose phy events */
  922. efx_stop_port(efx);
  923. /* Flush reconfigure_work, refill_workqueue, monitor_work */
  924. efx_flush_all(efx);
  925. /* Isolate the MAC from the TX and RX engines, so that queue
  926. * flushes will complete in a timely fashion. */
  927. falcon_deconfigure_mac_wrapper(efx);
  928. falcon_drain_tx_fifo(efx);
  929. /* Stop the kernel transmit interface late, so the watchdog
  930. * timer isn't ticking over the flush */
  931. efx_stop_queue(efx);
  932. if (NET_DEV_REGISTERED(efx)) {
  933. netif_tx_lock_bh(efx->net_dev);
  934. netif_tx_unlock_bh(efx->net_dev);
  935. }
  936. }
  937. static void efx_remove_all(struct efx_nic *efx)
  938. {
  939. struct efx_channel *channel;
  940. efx_for_each_channel(channel, efx)
  941. efx_remove_channel(channel);
  942. efx_remove_port(efx);
  943. efx_remove_nic(efx);
  944. }
  945. /* A convinience function to safely flush all the queues */
  946. int efx_flush_queues(struct efx_nic *efx)
  947. {
  948. int rc;
  949. EFX_ASSERT_RESET_SERIALISED(efx);
  950. efx_stop_all(efx);
  951. efx_fini_channels(efx);
  952. rc = efx_init_channels(efx);
  953. if (rc) {
  954. efx_schedule_reset(efx, RESET_TYPE_DISABLE);
  955. return rc;
  956. }
  957. efx_start_all(efx);
  958. return 0;
  959. }
  960. /**************************************************************************
  961. *
  962. * Interrupt moderation
  963. *
  964. **************************************************************************/
  965. /* Set interrupt moderation parameters */
  966. void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs)
  967. {
  968. struct efx_tx_queue *tx_queue;
  969. struct efx_rx_queue *rx_queue;
  970. EFX_ASSERT_RESET_SERIALISED(efx);
  971. efx_for_each_tx_queue(tx_queue, efx)
  972. tx_queue->channel->irq_moderation = tx_usecs;
  973. efx_for_each_rx_queue(rx_queue, efx)
  974. rx_queue->channel->irq_moderation = rx_usecs;
  975. }
  976. /**************************************************************************
  977. *
  978. * Hardware monitor
  979. *
  980. **************************************************************************/
  981. /* Run periodically off the general workqueue. Serialised against
  982. * efx_reconfigure_port via the mac_lock */
  983. static void efx_monitor(struct work_struct *data)
  984. {
  985. struct efx_nic *efx = container_of(data, struct efx_nic,
  986. monitor_work.work);
  987. int rc = 0;
  988. EFX_TRACE(efx, "hardware monitor executing on CPU %d\n",
  989. raw_smp_processor_id());
  990. /* If the mac_lock is already held then it is likely a port
  991. * reconfiguration is already in place, which will likely do
  992. * most of the work of check_hw() anyway. */
  993. if (!mutex_trylock(&efx->mac_lock)) {
  994. queue_delayed_work(efx->workqueue, &efx->monitor_work,
  995. efx_monitor_interval);
  996. return;
  997. }
  998. if (efx->port_enabled)
  999. rc = falcon_check_xmac(efx);
  1000. mutex_unlock(&efx->mac_lock);
  1001. if (rc) {
  1002. if (monitor_reset) {
  1003. EFX_ERR(efx, "hardware monitor detected a fault: "
  1004. "triggering reset\n");
  1005. efx_schedule_reset(efx, RESET_TYPE_MONITOR);
  1006. } else {
  1007. EFX_ERR(efx, "hardware monitor detected a fault, "
  1008. "skipping reset\n");
  1009. }
  1010. }
  1011. queue_delayed_work(efx->workqueue, &efx->monitor_work,
  1012. efx_monitor_interval);
  1013. }
  1014. /**************************************************************************
  1015. *
  1016. * ioctls
  1017. *
  1018. *************************************************************************/
  1019. /* Net device ioctl
  1020. * Context: process, rtnl_lock() held.
  1021. */
  1022. static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
  1023. {
  1024. struct efx_nic *efx = net_dev->priv;
  1025. EFX_ASSERT_RESET_SERIALISED(efx);
  1026. return generic_mii_ioctl(&efx->mii, if_mii(ifr), cmd, NULL);
  1027. }
  1028. /**************************************************************************
  1029. *
  1030. * NAPI interface
  1031. *
  1032. **************************************************************************/
  1033. static int efx_init_napi(struct efx_nic *efx)
  1034. {
  1035. struct efx_channel *channel;
  1036. int rc;
  1037. efx_for_each_channel(channel, efx) {
  1038. channel->napi_dev = efx->net_dev;
  1039. rc = efx_lro_init(&channel->lro_mgr, efx);
  1040. if (rc)
  1041. goto err;
  1042. }
  1043. return 0;
  1044. err:
  1045. efx_fini_napi(efx);
  1046. return rc;
  1047. }
  1048. static void efx_fini_napi(struct efx_nic *efx)
  1049. {
  1050. struct efx_channel *channel;
  1051. efx_for_each_channel(channel, efx) {
  1052. efx_lro_fini(&channel->lro_mgr);
  1053. channel->napi_dev = NULL;
  1054. }
  1055. }
  1056. /**************************************************************************
  1057. *
  1058. * Kernel netpoll interface
  1059. *
  1060. *************************************************************************/
  1061. #ifdef CONFIG_NET_POLL_CONTROLLER
  1062. /* Although in the common case interrupts will be disabled, this is not
  1063. * guaranteed. However, all our work happens inside the NAPI callback,
  1064. * so no locking is required.
  1065. */
  1066. static void efx_netpoll(struct net_device *net_dev)
  1067. {
  1068. struct efx_nic *efx = net_dev->priv;
  1069. struct efx_channel *channel;
  1070. efx_for_each_channel_with_interrupt(channel, efx)
  1071. efx_schedule_channel(channel);
  1072. }
  1073. #endif
  1074. /**************************************************************************
  1075. *
  1076. * Kernel net device interface
  1077. *
  1078. *************************************************************************/
  1079. /* Context: process, rtnl_lock() held. */
  1080. static int efx_net_open(struct net_device *net_dev)
  1081. {
  1082. struct efx_nic *efx = net_dev->priv;
  1083. EFX_ASSERT_RESET_SERIALISED(efx);
  1084. EFX_LOG(efx, "opening device %s on CPU %d\n", net_dev->name,
  1085. raw_smp_processor_id());
  1086. efx_start_all(efx);
  1087. return 0;
  1088. }
  1089. /* Context: process, rtnl_lock() held.
  1090. * Note that the kernel will ignore our return code; this method
  1091. * should really be a void.
  1092. */
  1093. static int efx_net_stop(struct net_device *net_dev)
  1094. {
  1095. struct efx_nic *efx = net_dev->priv;
  1096. int rc;
  1097. EFX_LOG(efx, "closing %s on CPU %d\n", net_dev->name,
  1098. raw_smp_processor_id());
  1099. /* Stop the device and flush all the channels */
  1100. efx_stop_all(efx);
  1101. efx_fini_channels(efx);
  1102. rc = efx_init_channels(efx);
  1103. if (rc)
  1104. efx_schedule_reset(efx, RESET_TYPE_DISABLE);
  1105. return 0;
  1106. }
  1107. /* Context: process, dev_base_lock held, non-blocking. */
  1108. static struct net_device_stats *efx_net_stats(struct net_device *net_dev)
  1109. {
  1110. struct efx_nic *efx = net_dev->priv;
  1111. struct efx_mac_stats *mac_stats = &efx->mac_stats;
  1112. struct net_device_stats *stats = &net_dev->stats;
  1113. if (!spin_trylock(&efx->stats_lock))
  1114. return stats;
  1115. if (efx->state == STATE_RUNNING) {
  1116. falcon_update_stats_xmac(efx);
  1117. falcon_update_nic_stats(efx);
  1118. }
  1119. spin_unlock(&efx->stats_lock);
  1120. stats->rx_packets = mac_stats->rx_packets;
  1121. stats->tx_packets = mac_stats->tx_packets;
  1122. stats->rx_bytes = mac_stats->rx_bytes;
  1123. stats->tx_bytes = mac_stats->tx_bytes;
  1124. stats->multicast = mac_stats->rx_multicast;
  1125. stats->collisions = mac_stats->tx_collision;
  1126. stats->rx_length_errors = (mac_stats->rx_gtjumbo +
  1127. mac_stats->rx_length_error);
  1128. stats->rx_over_errors = efx->n_rx_nodesc_drop_cnt;
  1129. stats->rx_crc_errors = mac_stats->rx_bad;
  1130. stats->rx_frame_errors = mac_stats->rx_align_error;
  1131. stats->rx_fifo_errors = mac_stats->rx_overflow;
  1132. stats->rx_missed_errors = mac_stats->rx_missed;
  1133. stats->tx_window_errors = mac_stats->tx_late_collision;
  1134. stats->rx_errors = (stats->rx_length_errors +
  1135. stats->rx_over_errors +
  1136. stats->rx_crc_errors +
  1137. stats->rx_frame_errors +
  1138. stats->rx_fifo_errors +
  1139. stats->rx_missed_errors +
  1140. mac_stats->rx_symbol_error);
  1141. stats->tx_errors = (stats->tx_window_errors +
  1142. mac_stats->tx_bad);
  1143. return stats;
  1144. }
  1145. /* Context: netif_tx_lock held, BHs disabled. */
  1146. static void efx_watchdog(struct net_device *net_dev)
  1147. {
  1148. struct efx_nic *efx = net_dev->priv;
  1149. EFX_ERR(efx, "TX stuck with stop_count=%d port_enabled=%d: %s\n",
  1150. atomic_read(&efx->netif_stop_count), efx->port_enabled,
  1151. monitor_reset ? "resetting channels" : "skipping reset");
  1152. if (monitor_reset)
  1153. efx_schedule_reset(efx, RESET_TYPE_MONITOR);
  1154. }
  1155. /* Context: process, rtnl_lock() held. */
  1156. static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
  1157. {
  1158. struct efx_nic *efx = net_dev->priv;
  1159. int rc = 0;
  1160. EFX_ASSERT_RESET_SERIALISED(efx);
  1161. if (new_mtu > EFX_MAX_MTU)
  1162. return -EINVAL;
  1163. efx_stop_all(efx);
  1164. EFX_LOG(efx, "changing MTU to %d\n", new_mtu);
  1165. efx_fini_channels(efx);
  1166. net_dev->mtu = new_mtu;
  1167. rc = efx_init_channels(efx);
  1168. if (rc)
  1169. goto fail;
  1170. efx_start_all(efx);
  1171. return rc;
  1172. fail:
  1173. efx_schedule_reset(efx, RESET_TYPE_DISABLE);
  1174. return rc;
  1175. }
  1176. static int efx_set_mac_address(struct net_device *net_dev, void *data)
  1177. {
  1178. struct efx_nic *efx = net_dev->priv;
  1179. struct sockaddr *addr = data;
  1180. char *new_addr = addr->sa_data;
  1181. EFX_ASSERT_RESET_SERIALISED(efx);
  1182. if (!is_valid_ether_addr(new_addr)) {
  1183. DECLARE_MAC_BUF(mac);
  1184. EFX_ERR(efx, "invalid ethernet MAC address requested: %s\n",
  1185. print_mac(mac, new_addr));
  1186. return -EINVAL;
  1187. }
  1188. memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);
  1189. /* Reconfigure the MAC */
  1190. efx_reconfigure_port(efx);
  1191. return 0;
  1192. }
  1193. /* Context: netif_tx_lock held, BHs disabled. */
  1194. static void efx_set_multicast_list(struct net_device *net_dev)
  1195. {
  1196. struct efx_nic *efx = net_dev->priv;
  1197. struct dev_mc_list *mc_list = net_dev->mc_list;
  1198. union efx_multicast_hash *mc_hash = &efx->multicast_hash;
  1199. int promiscuous;
  1200. u32 crc;
  1201. int bit;
  1202. int i;
  1203. /* Set per-MAC promiscuity flag and reconfigure MAC if necessary */
  1204. promiscuous = (net_dev->flags & IFF_PROMISC) ? 1 : 0;
  1205. if (efx->promiscuous != promiscuous) {
  1206. efx->promiscuous = promiscuous;
  1207. /* Close the window between efx_stop_port() and efx_flush_all()
  1208. * by only queuing work when the port is enabled. */
  1209. if (efx->port_enabled)
  1210. queue_work(efx->workqueue, &efx->reconfigure_work);
  1211. }
  1212. /* Build multicast hash table */
  1213. if (promiscuous || (net_dev->flags & IFF_ALLMULTI)) {
  1214. memset(mc_hash, 0xff, sizeof(*mc_hash));
  1215. } else {
  1216. memset(mc_hash, 0x00, sizeof(*mc_hash));
  1217. for (i = 0; i < net_dev->mc_count; i++) {
  1218. crc = ether_crc_le(ETH_ALEN, mc_list->dmi_addr);
  1219. bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
  1220. set_bit_le(bit, mc_hash->byte);
  1221. mc_list = mc_list->next;
  1222. }
  1223. }
  1224. /* Create and activate new global multicast hash table */
  1225. falcon_set_multicast_hash(efx);
  1226. }
  1227. static int efx_netdev_event(struct notifier_block *this,
  1228. unsigned long event, void *ptr)
  1229. {
  1230. struct net_device *net_dev = (struct net_device *)ptr;
  1231. if (net_dev->open == efx_net_open && event == NETDEV_CHANGENAME) {
  1232. struct efx_nic *efx = net_dev->priv;
  1233. strcpy(efx->name, net_dev->name);
  1234. }
  1235. return NOTIFY_DONE;
  1236. }
  1237. static struct notifier_block efx_netdev_notifier = {
  1238. .notifier_call = efx_netdev_event,
  1239. };
  1240. static int efx_register_netdev(struct efx_nic *efx)
  1241. {
  1242. struct net_device *net_dev = efx->net_dev;
  1243. int rc;
  1244. net_dev->watchdog_timeo = 5 * HZ;
  1245. net_dev->irq = efx->pci_dev->irq;
  1246. net_dev->open = efx_net_open;
  1247. net_dev->stop = efx_net_stop;
  1248. net_dev->get_stats = efx_net_stats;
  1249. net_dev->tx_timeout = &efx_watchdog;
  1250. net_dev->hard_start_xmit = efx_hard_start_xmit;
  1251. net_dev->do_ioctl = efx_ioctl;
  1252. net_dev->change_mtu = efx_change_mtu;
  1253. net_dev->set_mac_address = efx_set_mac_address;
  1254. net_dev->set_multicast_list = efx_set_multicast_list;
  1255. #ifdef CONFIG_NET_POLL_CONTROLLER
  1256. net_dev->poll_controller = efx_netpoll;
  1257. #endif
  1258. SET_NETDEV_DEV(net_dev, &efx->pci_dev->dev);
  1259. SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);
  1260. /* Always start with carrier off; PHY events will detect the link */
  1261. netif_carrier_off(efx->net_dev);
  1262. /* Clear MAC statistics */
  1263. falcon_update_stats_xmac(efx);
  1264. memset(&efx->mac_stats, 0, sizeof(efx->mac_stats));
  1265. rc = register_netdev(net_dev);
  1266. if (rc) {
  1267. EFX_ERR(efx, "could not register net dev\n");
  1268. return rc;
  1269. }
  1270. strcpy(efx->name, net_dev->name);
  1271. return 0;
  1272. }
  1273. static void efx_unregister_netdev(struct efx_nic *efx)
  1274. {
  1275. struct efx_tx_queue *tx_queue;
  1276. if (!efx->net_dev)
  1277. return;
  1278. BUG_ON(efx->net_dev->priv != efx);
  1279. /* Free up any skbs still remaining. This has to happen before
  1280. * we try to unregister the netdev as running their destructors
  1281. * may be needed to get the device ref. count to 0. */
  1282. efx_for_each_tx_queue(tx_queue, efx)
  1283. efx_release_tx_buffers(tx_queue);
  1284. if (NET_DEV_REGISTERED(efx)) {
  1285. strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
  1286. unregister_netdev(efx->net_dev);
  1287. }
  1288. }
  1289. /**************************************************************************
  1290. *
  1291. * Device reset and suspend
  1292. *
  1293. **************************************************************************/
  1294. /* The final hardware and software finalisation before reset. */
  1295. static int efx_reset_down(struct efx_nic *efx, struct ethtool_cmd *ecmd)
  1296. {
  1297. int rc;
  1298. EFX_ASSERT_RESET_SERIALISED(efx);
  1299. rc = falcon_xmac_get_settings(efx, ecmd);
  1300. if (rc) {
  1301. EFX_ERR(efx, "could not back up PHY settings\n");
  1302. goto fail;
  1303. }
  1304. efx_fini_channels(efx);
  1305. return 0;
  1306. fail:
  1307. return rc;
  1308. }
  1309. /* The first part of software initialisation after a hardware reset
  1310. * This function does not handle serialisation with the kernel, it
  1311. * assumes the caller has done this */
  1312. static int efx_reset_up(struct efx_nic *efx, struct ethtool_cmd *ecmd)
  1313. {
  1314. int rc;
  1315. rc = efx_init_channels(efx);
  1316. if (rc)
  1317. goto fail1;
  1318. /* Restore MAC and PHY settings. */
  1319. rc = falcon_xmac_set_settings(efx, ecmd);
  1320. if (rc) {
  1321. EFX_ERR(efx, "could not restore PHY settings\n");
  1322. goto fail2;
  1323. }
  1324. return 0;
  1325. fail2:
  1326. efx_fini_channels(efx);
  1327. fail1:
  1328. return rc;
  1329. }
  1330. /* Reset the NIC as transparently as possible. Do not reset the PHY
  1331. * Note that the reset may fail, in which case the card will be left
  1332. * in a most-probably-unusable state.
  1333. *
  1334. * This function will sleep. You cannot reset from within an atomic
  1335. * state; use efx_schedule_reset() instead.
  1336. *
  1337. * Grabs the rtnl_lock.
  1338. */
  1339. static int efx_reset(struct efx_nic *efx)
  1340. {
  1341. struct ethtool_cmd ecmd;
  1342. enum reset_type method = efx->reset_pending;
  1343. int rc;
  1344. /* Serialise with kernel interfaces */
  1345. rtnl_lock();
  1346. /* If we're not RUNNING then don't reset. Leave the reset_pending
  1347. * flag set so that efx_pci_probe_main will be retried */
  1348. if (efx->state != STATE_RUNNING) {
  1349. EFX_INFO(efx, "scheduled reset quenched. NIC not RUNNING\n");
  1350. goto unlock_rtnl;
  1351. }
  1352. efx->state = STATE_RESETTING;
  1353. EFX_INFO(efx, "resetting (%d)\n", method);
  1354. /* The net_dev->get_stats handler is quite slow, and will fail
  1355. * if a fetch is pending over reset. Serialise against it. */
  1356. spin_lock(&efx->stats_lock);
  1357. spin_unlock(&efx->stats_lock);
  1358. efx_stop_all(efx);
  1359. mutex_lock(&efx->mac_lock);
  1360. rc = efx_reset_down(efx, &ecmd);
  1361. if (rc)
  1362. goto fail1;
  1363. rc = falcon_reset_hw(efx, method);
  1364. if (rc) {
  1365. EFX_ERR(efx, "failed to reset hardware\n");
  1366. goto fail2;
  1367. }
  1368. /* Allow resets to be rescheduled. */
  1369. efx->reset_pending = RESET_TYPE_NONE;
  1370. /* Reinitialise bus-mastering, which may have been turned off before
  1371. * the reset was scheduled. This is still appropriate, even in the
  1372. * RESET_TYPE_DISABLE since this driver generally assumes the hardware
  1373. * can respond to requests. */
  1374. pci_set_master(efx->pci_dev);
  1375. /* Reinitialise device. This is appropriate in the RESET_TYPE_DISABLE
  1376. * case so the driver can talk to external SRAM */
  1377. rc = falcon_init_nic(efx);
  1378. if (rc) {
  1379. EFX_ERR(efx, "failed to initialise NIC\n");
  1380. goto fail3;
  1381. }
  1382. /* Leave device stopped if necessary */
  1383. if (method == RESET_TYPE_DISABLE) {
  1384. /* Reinitialise the device anyway so the driver unload sequence
  1385. * can talk to the external SRAM */
  1386. (void) falcon_init_nic(efx);
  1387. rc = -EIO;
  1388. goto fail4;
  1389. }
  1390. rc = efx_reset_up(efx, &ecmd);
  1391. if (rc)
  1392. goto fail5;
  1393. mutex_unlock(&efx->mac_lock);
  1394. EFX_LOG(efx, "reset complete\n");
  1395. efx->state = STATE_RUNNING;
  1396. efx_start_all(efx);
  1397. unlock_rtnl:
  1398. rtnl_unlock();
  1399. return 0;
  1400. fail5:
  1401. fail4:
  1402. fail3:
  1403. fail2:
  1404. fail1:
  1405. EFX_ERR(efx, "has been disabled\n");
  1406. efx->state = STATE_DISABLED;
  1407. mutex_unlock(&efx->mac_lock);
  1408. rtnl_unlock();
  1409. efx_unregister_netdev(efx);
  1410. efx_fini_port(efx);
  1411. return rc;
  1412. }
  1413. /* The worker thread exists so that code that cannot sleep can
  1414. * schedule a reset for later.
  1415. */
  1416. static void efx_reset_work(struct work_struct *data)
  1417. {
  1418. struct efx_nic *nic = container_of(data, struct efx_nic, reset_work);
  1419. efx_reset(nic);
  1420. }
  1421. void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
  1422. {
  1423. enum reset_type method;
  1424. if (efx->reset_pending != RESET_TYPE_NONE) {
  1425. EFX_INFO(efx, "quenching already scheduled reset\n");
  1426. return;
  1427. }
  1428. switch (type) {
  1429. case RESET_TYPE_INVISIBLE:
  1430. case RESET_TYPE_ALL:
  1431. case RESET_TYPE_WORLD:
  1432. case RESET_TYPE_DISABLE:
  1433. method = type;
  1434. break;
  1435. case RESET_TYPE_RX_RECOVERY:
  1436. case RESET_TYPE_RX_DESC_FETCH:
  1437. case RESET_TYPE_TX_DESC_FETCH:
  1438. case RESET_TYPE_TX_SKIP:
  1439. method = RESET_TYPE_INVISIBLE;
  1440. break;
  1441. default:
  1442. method = RESET_TYPE_ALL;
  1443. break;
  1444. }
  1445. if (method != type)
  1446. EFX_LOG(efx, "scheduling reset (%d:%d)\n", type, method);
  1447. else
  1448. EFX_LOG(efx, "scheduling reset (%d)\n", method);
  1449. efx->reset_pending = method;
  1450. queue_work(efx->workqueue, &efx->reset_work);
  1451. }
  1452. /**************************************************************************
  1453. *
  1454. * List of NICs we support
  1455. *
  1456. **************************************************************************/
  1457. /* PCI device ID table */
  1458. static struct pci_device_id efx_pci_table[] __devinitdata = {
  1459. {PCI_DEVICE(EFX_VENDID_SFC, FALCON_A_P_DEVID),
  1460. .driver_data = (unsigned long) &falcon_a_nic_type},
  1461. {PCI_DEVICE(EFX_VENDID_SFC, FALCON_B_P_DEVID),
  1462. .driver_data = (unsigned long) &falcon_b_nic_type},
  1463. {0} /* end of list */
  1464. };
  1465. /**************************************************************************
  1466. *
  1467. * Dummy PHY/MAC/Board operations
  1468. *
  1469. * Can be used where the MAC does not implement this operation
  1470. * Needed so all function pointers are valid and do not have to be tested
  1471. * before use
  1472. *
  1473. **************************************************************************/
  1474. int efx_port_dummy_op_int(struct efx_nic *efx)
  1475. {
  1476. return 0;
  1477. }
  1478. void efx_port_dummy_op_void(struct efx_nic *efx) {}
  1479. void efx_port_dummy_op_blink(struct efx_nic *efx, int blink) {}
  1480. static struct efx_phy_operations efx_dummy_phy_operations = {
  1481. .init = efx_port_dummy_op_int,
  1482. .reconfigure = efx_port_dummy_op_void,
  1483. .check_hw = efx_port_dummy_op_int,
  1484. .fini = efx_port_dummy_op_void,
  1485. .clear_interrupt = efx_port_dummy_op_void,
  1486. .reset_xaui = efx_port_dummy_op_void,
  1487. };
  1488. /* Dummy board operations */
  1489. static int efx_nic_dummy_op_int(struct efx_nic *nic)
  1490. {
  1491. return 0;
  1492. }
  1493. static struct efx_board efx_dummy_board_info = {
  1494. .init = efx_nic_dummy_op_int,
  1495. .init_leds = efx_port_dummy_op_int,
  1496. .set_fault_led = efx_port_dummy_op_blink,
  1497. };
  1498. /**************************************************************************
  1499. *
  1500. * Data housekeeping
  1501. *
  1502. **************************************************************************/
  1503. /* This zeroes out and then fills in the invariants in a struct
  1504. * efx_nic (including all sub-structures).
  1505. */
  1506. static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
  1507. struct pci_dev *pci_dev, struct net_device *net_dev)
  1508. {
  1509. struct efx_channel *channel;
  1510. struct efx_tx_queue *tx_queue;
  1511. struct efx_rx_queue *rx_queue;
  1512. int i, rc;
  1513. /* Initialise common structures */
  1514. memset(efx, 0, sizeof(*efx));
  1515. spin_lock_init(&efx->biu_lock);
  1516. spin_lock_init(&efx->phy_lock);
  1517. INIT_WORK(&efx->reset_work, efx_reset_work);
  1518. INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
  1519. efx->pci_dev = pci_dev;
  1520. efx->state = STATE_INIT;
  1521. efx->reset_pending = RESET_TYPE_NONE;
  1522. strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
  1523. efx->board_info = efx_dummy_board_info;
  1524. efx->net_dev = net_dev;
  1525. efx->rx_checksum_enabled = 1;
  1526. spin_lock_init(&efx->netif_stop_lock);
  1527. spin_lock_init(&efx->stats_lock);
  1528. mutex_init(&efx->mac_lock);
  1529. efx->phy_op = &efx_dummy_phy_operations;
  1530. efx->mii.dev = net_dev;
  1531. INIT_WORK(&efx->reconfigure_work, efx_reconfigure_work);
  1532. atomic_set(&efx->netif_stop_count, 1);
  1533. for (i = 0; i < EFX_MAX_CHANNELS; i++) {
  1534. channel = &efx->channel[i];
  1535. channel->efx = efx;
  1536. channel->channel = i;
  1537. channel->evqnum = i;
  1538. channel->work_pending = 0;
  1539. }
  1540. for (i = 0; i < EFX_MAX_TX_QUEUES; i++) {
  1541. tx_queue = &efx->tx_queue[i];
  1542. tx_queue->efx = efx;
  1543. tx_queue->queue = i;
  1544. tx_queue->buffer = NULL;
  1545. tx_queue->channel = &efx->channel[0]; /* for safety */
  1546. tx_queue->tso_headers_free = NULL;
  1547. }
  1548. for (i = 0; i < EFX_MAX_RX_QUEUES; i++) {
  1549. rx_queue = &efx->rx_queue[i];
  1550. rx_queue->efx = efx;
  1551. rx_queue->queue = i;
  1552. rx_queue->channel = &efx->channel[0]; /* for safety */
  1553. rx_queue->buffer = NULL;
  1554. spin_lock_init(&rx_queue->add_lock);
  1555. INIT_DELAYED_WORK(&rx_queue->work, efx_rx_work);
  1556. }
  1557. efx->type = type;
  1558. /* Sanity-check NIC type */
  1559. EFX_BUG_ON_PARANOID(efx->type->txd_ring_mask &
  1560. (efx->type->txd_ring_mask + 1));
  1561. EFX_BUG_ON_PARANOID(efx->type->rxd_ring_mask &
  1562. (efx->type->rxd_ring_mask + 1));
  1563. EFX_BUG_ON_PARANOID(efx->type->evq_size &
  1564. (efx->type->evq_size - 1));
  1565. /* As close as we can get to guaranteeing that we don't overflow */
  1566. EFX_BUG_ON_PARANOID(efx->type->evq_size <
  1567. (efx->type->txd_ring_mask + 1 +
  1568. efx->type->rxd_ring_mask + 1));
  1569. EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);
  1570. /* Higher numbered interrupt modes are less capable! */
  1571. efx->interrupt_mode = max(efx->type->max_interrupt_mode,
  1572. interrupt_mode);
  1573. efx->workqueue = create_singlethread_workqueue("sfc_work");
  1574. if (!efx->workqueue) {
  1575. rc = -ENOMEM;
  1576. goto fail1;
  1577. }
  1578. return 0;
  1579. fail1:
  1580. return rc;
  1581. }
  1582. static void efx_fini_struct(struct efx_nic *efx)
  1583. {
  1584. if (efx->workqueue) {
  1585. destroy_workqueue(efx->workqueue);
  1586. efx->workqueue = NULL;
  1587. }
  1588. }
  1589. /**************************************************************************
  1590. *
  1591. * PCI interface
  1592. *
  1593. **************************************************************************/
  1594. /* Main body of final NIC shutdown code
  1595. * This is called only at module unload (or hotplug removal).
  1596. */
  1597. static void efx_pci_remove_main(struct efx_nic *efx)
  1598. {
  1599. EFX_ASSERT_RESET_SERIALISED(efx);
  1600. /* Skip everything if we never obtained a valid membase */
  1601. if (!efx->membase)
  1602. return;
  1603. efx_fini_channels(efx);
  1604. efx_fini_port(efx);
  1605. /* Shutdown the board, then the NIC and board state */
  1606. falcon_fini_interrupt(efx);
  1607. efx_fini_napi(efx);
  1608. efx_remove_all(efx);
  1609. }
  1610. /* Final NIC shutdown
  1611. * This is called only at module unload (or hotplug removal).
  1612. */
  1613. static void efx_pci_remove(struct pci_dev *pci_dev)
  1614. {
  1615. struct efx_nic *efx;
  1616. efx = pci_get_drvdata(pci_dev);
  1617. if (!efx)
  1618. return;
  1619. /* Mark the NIC as fini, then stop the interface */
  1620. rtnl_lock();
  1621. efx->state = STATE_FINI;
  1622. dev_close(efx->net_dev);
  1623. /* Allow any queued efx_resets() to complete */
  1624. rtnl_unlock();
  1625. if (efx->membase == NULL)
  1626. goto out;
  1627. efx_unregister_netdev(efx);
  1628. /* Wait for any scheduled resets to complete. No more will be
  1629. * scheduled from this point because efx_stop_all() has been
  1630. * called, we are no longer registered with driverlink, and
  1631. * the net_device's have been removed. */
  1632. flush_workqueue(efx->workqueue);
  1633. efx_pci_remove_main(efx);
  1634. out:
  1635. efx_fini_io(efx);
  1636. EFX_LOG(efx, "shutdown successful\n");
  1637. pci_set_drvdata(pci_dev, NULL);
  1638. efx_fini_struct(efx);
  1639. free_netdev(efx->net_dev);
  1640. };
  1641. /* Main body of NIC initialisation
  1642. * This is called at module load (or hotplug insertion, theoretically).
  1643. */
  1644. static int efx_pci_probe_main(struct efx_nic *efx)
  1645. {
  1646. int rc;
  1647. /* Do start-of-day initialisation */
  1648. rc = efx_probe_all(efx);
  1649. if (rc)
  1650. goto fail1;
  1651. rc = efx_init_napi(efx);
  1652. if (rc)
  1653. goto fail2;
  1654. /* Initialise the board */
  1655. rc = efx->board_info.init(efx);
  1656. if (rc) {
  1657. EFX_ERR(efx, "failed to initialise board\n");
  1658. goto fail3;
  1659. }
  1660. rc = falcon_init_nic(efx);
  1661. if (rc) {
  1662. EFX_ERR(efx, "failed to initialise NIC\n");
  1663. goto fail4;
  1664. }
  1665. rc = efx_init_port(efx);
  1666. if (rc) {
  1667. EFX_ERR(efx, "failed to initialise port\n");
  1668. goto fail5;
  1669. }
  1670. rc = efx_init_channels(efx);
  1671. if (rc)
  1672. goto fail6;
  1673. rc = falcon_init_interrupt(efx);
  1674. if (rc)
  1675. goto fail7;
  1676. return 0;
  1677. fail7:
  1678. efx_fini_channels(efx);
  1679. fail6:
  1680. efx_fini_port(efx);
  1681. fail5:
  1682. fail4:
  1683. fail3:
  1684. efx_fini_napi(efx);
  1685. fail2:
  1686. efx_remove_all(efx);
  1687. fail1:
  1688. return rc;
  1689. }
  1690. /* NIC initialisation
  1691. *
  1692. * This is called at module load (or hotplug insertion,
  1693. * theoretically). It sets up PCI mappings, tests and resets the NIC,
  1694. * sets up and registers the network devices with the kernel and hooks
  1695. * the interrupt service routine. It does not prepare the device for
  1696. * transmission; this is left to the first time one of the network
  1697. * interfaces is brought up (i.e. efx_net_open).
  1698. */
  1699. static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
  1700. const struct pci_device_id *entry)
  1701. {
  1702. struct efx_nic_type *type = (struct efx_nic_type *) entry->driver_data;
  1703. struct net_device *net_dev;
  1704. struct efx_nic *efx;
  1705. int i, rc;
  1706. /* Allocate and initialise a struct net_device and struct efx_nic */
  1707. net_dev = alloc_etherdev(sizeof(*efx));
  1708. if (!net_dev)
  1709. return -ENOMEM;
  1710. net_dev->features |= (NETIF_F_IP_CSUM | NETIF_F_SG |
  1711. NETIF_F_HIGHDMA | NETIF_F_TSO);
  1712. if (lro)
  1713. net_dev->features |= NETIF_F_LRO;
  1714. efx = net_dev->priv;
  1715. pci_set_drvdata(pci_dev, efx);
  1716. rc = efx_init_struct(efx, type, pci_dev, net_dev);
  1717. if (rc)
  1718. goto fail1;
  1719. EFX_INFO(efx, "Solarflare Communications NIC detected\n");
  1720. /* Set up basic I/O (BAR mappings etc) */
  1721. rc = efx_init_io(efx);
  1722. if (rc)
  1723. goto fail2;
  1724. /* No serialisation is required with the reset path because
  1725. * we're in STATE_INIT. */
  1726. for (i = 0; i < 5; i++) {
  1727. rc = efx_pci_probe_main(efx);
  1728. if (rc == 0)
  1729. break;
  1730. /* Serialise against efx_reset(). No more resets will be
  1731. * scheduled since efx_stop_all() has been called, and we
  1732. * have not and never have been registered with either
  1733. * the rtnetlink or driverlink layers. */
  1734. cancel_work_sync(&efx->reset_work);
  1735. /* Retry if a recoverably reset event has been scheduled */
  1736. if ((efx->reset_pending != RESET_TYPE_INVISIBLE) &&
  1737. (efx->reset_pending != RESET_TYPE_ALL))
  1738. goto fail3;
  1739. efx->reset_pending = RESET_TYPE_NONE;
  1740. }
  1741. if (rc) {
  1742. EFX_ERR(efx, "Could not reset NIC\n");
  1743. goto fail4;
  1744. }
  1745. /* Switch to the running state before we expose the device to
  1746. * the OS. This is to ensure that the initial gathering of
  1747. * MAC stats succeeds. */
  1748. rtnl_lock();
  1749. efx->state = STATE_RUNNING;
  1750. rtnl_unlock();
  1751. rc = efx_register_netdev(efx);
  1752. if (rc)
  1753. goto fail5;
  1754. EFX_LOG(efx, "initialisation successful\n");
  1755. return 0;
  1756. fail5:
  1757. efx_pci_remove_main(efx);
  1758. fail4:
  1759. fail3:
  1760. efx_fini_io(efx);
  1761. fail2:
  1762. efx_fini_struct(efx);
  1763. fail1:
  1764. EFX_LOG(efx, "initialisation failed. rc=%d\n", rc);
  1765. free_netdev(net_dev);
  1766. return rc;
  1767. }
  1768. static struct pci_driver efx_pci_driver = {
  1769. .name = EFX_DRIVER_NAME,
  1770. .id_table = efx_pci_table,
  1771. .probe = efx_pci_probe,
  1772. .remove = efx_pci_remove,
  1773. };
  1774. /**************************************************************************
  1775. *
  1776. * Kernel module interface
  1777. *
  1778. *************************************************************************/
  1779. module_param(interrupt_mode, uint, 0444);
  1780. MODULE_PARM_DESC(interrupt_mode,
  1781. "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
  1782. static int __init efx_init_module(void)
  1783. {
  1784. int rc;
  1785. printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
  1786. rc = register_netdevice_notifier(&efx_netdev_notifier);
  1787. if (rc)
  1788. goto err_notifier;
  1789. refill_workqueue = create_workqueue("sfc_refill");
  1790. if (!refill_workqueue) {
  1791. rc = -ENOMEM;
  1792. goto err_refill;
  1793. }
  1794. rc = pci_register_driver(&efx_pci_driver);
  1795. if (rc < 0)
  1796. goto err_pci;
  1797. return 0;
  1798. err_pci:
  1799. destroy_workqueue(refill_workqueue);
  1800. err_refill:
  1801. unregister_netdevice_notifier(&efx_netdev_notifier);
  1802. err_notifier:
  1803. return rc;
  1804. }
  1805. static void __exit efx_exit_module(void)
  1806. {
  1807. printk(KERN_INFO "Solarflare NET driver unloading\n");
  1808. pci_unregister_driver(&efx_pci_driver);
  1809. destroy_workqueue(refill_workqueue);
  1810. unregister_netdevice_notifier(&efx_netdev_notifier);
  1811. }
  1812. module_init(efx_init_module);
  1813. module_exit(efx_exit_module);
  1814. MODULE_AUTHOR("Michael Brown <mbrown@fensystems.co.uk> and "
  1815. "Solarflare Communications");
  1816. MODULE_DESCRIPTION("Solarflare Communications network driver");
  1817. MODULE_LICENSE("GPL");
  1818. MODULE_DEVICE_TABLE(pci, efx_pci_table);