tx.c 31 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157
  1. /****************************************************************************
  2. * Driver for Solarflare Solarstorm network controllers and boards
  3. * Copyright 2005-2006 Fen Systems Ltd.
  4. * Copyright 2005-2009 Solarflare Communications Inc.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation, incorporated herein by reference.
  9. */
  10. #include <linux/pci.h>
  11. #include <linux/tcp.h>
  12. #include <linux/ip.h>
  13. #include <linux/in.h>
  14. #include <linux/ipv6.h>
  15. #include <net/ipv6.h>
  16. #include <linux/if_ether.h>
  17. #include <linux/highmem.h>
  18. #include "net_driver.h"
  19. #include "efx.h"
  20. #include "nic.h"
  21. #include "workarounds.h"
  22. /*
  23. * TX descriptor ring full threshold
  24. *
  25. * The tx_queue descriptor ring fill-level must fall below this value
  26. * before we restart the netif queue
  27. */
  28. #define EFX_TXQ_THRESHOLD (EFX_TXQ_MASK / 2u)
  29. /* We want to be able to nest calls to netif_stop_queue(), since each
  30. * channel can have an individual stop on the queue.
  31. */
  32. void efx_stop_queue(struct efx_nic *efx)
  33. {
  34. spin_lock_bh(&efx->netif_stop_lock);
  35. EFX_TRACE(efx, "stop TX queue\n");
  36. atomic_inc(&efx->netif_stop_count);
  37. netif_stop_queue(efx->net_dev);
  38. spin_unlock_bh(&efx->netif_stop_lock);
  39. }
  40. /* Wake netif's TX queue
  41. * We want to be able to nest calls to netif_stop_queue(), since each
  42. * channel can have an individual stop on the queue.
  43. */
  44. void efx_wake_queue(struct efx_nic *efx)
  45. {
  46. local_bh_disable();
  47. if (atomic_dec_and_lock(&efx->netif_stop_count,
  48. &efx->netif_stop_lock)) {
  49. EFX_TRACE(efx, "waking TX queue\n");
  50. netif_wake_queue(efx->net_dev);
  51. spin_unlock(&efx->netif_stop_lock);
  52. }
  53. local_bh_enable();
  54. }
  55. static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
  56. struct efx_tx_buffer *buffer)
  57. {
  58. if (buffer->unmap_len) {
  59. struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
  60. dma_addr_t unmap_addr = (buffer->dma_addr + buffer->len -
  61. buffer->unmap_len);
  62. if (buffer->unmap_single)
  63. pci_unmap_single(pci_dev, unmap_addr, buffer->unmap_len,
  64. PCI_DMA_TODEVICE);
  65. else
  66. pci_unmap_page(pci_dev, unmap_addr, buffer->unmap_len,
  67. PCI_DMA_TODEVICE);
  68. buffer->unmap_len = 0;
  69. buffer->unmap_single = false;
  70. }
  71. if (buffer->skb) {
  72. dev_kfree_skb_any((struct sk_buff *) buffer->skb);
  73. buffer->skb = NULL;
  74. EFX_TRACE(tx_queue->efx, "TX queue %d transmission id %x "
  75. "complete\n", tx_queue->queue, read_ptr);
  76. }
  77. }
  78. /**
  79. * struct efx_tso_header - a DMA mapped buffer for packet headers
  80. * @next: Linked list of free ones.
  81. * The list is protected by the TX queue lock.
  82. * @dma_unmap_len: Length to unmap for an oversize buffer, or 0.
  83. * @dma_addr: The DMA address of the header below.
  84. *
  85. * This controls the memory used for a TSO header. Use TSOH_DATA()
  86. * to find the packet header data. Use TSOH_SIZE() to calculate the
  87. * total size required for a given packet header length. TSO headers
  88. * in the free list are exactly %TSOH_STD_SIZE bytes in size.
  89. */
  90. struct efx_tso_header {
  91. union {
  92. struct efx_tso_header *next;
  93. size_t unmap_len;
  94. };
  95. dma_addr_t dma_addr;
  96. };
  97. static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
  98. struct sk_buff *skb);
  99. static void efx_fini_tso(struct efx_tx_queue *tx_queue);
  100. static void efx_tsoh_heap_free(struct efx_tx_queue *tx_queue,
  101. struct efx_tso_header *tsoh);
  102. static void efx_tsoh_free(struct efx_tx_queue *tx_queue,
  103. struct efx_tx_buffer *buffer)
  104. {
  105. if (buffer->tsoh) {
  106. if (likely(!buffer->tsoh->unmap_len)) {
  107. buffer->tsoh->next = tx_queue->tso_headers_free;
  108. tx_queue->tso_headers_free = buffer->tsoh;
  109. } else {
  110. efx_tsoh_heap_free(tx_queue, buffer->tsoh);
  111. }
  112. buffer->tsoh = NULL;
  113. }
  114. }
  115. static inline unsigned
  116. efx_max_tx_len(struct efx_nic *efx, dma_addr_t dma_addr)
  117. {
  118. /* Depending on the NIC revision, we can use descriptor
  119. * lengths up to 8K or 8K-1. However, since PCI Express
  120. * devices must split read requests at 4K boundaries, there is
  121. * little benefit from using descriptors that cross those
  122. * boundaries and we keep things simple by not doing so.
  123. */
  124. unsigned len = (~dma_addr & 0xfff) + 1;
  125. /* Work around hardware bug for unaligned buffers. */
  126. if (EFX_WORKAROUND_5391(efx) && (dma_addr & 0xf))
  127. len = min_t(unsigned, len, 512 - (dma_addr & 0xf));
  128. return len;
  129. }
  130. /*
  131. * Add a socket buffer to a TX queue
  132. *
  133. * This maps all fragments of a socket buffer for DMA and adds them to
  134. * the TX queue. The queue's insert pointer will be incremented by
  135. * the number of fragments in the socket buffer.
  136. *
  137. * If any DMA mapping fails, any mapped fragments will be unmapped,
  138. * the queue's insert pointer will be restored to its original value.
  139. *
  140. * This function is split out from efx_hard_start_xmit to allow the
  141. * loopback test to direct packets via specific TX queues.
  142. *
  143. * Returns NETDEV_TX_OK or NETDEV_TX_BUSY
  144. * You must hold netif_tx_lock() to call this function.
  145. */
  146. netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
  147. {
  148. struct efx_nic *efx = tx_queue->efx;
  149. struct pci_dev *pci_dev = efx->pci_dev;
  150. struct efx_tx_buffer *buffer;
  151. skb_frag_t *fragment;
  152. struct page *page;
  153. int page_offset;
  154. unsigned int len, unmap_len = 0, fill_level, insert_ptr;
  155. dma_addr_t dma_addr, unmap_addr = 0;
  156. unsigned int dma_len;
  157. bool unmap_single;
  158. int q_space, i = 0;
  159. netdev_tx_t rc = NETDEV_TX_OK;
  160. EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
  161. if (skb_shinfo(skb)->gso_size)
  162. return efx_enqueue_skb_tso(tx_queue, skb);
  163. /* Get size of the initial fragment */
  164. len = skb_headlen(skb);
  165. /* Pad if necessary */
  166. if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
  167. EFX_BUG_ON_PARANOID(skb->data_len);
  168. len = 32 + 1;
  169. if (skb_pad(skb, len - skb->len))
  170. return NETDEV_TX_OK;
  171. }
  172. fill_level = tx_queue->insert_count - tx_queue->old_read_count;
  173. q_space = EFX_TXQ_MASK - 1 - fill_level;
  174. /* Map for DMA. Use pci_map_single rather than pci_map_page
  175. * since this is more efficient on machines with sparse
  176. * memory.
  177. */
  178. unmap_single = true;
  179. dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
  180. /* Process all fragments */
  181. while (1) {
  182. if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
  183. goto pci_err;
  184. /* Store fields for marking in the per-fragment final
  185. * descriptor */
  186. unmap_len = len;
  187. unmap_addr = dma_addr;
  188. /* Add to TX queue, splitting across DMA boundaries */
  189. do {
  190. if (unlikely(q_space-- <= 0)) {
  191. /* It might be that completions have
  192. * happened since the xmit path last
  193. * checked. Update the xmit path's
  194. * copy of read_count.
  195. */
  196. ++tx_queue->stopped;
  197. /* This memory barrier protects the
  198. * change of stopped from the access
  199. * of read_count. */
  200. smp_mb();
  201. tx_queue->old_read_count =
  202. *(volatile unsigned *)
  203. &tx_queue->read_count;
  204. fill_level = (tx_queue->insert_count
  205. - tx_queue->old_read_count);
  206. q_space = EFX_TXQ_MASK - 1 - fill_level;
  207. if (unlikely(q_space-- <= 0))
  208. goto stop;
  209. smp_mb();
  210. --tx_queue->stopped;
  211. }
  212. insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK;
  213. buffer = &tx_queue->buffer[insert_ptr];
  214. efx_tsoh_free(tx_queue, buffer);
  215. EFX_BUG_ON_PARANOID(buffer->tsoh);
  216. EFX_BUG_ON_PARANOID(buffer->skb);
  217. EFX_BUG_ON_PARANOID(buffer->len);
  218. EFX_BUG_ON_PARANOID(!buffer->continuation);
  219. EFX_BUG_ON_PARANOID(buffer->unmap_len);
  220. dma_len = efx_max_tx_len(efx, dma_addr);
  221. if (likely(dma_len >= len))
  222. dma_len = len;
  223. /* Fill out per descriptor fields */
  224. buffer->len = dma_len;
  225. buffer->dma_addr = dma_addr;
  226. len -= dma_len;
  227. dma_addr += dma_len;
  228. ++tx_queue->insert_count;
  229. } while (len);
  230. /* Transfer ownership of the unmapping to the final buffer */
  231. buffer->unmap_single = unmap_single;
  232. buffer->unmap_len = unmap_len;
  233. unmap_len = 0;
  234. /* Get address and size of next fragment */
  235. if (i >= skb_shinfo(skb)->nr_frags)
  236. break;
  237. fragment = &skb_shinfo(skb)->frags[i];
  238. len = fragment->size;
  239. page = fragment->page;
  240. page_offset = fragment->page_offset;
  241. i++;
  242. /* Map for DMA */
  243. unmap_single = false;
  244. dma_addr = pci_map_page(pci_dev, page, page_offset, len,
  245. PCI_DMA_TODEVICE);
  246. }
  247. /* Transfer ownership of the skb to the final buffer */
  248. buffer->skb = skb;
  249. buffer->continuation = false;
  250. /* Pass off to hardware */
  251. efx_nic_push_buffers(tx_queue);
  252. return NETDEV_TX_OK;
  253. pci_err:
  254. EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d "
  255. "fragments for DMA\n", tx_queue->queue, skb->len,
  256. skb_shinfo(skb)->nr_frags + 1);
  257. /* Mark the packet as transmitted, and free the SKB ourselves */
  258. dev_kfree_skb_any(skb);
  259. goto unwind;
  260. stop:
  261. rc = NETDEV_TX_BUSY;
  262. if (tx_queue->stopped == 1)
  263. efx_stop_queue(efx);
  264. unwind:
  265. /* Work backwards until we hit the original insert pointer value */
  266. while (tx_queue->insert_count != tx_queue->write_count) {
  267. --tx_queue->insert_count;
  268. insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK;
  269. buffer = &tx_queue->buffer[insert_ptr];
  270. efx_dequeue_buffer(tx_queue, buffer);
  271. buffer->len = 0;
  272. }
  273. /* Free the fragment we were mid-way through pushing */
  274. if (unmap_len) {
  275. if (unmap_single)
  276. pci_unmap_single(pci_dev, unmap_addr, unmap_len,
  277. PCI_DMA_TODEVICE);
  278. else
  279. pci_unmap_page(pci_dev, unmap_addr, unmap_len,
  280. PCI_DMA_TODEVICE);
  281. }
  282. return rc;
  283. }
  284. /* Remove packets from the TX queue
  285. *
  286. * This removes packets from the TX queue, up to and including the
  287. * specified index.
  288. */
  289. static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
  290. unsigned int index)
  291. {
  292. struct efx_nic *efx = tx_queue->efx;
  293. unsigned int stop_index, read_ptr;
  294. stop_index = (index + 1) & EFX_TXQ_MASK;
  295. read_ptr = tx_queue->read_count & EFX_TXQ_MASK;
  296. while (read_ptr != stop_index) {
  297. struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
  298. if (unlikely(buffer->len == 0)) {
  299. EFX_ERR(tx_queue->efx, "TX queue %d spurious TX "
  300. "completion id %x\n", tx_queue->queue,
  301. read_ptr);
  302. efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
  303. return;
  304. }
  305. efx_dequeue_buffer(tx_queue, buffer);
  306. buffer->continuation = true;
  307. buffer->len = 0;
  308. ++tx_queue->read_count;
  309. read_ptr = tx_queue->read_count & EFX_TXQ_MASK;
  310. }
  311. }
  312. /* Initiate a packet transmission. We use one channel per CPU
  313. * (sharing when we have more CPUs than channels). On Falcon, the TX
  314. * completion events will be directed back to the CPU that transmitted
  315. * the packet, which should be cache-efficient.
  316. *
  317. * Context: non-blocking.
  318. * Note that returning anything other than NETDEV_TX_OK will cause the
  319. * OS to free the skb.
  320. */
  321. netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
  322. struct net_device *net_dev)
  323. {
  324. struct efx_nic *efx = netdev_priv(net_dev);
  325. struct efx_tx_queue *tx_queue;
  326. if (unlikely(efx->port_inhibited))
  327. return NETDEV_TX_BUSY;
  328. if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
  329. tx_queue = &efx->tx_queue[EFX_TX_QUEUE_OFFLOAD_CSUM];
  330. else
  331. tx_queue = &efx->tx_queue[EFX_TX_QUEUE_NO_CSUM];
  332. return efx_enqueue_skb(tx_queue, skb);
  333. }
  334. void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
  335. {
  336. unsigned fill_level;
  337. struct efx_nic *efx = tx_queue->efx;
  338. EFX_BUG_ON_PARANOID(index > EFX_TXQ_MASK);
  339. efx_dequeue_buffers(tx_queue, index);
  340. /* See if we need to restart the netif queue. This barrier
  341. * separates the update of read_count from the test of
  342. * stopped. */
  343. smp_mb();
  344. if (unlikely(tx_queue->stopped) && likely(efx->port_enabled)) {
  345. fill_level = tx_queue->insert_count - tx_queue->read_count;
  346. if (fill_level < EFX_TXQ_THRESHOLD) {
  347. EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
  348. /* Do this under netif_tx_lock(), to avoid racing
  349. * with efx_xmit(). */
  350. netif_tx_lock(efx->net_dev);
  351. if (tx_queue->stopped) {
  352. tx_queue->stopped = 0;
  353. efx_wake_queue(efx);
  354. }
  355. netif_tx_unlock(efx->net_dev);
  356. }
  357. }
  358. }
  359. int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
  360. {
  361. struct efx_nic *efx = tx_queue->efx;
  362. unsigned int txq_size;
  363. int i, rc;
  364. EFX_LOG(efx, "creating TX queue %d\n", tx_queue->queue);
  365. /* Allocate software ring */
  366. txq_size = EFX_TXQ_SIZE * sizeof(*tx_queue->buffer);
  367. tx_queue->buffer = kzalloc(txq_size, GFP_KERNEL);
  368. if (!tx_queue->buffer)
  369. return -ENOMEM;
  370. for (i = 0; i <= EFX_TXQ_MASK; ++i)
  371. tx_queue->buffer[i].continuation = true;
  372. /* Allocate hardware ring */
  373. rc = efx_nic_probe_tx(tx_queue);
  374. if (rc)
  375. goto fail;
  376. return 0;
  377. fail:
  378. kfree(tx_queue->buffer);
  379. tx_queue->buffer = NULL;
  380. return rc;
  381. }
  382. void efx_init_tx_queue(struct efx_tx_queue *tx_queue)
  383. {
  384. EFX_LOG(tx_queue->efx, "initialising TX queue %d\n", tx_queue->queue);
  385. tx_queue->insert_count = 0;
  386. tx_queue->write_count = 0;
  387. tx_queue->read_count = 0;
  388. tx_queue->old_read_count = 0;
  389. BUG_ON(tx_queue->stopped);
  390. /* Set up TX descriptor ring */
  391. efx_nic_init_tx(tx_queue);
  392. }
  393. void efx_release_tx_buffers(struct efx_tx_queue *tx_queue)
  394. {
  395. struct efx_tx_buffer *buffer;
  396. if (!tx_queue->buffer)
  397. return;
  398. /* Free any buffers left in the ring */
  399. while (tx_queue->read_count != tx_queue->write_count) {
  400. buffer = &tx_queue->buffer[tx_queue->read_count & EFX_TXQ_MASK];
  401. efx_dequeue_buffer(tx_queue, buffer);
  402. buffer->continuation = true;
  403. buffer->len = 0;
  404. ++tx_queue->read_count;
  405. }
  406. }
  407. void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
  408. {
  409. EFX_LOG(tx_queue->efx, "shutting down TX queue %d\n", tx_queue->queue);
  410. /* Flush TX queue, remove descriptor ring */
  411. efx_nic_fini_tx(tx_queue);
  412. efx_release_tx_buffers(tx_queue);
  413. /* Free up TSO header cache */
  414. efx_fini_tso(tx_queue);
  415. /* Release queue's stop on port, if any */
  416. if (tx_queue->stopped) {
  417. tx_queue->stopped = 0;
  418. efx_wake_queue(tx_queue->efx);
  419. }
  420. }
  421. void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
  422. {
  423. EFX_LOG(tx_queue->efx, "destroying TX queue %d\n", tx_queue->queue);
  424. efx_nic_remove_tx(tx_queue);
  425. kfree(tx_queue->buffer);
  426. tx_queue->buffer = NULL;
  427. }
  428. /* Efx TCP segmentation acceleration.
  429. *
  430. * Why? Because by doing it here in the driver we can go significantly
  431. * faster than the GSO.
  432. *
  433. * Requires TX checksum offload support.
  434. */
  435. /* Number of bytes inserted at the start of a TSO header buffer,
  436. * similar to NET_IP_ALIGN.
  437. */
  438. #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
  439. #define TSOH_OFFSET 0
  440. #else
  441. #define TSOH_OFFSET NET_IP_ALIGN
  442. #endif
  443. #define TSOH_BUFFER(tsoh) ((u8 *)(tsoh + 1) + TSOH_OFFSET)
  444. /* Total size of struct efx_tso_header, buffer and padding */
  445. #define TSOH_SIZE(hdr_len) \
  446. (sizeof(struct efx_tso_header) + TSOH_OFFSET + hdr_len)
  447. /* Size of blocks on free list. Larger blocks must be allocated from
  448. * the heap.
  449. */
  450. #define TSOH_STD_SIZE 128
  451. #define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2))
  452. #define ETH_HDR_LEN(skb) (skb_network_header(skb) - (skb)->data)
  453. #define SKB_TCP_OFF(skb) PTR_DIFF(tcp_hdr(skb), (skb)->data)
  454. #define SKB_IPV4_OFF(skb) PTR_DIFF(ip_hdr(skb), (skb)->data)
  455. #define SKB_IPV6_OFF(skb) PTR_DIFF(ipv6_hdr(skb), (skb)->data)
  456. /**
  457. * struct tso_state - TSO state for an SKB
  458. * @out_len: Remaining length in current segment
  459. * @seqnum: Current sequence number
  460. * @ipv4_id: Current IPv4 ID, host endian
  461. * @packet_space: Remaining space in current packet
  462. * @dma_addr: DMA address of current position
  463. * @in_len: Remaining length in current SKB fragment
  464. * @unmap_len: Length of SKB fragment
  465. * @unmap_addr: DMA address of SKB fragment
  466. * @unmap_single: DMA single vs page mapping flag
  467. * @protocol: Network protocol (after any VLAN header)
  468. * @header_len: Number of bytes of header
  469. * @full_packet_size: Number of bytes to put in each outgoing segment
  470. *
  471. * The state used during segmentation. It is put into this data structure
  472. * just to make it easy to pass into inline functions.
  473. */
  474. struct tso_state {
  475. /* Output position */
  476. unsigned out_len;
  477. unsigned seqnum;
  478. unsigned ipv4_id;
  479. unsigned packet_space;
  480. /* Input position */
  481. dma_addr_t dma_addr;
  482. unsigned in_len;
  483. unsigned unmap_len;
  484. dma_addr_t unmap_addr;
  485. bool unmap_single;
  486. __be16 protocol;
  487. unsigned header_len;
  488. int full_packet_size;
  489. };
  490. /*
  491. * Verify that our various assumptions about sk_buffs and the conditions
  492. * under which TSO will be attempted hold true. Return the protocol number.
  493. */
  494. static __be16 efx_tso_check_protocol(struct sk_buff *skb)
  495. {
  496. __be16 protocol = skb->protocol;
  497. EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
  498. protocol);
  499. if (protocol == htons(ETH_P_8021Q)) {
  500. /* Find the encapsulated protocol; reset network header
  501. * and transport header based on that. */
  502. struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
  503. protocol = veh->h_vlan_encapsulated_proto;
  504. skb_set_network_header(skb, sizeof(*veh));
  505. if (protocol == htons(ETH_P_IP))
  506. skb_set_transport_header(skb, sizeof(*veh) +
  507. 4 * ip_hdr(skb)->ihl);
  508. else if (protocol == htons(ETH_P_IPV6))
  509. skb_set_transport_header(skb, sizeof(*veh) +
  510. sizeof(struct ipv6hdr));
  511. }
  512. if (protocol == htons(ETH_P_IP)) {
  513. EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
  514. } else {
  515. EFX_BUG_ON_PARANOID(protocol != htons(ETH_P_IPV6));
  516. EFX_BUG_ON_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP);
  517. }
  518. EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
  519. + (tcp_hdr(skb)->doff << 2u)) >
  520. skb_headlen(skb));
  521. return protocol;
  522. }
  523. /*
  524. * Allocate a page worth of efx_tso_header structures, and string them
  525. * into the tx_queue->tso_headers_free linked list. Return 0 or -ENOMEM.
  526. */
  527. static int efx_tsoh_block_alloc(struct efx_tx_queue *tx_queue)
  528. {
  529. struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
  530. struct efx_tso_header *tsoh;
  531. dma_addr_t dma_addr;
  532. u8 *base_kva, *kva;
  533. base_kva = pci_alloc_consistent(pci_dev, PAGE_SIZE, &dma_addr);
  534. if (base_kva == NULL) {
  535. EFX_ERR(tx_queue->efx, "Unable to allocate page for TSO"
  536. " headers\n");
  537. return -ENOMEM;
  538. }
  539. /* pci_alloc_consistent() allocates pages. */
  540. EFX_BUG_ON_PARANOID(dma_addr & (PAGE_SIZE - 1u));
  541. for (kva = base_kva; kva < base_kva + PAGE_SIZE; kva += TSOH_STD_SIZE) {
  542. tsoh = (struct efx_tso_header *)kva;
  543. tsoh->dma_addr = dma_addr + (TSOH_BUFFER(tsoh) - base_kva);
  544. tsoh->next = tx_queue->tso_headers_free;
  545. tx_queue->tso_headers_free = tsoh;
  546. }
  547. return 0;
  548. }
  549. /* Free up a TSO header, and all others in the same page. */
  550. static void efx_tsoh_block_free(struct efx_tx_queue *tx_queue,
  551. struct efx_tso_header *tsoh,
  552. struct pci_dev *pci_dev)
  553. {
  554. struct efx_tso_header **p;
  555. unsigned long base_kva;
  556. dma_addr_t base_dma;
  557. base_kva = (unsigned long)tsoh & PAGE_MASK;
  558. base_dma = tsoh->dma_addr & PAGE_MASK;
  559. p = &tx_queue->tso_headers_free;
  560. while (*p != NULL) {
  561. if (((unsigned long)*p & PAGE_MASK) == base_kva)
  562. *p = (*p)->next;
  563. else
  564. p = &(*p)->next;
  565. }
  566. pci_free_consistent(pci_dev, PAGE_SIZE, (void *)base_kva, base_dma);
  567. }
  568. static struct efx_tso_header *
  569. efx_tsoh_heap_alloc(struct efx_tx_queue *tx_queue, size_t header_len)
  570. {
  571. struct efx_tso_header *tsoh;
  572. tsoh = kmalloc(TSOH_SIZE(header_len), GFP_ATOMIC | GFP_DMA);
  573. if (unlikely(!tsoh))
  574. return NULL;
  575. tsoh->dma_addr = pci_map_single(tx_queue->efx->pci_dev,
  576. TSOH_BUFFER(tsoh), header_len,
  577. PCI_DMA_TODEVICE);
  578. if (unlikely(pci_dma_mapping_error(tx_queue->efx->pci_dev,
  579. tsoh->dma_addr))) {
  580. kfree(tsoh);
  581. return NULL;
  582. }
  583. tsoh->unmap_len = header_len;
  584. return tsoh;
  585. }
  586. static void
  587. efx_tsoh_heap_free(struct efx_tx_queue *tx_queue, struct efx_tso_header *tsoh)
  588. {
  589. pci_unmap_single(tx_queue->efx->pci_dev,
  590. tsoh->dma_addr, tsoh->unmap_len,
  591. PCI_DMA_TODEVICE);
  592. kfree(tsoh);
  593. }
  594. /**
  595. * efx_tx_queue_insert - push descriptors onto the TX queue
  596. * @tx_queue: Efx TX queue
  597. * @dma_addr: DMA address of fragment
  598. * @len: Length of fragment
  599. * @final_buffer: The final buffer inserted into the queue
  600. *
  601. * Push descriptors onto the TX queue. Return 0 on success or 1 if
  602. * @tx_queue full.
  603. */
  604. static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
  605. dma_addr_t dma_addr, unsigned len,
  606. struct efx_tx_buffer **final_buffer)
  607. {
  608. struct efx_tx_buffer *buffer;
  609. struct efx_nic *efx = tx_queue->efx;
  610. unsigned dma_len, fill_level, insert_ptr;
  611. int q_space;
  612. EFX_BUG_ON_PARANOID(len <= 0);
  613. fill_level = tx_queue->insert_count - tx_queue->old_read_count;
  614. /* -1 as there is no way to represent all descriptors used */
  615. q_space = EFX_TXQ_MASK - 1 - fill_level;
  616. while (1) {
  617. if (unlikely(q_space-- <= 0)) {
  618. /* It might be that completions have happened
  619. * since the xmit path last checked. Update
  620. * the xmit path's copy of read_count.
  621. */
  622. ++tx_queue->stopped;
  623. /* This memory barrier protects the change of
  624. * stopped from the access of read_count. */
  625. smp_mb();
  626. tx_queue->old_read_count =
  627. *(volatile unsigned *)&tx_queue->read_count;
  628. fill_level = (tx_queue->insert_count
  629. - tx_queue->old_read_count);
  630. q_space = EFX_TXQ_MASK - 1 - fill_level;
  631. if (unlikely(q_space-- <= 0)) {
  632. *final_buffer = NULL;
  633. return 1;
  634. }
  635. smp_mb();
  636. --tx_queue->stopped;
  637. }
  638. insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK;
  639. buffer = &tx_queue->buffer[insert_ptr];
  640. ++tx_queue->insert_count;
  641. EFX_BUG_ON_PARANOID(tx_queue->insert_count -
  642. tx_queue->read_count >
  643. EFX_TXQ_MASK);
  644. efx_tsoh_free(tx_queue, buffer);
  645. EFX_BUG_ON_PARANOID(buffer->len);
  646. EFX_BUG_ON_PARANOID(buffer->unmap_len);
  647. EFX_BUG_ON_PARANOID(buffer->skb);
  648. EFX_BUG_ON_PARANOID(!buffer->continuation);
  649. EFX_BUG_ON_PARANOID(buffer->tsoh);
  650. buffer->dma_addr = dma_addr;
  651. dma_len = efx_max_tx_len(efx, dma_addr);
  652. /* If there is enough space to send then do so */
  653. if (dma_len >= len)
  654. break;
  655. buffer->len = dma_len; /* Don't set the other members */
  656. dma_addr += dma_len;
  657. len -= dma_len;
  658. }
  659. EFX_BUG_ON_PARANOID(!len);
  660. buffer->len = len;
  661. *final_buffer = buffer;
  662. return 0;
  663. }
  664. /*
  665. * Put a TSO header into the TX queue.
  666. *
  667. * This is special-cased because we know that it is small enough to fit in
  668. * a single fragment, and we know it doesn't cross a page boundary. It
  669. * also allows us to not worry about end-of-packet etc.
  670. */
  671. static void efx_tso_put_header(struct efx_tx_queue *tx_queue,
  672. struct efx_tso_header *tsoh, unsigned len)
  673. {
  674. struct efx_tx_buffer *buffer;
  675. buffer = &tx_queue->buffer[tx_queue->insert_count & EFX_TXQ_MASK];
  676. efx_tsoh_free(tx_queue, buffer);
  677. EFX_BUG_ON_PARANOID(buffer->len);
  678. EFX_BUG_ON_PARANOID(buffer->unmap_len);
  679. EFX_BUG_ON_PARANOID(buffer->skb);
  680. EFX_BUG_ON_PARANOID(!buffer->continuation);
  681. EFX_BUG_ON_PARANOID(buffer->tsoh);
  682. buffer->len = len;
  683. buffer->dma_addr = tsoh->dma_addr;
  684. buffer->tsoh = tsoh;
  685. ++tx_queue->insert_count;
  686. }
  687. /* Remove descriptors put into a tx_queue. */
  688. static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
  689. {
  690. struct efx_tx_buffer *buffer;
  691. dma_addr_t unmap_addr;
  692. /* Work backwards until we hit the original insert pointer value */
  693. while (tx_queue->insert_count != tx_queue->write_count) {
  694. --tx_queue->insert_count;
  695. buffer = &tx_queue->buffer[tx_queue->insert_count &
  696. EFX_TXQ_MASK];
  697. efx_tsoh_free(tx_queue, buffer);
  698. EFX_BUG_ON_PARANOID(buffer->skb);
  699. if (buffer->unmap_len) {
  700. unmap_addr = (buffer->dma_addr + buffer->len -
  701. buffer->unmap_len);
  702. if (buffer->unmap_single)
  703. pci_unmap_single(tx_queue->efx->pci_dev,
  704. unmap_addr, buffer->unmap_len,
  705. PCI_DMA_TODEVICE);
  706. else
  707. pci_unmap_page(tx_queue->efx->pci_dev,
  708. unmap_addr, buffer->unmap_len,
  709. PCI_DMA_TODEVICE);
  710. buffer->unmap_len = 0;
  711. }
  712. buffer->len = 0;
  713. buffer->continuation = true;
  714. }
  715. }
  716. /* Parse the SKB header and initialise state. */
  717. static void tso_start(struct tso_state *st, const struct sk_buff *skb)
  718. {
  719. /* All ethernet/IP/TCP headers combined size is TCP header size
  720. * plus offset of TCP header relative to start of packet.
  721. */
  722. st->header_len = ((tcp_hdr(skb)->doff << 2u)
  723. + PTR_DIFF(tcp_hdr(skb), skb->data));
  724. st->full_packet_size = st->header_len + skb_shinfo(skb)->gso_size;
  725. if (st->protocol == htons(ETH_P_IP))
  726. st->ipv4_id = ntohs(ip_hdr(skb)->id);
  727. else
  728. st->ipv4_id = 0;
  729. st->seqnum = ntohl(tcp_hdr(skb)->seq);
  730. EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
  731. EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
  732. EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
  733. st->packet_space = st->full_packet_size;
  734. st->out_len = skb->len - st->header_len;
  735. st->unmap_len = 0;
  736. st->unmap_single = false;
  737. }
  738. static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
  739. skb_frag_t *frag)
  740. {
  741. st->unmap_addr = pci_map_page(efx->pci_dev, frag->page,
  742. frag->page_offset, frag->size,
  743. PCI_DMA_TODEVICE);
  744. if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
  745. st->unmap_single = false;
  746. st->unmap_len = frag->size;
  747. st->in_len = frag->size;
  748. st->dma_addr = st->unmap_addr;
  749. return 0;
  750. }
  751. return -ENOMEM;
  752. }
  753. static int tso_get_head_fragment(struct tso_state *st, struct efx_nic *efx,
  754. const struct sk_buff *skb)
  755. {
  756. int hl = st->header_len;
  757. int len = skb_headlen(skb) - hl;
  758. st->unmap_addr = pci_map_single(efx->pci_dev, skb->data + hl,
  759. len, PCI_DMA_TODEVICE);
  760. if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
  761. st->unmap_single = true;
  762. st->unmap_len = len;
  763. st->in_len = len;
  764. st->dma_addr = st->unmap_addr;
  765. return 0;
  766. }
  767. return -ENOMEM;
  768. }
  769. /**
  770. * tso_fill_packet_with_fragment - form descriptors for the current fragment
  771. * @tx_queue: Efx TX queue
  772. * @skb: Socket buffer
  773. * @st: TSO state
  774. *
  775. * Form descriptors for the current fragment, until we reach the end
  776. * of fragment or end-of-packet. Return 0 on success, 1 if not enough
  777. * space in @tx_queue.
  778. */
  779. static int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
  780. const struct sk_buff *skb,
  781. struct tso_state *st)
  782. {
  783. struct efx_tx_buffer *buffer;
  784. int n, end_of_packet, rc;
  785. if (st->in_len == 0)
  786. return 0;
  787. if (st->packet_space == 0)
  788. return 0;
  789. EFX_BUG_ON_PARANOID(st->in_len <= 0);
  790. EFX_BUG_ON_PARANOID(st->packet_space <= 0);
  791. n = min(st->in_len, st->packet_space);
  792. st->packet_space -= n;
  793. st->out_len -= n;
  794. st->in_len -= n;
  795. rc = efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);
  796. if (likely(rc == 0)) {
  797. if (st->out_len == 0)
  798. /* Transfer ownership of the skb */
  799. buffer->skb = skb;
  800. end_of_packet = st->out_len == 0 || st->packet_space == 0;
  801. buffer->continuation = !end_of_packet;
  802. if (st->in_len == 0) {
  803. /* Transfer ownership of the pci mapping */
  804. buffer->unmap_len = st->unmap_len;
  805. buffer->unmap_single = st->unmap_single;
  806. st->unmap_len = 0;
  807. }
  808. }
  809. st->dma_addr += n;
  810. return rc;
  811. }
  812. /**
  813. * tso_start_new_packet - generate a new header and prepare for the new packet
  814. * @tx_queue: Efx TX queue
  815. * @skb: Socket buffer
  816. * @st: TSO state
  817. *
  818. * Generate a new header and prepare for the new packet. Return 0 on
  819. * success, or -1 if failed to alloc header.
  820. */
  821. static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
  822. const struct sk_buff *skb,
  823. struct tso_state *st)
  824. {
  825. struct efx_tso_header *tsoh;
  826. struct tcphdr *tsoh_th;
  827. unsigned ip_length;
  828. u8 *header;
  829. /* Allocate a DMA-mapped header buffer. */
  830. if (likely(TSOH_SIZE(st->header_len) <= TSOH_STD_SIZE)) {
  831. if (tx_queue->tso_headers_free == NULL) {
  832. if (efx_tsoh_block_alloc(tx_queue))
  833. return -1;
  834. }
  835. EFX_BUG_ON_PARANOID(!tx_queue->tso_headers_free);
  836. tsoh = tx_queue->tso_headers_free;
  837. tx_queue->tso_headers_free = tsoh->next;
  838. tsoh->unmap_len = 0;
  839. } else {
  840. tx_queue->tso_long_headers++;
  841. tsoh = efx_tsoh_heap_alloc(tx_queue, st->header_len);
  842. if (unlikely(!tsoh))
  843. return -1;
  844. }
  845. header = TSOH_BUFFER(tsoh);
  846. tsoh_th = (struct tcphdr *)(header + SKB_TCP_OFF(skb));
  847. /* Copy and update the headers. */
  848. memcpy(header, skb->data, st->header_len);
  849. tsoh_th->seq = htonl(st->seqnum);
  850. st->seqnum += skb_shinfo(skb)->gso_size;
  851. if (st->out_len > skb_shinfo(skb)->gso_size) {
  852. /* This packet will not finish the TSO burst. */
  853. ip_length = st->full_packet_size - ETH_HDR_LEN(skb);
  854. tsoh_th->fin = 0;
  855. tsoh_th->psh = 0;
  856. } else {
  857. /* This packet will be the last in the TSO burst. */
  858. ip_length = st->header_len - ETH_HDR_LEN(skb) + st->out_len;
  859. tsoh_th->fin = tcp_hdr(skb)->fin;
  860. tsoh_th->psh = tcp_hdr(skb)->psh;
  861. }
  862. if (st->protocol == htons(ETH_P_IP)) {
  863. struct iphdr *tsoh_iph =
  864. (struct iphdr *)(header + SKB_IPV4_OFF(skb));
  865. tsoh_iph->tot_len = htons(ip_length);
  866. /* Linux leaves suitable gaps in the IP ID space for us to fill. */
  867. tsoh_iph->id = htons(st->ipv4_id);
  868. st->ipv4_id++;
  869. } else {
  870. struct ipv6hdr *tsoh_iph =
  871. (struct ipv6hdr *)(header + SKB_IPV6_OFF(skb));
  872. tsoh_iph->payload_len = htons(ip_length - sizeof(*tsoh_iph));
  873. }
  874. st->packet_space = skb_shinfo(skb)->gso_size;
  875. ++tx_queue->tso_packets;
  876. /* Form a descriptor for this header. */
  877. efx_tso_put_header(tx_queue, tsoh, st->header_len);
  878. return 0;
  879. }
  880. /**
  881. * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
  882. * @tx_queue: Efx TX queue
  883. * @skb: Socket buffer
  884. *
  885. * Context: You must hold netif_tx_lock() to call this function.
  886. *
  887. * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
  888. * @skb was not enqueued. In all cases @skb is consumed. Return
  889. * %NETDEV_TX_OK or %NETDEV_TX_BUSY.
  890. */
  891. static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
  892. struct sk_buff *skb)
  893. {
  894. struct efx_nic *efx = tx_queue->efx;
  895. int frag_i, rc, rc2 = NETDEV_TX_OK;
  896. struct tso_state state;
  897. /* Find the packet protocol and sanity-check it */
  898. state.protocol = efx_tso_check_protocol(skb);
  899. EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
  900. tso_start(&state, skb);
  901. /* Assume that skb header area contains exactly the headers, and
  902. * all payload is in the frag list.
  903. */
  904. if (skb_headlen(skb) == state.header_len) {
  905. /* Grab the first payload fragment. */
  906. EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
  907. frag_i = 0;
  908. rc = tso_get_fragment(&state, efx,
  909. skb_shinfo(skb)->frags + frag_i);
  910. if (rc)
  911. goto mem_err;
  912. } else {
  913. rc = tso_get_head_fragment(&state, efx, skb);
  914. if (rc)
  915. goto mem_err;
  916. frag_i = -1;
  917. }
  918. if (tso_start_new_packet(tx_queue, skb, &state) < 0)
  919. goto mem_err;
  920. while (1) {
  921. rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
  922. if (unlikely(rc))
  923. goto stop;
  924. /* Move onto the next fragment? */
  925. if (state.in_len == 0) {
  926. if (++frag_i >= skb_shinfo(skb)->nr_frags)
  927. /* End of payload reached. */
  928. break;
  929. rc = tso_get_fragment(&state, efx,
  930. skb_shinfo(skb)->frags + frag_i);
  931. if (rc)
  932. goto mem_err;
  933. }
  934. /* Start at new packet? */
  935. if (state.packet_space == 0 &&
  936. tso_start_new_packet(tx_queue, skb, &state) < 0)
  937. goto mem_err;
  938. }
  939. /* Pass off to hardware */
  940. efx_nic_push_buffers(tx_queue);
  941. tx_queue->tso_bursts++;
  942. return NETDEV_TX_OK;
  943. mem_err:
  944. EFX_ERR(efx, "Out of memory for TSO headers, or PCI mapping error\n");
  945. dev_kfree_skb_any(skb);
  946. goto unwind;
  947. stop:
  948. rc2 = NETDEV_TX_BUSY;
  949. /* Stop the queue if it wasn't stopped before. */
  950. if (tx_queue->stopped == 1)
  951. efx_stop_queue(efx);
  952. unwind:
  953. /* Free the DMA mapping we were in the process of writing out */
  954. if (state.unmap_len) {
  955. if (state.unmap_single)
  956. pci_unmap_single(efx->pci_dev, state.unmap_addr,
  957. state.unmap_len, PCI_DMA_TODEVICE);
  958. else
  959. pci_unmap_page(efx->pci_dev, state.unmap_addr,
  960. state.unmap_len, PCI_DMA_TODEVICE);
  961. }
  962. efx_enqueue_unwind(tx_queue);
  963. return rc2;
  964. }
  965. /*
  966. * Free up all TSO datastructures associated with tx_queue. This
  967. * routine should be called only once the tx_queue is both empty and
  968. * will no longer be used.
  969. */
  970. static void efx_fini_tso(struct efx_tx_queue *tx_queue)
  971. {
  972. unsigned i;
  973. if (tx_queue->buffer) {
  974. for (i = 0; i <= EFX_TXQ_MASK; ++i)
  975. efx_tsoh_free(tx_queue, &tx_queue->buffer[i]);
  976. }
  977. while (tx_queue->tso_headers_free != NULL)
  978. efx_tsoh_block_free(tx_queue, tx_queue->tso_headers_free,
  979. tx_queue->efx->pci_dev);
  980. }