[PATCH] e1000: Added RX buffer enhancements

Align the prefetches to a dword to help speed them up.
Recycle skb's and early replenish.
Force memory writes to complete before fetching more descriptors.

Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: John Ronciak <john.ronciak@intel.com>
Signed-off-by: Jeff Garzik <jgarzik@pobox.com>

drivers/net/e1000/e1000_main.c

@@ -1653,23 +1653,8 @@ e1000_setup_rctl(struct e1000_adapter *adapter)
 		rctl |= adapter->rx_buffer_len << 0x11;
 	} else {
 		rctl &= ~E1000_RCTL_SZ_4096;
-		rctl |= E1000_RCTL_BSEX; 
-		switch (adapter->rx_buffer_len) {
-		case E1000_RXBUFFER_2048:
-		default:
-			rctl |= E1000_RCTL_SZ_2048;
-			rctl &= ~E1000_RCTL_BSEX;
-			break;
-		case E1000_RXBUFFER_4096:
-			rctl |= E1000_RCTL_SZ_4096;
-			break;
-		case E1000_RXBUFFER_8192:
-			rctl |= E1000_RCTL_SZ_8192;
-			break;
-		case E1000_RXBUFFER_16384:
-			rctl |= E1000_RCTL_SZ_16384;
-			break;
-		}
+		rctl &= ~E1000_RCTL_BSEX;
+		rctl |= E1000_RCTL_SZ_2048;
 	}
 
 #ifndef CONFIG_E1000_DISABLE_PACKET_SPLIT
@@ -3571,7 +3556,6 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
 	struct pci_dev *pdev = adapter->pdev;
 	struct e1000_rx_desc *rx_desc;
 	struct e1000_buffer *buffer_info;
-	struct sk_buff *skb;
 	unsigned long flags;
 	uint32_t length;
 	uint8_t last_byte;
@@ -3581,9 +3565,10 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
 
 	i = rx_ring->next_to_clean;
 	rx_desc = E1000_RX_DESC(*rx_ring, i);
+	buffer_info = &rx_ring->buffer_info[i];
 
-	while(rx_desc->status & E1000_RXD_STAT_DD) {
-		buffer_info = &rx_ring->buffer_info[i];
+	while (rx_desc->status & E1000_RXD_STAT_DD) {
+		struct sk_buff *skb;
 		u8 status;
 #ifdef CONFIG_E1000_NAPI
 		if(*work_done >= work_to_do)
@@ -3591,6 +3576,7 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
 		(*work_done)++;
 #endif
 		status = rx_desc->status;
+		skb = buffer_info->skb;
 		cleaned = TRUE;
 		cleaned_count++;
 		pci_unmap_single(pdev,
@@ -3598,20 +3584,50 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
 		                 buffer_info->length,
 		                 PCI_DMA_FROMDEVICE);
 
-		skb = buffer_info->skb;
 		length = le16_to_cpu(rx_desc->length);
 
-		if(unlikely(!(rx_desc->status & E1000_RXD_STAT_EOP))) {
-			/* All receives must fit into a single buffer */
-			E1000_DBG("%s: Receive packet consumed multiple"
-				  " buffers\n", netdev->name);
-			dev_kfree_skb_irq(skb);
+		skb_put(skb, length);
+
+		if (!(status & E1000_RXD_STAT_EOP)) {
+			if (!rx_ring->rx_skb_top) {
+				rx_ring->rx_skb_top = skb;
+				rx_ring->rx_skb_top->len = length;
+				rx_ring->rx_skb_prev = skb;
+			} else {
+				if (skb_shinfo(rx_ring->rx_skb_top)->frag_list) {
+					rx_ring->rx_skb_prev->next = skb;
+					skb->prev = rx_ring->rx_skb_prev;
+				} else {
+					skb_shinfo(rx_ring->rx_skb_top)->frag_list = skb;
+				}
+				rx_ring->rx_skb_prev = skb;
+				rx_ring->rx_skb_top->data_len += length;
+			}
 			goto next_desc;
+		} else {
+			if (rx_ring->rx_skb_top) {
+				if (skb_shinfo(rx_ring->rx_skb_top)
+							->frag_list) {
+					rx_ring->rx_skb_prev->next = skb;
+					skb->prev = rx_ring->rx_skb_prev;
+				} else
+					skb_shinfo(rx_ring->rx_skb_top)
+							->frag_list = skb;
+
+				rx_ring->rx_skb_top->data_len += length;
+				rx_ring->rx_skb_top->len +=
+					rx_ring->rx_skb_top->data_len;
+
+				skb = rx_ring->rx_skb_top;
+				multi_descriptor = TRUE;
+				rx_ring->rx_skb_top = NULL;
+				rx_ring->rx_skb_prev = NULL;
+			}
 		}
 
 		if(unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
 			last_byte = *(skb->data + length - 1);
-			if(TBI_ACCEPT(&adapter->hw, rx_desc->status,
+			if (TBI_ACCEPT(&adapter->hw, status,
 			              rx_desc->errors, length, last_byte)) {
 				spin_lock_irqsave(&adapter->stats_lock, flags);
 				e1000_tbi_adjust_stats(&adapter->hw,
@@ -3668,7 +3684,7 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
 		}
 #else /* CONFIG_E1000_NAPI */
 		if(unlikely(adapter->vlgrp &&
-			    (rx_desc->status & E1000_RXD_STAT_VP))) {
+			    (status & E1000_RXD_STAT_VP))) {
 			vlan_hwaccel_rx(skb, adapter->vlgrp,
 					le16_to_cpu(rx_desc->special) &
 					E1000_RXD_SPC_VLAN_MASK);
@@ -3795,12 +3811,8 @@ e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
 		skb->protocol = eth_type_trans(skb, netdev);
 
 		if(likely(rx_desc->wb.upper.header_status &
-			  E1000_RXDPS_HDRSTAT_HDRSP)) {
+			  E1000_RXDPS_HDRSTAT_HDRSP))
 			adapter->rx_hdr_split++;
-#ifdef HAVE_RX_ZERO_COPY
-			skb_shinfo(skb)->zero_copy = TRUE;
-#endif
-	        }
 #ifdef CONFIG_E1000_NAPI
 		if(unlikely(adapter->vlgrp && (staterr & E1000_RXD_STAT_VP))) {
 			vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
@@ -3940,20 +3952,22 @@ map_skb:
 		rx_desc = E1000_RX_DESC(*rx_ring, i);
 		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
 
-		if(unlikely((i & ~(E1000_RX_BUFFER_WRITE - 1)) == i)) {
-			/* Force memory writes to complete before letting h/w
-			 * know there are new descriptors to fetch.  (Only
-			 * applicable for weak-ordered memory model archs,
-			 * such as IA-64). */
-			wmb();
-			writel(i, adapter->hw.hw_addr + rx_ring->rdt);
-		}
-
 		if(unlikely(++i == rx_ring->count)) i = 0;
 		buffer_info = &rx_ring->buffer_info[i];
 	}
 
-	rx_ring->next_to_use = i;
+	if (likely(rx_ring->next_to_use != i)) {
+		rx_ring->next_to_use = i;
+		if (unlikely(i-- == 0))
+			i = (rx_ring->count - 1);
+
+		/* Force memory writes to complete before letting h/w
+		 * know there are new descriptors to fetch.  (Only
+		 * applicable for weak-ordered memory model archs,
+		 * such as IA-64). */
+		wmb();
+		writel(i, adapter->hw.hw_addr + rx_ring->rdt);
+	}
 }
 
 /**
@@ -3988,8 +4002,10 @@ e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
 				if (likely(!ps_page->ps_page[j])) {
 					ps_page->ps_page[j] =
 						alloc_page(GFP_ATOMIC);
-					if (unlikely(!ps_page->ps_page[j]))
+					if (unlikely(!ps_page->ps_page[j])) {
+						adapter->alloc_rx_buff_failed++;
 						goto no_buffers;
+					}
 					ps_page_dma->ps_page_dma[j] =
 						pci_map_page(pdev,
 							    ps_page->ps_page[j],
@@ -4008,8 +4024,10 @@ e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
 
 		skb = dev_alloc_skb(adapter->rx_ps_bsize0 + NET_IP_ALIGN);
 
-		if(unlikely(!skb))
+		if (unlikely(!skb)) {
+			adapter->alloc_rx_buff_failed++;
 			break;
+		}
 
 		/* Make buffer alignment 2 beyond a 16 byte boundary
 		 * this will result in a 16 byte aligned IP header after
@@ -4027,19 +4045,6 @@ e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
 
 		rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
 
-		if(unlikely((i & ~(E1000_RX_BUFFER_WRITE - 1)) == i)) {
-			/* Force memory writes to complete before letting h/w
-			 * know there are new descriptors to fetch.  (Only
-			 * applicable for weak-ordered memory model archs,
-			 * such as IA-64). */
-			wmb();
-			/* Hardware increments by 16 bytes, but packet split
-			 * descriptors are 32 bytes...so we increment tail
-			 * twice as much.
-			 */
-			writel(i<<1, adapter->hw.hw_addr + rx_ring->rdt);
-		}
-
 		if(unlikely(++i == rx_ring->count)) i = 0;
 		buffer_info = &rx_ring->buffer_info[i];
 		ps_page = &rx_ring->ps_page[i];
@@ -4047,7 +4052,21 @@ e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
 	}
 
 no_buffers:
-	rx_ring->next_to_use = i;
+	if (likely(rx_ring->next_to_use != i)) {
+		rx_ring->next_to_use = i;
+		if (unlikely(i-- == 0)) i = (rx_ring->count - 1);
+
+		/* Force memory writes to complete before letting h/w
+		 * know there are new descriptors to fetch.  (Only
+		 * applicable for weak-ordered memory model archs,
+		 * such as IA-64). */
+		wmb();
+		/* Hardware increments by 16 bytes, but packet split
+		 * descriptors are 32 bytes...so we increment tail
+		 * twice as much.
+		 */
+		writel(i<<1, adapter->hw.hw_addr + rx_ring->rdt);
+	}
 }
 
 /**