igb: Support using build_skb in the case that jumbo frames are disabled

This change makes it so that we can enable the use of build_skb for cases where jumbo frames are disabled. The advantage to this is that we do not have to perform a memcpy to populate the header and as a result we see a significant performance improvement. Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com> Tested-by: Aaron Brown <aaron.f.brown@intel.com> Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
author: Alexander Duyck <alexander.h.duyck@intel.com> 2013-02-02 00:07:11 -0500
committer: Jeff Kirsher <jeffrey.t.kirsher@intel.com> 2013-02-15 04:11:29 -0500
commit: 74e238eada573540f82530f8ae4f3343c023d774 (patch)
tree: 69086119adb9bea30dd8e866aded4e0190521332 /drivers/net/ethernet
parent: 9754e293491e3a4e6c1ac020d25140b1ed3d9cd2 (diff)
2 files changed, 161 insertions, 34 deletions
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 4b78053592ba..afdb8bbcf6ce 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -275,10 +275,18 @@ struct igb_q_vector {
 enum e1000_ring_flags_t {
        IGB_RING_FLAG_RX_SCTP_CSUM,
        IGB_RING_FLAG_RX_LB_VLAN_BSWAP,
+        IGB_RING_FLAG_RX_BUILD_SKB_ENABLED,
        IGB_RING_FLAG_TX_CTX_IDX,
        IGB_RING_FLAG_TX_DETECT_HANG
 };
+#define ring_uses_build_skb(ring) \
+        test_bit(IGB_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
+#define set_ring_build_skb_enabled(ring) \
+        set_bit(IGB_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
+#define clear_ring_build_skb_enabled(ring) \
+        clear_bit(IGB_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
 #define IGB_TXD_DCMD (E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS)
 #define IGB_RX_DESC(R, i)           \
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 1aaf19351863..b070a97376c8 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -3354,6 +3354,20 @@ void igb_configure_rx_ring(struct igb_adapter *adapter,
        wr32(E1000_RXDCTL(reg_idx), rxdctl);
 }
+static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
+                                  struct igb_ring *rx_ring)
+{
+#define IGB_MAX_BUILD_SKB_SIZE \
+        (SKB_WITH_OVERHEAD(IGB_RX_BUFSZ) - \
+         (NET_SKB_PAD + NET_IP_ALIGN + IGB_TS_HDR_LEN))
+        /* set build_skb flag */
+        if (adapter->max_frame_size <= IGB_MAX_BUILD_SKB_SIZE)
+                set_ring_build_skb_enabled(rx_ring);
+        else
+                clear_ring_build_skb_enabled(rx_ring);
+}
 /**
 * igb_configure_rx - Configure receive Unit after Reset
 * @adapter: board private structure
@@ -3373,8 +3387,11 @@ static void igb_configure_rx(struct igb_adapter *adapter)
        /* Setup the HW Rx Head and Tail Descriptor Pointers and
         * the Base and Length of the Rx Descriptor Ring */
-        for (i = 0; i < adapter->num_rx_queues; i++)
+        for (i = 0; i < adapter->num_rx_queues; i++) {
-                igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
+                struct igb_ring *rx_ring = adapter->rx_ring[i];
+                igb_set_rx_buffer_len(adapter, rx_ring);
+                igb_configure_rx_ring(adapter, rx_ring);
+        }
 }
 /**
@@ -6097,6 +6114,41 @@ static void igb_reuse_rx_page(struct igb_ring *rx_ring,
                                         DMA_FROM_DEVICE);
 }
+static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
+                                  struct page *page,
+                                  unsigned int truesize)
+{
+        /* avoid re-using remote pages */
+        if (unlikely(page_to_nid(page) != numa_node_id()))
+                return false;
+#if (PAGE_SIZE < 8192)
+        /* if we are only owner of page we can reuse it */
+        if (unlikely(page_count(page) != 1))
+                return false;
+        /* flip page offset to other buffer */
+        rx_buffer->page_offset ^= IGB_RX_BUFSZ;
+        /* since we are the only owner of the page and we need to
+         * increment it, just set the value to 2 in order to avoid
+         * an unnecessary locked operation
+         */
+        atomic_set(&page->_count, 2);
+#else
+        /* move offset up to the next cache line */
+        rx_buffer->page_offset += truesize;
+        if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))
+                return false;
+        /* bump ref count on page before it is given to the stack */
+        get_page(page);
+#endif
+        return true;
+}
 /**
 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
 * @rx_ring: rx descriptor ring to transact packets on
@@ -6119,6 +6171,11 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,
 {
        struct page *page = rx_buffer->page;
        unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
+#if (PAGE_SIZE < 8192)
+        unsigned int truesize = IGB_RX_BUFSZ;
+#else
+        unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
+#endif
        if ((size <= IGB_RX_HDR_LEN) && !skb_is_nonlinear(skb)) {
                unsigned char *va = page_address(page) + rx_buffer->page_offset;
@@ -6141,38 +6198,88 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,
        }
        skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
-                        rx_buffer->page_offset, size, IGB_RX_BUFSZ);
+                        rx_buffer->page_offset, size, truesize);
-        /* avoid re-using remote pages */
+        return igb_can_reuse_rx_page(rx_buffer, page, truesize);
-        if (unlikely(page_to_nid(page) != numa_node_id()))
+}
-                return false;
+static struct sk_buff *igb_build_rx_buffer(struct igb_ring *rx_ring,
+                                           union e1000_adv_rx_desc *rx_desc)
+{
+        struct igb_rx_buffer *rx_buffer;
+        struct sk_buff *skb;
+        struct page *page;
+        void *page_addr;
+        unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
 #if (PAGE_SIZE < 8192)
-        /* if we are only owner of page we can reuse it */
+        unsigned int truesize = IGB_RX_BUFSZ;
-        if (unlikely(page_count(page) != 1))
+#else
-                return false;
+        unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
+                                SKB_DATA_ALIGN(NET_SKB_PAD +
+                                               NET_IP_ALIGN +
+                                               size);
+#endif
-        /* flip page offset to other buffer */
+        /* If we spanned a buffer we have a huge mess so test for it */
-        rx_buffer->page_offset ^= IGB_RX_BUFSZ;
+        BUG_ON(unlikely(!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)));
-        /*
+        /* Guarantee this function can be used by verifying buffer sizes */
-         * since we are the only owner of the page and we need to
+        BUILD_BUG_ON(SKB_WITH_OVERHEAD(IGB_RX_BUFSZ) < (NET_SKB_PAD +
-         * increment it, just set the value to 2 in order to avoid
+                                                        NET_IP_ALIGN +
-         * an unnecessary locked operation
+                                                        IGB_TS_HDR_LEN +
-         */
+                                                        ETH_FRAME_LEN +
-        atomic_set(&page->_count, 2);
+                                                        ETH_FCS_LEN));
-#else
-        /* move offset up to the next cache line */
-        rx_buffer->page_offset += SKB_DATA_ALIGN(size);
-        if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))
+        rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
-                return false;
+        page = rx_buffer->page;
+        prefetchw(page);
-        /* bump ref count on page before it is given to the stack */
+        page_addr = page_address(page) + rx_buffer->page_offset;
-        get_page(page);
+        /* prefetch first cache line of first page */
+        prefetch(page_addr + NET_SKB_PAD + NET_IP_ALIGN);
+#if L1_CACHE_BYTES < 128
+        prefetch(page_addr + L1_CACHE_BYTES + NET_SKB_PAD + NET_IP_ALIGN);
 #endif
-        return true;
+        /* build an skb to around the page buffer */
+        skb = build_skb(page_addr, truesize);
+        if (unlikely(!skb)) {
+                rx_ring->rx_stats.alloc_failed++;
+                return NULL;
+        }
+        /* we are reusing so sync this buffer for CPU use */
+        dma_sync_single_range_for_cpu(rx_ring->dev,
+                                      rx_buffer->dma,
+                                      rx_buffer->page_offset,
+                                      IGB_RX_BUFSZ,
+                                      DMA_FROM_DEVICE);
+        /* update pointers within the skb to store the data */
+        skb_reserve(skb, NET_IP_ALIGN + NET_SKB_PAD);
+        __skb_put(skb, size);
+        /* pull timestamp out of packet data */
+        if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
+                igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb);
+                __skb_pull(skb, IGB_TS_HDR_LEN);
+        }
+        if (igb_can_reuse_rx_page(rx_buffer, page, truesize)) {
+                /* hand second half of page back to the ring */
+                igb_reuse_rx_page(rx_ring, rx_buffer);
+        } else {
+                /* we are not reusing the buffer so unmap it */
+                dma_unmap_page(rx_ring->dev, rx_buffer->dma,
+                               PAGE_SIZE, DMA_FROM_DEVICE);
+        }
+        /* clear contents of buffer_info */
+        rx_buffer->dma = 0;
+        rx_buffer->page = NULL;
+        return skb;
 }
 static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
@@ -6184,13 +6291,6 @@ static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
        rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
-        /*
-         * This memory barrier is needed to keep us from reading
-         * any other fields out of the rx_desc until we know the
-         * RXD_STAT_DD bit is set
-         */
-        rmb();
        page = rx_buffer->page;
        prefetchw(page);
@@ -6590,8 +6690,17 @@ static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
                if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))
                        break;
+                /* This memory barrier is needed to keep us from reading
+                 * any other fields out of the rx_desc until we know the
+                 * RXD_STAT_DD bit is set
+                 */
+                rmb();
                /* retrieve a buffer from the ring */
-                skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);
+                if (ring_uses_build_skb(rx_ring))
+                        skb = igb_build_rx_buffer(rx_ring, rx_desc);
+                else
+                        skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);
                /* exit if we failed to retrieve a buffer */
                if (!skb)
@@ -6678,6 +6787,14 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
        return true;
 }
+static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring)
+{
+        if (ring_uses_build_skb(rx_ring))
+                return NET_SKB_PAD + NET_IP_ALIGN;
+        else
+                return 0;
+}
 /**
 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
 * @adapter: address of board private structure
@@ -6704,7 +6821,9 @@ void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
                 * Refresh the desc even if buffer_addrs didn't change
                 * because each write-back erases this info.
                 */
-                rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
+                rx_desc->read.pkt_addr = cpu_to_le64(bi->dma +
+                                                     bi->page_offset +
+                                                     igb_rx_offset(rx_ring));
                rx_desc++;
                bi++;
author	Alexander Duyck <alexander.h.duyck@intel.com>	2013-02-02 00:07:11 -0500
committer	Jeff Kirsher <jeffrey.t.kirsher@intel.com>	2013-02-15 04:11:29 -0500
commit	74e238eada573540f82530f8ae4f3343c023d774 (patch)
tree	69086119adb9bea30dd8e866aded4e0190521332 /drivers/net/ethernet
parent	9754e293491e3a4e6c1ac020d25140b1ed3d9cd2 (diff)

diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h index 4b78053592ba..afdb8bbcf6ce 100644 --- a/drivers/net/ethernet/intel/igb/igb.h +++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -275,10 +275,18 @@ struct igb_q_vector {
275	enum e1000_ring_flags_t {	275	enum e1000_ring_flags_t {
276	IGB_RING_FLAG_RX_SCTP_CSUM,	276	IGB_RING_FLAG_RX_SCTP_CSUM,
277	IGB_RING_FLAG_RX_LB_VLAN_BSWAP,	277	IGB_RING_FLAG_RX_LB_VLAN_BSWAP,
		278	IGB_RING_FLAG_RX_BUILD_SKB_ENABLED,
278	IGB_RING_FLAG_TX_CTX_IDX,	279	IGB_RING_FLAG_TX_CTX_IDX,
279	IGB_RING_FLAG_TX_DETECT_HANG	280	IGB_RING_FLAG_TX_DETECT_HANG
280	};	281	};
281		282
		283	#define ring_uses_build_skb(ring) \
		284	test_bit(IGB_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
		285	#define set_ring_build_skb_enabled(ring) \
		286	set_bit(IGB_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
		287	#define clear_ring_build_skb_enabled(ring) \
		288	clear_bit(IGB_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags)
		289
282	#define IGB_TXD_DCMD (E1000_ADVTXD_DCMD_EOP \| E1000_ADVTXD_DCMD_RS)	290	#define IGB_TXD_DCMD (E1000_ADVTXD_DCMD_EOP \| E1000_ADVTXD_DCMD_RS)
283		291
284	#define IGB_RX_DESC(R, i) \	292	#define IGB_RX_DESC(R, i) \


diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 1aaf19351863..b070a97376c8 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -3354,6 +3354,20 @@ void igb_configure_rx_ring(struct igb_adapter *adapter,
3354	wr32(E1000_RXDCTL(reg_idx), rxdctl);	3354	wr32(E1000_RXDCTL(reg_idx), rxdctl);
3355	}	3355	}
3356		3356
		3357	static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
		3358	struct igb_ring *rx_ring)
		3359	{
		3360	#define IGB_MAX_BUILD_SKB_SIZE \
		3361	(SKB_WITH_OVERHEAD(IGB_RX_BUFSZ) - \
		3362	(NET_SKB_PAD + NET_IP_ALIGN + IGB_TS_HDR_LEN))
		3363
		3364	/* set build_skb flag */
		3365	if (adapter->max_frame_size <= IGB_MAX_BUILD_SKB_SIZE)
		3366	set_ring_build_skb_enabled(rx_ring);
		3367	else
		3368	clear_ring_build_skb_enabled(rx_ring);
		3369	}
		3370
3357	/**	3371	/**
3358	* igb_configure_rx - Configure receive Unit after Reset	3372	* igb_configure_rx - Configure receive Unit after Reset
3359	* @adapter: board private structure	3373	* @adapter: board private structure
@@ -3373,8 +3387,11 @@ static void igb_configure_rx(struct igb_adapter *adapter)
3373		3387
3374	/* Setup the HW Rx Head and Tail Descriptor Pointers and	3388	/* Setup the HW Rx Head and Tail Descriptor Pointers and
3375	* the Base and Length of the Rx Descriptor Ring */	3389	* the Base and Length of the Rx Descriptor Ring */
3376	for (i = 0; i < adapter->num_rx_queues; i++)	3390	for (i = 0; i < adapter->num_rx_queues; i++) {
3377	igb_configure_rx_ring(adapter, adapter->rx_ring[i]);	3391	struct igb_ring *rx_ring = adapter->rx_ring[i];
		3392	igb_set_rx_buffer_len(adapter, rx_ring);
		3393	igb_configure_rx_ring(adapter, rx_ring);
		3394	}
3378	}	3395	}
3379		3396
3380	/**	3397	/**
@@ -6097,6 +6114,41 @@ static void igb_reuse_rx_page(struct igb_ring *rx_ring,
6097	DMA_FROM_DEVICE);	6114	DMA_FROM_DEVICE);
6098	}	6115	}
6099		6116
		6117	static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
		6118	struct page *page,
		6119	unsigned int truesize)
		6120	{
		6121	/* avoid re-using remote pages */
		6122	if (unlikely(page_to_nid(page) != numa_node_id()))
		6123	return false;
		6124
		6125	#if (PAGE_SIZE < 8192)
		6126	/* if we are only owner of page we can reuse it */
		6127	if (unlikely(page_count(page) != 1))
		6128	return false;
		6129
		6130	/* flip page offset to other buffer */
		6131	rx_buffer->page_offset ^= IGB_RX_BUFSZ;
		6132
		6133	/* since we are the only owner of the page and we need to
		6134	* increment it, just set the value to 2 in order to avoid
		6135	* an unnecessary locked operation
		6136	*/
		6137	atomic_set(&page->_count, 2);
		6138	#else
		6139	/* move offset up to the next cache line */
		6140	rx_buffer->page_offset += truesize;
		6141
		6142	if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))
		6143	return false;
		6144
		6145	/* bump ref count on page before it is given to the stack */
		6146	get_page(page);
		6147	#endif
		6148
		6149	return true;
		6150	}
		6151
6100	/**	6152	/**
6101	* igb_add_rx_frag - Add contents of Rx buffer to sk_buff	6153	* igb_add_rx_frag - Add contents of Rx buffer to sk_buff
6102	* @rx_ring: rx descriptor ring to transact packets on	6154	* @rx_ring: rx descriptor ring to transact packets on
@@ -6119,6 +6171,11 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,
6119	{	6171	{
6120	struct page *page = rx_buffer->page;	6172	struct page *page = rx_buffer->page;
6121	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);	6173	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
		6174	#if (PAGE_SIZE < 8192)
		6175	unsigned int truesize = IGB_RX_BUFSZ;
		6176	#else
		6177	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
		6178	#endif
6122		6179
6123	if ((size <= IGB_RX_HDR_LEN) && !skb_is_nonlinear(skb)) {	6180	if ((size <= IGB_RX_HDR_LEN) && !skb_is_nonlinear(skb)) {
6124	unsigned char *va = page_address(page) + rx_buffer->page_offset;	6181	unsigned char *va = page_address(page) + rx_buffer->page_offset;
@@ -6141,38 +6198,88 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,
6141	}	6198	}
6142		6199
6143	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,	6200	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
6144	rx_buffer->page_offset, size, IGB_RX_BUFSZ);	6201	rx_buffer->page_offset, size, truesize);
6145		6202
6146	/* avoid re-using remote pages */	6203	return igb_can_reuse_rx_page(rx_buffer, page, truesize);
6147	if (unlikely(page_to_nid(page) != numa_node_id()))	6204	}
6148	return false;
6149		6205
		6206	static struct sk_buff igb_build_rx_buffer(struct igb_ring rx_ring,
		6207	union e1000_adv_rx_desc *rx_desc)
		6208	{
		6209	struct igb_rx_buffer *rx_buffer;
		6210	struct sk_buff *skb;
		6211	struct page *page;
		6212	void *page_addr;
		6213	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
6150	#if (PAGE_SIZE < 8192)	6214	#if (PAGE_SIZE < 8192)
6151	/* if we are only owner of page we can reuse it */	6215	unsigned int truesize = IGB_RX_BUFSZ;
6152	if (unlikely(page_count(page) != 1))	6216	#else
6153	return false;	6217	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
		6218	SKB_DATA_ALIGN(NET_SKB_PAD +
		6219	NET_IP_ALIGN +
		6220	size);
		6221	#endif
6154		6222
6155	/* flip page offset to other buffer */	6223	/* If we spanned a buffer we have a huge mess so test for it */
6156	rx_buffer->page_offset ^= IGB_RX_BUFSZ;	6224	BUG_ON(unlikely(!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)));
6157		6225
6158	/*	6226	/* Guarantee this function can be used by verifying buffer sizes */
6159	* since we are the only owner of the page and we need to	6227	BUILD_BUG_ON(SKB_WITH_OVERHEAD(IGB_RX_BUFSZ) < (NET_SKB_PAD +
6160	* increment it, just set the value to 2 in order to avoid	6228	NET_IP_ALIGN +
6161	* an unnecessary locked operation	6229	IGB_TS_HDR_LEN +
6162	*/	6230	ETH_FRAME_LEN +
6163	atomic_set(&page->_count, 2);	6231	ETH_FCS_LEN));
6164	#else
6165	/* move offset up to the next cache line */
6166	rx_buffer->page_offset += SKB_DATA_ALIGN(size);
6167		6232
6168	if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))	6233	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
6169	return false;	6234	page = rx_buffer->page;
		6235	prefetchw(page);
6170		6236
6171	/* bump ref count on page before it is given to the stack */	6237	page_addr = page_address(page) + rx_buffer->page_offset;
6172	get_page(page);	6238
		6239	/* prefetch first cache line of first page */
		6240	prefetch(page_addr + NET_SKB_PAD + NET_IP_ALIGN);
		6241	#if L1_CACHE_BYTES < 128
		6242	prefetch(page_addr + L1_CACHE_BYTES + NET_SKB_PAD + NET_IP_ALIGN);
6173	#endif	6243	#endif
6174		6244
6175	return true;	6245	/* build an skb to around the page buffer */
		6246	skb = build_skb(page_addr, truesize);
		6247	if (unlikely(!skb)) {
		6248	rx_ring->rx_stats.alloc_failed++;
		6249	return NULL;
		6250	}
		6251
		6252	/* we are reusing so sync this buffer for CPU use */
		6253	dma_sync_single_range_for_cpu(rx_ring->dev,
		6254	rx_buffer->dma,
		6255	rx_buffer->page_offset,
		6256	IGB_RX_BUFSZ,
		6257	DMA_FROM_DEVICE);
		6258
		6259	/* update pointers within the skb to store the data */
		6260	skb_reserve(skb, NET_IP_ALIGN + NET_SKB_PAD);
		6261	__skb_put(skb, size);
		6262
		6263	/* pull timestamp out of packet data */
		6264	if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
		6265	igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb);
		6266	__skb_pull(skb, IGB_TS_HDR_LEN);
		6267	}
		6268
		6269	if (igb_can_reuse_rx_page(rx_buffer, page, truesize)) {
		6270	/* hand second half of page back to the ring */
		6271	igb_reuse_rx_page(rx_ring, rx_buffer);
		6272	} else {
		6273	/* we are not reusing the buffer so unmap it */
		6274	dma_unmap_page(rx_ring->dev, rx_buffer->dma,
		6275	PAGE_SIZE, DMA_FROM_DEVICE);
		6276	}
		6277
		6278	/* clear contents of buffer_info */
		6279	rx_buffer->dma = 0;
		6280	rx_buffer->page = NULL;
		6281
		6282	return skb;
6176	}	6283	}
6177		6284
6178	static struct sk_buff igb_fetch_rx_buffer(struct igb_ring rx_ring,	6285	static struct sk_buff igb_fetch_rx_buffer(struct igb_ring rx_ring,
@@ -6184,13 +6291,6 @@ static struct sk_buff igb_fetch_rx_buffer(struct igb_ring rx_ring,
6184		6291
6185	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];	6292	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
6186		6293
6187	/*
6188	* This memory barrier is needed to keep us from reading
6189	* any other fields out of the rx_desc until we know the
6190	* RXD_STAT_DD bit is set
6191	*/
6192	rmb();
6193
6194	page = rx_buffer->page;	6294	page = rx_buffer->page;
6195	prefetchw(page);	6295	prefetchw(page);
6196		6296
@@ -6590,8 +6690,17 @@ static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
6590	if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))	6690	if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))
6591	break;	6691	break;
6592		6692
		6693	/* This memory barrier is needed to keep us from reading
		6694	* any other fields out of the rx_desc until we know the
		6695	* RXD_STAT_DD bit is set
		6696	*/
		6697	rmb();
		6698
6593	/* retrieve a buffer from the ring */	6699	/* retrieve a buffer from the ring */
6594	skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);	6700	if (ring_uses_build_skb(rx_ring))
		6701	skb = igb_build_rx_buffer(rx_ring, rx_desc);
		6702	else
		6703	skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);
6595		6704
6596	/* exit if we failed to retrieve a buffer */	6705	/* exit if we failed to retrieve a buffer */
6597	if (!skb)	6706	if (!skb)
@@ -6678,6 +6787,14 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6678	return true;	6787	return true;
6679	}	6788	}
6680		6789
		6790	static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring)
		6791	{
		6792	if (ring_uses_build_skb(rx_ring))
		6793	return NET_SKB_PAD + NET_IP_ALIGN;
		6794	else
		6795	return 0;
		6796	}
		6797
6681	/**	6798	/**
6682	* igb_alloc_rx_buffers - Replace used receive buffers; packet split	6799	* igb_alloc_rx_buffers - Replace used receive buffers; packet split
6683	* @adapter: address of board private structure	6800	* @adapter: address of board private structure
@@ -6704,7 +6821,9 @@ void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6704	* Refresh the desc even if buffer_addrs didn't change	6821	* Refresh the desc even if buffer_addrs didn't change
6705	* because each write-back erases this info.	6822	* because each write-back erases this info.
6706	*/	6823	*/
6707	rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);	6824	rx_desc->read.pkt_addr = cpu_to_le64(bi->dma +
		6825	bi->page_offset +
		6826	igb_rx_offset(rx_ring));
6708		6827
6709	rx_desc++;	6828	rx_desc++;
6710	bi++;	6829	bi++;