aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net
diff options
context:
space:
mode:
authorJesse Brandeburg <jesse.brandeburg@intel.com>2013-09-11 04:39:51 -0400
committerJeff Kirsher <jeffrey.t.kirsher@intel.com>2013-09-11 04:43:57 -0400
commitfd0a05ce74efc9aadd3b2e259ae1d16ee7c935c1 (patch)
tree91969cd31fffdeaea0d709afaf2a8d0ad829abdc /drivers/net
parent41c445ff0f482bb6e6b72dcee9e598e20575f743 (diff)
i40e: transmit, receive, and NAPI
This patch contains the transmit, receive, and NAPI routines, as well as ancillary routines. This file is code that is (will be) used by both the VF and PF drivers. Signed-off-by: Jesse Brandeburg <jesse.brandeburg@intel.com> Signed-off-by: Shannon Nelson <shannon.nelson@intel.com> CC: PJ Waskiewicz <peter.p.waskiewicz.jr@intel.com> CC: e1000-devel@lists.sourceforge.net Tested-by: Kavindya Deegala <kavindya.s.deegala@intel.com> Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Diffstat (limited to 'drivers/net')
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_txrx.c1817
1 files changed, 1817 insertions, 0 deletions
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
new file mode 100644
index 000000000000..49d2cfa9b0cc
--- /dev/null
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -0,0 +1,1817 @@
1/*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
4 * Copyright(c) 2013 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * The full GNU General Public License is included in this distribution in
20 * the file called "COPYING".
21 *
22 * Contact Information:
23 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25 *
26 ******************************************************************************/
27
28#include "i40e.h"
29
30static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
31 u32 td_tag)
32{
33 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
34 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
35 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
36 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
37 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
38}
39
40/**
41 * i40e_program_fdir_filter - Program a Flow Director filter
42 * @fdir_input: Packet data that will be filter parameters
43 * @pf: The pf pointer
44 * @add: True for add/update, False for remove
45 **/
46int i40e_program_fdir_filter(struct i40e_fdir_data *fdir_data,
47 struct i40e_pf *pf, bool add)
48{
49 struct i40e_filter_program_desc *fdir_desc;
50 struct i40e_tx_buffer *tx_buf;
51 struct i40e_tx_desc *tx_desc;
52 struct i40e_ring *tx_ring;
53 struct i40e_vsi *vsi;
54 struct device *dev;
55 dma_addr_t dma;
56 u32 td_cmd = 0;
57 u16 i;
58
59 /* find existing FDIR VSI */
60 vsi = NULL;
61 for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
62 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
63 vsi = pf->vsi[i];
64 if (!vsi)
65 return -ENOENT;
66
67 tx_ring = &vsi->tx_rings[0];
68 dev = tx_ring->dev;
69
70 dma = dma_map_single(dev, fdir_data->raw_packet,
71 I40E_FDIR_MAX_RAW_PACKET_LOOKUP, DMA_TO_DEVICE);
72 if (dma_mapping_error(dev, dma))
73 goto dma_fail;
74
75 /* grab the next descriptor */
76 fdir_desc = I40E_TX_FDIRDESC(tx_ring, tx_ring->next_to_use);
77 tx_buf = &tx_ring->tx_bi[tx_ring->next_to_use];
78 tx_ring->next_to_use++;
79 if (tx_ring->next_to_use == tx_ring->count)
80 tx_ring->next_to_use = 0;
81
82 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32((fdir_data->q_index
83 << I40E_TXD_FLTR_QW0_QINDEX_SHIFT)
84 & I40E_TXD_FLTR_QW0_QINDEX_MASK);
85
86 fdir_desc->qindex_flex_ptype_vsi |= cpu_to_le32((fdir_data->flex_off
87 << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT)
88 & I40E_TXD_FLTR_QW0_FLEXOFF_MASK);
89
90 fdir_desc->qindex_flex_ptype_vsi |= cpu_to_le32((fdir_data->pctype
91 << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT)
92 & I40E_TXD_FLTR_QW0_PCTYPE_MASK);
93
94 /* Use LAN VSI Id if not programmed by user */
95 if (fdir_data->dest_vsi == 0)
96 fdir_desc->qindex_flex_ptype_vsi |=
97 cpu_to_le32((pf->vsi[pf->lan_vsi]->id)
98 << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT);
99 else
100 fdir_desc->qindex_flex_ptype_vsi |=
101 cpu_to_le32((fdir_data->dest_vsi
102 << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT)
103 & I40E_TXD_FLTR_QW0_DEST_VSI_MASK);
104
105 fdir_desc->dtype_cmd_cntindex =
106 cpu_to_le32(I40E_TX_DESC_DTYPE_FILTER_PROG);
107
108 if (add)
109 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32(
110 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE
111 << I40E_TXD_FLTR_QW1_PCMD_SHIFT);
112 else
113 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32(
114 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE
115 << I40E_TXD_FLTR_QW1_PCMD_SHIFT);
116
117 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32((fdir_data->dest_ctl
118 << I40E_TXD_FLTR_QW1_DEST_SHIFT)
119 & I40E_TXD_FLTR_QW1_DEST_MASK);
120
121 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32(
122 (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT)
123 & I40E_TXD_FLTR_QW1_FD_STATUS_MASK);
124
125 if (fdir_data->cnt_index != 0) {
126 fdir_desc->dtype_cmd_cntindex |=
127 cpu_to_le32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK);
128 fdir_desc->dtype_cmd_cntindex |=
129 cpu_to_le32((fdir_data->cnt_index
130 << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT)
131 & I40E_TXD_FLTR_QW1_CNTINDEX_MASK);
132 }
133
134 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
135
136 /* Now program a dummy descriptor */
137 tx_desc = I40E_TX_DESC(tx_ring, tx_ring->next_to_use);
138 tx_buf = &tx_ring->tx_bi[tx_ring->next_to_use];
139 tx_ring->next_to_use++;
140 if (tx_ring->next_to_use == tx_ring->count)
141 tx_ring->next_to_use = 0;
142
143 tx_desc->buffer_addr = cpu_to_le64(dma);
144 td_cmd = I40E_TX_DESC_CMD_EOP |
145 I40E_TX_DESC_CMD_RS |
146 I40E_TX_DESC_CMD_DUMMY;
147
148 tx_desc->cmd_type_offset_bsz =
149 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_LOOKUP, 0);
150
151 /* Mark the data descriptor to be watched */
152 tx_buf->next_to_watch = tx_desc;
153
154 /* Force memory writes to complete before letting h/w
155 * know there are new descriptors to fetch. (Only
156 * applicable for weak-ordered memory model archs,
157 * such as IA-64).
158 */
159 wmb();
160
161 writel(tx_ring->next_to_use, tx_ring->tail);
162 return 0;
163
164dma_fail:
165 return -1;
166}
167
168/**
169 * i40e_fd_handle_status - check the Programming Status for FD
170 * @rx_ring: the Rx ring for this descriptor
171 * @qw: the descriptor data
172 * @prog_id: the id originally used for programming
173 *
174 * This is used to verify if the FD programming or invalidation
175 * requested by SW to the HW is successful or not and take actions accordingly.
176 **/
177static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u32 qw, u8 prog_id)
178{
179 struct pci_dev *pdev = rx_ring->vsi->back->pdev;
180 u32 error;
181
182 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
183 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
184
185 /* for now just print the Status */
186 dev_info(&pdev->dev, "FD programming id %02x, Status %08x\n",
187 prog_id, error);
188}
189
190/**
191 * i40e_unmap_tx_resource - Release a Tx buffer
192 * @ring: the ring that owns the buffer
193 * @tx_buffer: the buffer to free
194 **/
195static inline void i40e_unmap_tx_resource(struct i40e_ring *ring,
196 struct i40e_tx_buffer *tx_buffer)
197{
198 if (tx_buffer->dma) {
199 if (tx_buffer->tx_flags & I40E_TX_FLAGS_MAPPED_AS_PAGE)
200 dma_unmap_page(ring->dev,
201 tx_buffer->dma,
202 tx_buffer->length,
203 DMA_TO_DEVICE);
204 else
205 dma_unmap_single(ring->dev,
206 tx_buffer->dma,
207 tx_buffer->length,
208 DMA_TO_DEVICE);
209 }
210 tx_buffer->dma = 0;
211 tx_buffer->time_stamp = 0;
212}
213
214/**
215 * i40e_clean_tx_ring - Free any empty Tx buffers
216 * @tx_ring: ring to be cleaned
217 **/
218void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
219{
220 struct i40e_tx_buffer *tx_buffer;
221 unsigned long bi_size;
222 u16 i;
223
224 /* ring already cleared, nothing to do */
225 if (!tx_ring->tx_bi)
226 return;
227
228 /* Free all the Tx ring sk_buffs */
229 for (i = 0; i < tx_ring->count; i++) {
230 tx_buffer = &tx_ring->tx_bi[i];
231 i40e_unmap_tx_resource(tx_ring, tx_buffer);
232 if (tx_buffer->skb)
233 dev_kfree_skb_any(tx_buffer->skb);
234 tx_buffer->skb = NULL;
235 }
236
237 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
238 memset(tx_ring->tx_bi, 0, bi_size);
239
240 /* Zero out the descriptor ring */
241 memset(tx_ring->desc, 0, tx_ring->size);
242
243 tx_ring->next_to_use = 0;
244 tx_ring->next_to_clean = 0;
245}
246
247/**
248 * i40e_free_tx_resources - Free Tx resources per queue
249 * @tx_ring: Tx descriptor ring for a specific queue
250 *
251 * Free all transmit software resources
252 **/
253void i40e_free_tx_resources(struct i40e_ring *tx_ring)
254{
255 i40e_clean_tx_ring(tx_ring);
256 kfree(tx_ring->tx_bi);
257 tx_ring->tx_bi = NULL;
258
259 if (tx_ring->desc) {
260 dma_free_coherent(tx_ring->dev, tx_ring->size,
261 tx_ring->desc, tx_ring->dma);
262 tx_ring->desc = NULL;
263 }
264}
265
266/**
267 * i40e_get_tx_pending - how many tx descriptors not processed
268 * @tx_ring: the ring of descriptors
269 *
270 * Since there is no access to the ring head register
271 * in XL710, we need to use our local copies
272 **/
273static u32 i40e_get_tx_pending(struct i40e_ring *ring)
274{
275 u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
276 ? ring->next_to_use
277 : ring->next_to_use + ring->count);
278 return ntu - ring->next_to_clean;
279}
280
281/**
282 * i40e_check_tx_hang - Is there a hang in the Tx queue
283 * @tx_ring: the ring of descriptors
284 **/
285static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
286{
287 u32 tx_pending = i40e_get_tx_pending(tx_ring);
288 bool ret = false;
289
290 clear_check_for_tx_hang(tx_ring);
291
292 /* Check for a hung queue, but be thorough. This verifies
293 * that a transmit has been completed since the previous
294 * check AND there is at least one packet pending. The
295 * ARMED bit is set to indicate a potential hang. The
296 * bit is cleared if a pause frame is received to remove
297 * false hang detection due to PFC or 802.3x frames. By
298 * requiring this to fail twice we avoid races with
299 * PFC clearing the ARMED bit and conditions where we
300 * run the check_tx_hang logic with a transmit completion
301 * pending but without time to complete it yet.
302 */
303 if ((tx_ring->tx_stats.tx_done_old == tx_ring->tx_stats.packets) &&
304 tx_pending) {
305 /* make sure it is true for two checks in a row */
306 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
307 &tx_ring->state);
308 } else {
309 /* update completed stats and disarm the hang check */
310 tx_ring->tx_stats.tx_done_old = tx_ring->tx_stats.packets;
311 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
312 }
313
314 return ret;
315}
316
317/**
318 * i40e_clean_tx_irq - Reclaim resources after transmit completes
319 * @tx_ring: tx ring to clean
320 * @budget: how many cleans we're allowed
321 *
322 * Returns true if there's any budget left (e.g. the clean is finished)
323 **/
324static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
325{
326 u16 i = tx_ring->next_to_clean;
327 struct i40e_tx_buffer *tx_buf;
328 struct i40e_tx_desc *tx_desc;
329 unsigned int total_packets = 0;
330 unsigned int total_bytes = 0;
331
332 tx_buf = &tx_ring->tx_bi[i];
333 tx_desc = I40E_TX_DESC(tx_ring, i);
334
335 for (; budget; budget--) {
336 struct i40e_tx_desc *eop_desc;
337
338 eop_desc = tx_buf->next_to_watch;
339
340 /* if next_to_watch is not set then there is no work pending */
341 if (!eop_desc)
342 break;
343
344 /* if the descriptor isn't done, no work yet to do */
345 if (!(eop_desc->cmd_type_offset_bsz &
346 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
347 break;
348
349 /* count the packet as being completed */
350 tx_ring->tx_stats.completed++;
351 tx_buf->next_to_watch = NULL;
352 tx_buf->time_stamp = 0;
353
354 /* set memory barrier before eop_desc is verified */
355 rmb();
356
357 do {
358 i40e_unmap_tx_resource(tx_ring, tx_buf);
359
360 /* clear dtype status */
361 tx_desc->cmd_type_offset_bsz &=
362 ~cpu_to_le64(I40E_TXD_QW1_DTYPE_MASK);
363
364 if (likely(tx_desc == eop_desc)) {
365 eop_desc = NULL;
366
367 dev_kfree_skb_any(tx_buf->skb);
368 tx_buf->skb = NULL;
369
370 total_bytes += tx_buf->bytecount;
371 total_packets += tx_buf->gso_segs;
372 }
373
374 tx_buf++;
375 tx_desc++;
376 i++;
377 if (unlikely(i == tx_ring->count)) {
378 i = 0;
379 tx_buf = tx_ring->tx_bi;
380 tx_desc = I40E_TX_DESC(tx_ring, 0);
381 }
382 } while (eop_desc);
383 }
384
385 tx_ring->next_to_clean = i;
386 tx_ring->tx_stats.bytes += total_bytes;
387 tx_ring->tx_stats.packets += total_packets;
388 tx_ring->q_vector->tx.total_bytes += total_bytes;
389 tx_ring->q_vector->tx.total_packets += total_packets;
390 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
391 /* schedule immediate reset if we believe we hung */
392 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
393 " VSI <%d>\n"
394 " Tx Queue <%d>\n"
395 " next_to_use <%x>\n"
396 " next_to_clean <%x>\n",
397 tx_ring->vsi->seid,
398 tx_ring->queue_index,
399 tx_ring->next_to_use, i);
400 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
401 " time_stamp <%lx>\n"
402 " jiffies <%lx>\n",
403 tx_ring->tx_bi[i].time_stamp, jiffies);
404
405 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
406
407 dev_info(tx_ring->dev,
408 "tx hang detected on queue %d, resetting adapter\n",
409 tx_ring->queue_index);
410
411 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
412
413 /* the adapter is about to reset, no point in enabling stuff */
414 return true;
415 }
416
417#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
418 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
419 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
420 /* Make sure that anybody stopping the queue after this
421 * sees the new next_to_clean.
422 */
423 smp_mb();
424 if (__netif_subqueue_stopped(tx_ring->netdev,
425 tx_ring->queue_index) &&
426 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
427 netif_wake_subqueue(tx_ring->netdev,
428 tx_ring->queue_index);
429 ++tx_ring->tx_stats.restart_queue;
430 }
431 }
432
433 return budget > 0;
434}
435
436/**
437 * i40e_set_new_dynamic_itr - Find new ITR level
438 * @rc: structure containing ring performance data
439 *
440 * Stores a new ITR value based on packets and byte counts during
441 * the last interrupt. The advantage of per interrupt computation
442 * is faster updates and more accurate ITR for the current traffic
443 * pattern. Constants in this function were computed based on
444 * theoretical maximum wire speed and thresholds were set based on
445 * testing data as well as attempting to minimize response time
446 * while increasing bulk throughput.
447 **/
448static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
449{
450 enum i40e_latency_range new_latency_range = rc->latency_range;
451 u32 new_itr = rc->itr;
452 int bytes_per_int;
453
454 if (rc->total_packets == 0 || !rc->itr)
455 return;
456
457 /* simple throttlerate management
458 * 0-10MB/s lowest (100000 ints/s)
459 * 10-20MB/s low (20000 ints/s)
460 * 20-1249MB/s bulk (8000 ints/s)
461 */
462 bytes_per_int = rc->total_bytes / rc->itr;
463 switch (rc->itr) {
464 case I40E_LOWEST_LATENCY:
465 if (bytes_per_int > 10)
466 new_latency_range = I40E_LOW_LATENCY;
467 break;
468 case I40E_LOW_LATENCY:
469 if (bytes_per_int > 20)
470 new_latency_range = I40E_BULK_LATENCY;
471 else if (bytes_per_int <= 10)
472 new_latency_range = I40E_LOWEST_LATENCY;
473 break;
474 case I40E_BULK_LATENCY:
475 if (bytes_per_int <= 20)
476 rc->latency_range = I40E_LOW_LATENCY;
477 break;
478 }
479
480 switch (new_latency_range) {
481 case I40E_LOWEST_LATENCY:
482 new_itr = I40E_ITR_100K;
483 break;
484 case I40E_LOW_LATENCY:
485 new_itr = I40E_ITR_20K;
486 break;
487 case I40E_BULK_LATENCY:
488 new_itr = I40E_ITR_8K;
489 break;
490 default:
491 break;
492 }
493
494 if (new_itr != rc->itr) {
495 /* do an exponential smoothing */
496 new_itr = (10 * new_itr * rc->itr) /
497 ((9 * new_itr) + rc->itr);
498 rc->itr = new_itr & I40E_MAX_ITR;
499 }
500
501 rc->total_bytes = 0;
502 rc->total_packets = 0;
503}
504
505/**
506 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
507 * @q_vector: the vector to adjust
508 **/
509static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
510{
511 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
512 struct i40e_hw *hw = &q_vector->vsi->back->hw;
513 u32 reg_addr;
514 u16 old_itr;
515
516 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
517 old_itr = q_vector->rx.itr;
518 i40e_set_new_dynamic_itr(&q_vector->rx);
519 if (old_itr != q_vector->rx.itr)
520 wr32(hw, reg_addr, q_vector->rx.itr);
521
522 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
523 old_itr = q_vector->tx.itr;
524 i40e_set_new_dynamic_itr(&q_vector->tx);
525 if (old_itr != q_vector->tx.itr)
526 wr32(hw, reg_addr, q_vector->tx.itr);
527
528 i40e_flush(hw);
529}
530
531/**
532 * i40e_clean_programming_status - clean the programming status descriptor
533 * @rx_ring: the rx ring that has this descriptor
534 * @rx_desc: the rx descriptor written back by HW
535 *
536 * Flow director should handle FD_FILTER_STATUS to check its filter programming
537 * status being successful or not and take actions accordingly. FCoE should
538 * handle its context/filter programming/invalidation status and take actions.
539 *
540 **/
541static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
542 union i40e_rx_desc *rx_desc)
543{
544 u64 qw;
545 u8 id;
546
547 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
548 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
549 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
550
551 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
552 i40e_fd_handle_status(rx_ring, qw, id);
553}
554
555/**
556 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
557 * @tx_ring: the tx ring to set up
558 *
559 * Return 0 on success, negative on error
560 **/
561int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
562{
563 struct device *dev = tx_ring->dev;
564 int bi_size;
565
566 if (!dev)
567 return -ENOMEM;
568
569 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
570 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
571 if (!tx_ring->tx_bi)
572 goto err;
573
574 /* round up to nearest 4K */
575 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
576 tx_ring->size = ALIGN(tx_ring->size, 4096);
577 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
578 &tx_ring->dma, GFP_KERNEL);
579 if (!tx_ring->desc) {
580 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
581 tx_ring->size);
582 goto err;
583 }
584
585 tx_ring->next_to_use = 0;
586 tx_ring->next_to_clean = 0;
587 return 0;
588
589err:
590 kfree(tx_ring->tx_bi);
591 tx_ring->tx_bi = NULL;
592 return -ENOMEM;
593}
594
595/**
596 * i40e_clean_rx_ring - Free Rx buffers
597 * @rx_ring: ring to be cleaned
598 **/
599void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
600{
601 struct device *dev = rx_ring->dev;
602 struct i40e_rx_buffer *rx_bi;
603 unsigned long bi_size;
604 u16 i;
605
606 /* ring already cleared, nothing to do */
607 if (!rx_ring->rx_bi)
608 return;
609
610 /* Free all the Rx ring sk_buffs */
611 for (i = 0; i < rx_ring->count; i++) {
612 rx_bi = &rx_ring->rx_bi[i];
613 if (rx_bi->dma) {
614 dma_unmap_single(dev,
615 rx_bi->dma,
616 rx_ring->rx_buf_len,
617 DMA_FROM_DEVICE);
618 rx_bi->dma = 0;
619 }
620 if (rx_bi->skb) {
621 dev_kfree_skb(rx_bi->skb);
622 rx_bi->skb = NULL;
623 }
624 if (rx_bi->page) {
625 if (rx_bi->page_dma) {
626 dma_unmap_page(dev,
627 rx_bi->page_dma,
628 PAGE_SIZE / 2,
629 DMA_FROM_DEVICE);
630 rx_bi->page_dma = 0;
631 }
632 __free_page(rx_bi->page);
633 rx_bi->page = NULL;
634 rx_bi->page_offset = 0;
635 }
636 }
637
638 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
639 memset(rx_ring->rx_bi, 0, bi_size);
640
641 /* Zero out the descriptor ring */
642 memset(rx_ring->desc, 0, rx_ring->size);
643
644 rx_ring->next_to_clean = 0;
645 rx_ring->next_to_use = 0;
646}
647
648/**
649 * i40e_free_rx_resources - Free Rx resources
650 * @rx_ring: ring to clean the resources from
651 *
652 * Free all receive software resources
653 **/
654void i40e_free_rx_resources(struct i40e_ring *rx_ring)
655{
656 i40e_clean_rx_ring(rx_ring);
657 kfree(rx_ring->rx_bi);
658 rx_ring->rx_bi = NULL;
659
660 if (rx_ring->desc) {
661 dma_free_coherent(rx_ring->dev, rx_ring->size,
662 rx_ring->desc, rx_ring->dma);
663 rx_ring->desc = NULL;
664 }
665}
666
667/**
668 * i40e_setup_rx_descriptors - Allocate Rx descriptors
669 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
670 *
671 * Returns 0 on success, negative on failure
672 **/
673int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
674{
675 struct device *dev = rx_ring->dev;
676 int bi_size;
677
678 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
679 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
680 if (!rx_ring->rx_bi)
681 goto err;
682
683 /* Round up to nearest 4K */
684 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
685 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
686 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
687 rx_ring->size = ALIGN(rx_ring->size, 4096);
688 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
689 &rx_ring->dma, GFP_KERNEL);
690
691 if (!rx_ring->desc) {
692 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
693 rx_ring->size);
694 goto err;
695 }
696
697 rx_ring->next_to_clean = 0;
698 rx_ring->next_to_use = 0;
699
700 return 0;
701err:
702 kfree(rx_ring->rx_bi);
703 rx_ring->rx_bi = NULL;
704 return -ENOMEM;
705}
706
707/**
708 * i40e_release_rx_desc - Store the new tail and head values
709 * @rx_ring: ring to bump
710 * @val: new head index
711 **/
712static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
713{
714 rx_ring->next_to_use = val;
715 /* Force memory writes to complete before letting h/w
716 * know there are new descriptors to fetch. (Only
717 * applicable for weak-ordered memory model archs,
718 * such as IA-64).
719 */
720 wmb();
721 writel(val, rx_ring->tail);
722}
723
724/**
725 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
726 * @rx_ring: ring to place buffers on
727 * @cleaned_count: number of buffers to replace
728 **/
729void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
730{
731 u16 i = rx_ring->next_to_use;
732 union i40e_rx_desc *rx_desc;
733 struct i40e_rx_buffer *bi;
734 struct sk_buff *skb;
735
736 /* do nothing if no valid netdev defined */
737 if (!rx_ring->netdev || !cleaned_count)
738 return;
739
740 while (cleaned_count--) {
741 rx_desc = I40E_RX_DESC(rx_ring, i);
742 bi = &rx_ring->rx_bi[i];
743 skb = bi->skb;
744
745 if (!skb) {
746 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
747 rx_ring->rx_buf_len);
748 if (!skb) {
749 rx_ring->rx_stats.alloc_rx_buff_failed++;
750 goto no_buffers;
751 }
752 /* initialize queue mapping */
753 skb_record_rx_queue(skb, rx_ring->queue_index);
754 bi->skb = skb;
755 }
756
757 if (!bi->dma) {
758 bi->dma = dma_map_single(rx_ring->dev,
759 skb->data,
760 rx_ring->rx_buf_len,
761 DMA_FROM_DEVICE);
762 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
763 rx_ring->rx_stats.alloc_rx_buff_failed++;
764 bi->dma = 0;
765 goto no_buffers;
766 }
767 }
768
769 if (ring_is_ps_enabled(rx_ring)) {
770 if (!bi->page) {
771 bi->page = alloc_page(GFP_ATOMIC);
772 if (!bi->page) {
773 rx_ring->rx_stats.alloc_rx_page_failed++;
774 goto no_buffers;
775 }
776 }
777
778 if (!bi->page_dma) {
779 /* use a half page if we're re-using */
780 bi->page_offset ^= PAGE_SIZE / 2;
781 bi->page_dma = dma_map_page(rx_ring->dev,
782 bi->page,
783 bi->page_offset,
784 PAGE_SIZE / 2,
785 DMA_FROM_DEVICE);
786 if (dma_mapping_error(rx_ring->dev,
787 bi->page_dma)) {
788 rx_ring->rx_stats.alloc_rx_page_failed++;
789 bi->page_dma = 0;
790 goto no_buffers;
791 }
792 }
793
794 /* Refresh the desc even if buffer_addrs didn't change
795 * because each write-back erases this info.
796 */
797 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
798 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
799 } else {
800 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
801 rx_desc->read.hdr_addr = 0;
802 }
803 i++;
804 if (i == rx_ring->count)
805 i = 0;
806 }
807
808no_buffers:
809 if (rx_ring->next_to_use != i)
810 i40e_release_rx_desc(rx_ring, i);
811}
812
813/**
814 * i40e_receive_skb - Send a completed packet up the stack
815 * @rx_ring: rx ring in play
816 * @skb: packet to send up
817 * @vlan_tag: vlan tag for packet
818 **/
819static void i40e_receive_skb(struct i40e_ring *rx_ring,
820 struct sk_buff *skb, u16 vlan_tag)
821{
822 struct i40e_q_vector *q_vector = rx_ring->q_vector;
823 struct i40e_vsi *vsi = rx_ring->vsi;
824 u64 flags = vsi->back->flags;
825
826 if (vlan_tag & VLAN_VID_MASK)
827 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
828
829 if (flags & I40E_FLAG_IN_NETPOLL)
830 netif_rx(skb);
831 else
832 napi_gro_receive(&q_vector->napi, skb);
833}
834
835/**
836 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
837 * @vsi: the VSI we care about
838 * @skb: skb currently being received and modified
839 * @rx_status: status value of last descriptor in packet
840 * @rx_error: error value of last descriptor in packet
841 **/
842static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
843 struct sk_buff *skb,
844 u32 rx_status,
845 u32 rx_error)
846{
847 skb->ip_summed = CHECKSUM_NONE;
848
849 /* Rx csum enabled and ip headers found? */
850 if (!(vsi->netdev->features & NETIF_F_RXCSUM &&
851 rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
852 return;
853
854 /* IP or L4 checksum error */
855 if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
856 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) {
857 vsi->back->hw_csum_rx_error++;
858 return;
859 }
860
861 skb->ip_summed = CHECKSUM_UNNECESSARY;
862}
863
864/**
865 * i40e_rx_hash - returns the hash value from the Rx descriptor
866 * @ring: descriptor ring
867 * @rx_desc: specific descriptor
868 **/
869static inline u32 i40e_rx_hash(struct i40e_ring *ring,
870 union i40e_rx_desc *rx_desc)
871{
872 if (ring->netdev->features & NETIF_F_RXHASH) {
873 if ((le64_to_cpu(rx_desc->wb.qword1.status_error_len) >>
874 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) &
875 I40E_RX_DESC_FLTSTAT_RSS_HASH)
876 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
877 }
878 return 0;
879}
880
881/**
882 * i40e_clean_rx_irq - Reclaim resources after receive completes
883 * @rx_ring: rx ring to clean
884 * @budget: how many cleans we're allowed
885 *
886 * Returns true if there's any budget left (e.g. the clean is finished)
887 **/
888static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
889{
890 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
891 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
892 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
893 const int current_node = numa_node_id();
894 struct i40e_vsi *vsi = rx_ring->vsi;
895 u16 i = rx_ring->next_to_clean;
896 union i40e_rx_desc *rx_desc;
897 u32 rx_error, rx_status;
898 u64 qword;
899
900 rx_desc = I40E_RX_DESC(rx_ring, i);
901 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
902 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK)
903 >> I40E_RXD_QW1_STATUS_SHIFT;
904
905 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
906 union i40e_rx_desc *next_rxd;
907 struct i40e_rx_buffer *rx_bi;
908 struct sk_buff *skb;
909 u16 vlan_tag;
910 if (i40e_rx_is_programming_status(qword)) {
911 i40e_clean_programming_status(rx_ring, rx_desc);
912 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
913 goto next_desc;
914 }
915 rx_bi = &rx_ring->rx_bi[i];
916 skb = rx_bi->skb;
917 prefetch(skb->data);
918
919 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
920 >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
921 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK)
922 >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
923 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK)
924 >> I40E_RXD_QW1_LENGTH_SPH_SHIFT;
925
926 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK)
927 >> I40E_RXD_QW1_ERROR_SHIFT;
928 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
929 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
930
931 rx_bi->skb = NULL;
932
933 /* This memory barrier is needed to keep us from reading
934 * any other fields out of the rx_desc until we know the
935 * STATUS_DD bit is set
936 */
937 rmb();
938
939 /* Get the header and possibly the whole packet
940 * If this is an skb from previous receive dma will be 0
941 */
942 if (rx_bi->dma) {
943 u16 len;
944
945 if (rx_hbo)
946 len = I40E_RX_HDR_SIZE;
947 else if (rx_sph)
948 len = rx_header_len;
949 else if (rx_packet_len)
950 len = rx_packet_len; /* 1buf/no split found */
951 else
952 len = rx_header_len; /* split always mode */
953
954 skb_put(skb, len);
955 dma_unmap_single(rx_ring->dev,
956 rx_bi->dma,
957 rx_ring->rx_buf_len,
958 DMA_FROM_DEVICE);
959 rx_bi->dma = 0;
960 }
961
962 /* Get the rest of the data if this was a header split */
963 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
964
965 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
966 rx_bi->page,
967 rx_bi->page_offset,
968 rx_packet_len);
969
970 skb->len += rx_packet_len;
971 skb->data_len += rx_packet_len;
972 skb->truesize += rx_packet_len;
973
974 if ((page_count(rx_bi->page) == 1) &&
975 (page_to_nid(rx_bi->page) == current_node))
976 get_page(rx_bi->page);
977 else
978 rx_bi->page = NULL;
979
980 dma_unmap_page(rx_ring->dev,
981 rx_bi->page_dma,
982 PAGE_SIZE / 2,
983 DMA_FROM_DEVICE);
984 rx_bi->page_dma = 0;
985 }
986 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
987
988 if (unlikely(
989 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
990 struct i40e_rx_buffer *next_buffer;
991
992 next_buffer = &rx_ring->rx_bi[i];
993
994 if (ring_is_ps_enabled(rx_ring)) {
995 rx_bi->skb = next_buffer->skb;
996 rx_bi->dma = next_buffer->dma;
997 next_buffer->skb = skb;
998 next_buffer->dma = 0;
999 }
1000 rx_ring->rx_stats.non_eop_descs++;
1001 goto next_desc;
1002 }
1003
1004 /* ERR_MASK will only have valid bits if EOP set */
1005 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1006 dev_kfree_skb_any(skb);
1007 goto next_desc;
1008 }
1009
1010 skb->rxhash = i40e_rx_hash(rx_ring, rx_desc);
1011 i40e_rx_checksum(vsi, skb, rx_status, rx_error);
1012
1013 /* probably a little skewed due to removing CRC */
1014 total_rx_bytes += skb->len;
1015 total_rx_packets++;
1016
1017 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1018 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1019 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1020 : 0;
1021 i40e_receive_skb(rx_ring, skb, vlan_tag);
1022
1023 rx_ring->netdev->last_rx = jiffies;
1024 budget--;
1025next_desc:
1026 rx_desc->wb.qword1.status_error_len = 0;
1027 if (!budget)
1028 break;
1029
1030 cleaned_count++;
1031 /* return some buffers to hardware, one at a time is too slow */
1032 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1033 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1034 cleaned_count = 0;
1035 }
1036
1037 /* use prefetched values */
1038 rx_desc = next_rxd;
1039 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1040 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK)
1041 >> I40E_RXD_QW1_STATUS_SHIFT;
1042 }
1043
1044 rx_ring->next_to_clean = i;
1045 rx_ring->rx_stats.packets += total_rx_packets;
1046 rx_ring->rx_stats.bytes += total_rx_bytes;
1047 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1048 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1049
1050 if (cleaned_count)
1051 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1052
1053 return budget > 0;
1054}
1055
1056/**
1057 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1058 * @napi: napi struct with our devices info in it
1059 * @budget: amount of work driver is allowed to do this pass, in packets
1060 *
1061 * This function will clean all queues associated with a q_vector.
1062 *
1063 * Returns the amount of work done
1064 **/
1065int i40e_napi_poll(struct napi_struct *napi, int budget)
1066{
1067 struct i40e_q_vector *q_vector =
1068 container_of(napi, struct i40e_q_vector, napi);
1069 struct i40e_vsi *vsi = q_vector->vsi;
1070 bool clean_complete = true;
1071 int budget_per_ring;
1072 int i;
1073
1074 if (test_bit(__I40E_DOWN, &vsi->state)) {
1075 napi_complete(napi);
1076 return 0;
1077 }
1078
1079 /* We attempt to distribute budget to each Rx queue fairly, but don't
1080 * allow the budget to go below 1 because that would exit polling early.
1081 * Since the actual Tx work is minimal, we can give the Tx a larger
1082 * budget and be more aggressive about cleaning up the Tx descriptors.
1083 */
1084 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1085 for (i = 0; i < q_vector->num_ringpairs; i++) {
1086 clean_complete &= i40e_clean_tx_irq(q_vector->tx.ring[i],
1087 vsi->work_limit);
1088 clean_complete &= i40e_clean_rx_irq(q_vector->rx.ring[i],
1089 budget_per_ring);
1090 }
1091
1092 /* If work not completed, return budget and polling will return */
1093 if (!clean_complete)
1094 return budget;
1095
1096 /* Work is done so exit the polling mode and re-enable the interrupt */
1097 napi_complete(napi);
1098 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1099 ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1100 i40e_update_dynamic_itr(q_vector);
1101
1102 if (!test_bit(__I40E_DOWN, &vsi->state)) {
1103 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1104 i40e_irq_dynamic_enable(vsi,
1105 q_vector->v_idx + vsi->base_vector);
1106 } else {
1107 struct i40e_hw *hw = &vsi->back->hw;
1108 /* We re-enable the queue 0 cause, but
1109 * don't worry about dynamic_enable
1110 * because we left it on for the other
1111 * possible interrupts during napi
1112 */
1113 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1114 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1115 wr32(hw, I40E_QINT_RQCTL(0), qval);
1116
1117 qval = rd32(hw, I40E_QINT_TQCTL(0));
1118 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1119 wr32(hw, I40E_QINT_TQCTL(0), qval);
1120 i40e_flush(hw);
1121 }
1122 }
1123
1124 return 0;
1125}
1126
1127/**
1128 * i40e_atr - Add a Flow Director ATR filter
1129 * @tx_ring: ring to add programming descriptor to
1130 * @skb: send buffer
1131 * @flags: send flags
1132 * @protocol: wire protocol
1133 **/
1134static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1135 u32 flags, __be16 protocol)
1136{
1137 struct i40e_filter_program_desc *fdir_desc;
1138 struct i40e_pf *pf = tx_ring->vsi->back;
1139 union {
1140 unsigned char *network;
1141 struct iphdr *ipv4;
1142 struct ipv6hdr *ipv6;
1143 } hdr;
1144 struct tcphdr *th;
1145 unsigned int hlen;
1146 u32 flex_ptype, dtype_cmd;
1147
1148 /* make sure ATR is enabled */
1149 if (!(pf->flags & I40E_FLAG_FDIR_ATR_ENABLED))
1150 return;
1151
1152 /* if sampling is disabled do nothing */
1153 if (!tx_ring->atr_sample_rate)
1154 return;
1155
1156 tx_ring->atr_count++;
1157
1158 /* snag network header to get L4 type and address */
1159 hdr.network = skb_network_header(skb);
1160
1161 /* Currently only IPv4/IPv6 with TCP is supported */
1162 if (protocol == htons(ETH_P_IP)) {
1163 if (hdr.ipv4->protocol != IPPROTO_TCP)
1164 return;
1165
1166 /* access ihl as a u8 to avoid unaligned access on ia64 */
1167 hlen = (hdr.network[0] & 0x0F) << 2;
1168 } else if (protocol == htons(ETH_P_IPV6)) {
1169 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1170 return;
1171
1172 hlen = sizeof(struct ipv6hdr);
1173 } else {
1174 return;
1175 }
1176
1177 th = (struct tcphdr *)(hdr.network + hlen);
1178
1179 /* sample on all syn/fin packets or once every atr sample rate */
1180 if (!th->fin && !th->syn && (tx_ring->atr_count < tx_ring->atr_sample_rate))
1181 return;
1182
1183 tx_ring->atr_count = 0;
1184
1185 /* grab the next descriptor */
1186 fdir_desc = I40E_TX_FDIRDESC(tx_ring, tx_ring->next_to_use);
1187 tx_ring->next_to_use++;
1188 if (tx_ring->next_to_use == tx_ring->count)
1189 tx_ring->next_to_use = 0;
1190
1191 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1192 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1193 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1194 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1195 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1196 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1197 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1198
1199 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1200
1201 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1202
1203 dtype_cmd |= th->fin ?
1204 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1205 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1206 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1207 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1208
1209 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1210 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1211
1212 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1213 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1214
1215 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1216 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1217}
1218
1219#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
1220/**
1221 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1222 * @skb: send buffer
1223 * @tx_ring: ring to send buffer on
1224 * @flags: the tx flags to be set
1225 *
1226 * Checks the skb and set up correspondingly several generic transmit flags
1227 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1228 *
1229 * Returns error code indicate the frame should be dropped upon error and the
1230 * otherwise returns 0 to indicate the flags has been set properly.
1231 **/
1232static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1233 struct i40e_ring *tx_ring,
1234 u32 *flags)
1235{
1236 __be16 protocol = skb->protocol;
1237 u32 tx_flags = 0;
1238
1239 /* if we have a HW VLAN tag being added, default to the HW one */
1240 if (vlan_tx_tag_present(skb)) {
1241 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1242 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1243 /* else if it is a SW VLAN, check the next protocol and store the tag */
1244 } else if (protocol == __constant_htons(ETH_P_8021Q)) {
1245 struct vlan_hdr *vhdr, _vhdr;
1246 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1247 if (!vhdr)
1248 return -EINVAL;
1249
1250 protocol = vhdr->h_vlan_encapsulated_proto;
1251 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1252 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1253 }
1254
1255 /* Insert 802.1p priority into VLAN header */
1256 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1257 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1258 (skb->priority != TC_PRIO_CONTROL))) {
1259 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1260 tx_flags |= (skb->priority & 0x7) <<
1261 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1262 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1263 struct vlan_ethhdr *vhdr;
1264 if (skb_header_cloned(skb) &&
1265 pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
1266 return -ENOMEM;
1267 vhdr = (struct vlan_ethhdr *)skb->data;
1268 vhdr->h_vlan_TCI = htons(tx_flags >>
1269 I40E_TX_FLAGS_VLAN_SHIFT);
1270 } else {
1271 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1272 }
1273 }
1274 *flags = tx_flags;
1275 return 0;
1276}
1277
1278/**
1279 * i40e_tx_csum - is checksum offload requested
1280 * @tx_ring: ptr to the ring to send
1281 * @skb: ptr to the skb we're sending
1282 * @tx_flags: the collected send information
1283 * @protocol: the send protocol
1284 *
1285 * Returns true if checksum offload is requested
1286 **/
1287static bool i40e_tx_csum(struct i40e_ring *tx_ring, struct sk_buff *skb,
1288 u32 tx_flags, __be16 protocol)
1289{
1290 if ((skb->ip_summed != CHECKSUM_PARTIAL) &&
1291 !(tx_flags & I40E_TX_FLAGS_TXSW)) {
1292 if (!(tx_flags & I40E_TX_FLAGS_HW_VLAN))
1293 return false;
1294 }
1295
1296 return skb->ip_summed == CHECKSUM_PARTIAL;
1297}
1298
1299/**
1300 * i40e_tso - set up the tso context descriptor
1301 * @tx_ring: ptr to the ring to send
1302 * @skb: ptr to the skb we're sending
1303 * @tx_flags: the collected send information
1304 * @protocol: the send protocol
1305 * @hdr_len: ptr to the size of the packet header
1306 * @cd_tunneling: ptr to context descriptor bits
1307 *
1308 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1309 **/
1310static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1311 u32 tx_flags, __be16 protocol, u8 *hdr_len,
1312 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1313{
1314 u32 cd_cmd, cd_tso_len, cd_mss;
1315 struct tcphdr *tcph;
1316 struct iphdr *iph;
1317 u32 l4len;
1318 int err;
1319 struct ipv6hdr *ipv6h;
1320
1321 if (!skb_is_gso(skb))
1322 return 0;
1323
1324 if (skb_header_cloned(skb)) {
1325 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1326 if (err)
1327 return err;
1328 }
1329
1330 if (protocol == __constant_htons(ETH_P_IP)) {
1331 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1332 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1333 iph->tot_len = 0;
1334 iph->check = 0;
1335 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1336 0, IPPROTO_TCP, 0);
1337 } else if (skb_is_gso_v6(skb)) {
1338
1339 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1340 : ipv6_hdr(skb);
1341 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1342 ipv6h->payload_len = 0;
1343 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1344 0, IPPROTO_TCP, 0);
1345 }
1346
1347 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1348 *hdr_len = (skb->encapsulation
1349 ? (skb_inner_transport_header(skb) - skb->data)
1350 : skb_transport_offset(skb)) + l4len;
1351
1352 /* find the field values */
1353 cd_cmd = I40E_TX_CTX_DESC_TSO;
1354 cd_tso_len = skb->len - *hdr_len;
1355 cd_mss = skb_shinfo(skb)->gso_size;
1356 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT)
1357 | ((u64)cd_tso_len
1358 << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT)
1359 | ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1360 return 1;
1361}
1362
1363/**
1364 * i40e_tx_enable_csum - Enable Tx checksum offloads
1365 * @skb: send buffer
1366 * @tx_flags: Tx flags currently set
1367 * @td_cmd: Tx descriptor command bits to set
1368 * @td_offset: Tx descriptor header offsets to set
1369 * @cd_tunneling: ptr to context desc bits
1370 **/
1371static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1372 u32 *td_cmd, u32 *td_offset,
1373 struct i40e_ring *tx_ring,
1374 u32 *cd_tunneling)
1375{
1376 struct ipv6hdr *this_ipv6_hdr;
1377 unsigned int this_tcp_hdrlen;
1378 struct iphdr *this_ip_hdr;
1379 u32 network_hdr_len;
1380 u8 l4_hdr = 0;
1381
1382 if (skb->encapsulation) {
1383 network_hdr_len = skb_inner_network_header_len(skb);
1384 this_ip_hdr = inner_ip_hdr(skb);
1385 this_ipv6_hdr = inner_ipv6_hdr(skb);
1386 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1387
1388 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1389
1390 if (tx_flags & I40E_TX_FLAGS_TSO) {
1391 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1392 ip_hdr(skb)->check = 0;
1393 } else {
1394 *cd_tunneling |=
1395 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1396 }
1397 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1398 if (tx_flags & I40E_TX_FLAGS_TSO) {
1399 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1400 ip_hdr(skb)->check = 0;
1401 } else {
1402 *cd_tunneling |=
1403 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1404 }
1405 }
1406
1407 /* Now set the ctx descriptor fields */
1408 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1409 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1410 I40E_TXD_CTX_UDP_TUNNELING |
1411 ((skb_inner_network_offset(skb) -
1412 skb_transport_offset(skb)) >> 1) <<
1413 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1414
1415 } else {
1416 network_hdr_len = skb_network_header_len(skb);
1417 this_ip_hdr = ip_hdr(skb);
1418 this_ipv6_hdr = ipv6_hdr(skb);
1419 this_tcp_hdrlen = tcp_hdrlen(skb);
1420 }
1421
1422 /* Enable IP checksum offloads */
1423 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1424 l4_hdr = this_ip_hdr->protocol;
1425 /* the stack computes the IP header already, the only time we
1426 * need the hardware to recompute it is in the case of TSO.
1427 */
1428 if (tx_flags & I40E_TX_FLAGS_TSO) {
1429 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1430 this_ip_hdr->check = 0;
1431 } else {
1432 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1433 }
1434 /* Now set the td_offset for IP header length */
1435 *td_offset = (network_hdr_len >> 2) <<
1436 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1437 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1438 l4_hdr = this_ipv6_hdr->nexthdr;
1439 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1440 /* Now set the td_offset for IP header length */
1441 *td_offset = (network_hdr_len >> 2) <<
1442 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1443 }
1444 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1445 *td_offset |= (skb_network_offset(skb) >> 1) <<
1446 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1447
1448 /* Enable L4 checksum offloads */
1449 switch (l4_hdr) {
1450 case IPPROTO_TCP:
1451 /* enable checksum offloads */
1452 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1453 *td_offset |= (this_tcp_hdrlen >> 2) <<
1454 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1455 break;
1456 case IPPROTO_SCTP:
1457 /* enable SCTP checksum offload */
1458 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1459 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1460 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1461 break;
1462 case IPPROTO_UDP:
1463 /* enable UDP checksum offload */
1464 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1465 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1466 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1467 break;
1468 default:
1469 break;
1470 }
1471}
1472
1473/**
1474 * i40e_create_tx_ctx Build the Tx context descriptor
1475 * @tx_ring: ring to create the descriptor on
1476 * @cd_type_cmd_tso_mss: Quad Word 1
1477 * @cd_tunneling: Quad Word 0 - bits 0-31
1478 * @cd_l2tag2: Quad Word 0 - bits 32-63
1479 **/
1480static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1481 const u64 cd_type_cmd_tso_mss,
1482 const u32 cd_tunneling, const u32 cd_l2tag2)
1483{
1484 struct i40e_tx_context_desc *context_desc;
1485
1486 if (!cd_type_cmd_tso_mss && !cd_tunneling && !cd_l2tag2)
1487 return;
1488
1489 /* grab the next descriptor */
1490 context_desc = I40E_TX_CTXTDESC(tx_ring, tx_ring->next_to_use);
1491 tx_ring->next_to_use++;
1492 if (tx_ring->next_to_use == tx_ring->count)
1493 tx_ring->next_to_use = 0;
1494
1495 /* cpu_to_le32 and assign to struct fields */
1496 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1497 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1498 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1499}
1500
1501/**
1502 * i40e_tx_map - Build the Tx descriptor
1503 * @tx_ring: ring to send buffer on
1504 * @skb: send buffer
1505 * @first: first buffer info buffer to use
1506 * @tx_flags: collected send information
1507 * @hdr_len: size of the packet header
1508 * @td_cmd: the command field in the descriptor
1509 * @td_offset: offset for checksum or crc
1510 **/
1511static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
1512 struct i40e_tx_buffer *first, u32 tx_flags,
1513 const u8 hdr_len, u32 td_cmd, u32 td_offset)
1514{
1515 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1516 unsigned int data_len = skb->data_len;
1517 unsigned int size = skb_headlen(skb);
1518 struct device *dev = tx_ring->dev;
1519 u32 paylen = skb->len - hdr_len;
1520 u16 i = tx_ring->next_to_use;
1521 struct i40e_tx_buffer *tx_bi;
1522 struct i40e_tx_desc *tx_desc;
1523 u32 buf_offset = 0;
1524 u32 td_tag = 0;
1525 dma_addr_t dma;
1526 u16 gso_segs;
1527
1528 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1529 if (dma_mapping_error(dev, dma))
1530 goto dma_error;
1531
1532 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
1533 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1534 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
1535 I40E_TX_FLAGS_VLAN_SHIFT;
1536 }
1537
1538 tx_desc = I40E_TX_DESC(tx_ring, i);
1539 for (;;) {
1540 while (size > I40E_MAX_DATA_PER_TXD) {
1541 tx_desc->buffer_addr = cpu_to_le64(dma + buf_offset);
1542 tx_desc->cmd_type_offset_bsz =
1543 build_ctob(td_cmd, td_offset,
1544 I40E_MAX_DATA_PER_TXD, td_tag);
1545
1546 buf_offset += I40E_MAX_DATA_PER_TXD;
1547 size -= I40E_MAX_DATA_PER_TXD;
1548
1549 tx_desc++;
1550 i++;
1551 if (i == tx_ring->count) {
1552 tx_desc = I40E_TX_DESC(tx_ring, 0);
1553 i = 0;
1554 }
1555 }
1556
1557 tx_bi = &tx_ring->tx_bi[i];
1558 tx_bi->length = buf_offset + size;
1559 tx_bi->tx_flags = tx_flags;
1560 tx_bi->dma = dma;
1561
1562 tx_desc->buffer_addr = cpu_to_le64(dma + buf_offset);
1563 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
1564 size, td_tag);
1565
1566 if (likely(!data_len))
1567 break;
1568
1569 size = skb_frag_size(frag);
1570 data_len -= size;
1571 buf_offset = 0;
1572 tx_flags |= I40E_TX_FLAGS_MAPPED_AS_PAGE;
1573
1574 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1575 if (dma_mapping_error(dev, dma))
1576 goto dma_error;
1577
1578 tx_desc++;
1579 i++;
1580 if (i == tx_ring->count) {
1581 tx_desc = I40E_TX_DESC(tx_ring, 0);
1582 i = 0;
1583 }
1584
1585 frag++;
1586 }
1587
1588 tx_desc->cmd_type_offset_bsz |=
1589 cpu_to_le64((u64)I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT);
1590
1591 i++;
1592 if (i == tx_ring->count)
1593 i = 0;
1594
1595 tx_ring->next_to_use = i;
1596
1597 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
1598 gso_segs = skb_shinfo(skb)->gso_segs;
1599 else
1600 gso_segs = 1;
1601
1602 /* multiply data chunks by size of headers */
1603 tx_bi->bytecount = paylen + (gso_segs * hdr_len);
1604 tx_bi->gso_segs = gso_segs;
1605 tx_bi->skb = skb;
1606
1607 /* set the timestamp and next to watch values */
1608 first->time_stamp = jiffies;
1609 first->next_to_watch = tx_desc;
1610
1611 /* Force memory writes to complete before letting h/w
1612 * know there are new descriptors to fetch. (Only
1613 * applicable for weak-ordered memory model archs,
1614 * such as IA-64).
1615 */
1616 wmb();
1617
1618 writel(i, tx_ring->tail);
1619 return;
1620
1621dma_error:
1622 dev_info(dev, "TX DMA map failed\n");
1623
1624 /* clear dma mappings for failed tx_bi map */
1625 for (;;) {
1626 tx_bi = &tx_ring->tx_bi[i];
1627 i40e_unmap_tx_resource(tx_ring, tx_bi);
1628 if (tx_bi == first)
1629 break;
1630 if (i == 0)
1631 i = tx_ring->count;
1632 i--;
1633 }
1634
1635 dev_kfree_skb_any(skb);
1636
1637 tx_ring->next_to_use = i;
1638}
1639
1640/**
1641 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
1642 * @tx_ring: the ring to be checked
1643 * @size: the size buffer we want to assure is available
1644 *
1645 * Returns -EBUSY if a stop is needed, else 0
1646 **/
1647static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
1648{
1649 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
1650 smp_mb();
1651
1652 /* Check again in a case another CPU has just made room available. */
1653 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
1654 return -EBUSY;
1655
1656 /* A reprieve! - use start_queue because it doesn't call schedule */
1657 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
1658 ++tx_ring->tx_stats.restart_queue;
1659 return 0;
1660}
1661
1662/**
1663 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
1664 * @tx_ring: the ring to be checked
1665 * @size: the size buffer we want to assure is available
1666 *
1667 * Returns 0 if stop is not needed
1668 **/
1669static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
1670{
1671 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
1672 return 0;
1673 return __i40e_maybe_stop_tx(tx_ring, size);
1674}
1675
1676/**
1677 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
1678 * @skb: send buffer
1679 * @tx_ring: ring to send buffer on
1680 *
1681 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
1682 * there is not enough descriptors available in this ring since we need at least
1683 * one descriptor.
1684 **/
1685static int i40e_xmit_descriptor_count(struct sk_buff *skb,
1686 struct i40e_ring *tx_ring)
1687{
1688#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
1689 unsigned int f;
1690#endif
1691 int count = 0;
1692
1693 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
1694 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
1695 * + 2 desc gap to keep tail from touching head,
1696 * + 1 desc for context descriptor,
1697 * otherwise try next time
1698 */
1699#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
1700 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1701 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
1702#else
1703 count += skb_shinfo(skb)->nr_frags;
1704#endif
1705 count += TXD_USE_COUNT(skb_headlen(skb));
1706 if (i40e_maybe_stop_tx(tx_ring, count + 3)) {
1707 tx_ring->tx_stats.tx_busy++;
1708 return 0;
1709 }
1710 return count;
1711}
1712
1713/**
1714 * i40e_xmit_frame_ring - Sends buffer on Tx ring
1715 * @skb: send buffer
1716 * @tx_ring: ring to send buffer on
1717 *
1718 * Returns NETDEV_TX_OK if sent, else an error code
1719 **/
1720static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
1721 struct i40e_ring *tx_ring)
1722{
1723 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
1724 u32 cd_tunneling = 0, cd_l2tag2 = 0;
1725 struct i40e_tx_buffer *first;
1726 u32 td_offset = 0;
1727 u32 tx_flags = 0;
1728 __be16 protocol;
1729 u32 td_cmd = 0;
1730 u8 hdr_len = 0;
1731 int tso;
1732 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
1733 return NETDEV_TX_BUSY;
1734
1735 /* prepare the xmit flags */
1736 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
1737 goto out_drop;
1738
1739 /* obtain protocol of skb */
1740 protocol = skb->protocol;
1741
1742 /* record the location of the first descriptor for this packet */
1743 first = &tx_ring->tx_bi[tx_ring->next_to_use];
1744
1745 /* setup IPv4/IPv6 offloads */
1746 if (protocol == __constant_htons(ETH_P_IP))
1747 tx_flags |= I40E_TX_FLAGS_IPV4;
1748 else if (protocol == __constant_htons(ETH_P_IPV6))
1749 tx_flags |= I40E_TX_FLAGS_IPV6;
1750
1751 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
1752 &cd_type_cmd_tso_mss, &cd_tunneling);
1753
1754 if (tso < 0)
1755 goto out_drop;
1756 else if (tso)
1757 tx_flags |= I40E_TX_FLAGS_TSO;
1758
1759 skb_tx_timestamp(skb);
1760
1761 /* Always offload the checksum, since it's in the data descriptor */
1762 if (i40e_tx_csum(tx_ring, skb, tx_flags, protocol))
1763 tx_flags |= I40E_TX_FLAGS_CSUM;
1764
1765 /* always enable offload insertion */
1766 td_cmd |= I40E_TX_DESC_CMD_ICRC;
1767
1768 if (tx_flags & I40E_TX_FLAGS_CSUM)
1769 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
1770 tx_ring, &cd_tunneling);
1771
1772 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
1773 cd_tunneling, cd_l2tag2);
1774
1775 /* Add Flow Director ATR if it's enabled.
1776 *
1777 * NOTE: this must always be directly before the data descriptor.
1778 */
1779 i40e_atr(tx_ring, skb, tx_flags, protocol);
1780
1781 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
1782 td_cmd, td_offset);
1783
1784 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
1785
1786 return NETDEV_TX_OK;
1787
1788out_drop:
1789 dev_kfree_skb_any(skb);
1790 return NETDEV_TX_OK;
1791}
1792
1793/**
1794 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
1795 * @skb: send buffer
1796 * @netdev: network interface device structure
1797 *
1798 * Returns NETDEV_TX_OK if sent, else an error code
1799 **/
1800netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1801{
1802 struct i40e_netdev_priv *np = netdev_priv(netdev);
1803 struct i40e_vsi *vsi = np->vsi;
1804 struct i40e_ring *tx_ring = &vsi->tx_rings[skb->queue_mapping];
1805
1806 /* hardware can't handle really short frames, hardware padding works
1807 * beyond this point
1808 */
1809 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
1810 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
1811 return NETDEV_TX_OK;
1812 skb->len = I40E_MIN_TX_LEN;
1813 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
1814 }
1815
1816 return i40e_xmit_frame_ring(skb, tx_ring);
1817}
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-02 13:44:00 -0500