aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/cavium/thunder/nic.h10
-rw-r--r--drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c29
-rw-r--r--drivers/net/ethernet/cavium/thunder/nicvf_main.c313
-rw-r--r--drivers/net/ethernet/cavium/thunder/nicvf_queues.c387
-rw-r--r--drivers/net/ethernet/cavium/thunder/nicvf_queues.h32
-rw-r--r--drivers/net/ethernet/cavium/thunder/q_struct.h10
6 files changed, 657 insertions, 124 deletions
diff --git a/drivers/net/ethernet/cavium/thunder/nic.h b/drivers/net/ethernet/cavium/thunder/nic.h
index 6fb44218bf55..4a02e618e318 100644
--- a/drivers/net/ethernet/cavium/thunder/nic.h
+++ b/drivers/net/ethernet/cavium/thunder/nic.h
@@ -252,12 +252,14 @@ struct nicvf_drv_stats {
252 u64 tx_csum_overflow; 252 u64 tx_csum_overflow;
253 253
254 /* driver debug stats */ 254 /* driver debug stats */
255 u64 rcv_buffer_alloc_failures;
256 u64 tx_tso; 255 u64 tx_tso;
257 u64 tx_timeout; 256 u64 tx_timeout;
258 u64 txq_stop; 257 u64 txq_stop;
259 u64 txq_wake; 258 u64 txq_wake;
260 259
260 u64 rcv_buffer_alloc_failures;
261 u64 page_alloc;
262
261 struct u64_stats_sync syncp; 263 struct u64_stats_sync syncp;
262}; 264};
263 265
@@ -266,9 +268,9 @@ struct nicvf {
266 struct net_device *netdev; 268 struct net_device *netdev;
267 struct pci_dev *pdev; 269 struct pci_dev *pdev;
268 void __iomem *reg_base; 270 void __iomem *reg_base;
271 struct bpf_prog *xdp_prog;
269#define MAX_QUEUES_PER_QSET 8 272#define MAX_QUEUES_PER_QSET 8
270 struct queue_set *qs; 273 struct queue_set *qs;
271 struct nicvf_cq_poll *napi[8];
272 void *iommu_domain; 274 void *iommu_domain;
273 u8 vf_id; 275 u8 vf_id;
274 u8 sqs_id; 276 u8 sqs_id;
@@ -294,6 +296,7 @@ struct nicvf {
294 /* Queue count */ 296 /* Queue count */
295 u8 rx_queues; 297 u8 rx_queues;
296 u8 tx_queues; 298 u8 tx_queues;
299 u8 xdp_tx_queues;
297 u8 max_queues; 300 u8 max_queues;
298 301
299 u8 node; 302 u8 node;
@@ -318,6 +321,9 @@ struct nicvf {
318 struct nicvf_drv_stats __percpu *drv_stats; 321 struct nicvf_drv_stats __percpu *drv_stats;
319 struct bgx_stats bgx_stats; 322 struct bgx_stats bgx_stats;
320 323
324 /* Napi */
325 struct nicvf_cq_poll *napi[8];
326
321 /* MSI-X */ 327 /* MSI-X */
322 u8 num_vec; 328 u8 num_vec;
323 char irq_name[NIC_VF_MSIX_VECTORS][IFNAMSIZ + 15]; 329 char irq_name[NIC_VF_MSIX_VECTORS][IFNAMSIZ + 15];
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
index 02a986cdbb39..b9ece9cbf98b 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
@@ -100,11 +100,12 @@ static const struct nicvf_stat nicvf_drv_stats[] = {
100 NICVF_DRV_STAT(tx_csum_overlap), 100 NICVF_DRV_STAT(tx_csum_overlap),
101 NICVF_DRV_STAT(tx_csum_overflow), 101 NICVF_DRV_STAT(tx_csum_overflow),
102 102
103 NICVF_DRV_STAT(rcv_buffer_alloc_failures),
104 NICVF_DRV_STAT(tx_tso), 103 NICVF_DRV_STAT(tx_tso),
105 NICVF_DRV_STAT(tx_timeout), 104 NICVF_DRV_STAT(tx_timeout),
106 NICVF_DRV_STAT(txq_stop), 105 NICVF_DRV_STAT(txq_stop),
107 NICVF_DRV_STAT(txq_wake), 106 NICVF_DRV_STAT(txq_wake),
107 NICVF_DRV_STAT(rcv_buffer_alloc_failures),
108 NICVF_DRV_STAT(page_alloc),
108}; 109};
109 110
110static const struct nicvf_stat nicvf_queue_stats[] = { 111static const struct nicvf_stat nicvf_queue_stats[] = {
@@ -720,7 +721,7 @@ static int nicvf_set_channels(struct net_device *dev,
720 struct nicvf *nic = netdev_priv(dev); 721 struct nicvf *nic = netdev_priv(dev);
721 int err = 0; 722 int err = 0;
722 bool if_up = netif_running(dev); 723 bool if_up = netif_running(dev);
723 int cqcount; 724 u8 cqcount, txq_count;
724 725
725 if (!channel->rx_count || !channel->tx_count) 726 if (!channel->rx_count || !channel->tx_count)
726 return -EINVAL; 727 return -EINVAL;
@@ -729,10 +730,26 @@ static int nicvf_set_channels(struct net_device *dev,
729 if (channel->tx_count > nic->max_queues) 730 if (channel->tx_count > nic->max_queues)
730 return -EINVAL; 731 return -EINVAL;
731 732
733 if (nic->xdp_prog &&
734 ((channel->tx_count + channel->rx_count) > nic->max_queues)) {
735 netdev_err(nic->netdev,
736 "XDP mode, RXQs + TXQs > Max %d\n",
737 nic->max_queues);
738 return -EINVAL;
739 }
740
732 if (if_up) 741 if (if_up)
733 nicvf_stop(dev); 742 nicvf_stop(dev);
734 743
735 cqcount = max(channel->rx_count, channel->tx_count); 744 nic->rx_queues = channel->rx_count;
745 nic->tx_queues = channel->tx_count;
746 if (!nic->xdp_prog)
747 nic->xdp_tx_queues = 0;
748 else
749 nic->xdp_tx_queues = channel->rx_count;
750
751 txq_count = nic->xdp_tx_queues + nic->tx_queues;
752 cqcount = max(nic->rx_queues, txq_count);
736 753
737 if (cqcount > MAX_CMP_QUEUES_PER_QS) { 754 if (cqcount > MAX_CMP_QUEUES_PER_QS) {
738 nic->sqs_count = roundup(cqcount, MAX_CMP_QUEUES_PER_QS); 755 nic->sqs_count = roundup(cqcount, MAX_CMP_QUEUES_PER_QS);
@@ -741,12 +758,10 @@ static int nicvf_set_channels(struct net_device *dev,
741 nic->sqs_count = 0; 758 nic->sqs_count = 0;
742 } 759 }
743 760
744 nic->qs->rq_cnt = min_t(u32, channel->rx_count, MAX_RCV_QUEUES_PER_QS); 761 nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS);
745 nic->qs->sq_cnt = min_t(u32, channel->tx_count, MAX_SND_QUEUES_PER_QS); 762 nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS);
746 nic->qs->cq_cnt = max(nic->qs->rq_cnt, nic->qs->sq_cnt); 763 nic->qs->cq_cnt = max(nic->qs->rq_cnt, nic->qs->sq_cnt);
747 764
748 nic->rx_queues = channel->rx_count;
749 nic->tx_queues = channel->tx_count;
750 err = nicvf_set_real_num_queues(dev, nic->tx_queues, nic->rx_queues); 765 err = nicvf_set_real_num_queues(dev, nic->tx_queues, nic->rx_queues);
751 if (err) 766 if (err)
752 return err; 767 return err;
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
index 81a2fcb3cb1b..d6477af88085 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
@@ -17,6 +17,9 @@
17#include <linux/prefetch.h> 17#include <linux/prefetch.h>
18#include <linux/irq.h> 18#include <linux/irq.h>
19#include <linux/iommu.h> 19#include <linux/iommu.h>
20#include <linux/bpf.h>
21#include <linux/bpf_trace.h>
22#include <linux/filter.h>
20 23
21#include "nic_reg.h" 24#include "nic_reg.h"
22#include "nic.h" 25#include "nic.h"
@@ -397,8 +400,10 @@ static void nicvf_request_sqs(struct nicvf *nic)
397 400
398 if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS) 401 if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
399 rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS; 402 rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
400 if (nic->tx_queues > MAX_SND_QUEUES_PER_QS) 403
401 tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS; 404 tx_queues = nic->tx_queues + nic->xdp_tx_queues;
405 if (tx_queues > MAX_SND_QUEUES_PER_QS)
406 tx_queues = tx_queues - MAX_SND_QUEUES_PER_QS;
402 407
403 /* Set no of Rx/Tx queues in each of the SQsets */ 408 /* Set no of Rx/Tx queues in each of the SQsets */
404 for (sqs = 0; sqs < nic->sqs_count; sqs++) { 409 for (sqs = 0; sqs < nic->sqs_count; sqs++) {
@@ -496,12 +501,99 @@ static int nicvf_init_resources(struct nicvf *nic)
496 return 0; 501 return 0;
497} 502}
498 503
504static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
505 struct cqe_rx_t *cqe_rx, struct snd_queue *sq,
506 struct sk_buff **skb)
507{
508 struct xdp_buff xdp;
509 struct page *page;
510 u32 action;
511 u16 len, offset = 0;
512 u64 dma_addr, cpu_addr;
513 void *orig_data;
514
515 /* Retrieve packet buffer's DMA address and length */
516 len = *((u16 *)((void *)cqe_rx + (3 * sizeof(u64))));
517 dma_addr = *((u64 *)((void *)cqe_rx + (7 * sizeof(u64))));
518
519 cpu_addr = nicvf_iova_to_phys(nic, dma_addr);
520 if (!cpu_addr)
521 return false;
522 cpu_addr = (u64)phys_to_virt(cpu_addr);
523 page = virt_to_page((void *)cpu_addr);
524
525 xdp.data_hard_start = page_address(page);
526 xdp.data = (void *)cpu_addr;
527 xdp.data_end = xdp.data + len;
528 orig_data = xdp.data;
529
530 rcu_read_lock();
531 action = bpf_prog_run_xdp(prog, &xdp);
532 rcu_read_unlock();
533
534 /* Check if XDP program has changed headers */
535 if (orig_data != xdp.data) {
536 len = xdp.data_end - xdp.data;
537 offset = orig_data - xdp.data;
538 dma_addr -= offset;
539 }
540
541 switch (action) {
542 case XDP_PASS:
543 /* Check if it's a recycled page, if not
544 * unmap the DMA mapping.
545 *
546 * Recycled page holds an extra reference.
547 */
548 if (page_ref_count(page) == 1) {
549 dma_addr &= PAGE_MASK;
550 dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
551 RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
552 DMA_FROM_DEVICE,
553 DMA_ATTR_SKIP_CPU_SYNC);
554 }
555
556 /* Build SKB and pass on packet to network stack */
557 *skb = build_skb(xdp.data,
558 RCV_FRAG_LEN - cqe_rx->align_pad + offset);
559 if (!*skb)
560 put_page(page);
561 else
562 skb_put(*skb, len);
563 return false;
564 case XDP_TX:
565 nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len);
566 return true;
567 default:
568 bpf_warn_invalid_xdp_action(action);
569 case XDP_ABORTED:
570 trace_xdp_exception(nic->netdev, prog, action);
571 case XDP_DROP:
572 /* Check if it's a recycled page, if not
573 * unmap the DMA mapping.
574 *
575 * Recycled page holds an extra reference.
576 */
577 if (page_ref_count(page) == 1) {
578 dma_addr &= PAGE_MASK;
579 dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
580 RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
581 DMA_FROM_DEVICE,
582 DMA_ATTR_SKIP_CPU_SYNC);
583 }
584 put_page(page);
585 return true;
586 }
587 return false;
588}
589
499static void nicvf_snd_pkt_handler(struct net_device *netdev, 590static void nicvf_snd_pkt_handler(struct net_device *netdev,
500 struct cqe_send_t *cqe_tx, 591 struct cqe_send_t *cqe_tx,
501 int cqe_type, int budget, 592 int budget, int *subdesc_cnt,
502 unsigned int *tx_pkts, unsigned int *tx_bytes) 593 unsigned int *tx_pkts, unsigned int *tx_bytes)
503{ 594{
504 struct sk_buff *skb = NULL; 595 struct sk_buff *skb = NULL;
596 struct page *page;
505 struct nicvf *nic = netdev_priv(netdev); 597 struct nicvf *nic = netdev_priv(netdev);
506 struct snd_queue *sq; 598 struct snd_queue *sq;
507 struct sq_hdr_subdesc *hdr; 599 struct sq_hdr_subdesc *hdr;
@@ -513,12 +605,26 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
513 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) 605 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
514 return; 606 return;
515 607
516 netdev_dbg(nic->netdev, 608 /* Check for errors */
517 "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n", 609 if (cqe_tx->send_status)
518 __func__, cqe_tx->sq_qs, cqe_tx->sq_idx, 610 nicvf_check_cqe_tx_errs(nic->pnicvf, cqe_tx);
519 cqe_tx->sqe_ptr, hdr->subdesc_cnt); 611
612 /* Is this a XDP designated Tx queue */
613 if (sq->is_xdp) {
614 page = (struct page *)sq->xdp_page[cqe_tx->sqe_ptr];
615 /* Check if it's recycled page or else unmap DMA mapping */
616 if (page && (page_ref_count(page) == 1))
617 nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
618 hdr->subdesc_cnt);
619
620 /* Release page reference for recycling */
621 if (page)
622 put_page(page);
623 sq->xdp_page[cqe_tx->sqe_ptr] = (u64)NULL;
624 *subdesc_cnt += hdr->subdesc_cnt + 1;
625 return;
626 }
520 627
521 nicvf_check_cqe_tx_errs(nic, cqe_tx);
522 skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr]; 628 skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
523 if (skb) { 629 if (skb) {
524 /* Check for dummy descriptor used for HW TSO offload on 88xx */ 630 /* Check for dummy descriptor used for HW TSO offload on 88xx */
@@ -528,12 +634,12 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
528 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2); 634 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
529 nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2, 635 nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,
530 tso_sqe->subdesc_cnt); 636 tso_sqe->subdesc_cnt);
531 nicvf_put_sq_desc(sq, tso_sqe->subdesc_cnt + 1); 637 *subdesc_cnt += tso_sqe->subdesc_cnt + 1;
532 } else { 638 } else {
533 nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr, 639 nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
534 hdr->subdesc_cnt); 640 hdr->subdesc_cnt);
535 } 641 }
536 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 642 *subdesc_cnt += hdr->subdesc_cnt + 1;
537 prefetch(skb); 643 prefetch(skb);
538 (*tx_pkts)++; 644 (*tx_pkts)++;
539 *tx_bytes += skb->len; 645 *tx_bytes += skb->len;
@@ -544,7 +650,7 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
544 * a SKB attached, so just free SQEs here. 650 * a SKB attached, so just free SQEs here.
545 */ 651 */
546 if (!nic->hw_tso) 652 if (!nic->hw_tso)
547 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 653 *subdesc_cnt += hdr->subdesc_cnt + 1;
548 } 654 }
549} 655}
550 656
@@ -578,9 +684,9 @@ static inline void nicvf_set_rxhash(struct net_device *netdev,
578 684
579static void nicvf_rcv_pkt_handler(struct net_device *netdev, 685static void nicvf_rcv_pkt_handler(struct net_device *netdev,
580 struct napi_struct *napi, 686 struct napi_struct *napi,
581 struct cqe_rx_t *cqe_rx) 687 struct cqe_rx_t *cqe_rx, struct snd_queue *sq)
582{ 688{
583 struct sk_buff *skb; 689 struct sk_buff *skb = NULL;
584 struct nicvf *nic = netdev_priv(netdev); 690 struct nicvf *nic = netdev_priv(netdev);
585 struct nicvf *snic = nic; 691 struct nicvf *snic = nic;
586 int err = 0; 692 int err = 0;
@@ -595,16 +701,25 @@ static void nicvf_rcv_pkt_handler(struct net_device *netdev,
595 } 701 }
596 702
597 /* Check for errors */ 703 /* Check for errors */
598 err = nicvf_check_cqe_rx_errs(nic, cqe_rx); 704 if (cqe_rx->err_level || cqe_rx->err_opcode) {
599 if (err && !cqe_rx->rb_cnt) 705 err = nicvf_check_cqe_rx_errs(nic, cqe_rx);
600 return; 706 if (err && !cqe_rx->rb_cnt)
707 return;
708 }
601 709
602 skb = nicvf_get_rcv_skb(snic, cqe_rx); 710 /* For XDP, ignore pkts spanning multiple pages */
603 if (!skb) { 711 if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) {
604 netdev_dbg(nic->netdev, "Packet not received\n"); 712 /* Packet consumed by XDP */
605 return; 713 if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, &skb))
714 return;
715 } else {
716 skb = nicvf_get_rcv_skb(snic, cqe_rx,
717 nic->xdp_prog ? true : false);
606 } 718 }
607 719
720 if (!skb)
721 return;
722
608 if (netif_msg_pktdata(nic)) { 723 if (netif_msg_pktdata(nic)) {
609 netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name, 724 netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name,
610 skb, skb->len); 725 skb, skb->len);
@@ -646,13 +761,14 @@ static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
646{ 761{
647 int processed_cqe, work_done = 0, tx_done = 0; 762 int processed_cqe, work_done = 0, tx_done = 0;
648 int cqe_count, cqe_head; 763 int cqe_count, cqe_head;
764 int subdesc_cnt = 0;
649 struct nicvf *nic = netdev_priv(netdev); 765 struct nicvf *nic = netdev_priv(netdev);
650 struct queue_set *qs = nic->qs; 766 struct queue_set *qs = nic->qs;
651 struct cmp_queue *cq = &qs->cq[cq_idx]; 767 struct cmp_queue *cq = &qs->cq[cq_idx];
652 struct cqe_rx_t *cq_desc; 768 struct cqe_rx_t *cq_desc;
653 struct netdev_queue *txq; 769 struct netdev_queue *txq;
654 struct snd_queue *sq; 770 struct snd_queue *sq = &qs->sq[cq_idx];
655 unsigned int tx_pkts = 0, tx_bytes = 0; 771 unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx;
656 772
657 spin_lock_bh(&cq->lock); 773 spin_lock_bh(&cq->lock);
658loop: 774loop:
@@ -667,8 +783,6 @@ loop:
667 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9; 783 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
668 cqe_head &= 0xFFFF; 784 cqe_head &= 0xFFFF;
669 785
670 netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n",
671 __func__, cq_idx, cqe_count, cqe_head);
672 while (processed_cqe < cqe_count) { 786 while (processed_cqe < cqe_count) {
673 /* Get the CQ descriptor */ 787 /* Get the CQ descriptor */
674 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head); 788 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
@@ -682,17 +796,15 @@ loop:
682 break; 796 break;
683 } 797 }
684 798
685 netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n",
686 cq_idx, cq_desc->cqe_type);
687 switch (cq_desc->cqe_type) { 799 switch (cq_desc->cqe_type) {
688 case CQE_TYPE_RX: 800 case CQE_TYPE_RX:
689 nicvf_rcv_pkt_handler(netdev, napi, cq_desc); 801 nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq);
690 work_done++; 802 work_done++;
691 break; 803 break;
692 case CQE_TYPE_SEND: 804 case CQE_TYPE_SEND:
693 nicvf_snd_pkt_handler(netdev, 805 nicvf_snd_pkt_handler(netdev, (void *)cq_desc,
694 (void *)cq_desc, CQE_TYPE_SEND, 806 budget, &subdesc_cnt,
695 budget, &tx_pkts, &tx_bytes); 807 &tx_pkts, &tx_bytes);
696 tx_done++; 808 tx_done++;
697 break; 809 break;
698 case CQE_TYPE_INVALID: 810 case CQE_TYPE_INVALID:
@@ -704,9 +816,6 @@ loop:
704 } 816 }
705 processed_cqe++; 817 processed_cqe++;
706 } 818 }
707 netdev_dbg(nic->netdev,
708 "%s CQ%d processed_cqe %d work_done %d budget %d\n",
709 __func__, cq_idx, processed_cqe, work_done, budget);
710 819
711 /* Ring doorbell to inform H/W to reuse processed CQEs */ 820 /* Ring doorbell to inform H/W to reuse processed CQEs */
712 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, 821 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
@@ -716,13 +825,26 @@ loop:
716 goto loop; 825 goto loop;
717 826
718done: 827done:
828 /* Update SQ's descriptor free count */
829 if (subdesc_cnt)
830 nicvf_put_sq_desc(sq, subdesc_cnt);
831
832 txq_idx = nicvf_netdev_qidx(nic, cq_idx);
833 /* Handle XDP TX queues */
834 if (nic->pnicvf->xdp_prog) {
835 if (txq_idx < nic->pnicvf->xdp_tx_queues) {
836 nicvf_xdp_sq_doorbell(nic, sq, cq_idx);
837 goto out;
838 }
839 nic = nic->pnicvf;
840 txq_idx -= nic->pnicvf->xdp_tx_queues;
841 }
842
719 /* Wakeup TXQ if its stopped earlier due to SQ full */ 843 /* Wakeup TXQ if its stopped earlier due to SQ full */
720 sq = &nic->qs->sq[cq_idx];
721 if (tx_done || 844 if (tx_done ||
722 (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) { 845 (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) {
723 netdev = nic->pnicvf->netdev; 846 netdev = nic->pnicvf->netdev;
724 txq = netdev_get_tx_queue(netdev, 847 txq = netdev_get_tx_queue(netdev, txq_idx);
725 nicvf_netdev_qidx(nic, cq_idx));
726 if (tx_pkts) 848 if (tx_pkts)
727 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes); 849 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
728 850
@@ -735,10 +857,11 @@ done:
735 if (netif_msg_tx_err(nic)) 857 if (netif_msg_tx_err(nic))
736 netdev_warn(netdev, 858 netdev_warn(netdev,
737 "%s: Transmit queue wakeup SQ%d\n", 859 "%s: Transmit queue wakeup SQ%d\n",
738 netdev->name, cq_idx); 860 netdev->name, txq_idx);
739 } 861 }
740 } 862 }
741 863
864out:
742 spin_unlock_bh(&cq->lock); 865 spin_unlock_bh(&cq->lock);
743 return work_done; 866 return work_done;
744} 867}
@@ -1054,6 +1177,13 @@ static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
1054 return NETDEV_TX_OK; 1177 return NETDEV_TX_OK;
1055 } 1178 }
1056 1179
1180 /* In XDP case, initial HW tx queues are used for XDP,
1181 * but stack's queue mapping starts at '0', so skip the
1182 * Tx queues attached to Rx queues for XDP.
1183 */
1184 if (nic->xdp_prog)
1185 qid += nic->xdp_tx_queues;
1186
1057 snic = nic; 1187 snic = nic;
1058 /* Get secondary Qset's SQ structure */ 1188 /* Get secondary Qset's SQ structure */
1059 if (qid >= MAX_SND_QUEUES_PER_QS) { 1189 if (qid >= MAX_SND_QUEUES_PER_QS) {
@@ -1531,6 +1661,114 @@ static int nicvf_set_features(struct net_device *netdev,
1531 return 0; 1661 return 0;
1532} 1662}
1533 1663
1664static void nicvf_set_xdp_queues(struct nicvf *nic, bool bpf_attached)
1665{
1666 u8 cq_count, txq_count;
1667
1668 /* Set XDP Tx queue count same as Rx queue count */
1669 if (!bpf_attached)
1670 nic->xdp_tx_queues = 0;
1671 else
1672 nic->xdp_tx_queues = nic->rx_queues;
1673
1674 /* If queue count > MAX_CMP_QUEUES_PER_QS, then additional qsets
1675 * needs to be allocated, check how many.
1676 */
1677 txq_count = nic->xdp_tx_queues + nic->tx_queues;
1678 cq_count = max(nic->rx_queues, txq_count);
1679 if (cq_count > MAX_CMP_QUEUES_PER_QS) {
1680 nic->sqs_count = roundup(cq_count, MAX_CMP_QUEUES_PER_QS);
1681 nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1;
1682 } else {
1683 nic->sqs_count = 0;
1684 }
1685
1686 /* Set primary Qset's resources */
1687 nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS);
1688 nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS);
1689 nic->qs->cq_cnt = max_t(u8, nic->qs->rq_cnt, nic->qs->sq_cnt);
1690
1691 /* Update stack */
1692 nicvf_set_real_num_queues(nic->netdev, nic->tx_queues, nic->rx_queues);
1693}
1694
1695static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog)
1696{
1697 struct net_device *dev = nic->netdev;
1698 bool if_up = netif_running(nic->netdev);
1699 struct bpf_prog *old_prog;
1700 bool bpf_attached = false;
1701
1702 /* For now just support only the usual MTU sized frames */
1703 if (prog && (dev->mtu > 1500)) {
1704 netdev_warn(dev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
1705 dev->mtu);
1706 return -EOPNOTSUPP;
1707 }
1708
1709 /* ALL SQs attached to CQs i.e same as RQs, are treated as
1710 * XDP Tx queues and more Tx queues are allocated for
1711 * network stack to send pkts out.
1712 *
1713 * No of Tx queues are either same as Rx queues or whatever
1714 * is left in max no of queues possible.
1715 */
1716 if ((nic->rx_queues + nic->tx_queues) > nic->max_queues) {
1717 netdev_warn(dev,
1718 "Failed to attach BPF prog, RXQs + TXQs > Max %d\n",
1719 nic->max_queues);
1720 return -ENOMEM;
1721 }
1722
1723 if (if_up)
1724 nicvf_stop(nic->netdev);
1725
1726 old_prog = xchg(&nic->xdp_prog, prog);
1727 /* Detach old prog, if any */
1728 if (old_prog)
1729 bpf_prog_put(old_prog);
1730
1731 if (nic->xdp_prog) {
1732 /* Attach BPF program */
1733 nic->xdp_prog = bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
1734 if (!IS_ERR(nic->xdp_prog))
1735 bpf_attached = true;
1736 }
1737
1738 /* Calculate Tx queues needed for XDP and network stack */
1739 nicvf_set_xdp_queues(nic, bpf_attached);
1740
1741 if (if_up) {
1742 /* Reinitialize interface, clean slate */
1743 nicvf_open(nic->netdev);
1744 netif_trans_update(nic->netdev);
1745 }
1746
1747 return 0;
1748}
1749
1750static int nicvf_xdp(struct net_device *netdev, struct netdev_xdp *xdp)
1751{
1752 struct nicvf *nic = netdev_priv(netdev);
1753
1754 /* To avoid checks while retrieving buffer address from CQE_RX,
1755 * do not support XDP for T88 pass1.x silicons which are anyway
1756 * not in use widely.
1757 */
1758 if (pass1_silicon(nic->pdev))
1759 return -EOPNOTSUPP;
1760
1761 switch (xdp->command) {
1762 case XDP_SETUP_PROG:
1763 return nicvf_xdp_setup(nic, xdp->prog);
1764 case XDP_QUERY_PROG:
1765 xdp->prog_attached = !!nic->xdp_prog;
1766 return 0;
1767 default:
1768 return -EINVAL;
1769 }
1770}
1771
1534static const struct net_device_ops nicvf_netdev_ops = { 1772static const struct net_device_ops nicvf_netdev_ops = {
1535 .ndo_open = nicvf_open, 1773 .ndo_open = nicvf_open,
1536 .ndo_stop = nicvf_stop, 1774 .ndo_stop = nicvf_stop,
@@ -1541,6 +1779,7 @@ static const struct net_device_ops nicvf_netdev_ops = {
1541 .ndo_tx_timeout = nicvf_tx_timeout, 1779 .ndo_tx_timeout = nicvf_tx_timeout,
1542 .ndo_fix_features = nicvf_fix_features, 1780 .ndo_fix_features = nicvf_fix_features,
1543 .ndo_set_features = nicvf_set_features, 1781 .ndo_set_features = nicvf_set_features,
1782 .ndo_xdp = nicvf_xdp,
1544}; 1783};
1545 1784
1546static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1785static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
index 7b0fd8d871cc..2b181762ad49 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
@@ -19,16 +19,8 @@
19#include "q_struct.h" 19#include "q_struct.h"
20#include "nicvf_queues.h" 20#include "nicvf_queues.h"
21 21
22#define NICVF_PAGE_ORDER ((PAGE_SIZE <= 4096) ? PAGE_ALLOC_COSTLY_ORDER : 0) 22static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
23 23 int size, u64 data);
24static inline u64 nicvf_iova_to_phys(struct nicvf *nic, dma_addr_t dma_addr)
25{
26 /* Translation is installed only when IOMMU is present */
27 if (nic->iommu_domain)
28 return iommu_iova_to_phys(nic->iommu_domain, dma_addr);
29 return dma_addr;
30}
31
32static void nicvf_get_page(struct nicvf *nic) 24static void nicvf_get_page(struct nicvf *nic)
33{ 25{
34 if (!nic->rb_pageref || !nic->rb_page) 26 if (!nic->rb_pageref || !nic->rb_page)
@@ -90,46 +82,152 @@ static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem)
90 dmem->base = NULL; 82 dmem->base = NULL;
91} 83}
92 84
93/* Allocate buffer for packet reception 85#define XDP_PAGE_REFCNT_REFILL 256
94 * HW returns memory address where packet is DMA'ed but not a pointer 86
95 * into RBDR ring, so save buffer address at the start of fragment and 87/* Allocate a new page or recycle one if possible
96 * align the start address to a cache aligned address 88 *
89 * We cannot optimize dma mapping here, since
90 * 1. It's only one RBDR ring for 8 Rx queues.
91 * 2. CQE_RX gives address of the buffer where pkt has been DMA'ed
92 * and not idx into RBDR ring, so can't refer to saved info.
93 * 3. There are multiple receive buffers per page
97 */ 94 */
98static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, gfp_t gfp, 95static inline struct pgcache *nicvf_alloc_page(struct nicvf *nic,
99 u32 buf_len, u64 **rbuf) 96 struct rbdr *rbdr, gfp_t gfp)
97{
98 int ref_count;
99 struct page *page = NULL;
100 struct pgcache *pgcache, *next;
101
102 /* Check if page is already allocated */
103 pgcache = &rbdr->pgcache[rbdr->pgidx];
104 page = pgcache->page;
105 /* Check if page can be recycled */
106 if (page) {
107 ref_count = page_ref_count(page);
108 /* Check if this page has been used once i.e 'put_page'
109 * called after packet transmission i.e internal ref_count
110 * and page's ref_count are equal i.e page can be recycled.
111 */
112 if (rbdr->is_xdp && (ref_count == pgcache->ref_count))
113 pgcache->ref_count--;
114 else
115 page = NULL;
116
117 /* In non-XDP mode, page's ref_count needs to be '1' for it
118 * to be recycled.
119 */
120 if (!rbdr->is_xdp && (ref_count != 1))
121 page = NULL;
122 }
123
124 if (!page) {
125 page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, 0);
126 if (!page)
127 return NULL;
128
129 this_cpu_inc(nic->pnicvf->drv_stats->page_alloc);
130
131 /* Check for space */
132 if (rbdr->pgalloc >= rbdr->pgcnt) {
133 /* Page can still be used */
134 nic->rb_page = page;
135 return NULL;
136 }
137
138 /* Save the page in page cache */
139 pgcache->page = page;
140 pgcache->dma_addr = 0;
141 pgcache->ref_count = 0;
142 rbdr->pgalloc++;
143 }
144
145 /* Take additional page references for recycling */
146 if (rbdr->is_xdp) {
147 /* Since there is single RBDR (i.e single core doing
148 * page recycling) per 8 Rx queues, in XDP mode adjusting
149 * page references atomically is the biggest bottleneck, so
150 * take bunch of references at a time.
151 *
152 * So here, below reference counts defer by '1'.
153 */
154 if (!pgcache->ref_count) {
155 pgcache->ref_count = XDP_PAGE_REFCNT_REFILL;
156 page_ref_add(page, XDP_PAGE_REFCNT_REFILL);
157 }
158 } else {
159 /* In non-XDP case, single 64K page is divided across multiple
160 * receive buffers, so cost of recycling is less anyway.
161 * So we can do with just one extra reference.
162 */
163 page_ref_add(page, 1);
164 }
165
166 rbdr->pgidx++;
167 rbdr->pgidx &= (rbdr->pgcnt - 1);
168
169 /* Prefetch refcount of next page in page cache */
170 next = &rbdr->pgcache[rbdr->pgidx];
171 page = next->page;
172 if (page)
173 prefetch(&page->_refcount);
174
175 return pgcache;
176}
177
178/* Allocate buffer for packet reception */
179static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr,
180 gfp_t gfp, u32 buf_len, u64 *rbuf)
100{ 181{
101 int order = NICVF_PAGE_ORDER; 182 struct pgcache *pgcache = NULL;
102 183
103 /* Check if request can be accomodated in previous allocated page */ 184 /* Check if request can be accomodated in previous allocated page.
104 if (nic->rb_page && 185 * But in XDP mode only one buffer per page is permitted.
105 ((nic->rb_page_offset + buf_len) < (PAGE_SIZE << order))) { 186 */
187 if (!rbdr->is_xdp && nic->rb_page &&
188 ((nic->rb_page_offset + buf_len) <= PAGE_SIZE)) {
106 nic->rb_pageref++; 189 nic->rb_pageref++;
107 goto ret; 190 goto ret;
108 } 191 }
109 192
110 nicvf_get_page(nic); 193 nicvf_get_page(nic);
194 nic->rb_page = NULL;
111 195
112 /* Allocate a new page */ 196 /* Get new page, either recycled or new one */
113 nic->rb_page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, 197 pgcache = nicvf_alloc_page(nic, rbdr, gfp);
114 order); 198 if (!pgcache && !nic->rb_page) {
115 if (!nic->rb_page) {
116 this_cpu_inc(nic->pnicvf->drv_stats->rcv_buffer_alloc_failures); 199 this_cpu_inc(nic->pnicvf->drv_stats->rcv_buffer_alloc_failures);
117 return -ENOMEM; 200 return -ENOMEM;
118 } 201 }
202
119 nic->rb_page_offset = 0; 203 nic->rb_page_offset = 0;
204
205 /* Reserve space for header modifications by BPF program */
206 if (rbdr->is_xdp)
207 buf_len += XDP_PACKET_HEADROOM;
208
209 /* Check if it's recycled */
210 if (pgcache)
211 nic->rb_page = pgcache->page;
120ret: 212ret:
121 /* HW will ensure data coherency, CPU sync not required */ 213 if (rbdr->is_xdp && pgcache && pgcache->dma_addr) {
122 *rbuf = (u64 *)((u64)dma_map_page_attrs(&nic->pdev->dev, nic->rb_page, 214 *rbuf = pgcache->dma_addr;
215 } else {
216 /* HW will ensure data coherency, CPU sync not required */
217 *rbuf = (u64)dma_map_page_attrs(&nic->pdev->dev, nic->rb_page,
123 nic->rb_page_offset, buf_len, 218 nic->rb_page_offset, buf_len,
124 DMA_FROM_DEVICE, 219 DMA_FROM_DEVICE,
125 DMA_ATTR_SKIP_CPU_SYNC)); 220 DMA_ATTR_SKIP_CPU_SYNC);
126 if (dma_mapping_error(&nic->pdev->dev, (dma_addr_t)*rbuf)) { 221 if (dma_mapping_error(&nic->pdev->dev, (dma_addr_t)*rbuf)) {
127 if (!nic->rb_page_offset) 222 if (!nic->rb_page_offset)
128 __free_pages(nic->rb_page, order); 223 __free_pages(nic->rb_page, 0);
129 nic->rb_page = NULL; 224 nic->rb_page = NULL;
130 return -ENOMEM; 225 return -ENOMEM;
226 }
227 if (pgcache)
228 pgcache->dma_addr = *rbuf + XDP_PACKET_HEADROOM;
229 nic->rb_page_offset += buf_len;
131 } 230 }
132 nic->rb_page_offset += buf_len;
133 231
134 return 0; 232 return 0;
135} 233}
@@ -159,7 +257,7 @@ static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,
159 int ring_len, int buf_size) 257 int ring_len, int buf_size)
160{ 258{
161 int idx; 259 int idx;
162 u64 *rbuf; 260 u64 rbuf;
163 struct rbdr_entry_t *desc; 261 struct rbdr_entry_t *desc;
164 int err; 262 int err;
165 263
@@ -177,10 +275,34 @@ static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,
177 rbdr->head = 0; 275 rbdr->head = 0;
178 rbdr->tail = 0; 276 rbdr->tail = 0;
179 277
278 /* Initialize page recycling stuff.
279 *
280 * Can't use single buffer per page especially with 64K pages.
281 * On embedded platforms i.e 81xx/83xx available memory itself
282 * is low and minimum ring size of RBDR is 8K, that takes away
283 * lots of memory.
284 *
285 * But for XDP it has to be a single buffer per page.
286 */
287 if (!nic->pnicvf->xdp_prog) {
288 rbdr->pgcnt = ring_len / (PAGE_SIZE / buf_size);
289 rbdr->is_xdp = false;
290 } else {
291 rbdr->pgcnt = ring_len;
292 rbdr->is_xdp = true;
293 }
294 rbdr->pgcnt = roundup_pow_of_two(rbdr->pgcnt);
295 rbdr->pgcache = kzalloc(sizeof(*rbdr->pgcache) *
296 rbdr->pgcnt, GFP_KERNEL);
297 if (!rbdr->pgcache)
298 return -ENOMEM;
299 rbdr->pgidx = 0;
300 rbdr->pgalloc = 0;
301
180 nic->rb_page = NULL; 302 nic->rb_page = NULL;
181 for (idx = 0; idx < ring_len; idx++) { 303 for (idx = 0; idx < ring_len; idx++) {
182 err = nicvf_alloc_rcv_buffer(nic, GFP_KERNEL, RCV_FRAG_LEN, 304 err = nicvf_alloc_rcv_buffer(nic, rbdr, GFP_KERNEL,
183 &rbuf); 305 RCV_FRAG_LEN, &rbuf);
184 if (err) { 306 if (err) {
185 /* To free already allocated and mapped ones */ 307 /* To free already allocated and mapped ones */
186 rbdr->tail = idx - 1; 308 rbdr->tail = idx - 1;
@@ -188,7 +310,7 @@ static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,
188 } 310 }
189 311
190 desc = GET_RBDR_DESC(rbdr, idx); 312 desc = GET_RBDR_DESC(rbdr, idx);
191 desc->buf_addr = (u64)rbuf >> NICVF_RCV_BUF_ALIGN; 313 desc->buf_addr = rbuf & ~(NICVF_RCV_BUF_ALIGN_BYTES - 1);
192 } 314 }
193 315
194 nicvf_get_page(nic); 316 nicvf_get_page(nic);
@@ -201,6 +323,7 @@ static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
201{ 323{
202 int head, tail; 324 int head, tail;
203 u64 buf_addr, phys_addr; 325 u64 buf_addr, phys_addr;
326 struct pgcache *pgcache;
204 struct rbdr_entry_t *desc; 327 struct rbdr_entry_t *desc;
205 328
206 if (!rbdr) 329 if (!rbdr)
@@ -216,7 +339,7 @@ static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
216 /* Release page references */ 339 /* Release page references */
217 while (head != tail) { 340 while (head != tail) {
218 desc = GET_RBDR_DESC(rbdr, head); 341 desc = GET_RBDR_DESC(rbdr, head);
219 buf_addr = ((u64)desc->buf_addr) << NICVF_RCV_BUF_ALIGN; 342 buf_addr = desc->buf_addr;
220 phys_addr = nicvf_iova_to_phys(nic, buf_addr); 343 phys_addr = nicvf_iova_to_phys(nic, buf_addr);
221 dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN, 344 dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN,
222 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 345 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
@@ -227,13 +350,31 @@ static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
227 } 350 }
228 /* Release buffer of tail desc */ 351 /* Release buffer of tail desc */
229 desc = GET_RBDR_DESC(rbdr, tail); 352 desc = GET_RBDR_DESC(rbdr, tail);
230 buf_addr = ((u64)desc->buf_addr) << NICVF_RCV_BUF_ALIGN; 353 buf_addr = desc->buf_addr;
231 phys_addr = nicvf_iova_to_phys(nic, buf_addr); 354 phys_addr = nicvf_iova_to_phys(nic, buf_addr);
232 dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN, 355 dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN,
233 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 356 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
234 if (phys_addr) 357 if (phys_addr)
235 put_page(virt_to_page(phys_to_virt(phys_addr))); 358 put_page(virt_to_page(phys_to_virt(phys_addr)));
236 359
360 /* Sync page cache info */
361 smp_rmb();
362
363 /* Release additional page references held for recycling */
364 head = 0;
365 while (head < rbdr->pgcnt) {
366 pgcache = &rbdr->pgcache[head];
367 if (pgcache->page && page_ref_count(pgcache->page) != 0) {
368 if (!rbdr->is_xdp) {
369 put_page(pgcache->page);
370 continue;
371 }
372 page_ref_sub(pgcache->page, pgcache->ref_count - 1);
373 put_page(pgcache->page);
374 }
375 head++;
376 }
377
237 /* Free RBDR ring */ 378 /* Free RBDR ring */
238 nicvf_free_q_desc_mem(nic, &rbdr->dmem); 379 nicvf_free_q_desc_mem(nic, &rbdr->dmem);
239} 380}
@@ -248,7 +389,7 @@ static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp)
248 int refill_rb_cnt; 389 int refill_rb_cnt;
249 struct rbdr *rbdr; 390 struct rbdr *rbdr;
250 struct rbdr_entry_t *desc; 391 struct rbdr_entry_t *desc;
251 u64 *rbuf; 392 u64 rbuf;
252 int new_rb = 0; 393 int new_rb = 0;
253 394
254refill: 395refill:
@@ -269,17 +410,20 @@ refill:
269 else 410 else
270 refill_rb_cnt = qs->rbdr_len - qcount - 1; 411 refill_rb_cnt = qs->rbdr_len - qcount - 1;
271 412
413 /* Sync page cache info */
414 smp_rmb();
415
272 /* Start filling descs from tail */ 416 /* Start filling descs from tail */
273 tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3; 417 tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3;
274 while (refill_rb_cnt) { 418 while (refill_rb_cnt) {
275 tail++; 419 tail++;
276 tail &= (rbdr->dmem.q_len - 1); 420 tail &= (rbdr->dmem.q_len - 1);
277 421
278 if (nicvf_alloc_rcv_buffer(nic, gfp, RCV_FRAG_LEN, &rbuf)) 422 if (nicvf_alloc_rcv_buffer(nic, rbdr, gfp, RCV_FRAG_LEN, &rbuf))
279 break; 423 break;
280 424
281 desc = GET_RBDR_DESC(rbdr, tail); 425 desc = GET_RBDR_DESC(rbdr, tail);
282 desc->buf_addr = (u64)rbuf >> NICVF_RCV_BUF_ALIGN; 426 desc->buf_addr = rbuf & ~(NICVF_RCV_BUF_ALIGN_BYTES - 1);
283 refill_rb_cnt--; 427 refill_rb_cnt--;
284 new_rb++; 428 new_rb++;
285 } 429 }
@@ -362,7 +506,7 @@ static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq)
362 506
363/* Initialize transmit queue */ 507/* Initialize transmit queue */
364static int nicvf_init_snd_queue(struct nicvf *nic, 508static int nicvf_init_snd_queue(struct nicvf *nic,
365 struct snd_queue *sq, int q_len) 509 struct snd_queue *sq, int q_len, int qidx)
366{ 510{
367 int err; 511 int err;
368 512
@@ -375,17 +519,38 @@ static int nicvf_init_snd_queue(struct nicvf *nic,
375 sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL); 519 sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL);
376 if (!sq->skbuff) 520 if (!sq->skbuff)
377 return -ENOMEM; 521 return -ENOMEM;
522
378 sq->head = 0; 523 sq->head = 0;
379 sq->tail = 0; 524 sq->tail = 0;
380 atomic_set(&sq->free_cnt, q_len - 1);
381 sq->thresh = SND_QUEUE_THRESH; 525 sq->thresh = SND_QUEUE_THRESH;
382 526
383 /* Preallocate memory for TSO segment's header */ 527 /* Check if this SQ is a XDP TX queue */
384 sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev, 528 if (nic->sqs_mode)
385 q_len * TSO_HEADER_SIZE, 529 qidx += ((nic->sqs_id + 1) * MAX_SND_QUEUES_PER_QS);
386 &sq->tso_hdrs_phys, GFP_KERNEL); 530 if (qidx < nic->pnicvf->xdp_tx_queues) {
387 if (!sq->tso_hdrs) 531 /* Alloc memory to save page pointers for XDP_TX */
388 return -ENOMEM; 532 sq->xdp_page = kcalloc(q_len, sizeof(u64), GFP_KERNEL);
533 if (!sq->xdp_page)
534 return -ENOMEM;
535 sq->xdp_desc_cnt = 0;
536 sq->xdp_free_cnt = q_len - 1;
537 sq->is_xdp = true;
538 } else {
539 sq->xdp_page = NULL;
540 sq->xdp_desc_cnt = 0;
541 sq->xdp_free_cnt = 0;
542 sq->is_xdp = false;
543
544 atomic_set(&sq->free_cnt, q_len - 1);
545
546 /* Preallocate memory for TSO segment's header */
547 sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev,
548 q_len * TSO_HEADER_SIZE,
549 &sq->tso_hdrs_phys,
550 GFP_KERNEL);
551 if (!sq->tso_hdrs)
552 return -ENOMEM;
553 }
389 554
390 return 0; 555 return 0;
391} 556}
@@ -411,6 +576,7 @@ void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq,
411static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) 576static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
412{ 577{
413 struct sk_buff *skb; 578 struct sk_buff *skb;
579 struct page *page;
414 struct sq_hdr_subdesc *hdr; 580 struct sq_hdr_subdesc *hdr;
415 struct sq_hdr_subdesc *tso_sqe; 581 struct sq_hdr_subdesc *tso_sqe;
416 582
@@ -428,8 +594,15 @@ static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
428 smp_rmb(); 594 smp_rmb();
429 while (sq->head != sq->tail) { 595 while (sq->head != sq->tail) {
430 skb = (struct sk_buff *)sq->skbuff[sq->head]; 596 skb = (struct sk_buff *)sq->skbuff[sq->head];
431 if (!skb) 597 if (!skb || !sq->xdp_page)
598 goto next;
599
600 page = (struct page *)sq->xdp_page[sq->head];
601 if (!page)
432 goto next; 602 goto next;
603 else
604 put_page(page);
605
433 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head); 606 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);
434 /* Check for dummy descriptor used for HW TSO offload on 88xx */ 607 /* Check for dummy descriptor used for HW TSO offload on 88xx */
435 if (hdr->dont_send) { 608 if (hdr->dont_send) {
@@ -442,12 +615,14 @@ static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
442 nicvf_unmap_sndq_buffers(nic, sq, sq->head, 615 nicvf_unmap_sndq_buffers(nic, sq, sq->head,
443 hdr->subdesc_cnt); 616 hdr->subdesc_cnt);
444 } 617 }
445 dev_kfree_skb_any(skb); 618 if (skb)
619 dev_kfree_skb_any(skb);
446next: 620next:
447 sq->head++; 621 sq->head++;
448 sq->head &= (sq->dmem.q_len - 1); 622 sq->head &= (sq->dmem.q_len - 1);
449 } 623 }
450 kfree(sq->skbuff); 624 kfree(sq->skbuff);
625 kfree(sq->xdp_page);
451 nicvf_free_q_desc_mem(nic, &sq->dmem); 626 nicvf_free_q_desc_mem(nic, &sq->dmem);
452} 627}
453 628
@@ -838,7 +1013,7 @@ static int nicvf_alloc_resources(struct nicvf *nic)
838 1013
839 /* Alloc send queue */ 1014 /* Alloc send queue */
840 for (qidx = 0; qidx < qs->sq_cnt; qidx++) { 1015 for (qidx = 0; qidx < qs->sq_cnt; qidx++) {
841 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len)) 1016 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx))
842 goto alloc_fail; 1017 goto alloc_fail;
843 } 1018 }
844 1019
@@ -876,6 +1051,7 @@ int nicvf_set_qset_resources(struct nicvf *nic)
876 1051
877 nic->rx_queues = qs->rq_cnt; 1052 nic->rx_queues = qs->rq_cnt;
878 nic->tx_queues = qs->sq_cnt; 1053 nic->tx_queues = qs->sq_cnt;
1054 nic->xdp_tx_queues = 0;
879 1055
880 return 0; 1056 return 0;
881} 1057}
@@ -940,7 +1116,10 @@ static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)
940 int qentry; 1116 int qentry;
941 1117
942 qentry = sq->tail; 1118 qentry = sq->tail;
943 atomic_sub(desc_cnt, &sq->free_cnt); 1119 if (!sq->is_xdp)
1120 atomic_sub(desc_cnt, &sq->free_cnt);
1121 else
1122 sq->xdp_free_cnt -= desc_cnt;
944 sq->tail += desc_cnt; 1123 sq->tail += desc_cnt;
945 sq->tail &= (sq->dmem.q_len - 1); 1124 sq->tail &= (sq->dmem.q_len - 1);
946 1125
@@ -958,7 +1137,10 @@ static inline void nicvf_rollback_sq_desc(struct snd_queue *sq,
958/* Free descriptor back to SQ for future use */ 1137/* Free descriptor back to SQ for future use */
959void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt) 1138void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)
960{ 1139{
961 atomic_add(desc_cnt, &sq->free_cnt); 1140 if (!sq->is_xdp)
1141 atomic_add(desc_cnt, &sq->free_cnt);
1142 else
1143 sq->xdp_free_cnt += desc_cnt;
962 sq->head += desc_cnt; 1144 sq->head += desc_cnt;
963 sq->head &= (sq->dmem.q_len - 1); 1145 sq->head &= (sq->dmem.q_len - 1);
964} 1146}
@@ -1016,6 +1198,58 @@ void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq,
1016 } 1198 }
1017} 1199}
1018 1200
1201/* XDP Transmit APIs */
1202void nicvf_xdp_sq_doorbell(struct nicvf *nic,
1203 struct snd_queue *sq, int sq_num)
1204{
1205 if (!sq->xdp_desc_cnt)
1206 return;
1207
1208 /* make sure all memory stores are done before ringing doorbell */
1209 wmb();
1210
1211 /* Inform HW to xmit all TSO segments */
1212 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
1213 sq_num, sq->xdp_desc_cnt);
1214 sq->xdp_desc_cnt = 0;
1215}
1216
1217static inline void
1218nicvf_xdp_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
1219 int subdesc_cnt, u64 data, int len)
1220{
1221 struct sq_hdr_subdesc *hdr;
1222
1223 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
1224 memset(hdr, 0, SND_QUEUE_DESC_SIZE);
1225 hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
1226 hdr->subdesc_cnt = subdesc_cnt;
1227 hdr->tot_len = len;
1228 hdr->post_cqe = 1;
1229 sq->xdp_page[qentry] = (u64)virt_to_page((void *)data);
1230}
1231
1232int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq,
1233 u64 bufaddr, u64 dma_addr, u16 len)
1234{
1235 int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT;
1236 int qentry;
1237
1238 if (subdesc_cnt > sq->xdp_free_cnt)
1239 return 0;
1240
1241 qentry = nicvf_get_sq_desc(sq, subdesc_cnt);
1242
1243 nicvf_xdp_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, bufaddr, len);
1244
1245 qentry = nicvf_get_nxt_sqentry(sq, qentry);
1246 nicvf_sq_add_gather_subdesc(sq, qentry, len, dma_addr);
1247
1248 sq->xdp_desc_cnt += subdesc_cnt;
1249
1250 return 1;
1251}
1252
1019/* Calculate no of SQ subdescriptors needed to transmit all 1253/* Calculate no of SQ subdescriptors needed to transmit all
1020 * segments of this TSO packet. 1254 * segments of this TSO packet.
1021 * Taken from 'Tilera network driver' with a minor modification. 1255 * Taken from 'Tilera network driver' with a minor modification.
@@ -1374,8 +1608,33 @@ static inline unsigned frag_num(unsigned i)
1374#endif 1608#endif
1375} 1609}
1376 1610
1611static void nicvf_unmap_rcv_buffer(struct nicvf *nic, u64 dma_addr,
1612 u64 buf_addr, bool xdp)
1613{
1614 struct page *page = NULL;
1615 int len = RCV_FRAG_LEN;
1616
1617 if (xdp) {
1618 page = virt_to_page(phys_to_virt(buf_addr));
1619 /* Check if it's a recycled page, if not
1620 * unmap the DMA mapping.
1621 *
1622 * Recycled page holds an extra reference.
1623 */
1624 if (page_ref_count(page) != 1)
1625 return;
1626
1627 len += XDP_PACKET_HEADROOM;
1628 /* Receive buffers in XDP mode are mapped from page start */
1629 dma_addr &= PAGE_MASK;
1630 }
1631 dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, len,
1632 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1633}
1634
1377/* Returns SKB for a received packet */ 1635/* Returns SKB for a received packet */
1378struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx) 1636struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic,
1637 struct cqe_rx_t *cqe_rx, bool xdp)
1379{ 1638{
1380 int frag; 1639 int frag;
1381 int payload_len = 0; 1640 int payload_len = 0;
@@ -1410,10 +1669,9 @@ struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
1410 1669
1411 if (!frag) { 1670 if (!frag) {
1412 /* First fragment */ 1671 /* First fragment */
1413 dma_unmap_page_attrs(&nic->pdev->dev, 1672 nicvf_unmap_rcv_buffer(nic,
1414 *rb_ptrs - cqe_rx->align_pad, 1673 *rb_ptrs - cqe_rx->align_pad,
1415 RCV_FRAG_LEN, DMA_FROM_DEVICE, 1674 phys_addr, xdp);
1416 DMA_ATTR_SKIP_CPU_SYNC);
1417 skb = nicvf_rb_ptr_to_skb(nic, 1675 skb = nicvf_rb_ptr_to_skb(nic,
1418 phys_addr - cqe_rx->align_pad, 1676 phys_addr - cqe_rx->align_pad,
1419 payload_len); 1677 payload_len);
@@ -1423,9 +1681,7 @@ struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
1423 skb_put(skb, payload_len); 1681 skb_put(skb, payload_len);
1424 } else { 1682 } else {
1425 /* Add fragments */ 1683 /* Add fragments */
1426 dma_unmap_page_attrs(&nic->pdev->dev, *rb_ptrs, 1684 nicvf_unmap_rcv_buffer(nic, *rb_ptrs, phys_addr, xdp);
1427 RCV_FRAG_LEN, DMA_FROM_DEVICE,
1428 DMA_ATTR_SKIP_CPU_SYNC);
1429 page = virt_to_page(phys_to_virt(phys_addr)); 1685 page = virt_to_page(phys_to_virt(phys_addr));
1430 offset = phys_to_virt(phys_addr) - page_address(page); 1686 offset = phys_to_virt(phys_addr) - page_address(page);
1431 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 1687 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
@@ -1555,9 +1811,6 @@ void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
1555/* Check for errors in the receive cmp.queue entry */ 1811/* Check for errors in the receive cmp.queue entry */
1556int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx) 1812int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
1557{ 1813{
1558 if (!cqe_rx->err_level && !cqe_rx->err_opcode)
1559 return 0;
1560
1561 if (netif_msg_rx_err(nic)) 1814 if (netif_msg_rx_err(nic))
1562 netdev_err(nic->netdev, 1815 netdev_err(nic->netdev,
1563 "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n", 1816 "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n",
@@ -1646,8 +1899,6 @@ int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
1646int nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cqe_send_t *cqe_tx) 1899int nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cqe_send_t *cqe_tx)
1647{ 1900{
1648 switch (cqe_tx->send_status) { 1901 switch (cqe_tx->send_status) {
1649 case CQ_TX_ERROP_GOOD:
1650 return 0;
1651 case CQ_TX_ERROP_DESC_FAULT: 1902 case CQ_TX_ERROP_DESC_FAULT:
1652 this_cpu_inc(nic->drv_stats->tx_desc_fault); 1903 this_cpu_inc(nic->drv_stats->tx_desc_fault);
1653 break; 1904 break;
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
index 10cb4b84625b..57858522c33c 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
@@ -10,6 +10,7 @@
10#define NICVF_QUEUES_H 10#define NICVF_QUEUES_H
11 11
12#include <linux/netdevice.h> 12#include <linux/netdevice.h>
13#include <linux/iommu.h>
13#include "q_struct.h" 14#include "q_struct.h"
14 15
15#define MAX_QUEUE_SET 128 16#define MAX_QUEUE_SET 128
@@ -213,6 +214,12 @@ struct q_desc_mem {
213 void *unalign_base; 214 void *unalign_base;
214}; 215};
215 216
217struct pgcache {
218 struct page *page;
219 int ref_count;
220 u64 dma_addr;
221};
222
216struct rbdr { 223struct rbdr {
217 bool enable; 224 bool enable;
218 u32 dma_size; 225 u32 dma_size;
@@ -222,6 +229,13 @@ struct rbdr {
222 u32 head; 229 u32 head;
223 u32 tail; 230 u32 tail;
224 struct q_desc_mem dmem; 231 struct q_desc_mem dmem;
232 bool is_xdp;
233
234 /* For page recycling */
235 int pgidx;
236 int pgcnt;
237 int pgalloc;
238 struct pgcache *pgcache;
225} ____cacheline_aligned_in_smp; 239} ____cacheline_aligned_in_smp;
226 240
227struct rcv_queue { 241struct rcv_queue {
@@ -258,6 +272,10 @@ struct snd_queue {
258 u32 tail; 272 u32 tail;
259 u64 *skbuff; 273 u64 *skbuff;
260 void *desc; 274 void *desc;
275 u64 *xdp_page;
276 u16 xdp_desc_cnt;
277 u16 xdp_free_cnt;
278 bool is_xdp;
261 279
262#define TSO_HEADER_SIZE 128 280#define TSO_HEADER_SIZE 128
263 /* For TSO segment's header */ 281 /* For TSO segment's header */
@@ -301,6 +319,14 @@ struct queue_set {
301 319
302#define CQ_ERR_MASK (CQ_WR_FULL | CQ_WR_DISABLE | CQ_WR_FAULT) 320#define CQ_ERR_MASK (CQ_WR_FULL | CQ_WR_DISABLE | CQ_WR_FAULT)
303 321
322static inline u64 nicvf_iova_to_phys(struct nicvf *nic, dma_addr_t dma_addr)
323{
324 /* Translation is installed only when IOMMU is present */
325 if (nic->iommu_domain)
326 return iommu_iova_to_phys(nic->iommu_domain, dma_addr);
327 return dma_addr;
328}
329
304void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq, 330void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq,
305 int hdr_sqe, u8 subdesc_cnt); 331 int hdr_sqe, u8 subdesc_cnt);
306void nicvf_config_vlan_stripping(struct nicvf *nic, 332void nicvf_config_vlan_stripping(struct nicvf *nic,
@@ -318,8 +344,12 @@ void nicvf_sq_free_used_descs(struct net_device *netdev,
318 struct snd_queue *sq, int qidx); 344 struct snd_queue *sq, int qidx);
319int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq, 345int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq,
320 struct sk_buff *skb, u8 sq_num); 346 struct sk_buff *skb, u8 sq_num);
347int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq,
348 u64 bufaddr, u64 dma_addr, u16 len);
349void nicvf_xdp_sq_doorbell(struct nicvf *nic, struct snd_queue *sq, int sq_num);
321 350
322struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx); 351struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic,
352 struct cqe_rx_t *cqe_rx, bool xdp);
323void nicvf_rbdr_task(unsigned long data); 353void nicvf_rbdr_task(unsigned long data);
324void nicvf_rbdr_work(struct work_struct *work); 354void nicvf_rbdr_work(struct work_struct *work);
325 355
diff --git a/drivers/net/ethernet/cavium/thunder/q_struct.h b/drivers/net/ethernet/cavium/thunder/q_struct.h
index f36347237a54..e47205aa87ea 100644
--- a/drivers/net/ethernet/cavium/thunder/q_struct.h
+++ b/drivers/net/ethernet/cavium/thunder/q_struct.h
@@ -359,15 +359,7 @@ union cq_desc_t {
359}; 359};
360 360
361struct rbdr_entry_t { 361struct rbdr_entry_t {
362#if defined(__BIG_ENDIAN_BITFIELD) 362 u64 buf_addr;
363 u64 rsvd0:15;
364 u64 buf_addr:42;
365 u64 cache_align:7;
366#elif defined(__LITTLE_ENDIAN_BITFIELD)
367 u64 cache_align:7;
368 u64 buf_addr:42;
369 u64 rsvd0:15;
370#endif
371}; 363};
372 364
373/* TCP reassembly context */ 365/* TCP reassembly context */
generated by cgit v1.2.2 (git 2.25.0) at 2026-07-07 22:13:25 -0400