aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/net/vmxnet3/vmxnet3_drv.c951
-rw-r--r--drivers/net/vmxnet3/vmxnet3_ethtool.c171
-rw-r--r--drivers/net/vmxnet3/vmxnet3_int.h73
3 files changed, 921 insertions, 274 deletions
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 21314e06e6d7..65860a998321 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -44,6 +44,9 @@ MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
44 44
45static atomic_t devices_found; 45static atomic_t devices_found;
46 46
47#define VMXNET3_MAX_DEVICES 10
48static int enable_mq = 1;
49static int irq_share_mode;
47 50
48/* 51/*
49 * Enable/Disable the given intr 52 * Enable/Disable the given intr
@@ -99,7 +102,7 @@ vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
99static bool 102static bool
100vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 103vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
101{ 104{
102 return netif_queue_stopped(adapter->netdev); 105 return tq->stopped;
103} 106}
104 107
105 108
@@ -107,7 +110,7 @@ static void
107vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 110vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
108{ 111{
109 tq->stopped = false; 112 tq->stopped = false;
110 netif_start_queue(adapter->netdev); 113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
111} 114}
112 115
113 116
@@ -115,7 +118,7 @@ static void
115vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 118vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
116{ 119{
117 tq->stopped = false; 120 tq->stopped = false;
118 netif_wake_queue(adapter->netdev); 121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
119} 122}
120 123
121 124
@@ -124,7 +127,7 @@ vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
124{ 127{
125 tq->stopped = true; 128 tq->stopped = true;
126 tq->num_stop++; 129 tq->num_stop++;
127 netif_stop_queue(adapter->netdev); 130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
128} 131}
129 132
130 133
@@ -135,6 +138,7 @@ static void
135vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 138vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
136{ 139{
137 u32 ret; 140 u32 ret;
141 int i;
138 142
139 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 143 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
140 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 144 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
@@ -145,22 +149,28 @@ vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
145 if (!netif_carrier_ok(adapter->netdev)) 149 if (!netif_carrier_ok(adapter->netdev))
146 netif_carrier_on(adapter->netdev); 150 netif_carrier_on(adapter->netdev);
147 151
148 if (affectTxQueue) 152 if (affectTxQueue) {
149 vmxnet3_tq_start(&adapter->tx_queue, adapter); 153 for (i = 0; i < adapter->num_tx_queues; i++)
154 vmxnet3_tq_start(&adapter->tx_queue[i],
155 adapter);
156 }
150 } else { 157 } else {
151 printk(KERN_INFO "%s: NIC Link is Down\n", 158 printk(KERN_INFO "%s: NIC Link is Down\n",
152 adapter->netdev->name); 159 adapter->netdev->name);
153 if (netif_carrier_ok(adapter->netdev)) 160 if (netif_carrier_ok(adapter->netdev))
154 netif_carrier_off(adapter->netdev); 161 netif_carrier_off(adapter->netdev);
155 162
156 if (affectTxQueue) 163 if (affectTxQueue) {
157 vmxnet3_tq_stop(&adapter->tx_queue, adapter); 164 for (i = 0; i < adapter->num_tx_queues; i++)
165 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
166 }
158 } 167 }
159} 168}
160 169
161static void 170static void
162vmxnet3_process_events(struct vmxnet3_adapter *adapter) 171vmxnet3_process_events(struct vmxnet3_adapter *adapter)
163{ 172{
173 int i;
164 u32 events = le32_to_cpu(adapter->shared->ecr); 174 u32 events = le32_to_cpu(adapter->shared->ecr);
165 if (!events) 175 if (!events)
166 return; 176 return;
@@ -176,16 +186,18 @@ vmxnet3_process_events(struct vmxnet3_adapter *adapter)
176 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 186 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
177 VMXNET3_CMD_GET_QUEUE_STATUS); 187 VMXNET3_CMD_GET_QUEUE_STATUS);
178 188
179 if (adapter->tqd_start->status.stopped) { 189 for (i = 0; i < adapter->num_tx_queues; i++)
180 printk(KERN_ERR "%s: tq error 0x%x\n", 190 if (adapter->tqd_start[i].status.stopped)
181 adapter->netdev->name, 191 dev_err(&adapter->netdev->dev,
182 le32_to_cpu(adapter->tqd_start->status.error)); 192 "%s: tq[%d] error 0x%x\n",
183 } 193 adapter->netdev->name, i, le32_to_cpu(
184 if (adapter->rqd_start->status.stopped) { 194 adapter->tqd_start[i].status.error));
185 printk(KERN_ERR "%s: rq error 0x%x\n", 195 for (i = 0; i < adapter->num_rx_queues; i++)
186 adapter->netdev->name, 196 if (adapter->rqd_start[i].status.stopped)
187 adapter->rqd_start->status.error); 197 dev_err(&adapter->netdev->dev,
188 } 198 "%s: rq[%d] error 0x%x\n",
199 adapter->netdev->name, i,
200 adapter->rqd_start[i].status.error);
189 201
190 schedule_work(&adapter->work); 202 schedule_work(&adapter->work);
191 } 203 }
@@ -410,7 +422,7 @@ vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
410} 422}
411 423
412 424
413void 425static void
414vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 426vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
415 struct vmxnet3_adapter *adapter) 427 struct vmxnet3_adapter *adapter)
416{ 428{
@@ -437,6 +449,17 @@ vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
437} 449}
438 450
439 451
452/* Destroy all tx queues */
453void
454vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
455{
456 int i;
457
458 for (i = 0; i < adapter->num_tx_queues; i++)
459 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
460}
461
462
440static void 463static void
441vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 464vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
442 struct vmxnet3_adapter *adapter) 465 struct vmxnet3_adapter *adapter)
@@ -518,6 +541,14 @@ err:
518 return -ENOMEM; 541 return -ENOMEM;
519} 542}
520 543
544static void
545vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
546{
547 int i;
548
549 for (i = 0; i < adapter->num_tx_queues; i++)
550 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
551}
521 552
522/* 553/*
523 * starting from ring->next2fill, allocate rx buffers for the given ring 554 * starting from ring->next2fill, allocate rx buffers for the given ring
@@ -732,6 +763,17 @@ vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
732} 763}
733 764
734 765
766/* Init all tx queues */
767static void
768vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
769{
770 int i;
771
772 for (i = 0; i < adapter->num_tx_queues; i++)
773 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
774}
775
776
735/* 777/*
736 * parse and copy relevant protocol headers: 778 * parse and copy relevant protocol headers:
737 * For a tso pkt, relevant headers are L2/3/4 including options 779 * For a tso pkt, relevant headers are L2/3/4 including options
@@ -903,6 +945,21 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
903 } 945 }
904 } 946 }
905 947
948 spin_lock_irqsave(&tq->tx_lock, flags);
949
950 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
951 tq->stats.tx_ring_full++;
952 dev_dbg(&adapter->netdev->dev,
953 "tx queue stopped on %s, next2comp %u"
954 " next2fill %u\n", adapter->netdev->name,
955 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
956
957 vmxnet3_tq_stop(tq, adapter);
958 spin_unlock_irqrestore(&tq->tx_lock, flags);
959 return NETDEV_TX_BUSY;
960 }
961
962
906 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter); 963 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
907 if (ret >= 0) { 964 if (ret >= 0) {
908 BUG_ON(ret <= 0 && ctx.copy_size != 0); 965 BUG_ON(ret <= 0 && ctx.copy_size != 0);
@@ -926,20 +983,6 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926 goto drop_pkt; 983 goto drop_pkt;
927 } 984 }
928 985
929 spin_lock_irqsave(&tq->tx_lock, flags);
930
931 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
932 tq->stats.tx_ring_full++;
933 dev_dbg(&adapter->netdev->dev,
934 "tx queue stopped on %s, next2comp %u"
935 " next2fill %u\n", adapter->netdev->name,
936 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
937
938 vmxnet3_tq_stop(tq, adapter);
939 spin_unlock_irqrestore(&tq->tx_lock, flags);
940 return NETDEV_TX_BUSY;
941 }
942
943 /* fill tx descs related to addr & len */ 986 /* fill tx descs related to addr & len */
944 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter); 987 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
945 988
@@ -1000,7 +1043,8 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
1000 if (le32_to_cpu(tq->shared->txNumDeferred) >= 1043 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1001 le32_to_cpu(tq->shared->txThreshold)) { 1044 le32_to_cpu(tq->shared->txThreshold)) {
1002 tq->shared->txNumDeferred = 0; 1045 tq->shared->txNumDeferred = 0;
1003 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_TXPROD, 1046 VMXNET3_WRITE_BAR0_REG(adapter,
1047 VMXNET3_REG_TXPROD + tq->qid * 8,
1004 tq->tx_ring.next2fill); 1048 tq->tx_ring.next2fill);
1005 } 1049 }
1006 1050
@@ -1020,7 +1064,10 @@ vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1020{ 1064{
1021 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1065 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1022 1066
1023 return vmxnet3_tq_xmit(skb, &adapter->tx_queue, adapter, netdev); 1067 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1068 return vmxnet3_tq_xmit(skb,
1069 &adapter->tx_queue[skb->queue_mapping],
1070 adapter, netdev);
1024} 1071}
1025 1072
1026 1073
@@ -1106,9 +1153,9 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1106 break; 1153 break;
1107 } 1154 }
1108 num_rxd++; 1155 num_rxd++;
1109 1156 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1110 idx = rcd->rxdIdx; 1157 idx = rcd->rxdIdx;
1111 ring_idx = rcd->rqID == rq->qid ? 0 : 1; 1158 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1112 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1159 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1113 &rxCmdDesc); 1160 &rxCmdDesc);
1114 rbi = rq->buf_info[ring_idx] + idx; 1161 rbi = rq->buf_info[ring_idx] + idx;
@@ -1260,6 +1307,16 @@ vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1260} 1307}
1261 1308
1262 1309
1310static void
1311vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1312{
1313 int i;
1314
1315 for (i = 0; i < adapter->num_rx_queues; i++)
1316 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1317}
1318
1319
1263void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1320void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1264 struct vmxnet3_adapter *adapter) 1321 struct vmxnet3_adapter *adapter)
1265{ 1322{
@@ -1351,6 +1408,25 @@ vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1351 1408
1352 1409
1353static int 1410static int
1411vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1412{
1413 int i, err = 0;
1414
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
1416 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1417 if (unlikely(err)) {
1418 dev_err(&adapter->netdev->dev, "%s: failed to "
1419 "initialize rx queue%i\n",
1420 adapter->netdev->name, i);
1421 break;
1422 }
1423 }
1424 return err;
1425
1426}
1427
1428
1429static int
1354vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 1430vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1355{ 1431{
1356 int i; 1432 int i;
@@ -1398,33 +1474,177 @@ err:
1398 1474
1399 1475
1400static int 1476static int
1477vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1478{
1479 int i, err = 0;
1480
1481 for (i = 0; i < adapter->num_rx_queues; i++) {
1482 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1483 if (unlikely(err)) {
1484 dev_err(&adapter->netdev->dev,
1485 "%s: failed to create rx queue%i\n",
1486 adapter->netdev->name, i);
1487 goto err_out;
1488 }
1489 }
1490 return err;
1491err_out:
1492 vmxnet3_rq_destroy_all(adapter);
1493 return err;
1494
1495}
1496
1497/* Multiple queue aware polling function for tx and rx */
1498
1499static int
1401vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 1500vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1402{ 1501{
1502 int rcd_done = 0, i;
1403 if (unlikely(adapter->shared->ecr)) 1503 if (unlikely(adapter->shared->ecr))
1404 vmxnet3_process_events(adapter); 1504 vmxnet3_process_events(adapter);
1505 for (i = 0; i < adapter->num_tx_queues; i++)
1506 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1405 1507
1406 vmxnet3_tq_tx_complete(&adapter->tx_queue, adapter); 1508 for (i = 0; i < adapter->num_rx_queues; i++)
1407 return vmxnet3_rq_rx_complete(&adapter->rx_queue, adapter, budget); 1509 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1510 adapter, budget);
1511 return rcd_done;
1408} 1512}
1409 1513
1410 1514
1411static int 1515static int
1412vmxnet3_poll(struct napi_struct *napi, int budget) 1516vmxnet3_poll(struct napi_struct *napi, int budget)
1413{ 1517{
1414 struct vmxnet3_adapter *adapter = container_of(napi, 1518 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1415 struct vmxnet3_adapter, napi); 1519 struct vmxnet3_rx_queue, napi);
1520 int rxd_done;
1521
1522 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1523
1524 if (rxd_done < budget) {
1525 napi_complete(napi);
1526 vmxnet3_enable_all_intrs(rx_queue->adapter);
1527 }
1528 return rxd_done;
1529}
1530
1531/*
1532 * NAPI polling function for MSI-X mode with multiple Rx queues
1533 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1534 */
1535
1536static int
1537vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1538{
1539 struct vmxnet3_rx_queue *rq = container_of(napi,
1540 struct vmxnet3_rx_queue, napi);
1541 struct vmxnet3_adapter *adapter = rq->adapter;
1416 int rxd_done; 1542 int rxd_done;
1417 1543
1418 rxd_done = vmxnet3_do_poll(adapter, budget); 1544 /* When sharing interrupt with corresponding tx queue, process
1545 * tx completions in that queue as well
1546 */
1547 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1548 struct vmxnet3_tx_queue *tq =
1549 &adapter->tx_queue[rq - adapter->rx_queue];
1550 vmxnet3_tq_tx_complete(tq, adapter);
1551 }
1552
1553 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1419 1554
1420 if (rxd_done < budget) { 1555 if (rxd_done < budget) {
1421 napi_complete(napi); 1556 napi_complete(napi);
1422 vmxnet3_enable_intr(adapter, 0); 1557 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1423 } 1558 }
1424 return rxd_done; 1559 return rxd_done;
1425} 1560}
1426 1561
1427 1562
1563#ifdef CONFIG_PCI_MSI
1564
1565/*
1566 * Handle completion interrupts on tx queues
1567 * Returns whether or not the intr is handled
1568 */
1569
1570static irqreturn_t
1571vmxnet3_msix_tx(int irq, void *data)
1572{
1573 struct vmxnet3_tx_queue *tq = data;
1574 struct vmxnet3_adapter *adapter = tq->adapter;
1575
1576 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1577 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1578
1579 /* Handle the case where only one irq is allocate for all tx queues */
1580 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1581 int i;
1582 for (i = 0; i < adapter->num_tx_queues; i++) {
1583 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1584 vmxnet3_tq_tx_complete(txq, adapter);
1585 }
1586 } else {
1587 vmxnet3_tq_tx_complete(tq, adapter);
1588 }
1589 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1590
1591 return IRQ_HANDLED;
1592}
1593
1594
1595/*
1596 * Handle completion interrupts on rx queues. Returns whether or not the
1597 * intr is handled
1598 */
1599
1600static irqreturn_t
1601vmxnet3_msix_rx(int irq, void *data)
1602{
1603 struct vmxnet3_rx_queue *rq = data;
1604 struct vmxnet3_adapter *adapter = rq->adapter;
1605
1606 /* disable intr if needed */
1607 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1608 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1609 napi_schedule(&rq->napi);
1610
1611 return IRQ_HANDLED;
1612}
1613
1614/*
1615 *----------------------------------------------------------------------------
1616 *
1617 * vmxnet3_msix_event --
1618 *
1619 * vmxnet3 msix event intr handler
1620 *
1621 * Result:
1622 * whether or not the intr is handled
1623 *
1624 *----------------------------------------------------------------------------
1625 */
1626
1627static irqreturn_t
1628vmxnet3_msix_event(int irq, void *data)
1629{
1630 struct net_device *dev = data;
1631 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1632
1633 /* disable intr if needed */
1634 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1635 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1636
1637 if (adapter->shared->ecr)
1638 vmxnet3_process_events(adapter);
1639
1640 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1641
1642 return IRQ_HANDLED;
1643}
1644
1645#endif /* CONFIG_PCI_MSI */
1646
1647
1428/* Interrupt handler for vmxnet3 */ 1648/* Interrupt handler for vmxnet3 */
1429static irqreturn_t 1649static irqreturn_t
1430vmxnet3_intr(int irq, void *dev_id) 1650vmxnet3_intr(int irq, void *dev_id)
@@ -1432,7 +1652,7 @@ vmxnet3_intr(int irq, void *dev_id)
1432 struct net_device *dev = dev_id; 1652 struct net_device *dev = dev_id;
1433 struct vmxnet3_adapter *adapter = netdev_priv(dev); 1653 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1434 1654
1435 if (unlikely(adapter->intr.type == VMXNET3_IT_INTX)) { 1655 if (adapter->intr.type == VMXNET3_IT_INTX) {
1436 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 1656 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1437 if (unlikely(icr == 0)) 1657 if (unlikely(icr == 0))
1438 /* not ours */ 1658 /* not ours */
@@ -1442,77 +1662,144 @@ vmxnet3_intr(int irq, void *dev_id)
1442 1662
1443 /* disable intr if needed */ 1663 /* disable intr if needed */
1444 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1664 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1445 vmxnet3_disable_intr(adapter, 0); 1665 vmxnet3_disable_all_intrs(adapter);
1446 1666
1447 napi_schedule(&adapter->napi); 1667 napi_schedule(&adapter->rx_queue[0].napi);
1448 1668
1449 return IRQ_HANDLED; 1669 return IRQ_HANDLED;
1450} 1670}
1451 1671
1452#ifdef CONFIG_NET_POLL_CONTROLLER 1672#ifdef CONFIG_NET_POLL_CONTROLLER
1453 1673
1454
1455/* netpoll callback. */ 1674/* netpoll callback. */
1456static void 1675static void
1457vmxnet3_netpoll(struct net_device *netdev) 1676vmxnet3_netpoll(struct net_device *netdev)
1458{ 1677{
1459 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1678 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1460 int irq;
1461 1679
1462#ifdef CONFIG_PCI_MSI 1680 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1463 if (adapter->intr.type == VMXNET3_IT_MSIX) 1681 vmxnet3_disable_all_intrs(adapter);
1464 irq = adapter->intr.msix_entries[0].vector; 1682
1465 else 1683 vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
1466#endif 1684 vmxnet3_enable_all_intrs(adapter);
1467 irq = adapter->pdev->irq;
1468 1685
1469 disable_irq(irq);
1470 vmxnet3_intr(irq, netdev);
1471 enable_irq(irq);
1472} 1686}
1473#endif 1687#endif /* CONFIG_NET_POLL_CONTROLLER */
1474 1688
1475static int 1689static int
1476vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 1690vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1477{ 1691{
1478 int err; 1692 struct vmxnet3_intr *intr = &adapter->intr;
1693 int err = 0, i;
1694 int vector = 0;
1479 1695
1480#ifdef CONFIG_PCI_MSI 1696#ifdef CONFIG_PCI_MSI
1481 if (adapter->intr.type == VMXNET3_IT_MSIX) { 1697 if (adapter->intr.type == VMXNET3_IT_MSIX) {
1482 /* we only use 1 MSI-X vector */ 1698 for (i = 0; i < adapter->num_tx_queues; i++) {
1483 err = request_irq(adapter->intr.msix_entries[0].vector, 1699 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1484 vmxnet3_intr, 0, adapter->netdev->name, 1700 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1485 adapter->netdev); 1701 adapter->netdev->name, vector);
1486 } else if (adapter->intr.type == VMXNET3_IT_MSI) { 1702 err = request_irq(
1703 intr->msix_entries[vector].vector,
1704 vmxnet3_msix_tx, 0,
1705 adapter->tx_queue[i].name,
1706 &adapter->tx_queue[i]);
1707 } else {
1708 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1709 adapter->netdev->name, vector);
1710 }
1711 if (err) {
1712 dev_err(&adapter->netdev->dev,
1713 "Failed to request irq for MSIX, %s, "
1714 "error %d\n",
1715 adapter->tx_queue[i].name, err);
1716 return err;
1717 }
1718
1719 /* Handle the case where only 1 MSIx was allocated for
1720 * all tx queues */
1721 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1722 for (; i < adapter->num_tx_queues; i++)
1723 adapter->tx_queue[i].comp_ring.intr_idx
1724 = vector;
1725 vector++;
1726 break;
1727 } else {
1728 adapter->tx_queue[i].comp_ring.intr_idx
1729 = vector++;
1730 }
1731 }
1732 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1733 vector = 0;
1734
1735 for (i = 0; i < adapter->num_rx_queues; i++) {
1736 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1737 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1738 adapter->netdev->name, vector);
1739 else
1740 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1741 adapter->netdev->name, vector);
1742 err = request_irq(intr->msix_entries[vector].vector,
1743 vmxnet3_msix_rx, 0,
1744 adapter->rx_queue[i].name,
1745 &(adapter->rx_queue[i]));
1746 if (err) {
1747 printk(KERN_ERR "Failed to request irq for MSIX"
1748 ", %s, error %d\n",
1749 adapter->rx_queue[i].name, err);
1750 return err;
1751 }
1752
1753 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1754 }
1755
1756 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1757 adapter->netdev->name, vector);
1758 err = request_irq(intr->msix_entries[vector].vector,
1759 vmxnet3_msix_event, 0,
1760 intr->event_msi_vector_name, adapter->netdev);
1761 intr->event_intr_idx = vector;
1762
1763 } else if (intr->type == VMXNET3_IT_MSI) {
1764 adapter->num_rx_queues = 1;
1487 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 1765 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1488 adapter->netdev->name, adapter->netdev); 1766 adapter->netdev->name, adapter->netdev);
1489 } else 1767 } else {
1490#endif 1768#endif
1491 { 1769 adapter->num_rx_queues = 1;
1492 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 1770 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1493 IRQF_SHARED, adapter->netdev->name, 1771 IRQF_SHARED, adapter->netdev->name,
1494 adapter->netdev); 1772 adapter->netdev);
1773#ifdef CONFIG_PCI_MSI
1495 } 1774 }
1496 1775#endif
1497 if (err) 1776 intr->num_intrs = vector + 1;
1777 if (err) {
1498 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error" 1778 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error"
1499 ":%d\n", adapter->netdev->name, adapter->intr.type, err); 1779 ":%d\n", adapter->netdev->name, intr->type, err);
1780 } else {
1781 /* Number of rx queues will not change after this */
1782 for (i = 0; i < adapter->num_rx_queues; i++) {
1783 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1784 rq->qid = i;
1785 rq->qid2 = i + adapter->num_rx_queues;
1786 }
1500 1787
1501 1788
1502 if (!err) {
1503 int i;
1504 /* init our intr settings */
1505 for (i = 0; i < adapter->intr.num_intrs; i++)
1506 adapter->intr.mod_levels[i] = UPT1_IML_ADAPTIVE;
1507 1789
1508 /* next setup intr index for all intr sources */ 1790 /* init our intr settings */
1509 adapter->tx_queue.comp_ring.intr_idx = 0; 1791 for (i = 0; i < intr->num_intrs; i++)
1510 adapter->rx_queue.comp_ring.intr_idx = 0; 1792 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1511 adapter->intr.event_intr_idx = 0; 1793 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1794 adapter->intr.event_intr_idx = 0;
1795 for (i = 0; i < adapter->num_tx_queues; i++)
1796 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1797 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1798 }
1512 1799
1513 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors " 1800 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors "
1514 "allocated\n", adapter->netdev->name, adapter->intr.type, 1801 "allocated\n", adapter->netdev->name, intr->type,
1515 adapter->intr.mask_mode, adapter->intr.num_intrs); 1802 intr->mask_mode, intr->num_intrs);
1516 } 1803 }
1517 1804
1518 return err; 1805 return err;
@@ -1522,18 +1809,32 @@ vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1522static void 1809static void
1523vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 1810vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1524{ 1811{
1525 BUG_ON(adapter->intr.type == VMXNET3_IT_AUTO || 1812 struct vmxnet3_intr *intr = &adapter->intr;
1526 adapter->intr.num_intrs <= 0); 1813 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1527 1814
1528 switch (adapter->intr.type) { 1815 switch (intr->type) {
1529#ifdef CONFIG_PCI_MSI 1816#ifdef CONFIG_PCI_MSI
1530 case VMXNET3_IT_MSIX: 1817 case VMXNET3_IT_MSIX:
1531 { 1818 {
1532 int i; 1819 int i, vector = 0;
1533 1820
1534 for (i = 0; i < adapter->intr.num_intrs; i++) 1821 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1535 free_irq(adapter->intr.msix_entries[i].vector, 1822 for (i = 0; i < adapter->num_tx_queues; i++) {
1536 adapter->netdev); 1823 free_irq(intr->msix_entries[vector++].vector,
1824 &(adapter->tx_queue[i]));
1825 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1826 break;
1827 }
1828 }
1829
1830 for (i = 0; i < adapter->num_rx_queues; i++) {
1831 free_irq(intr->msix_entries[vector++].vector,
1832 &(adapter->rx_queue[i]));
1833 }
1834
1835 free_irq(intr->msix_entries[vector].vector,
1836 adapter->netdev);
1837 BUG_ON(vector >= intr->num_intrs);
1537 break; 1838 break;
1538 } 1839 }
1539#endif 1840#endif
@@ -1727,6 +2028,15 @@ vmxnet3_set_mc(struct net_device *netdev)
1727 kfree(new_table); 2028 kfree(new_table);
1728} 2029}
1729 2030
2031void
2032vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2033{
2034 int i;
2035
2036 for (i = 0; i < adapter->num_rx_queues; i++)
2037 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2038}
2039
1730 2040
1731/* 2041/*
1732 * Set up driver_shared based on settings in adapter. 2042 * Set up driver_shared based on settings in adapter.
@@ -1774,40 +2084,72 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
1774 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2084 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
1775 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2085 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
1776 devRead->misc.queueDescLen = cpu_to_le32( 2086 devRead->misc.queueDescLen = cpu_to_le32(
1777 sizeof(struct Vmxnet3_TxQueueDesc) + 2087 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
1778 sizeof(struct Vmxnet3_RxQueueDesc)); 2088 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
1779 2089
1780 /* tx queue settings */ 2090 /* tx queue settings */
1781 BUG_ON(adapter->tx_queue.tx_ring.base == NULL); 2091 devRead->misc.numTxQueues = adapter->num_tx_queues;
1782 2092 for (i = 0; i < adapter->num_tx_queues; i++) {
1783 devRead->misc.numTxQueues = 1; 2093 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
1784 tqc = &adapter->tqd_start->conf; 2094 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
1785 tqc->txRingBasePA = cpu_to_le64(adapter->tx_queue.tx_ring.basePA); 2095 tqc = &adapter->tqd_start[i].conf;
1786 tqc->dataRingBasePA = cpu_to_le64(adapter->tx_queue.data_ring.basePA); 2096 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
1787 tqc->compRingBasePA = cpu_to_le64(adapter->tx_queue.comp_ring.basePA); 2097 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
1788 tqc->ddPA = cpu_to_le64(virt_to_phys( 2098 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
1789 adapter->tx_queue.buf_info)); 2099 tqc->ddPA = cpu_to_le64(virt_to_phys(tq->buf_info));
1790 tqc->txRingSize = cpu_to_le32(adapter->tx_queue.tx_ring.size); 2100 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
1791 tqc->dataRingSize = cpu_to_le32(adapter->tx_queue.data_ring.size); 2101 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
1792 tqc->compRingSize = cpu_to_le32(adapter->tx_queue.comp_ring.size); 2102 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
1793 tqc->ddLen = cpu_to_le32(sizeof(struct vmxnet3_tx_buf_info) * 2103 tqc->ddLen = cpu_to_le32(
1794 tqc->txRingSize); 2104 sizeof(struct vmxnet3_tx_buf_info) *
1795 tqc->intrIdx = adapter->tx_queue.comp_ring.intr_idx; 2105 tqc->txRingSize);
2106 tqc->intrIdx = tq->comp_ring.intr_idx;
2107 }
1796 2108
1797 /* rx queue settings */ 2109 /* rx queue settings */
1798 devRead->misc.numRxQueues = 1; 2110 devRead->misc.numRxQueues = adapter->num_rx_queues;
1799 rqc = &adapter->rqd_start->conf; 2111 for (i = 0; i < adapter->num_rx_queues; i++) {
1800 rqc->rxRingBasePA[0] = cpu_to_le64(adapter->rx_queue.rx_ring[0].basePA); 2112 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1801 rqc->rxRingBasePA[1] = cpu_to_le64(adapter->rx_queue.rx_ring[1].basePA); 2113 rqc = &adapter->rqd_start[i].conf;
1802 rqc->compRingBasePA = cpu_to_le64(adapter->rx_queue.comp_ring.basePA); 2114 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
1803 rqc->ddPA = cpu_to_le64(virt_to_phys( 2115 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
1804 adapter->rx_queue.buf_info)); 2116 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
1805 rqc->rxRingSize[0] = cpu_to_le32(adapter->rx_queue.rx_ring[0].size); 2117 rqc->ddPA = cpu_to_le64(virt_to_phys(
1806 rqc->rxRingSize[1] = cpu_to_le32(adapter->rx_queue.rx_ring[1].size); 2118 rq->buf_info));
1807 rqc->compRingSize = cpu_to_le32(adapter->rx_queue.comp_ring.size); 2119 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
1808 rqc->ddLen = cpu_to_le32(sizeof(struct vmxnet3_rx_buf_info) * 2120 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
1809 (rqc->rxRingSize[0] + rqc->rxRingSize[1])); 2121 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
1810 rqc->intrIdx = adapter->rx_queue.comp_ring.intr_idx; 2122 rqc->ddLen = cpu_to_le32(
2123 sizeof(struct vmxnet3_rx_buf_info) *
2124 (rqc->rxRingSize[0] +
2125 rqc->rxRingSize[1]));
2126 rqc->intrIdx = rq->comp_ring.intr_idx;
2127 }
2128
2129#ifdef VMXNET3_RSS
2130 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2131
2132 if (adapter->rss) {
2133 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2134 devRead->misc.uptFeatures |= UPT1_F_RSS;
2135 devRead->misc.numRxQueues = adapter->num_rx_queues;
2136 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2137 UPT1_RSS_HASH_TYPE_IPV4 |
2138 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2139 UPT1_RSS_HASH_TYPE_IPV6;
2140 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2141 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2142 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2143 get_random_bytes(&rssConf->hashKey[0], rssConf->hashKeySize);
2144 for (i = 0; i < rssConf->indTableSize; i++)
2145 rssConf->indTable[i] = i % adapter->num_rx_queues;
2146
2147 devRead->rssConfDesc.confVer = 1;
2148 devRead->rssConfDesc.confLen = sizeof(*rssConf);
2149 devRead->rssConfDesc.confPA = virt_to_phys(rssConf);
2150 }
2151
2152#endif /* VMXNET3_RSS */
1811 2153
1812 /* intr settings */ 2154 /* intr settings */
1813 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2155 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
@@ -1829,18 +2171,18 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
1829int 2171int
1830vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 2172vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1831{ 2173{
1832 int err; 2174 int err, i;
1833 u32 ret; 2175 u32 ret;
1834 2176
1835 dev_dbg(&adapter->netdev->dev, 2177 dev_dbg(&adapter->netdev->dev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
1836 "%s: skb_buf_size %d, rx_buf_per_pkt %d, ring sizes" 2178 " ring sizes %u %u %u\n", adapter->netdev->name,
1837 " %u %u %u\n", adapter->netdev->name, adapter->skb_buf_size, 2179 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
1838 adapter->rx_buf_per_pkt, adapter->tx_queue.tx_ring.size, 2180 adapter->tx_queue[0].tx_ring.size,
1839 adapter->rx_queue.rx_ring[0].size, 2181 adapter->rx_queue[0].rx_ring[0].size,
1840 adapter->rx_queue.rx_ring[1].size); 2182 adapter->rx_queue[0].rx_ring[1].size);
1841 2183
1842 vmxnet3_tq_init(&adapter->tx_queue, adapter); 2184 vmxnet3_tq_init_all(adapter);
1843 err = vmxnet3_rq_init(&adapter->rx_queue, adapter); 2185 err = vmxnet3_rq_init_all(adapter);
1844 if (err) { 2186 if (err) {
1845 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n", 2187 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n",
1846 adapter->netdev->name, err); 2188 adapter->netdev->name, err);
@@ -1870,10 +2212,15 @@ vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1870 err = -EINVAL; 2212 err = -EINVAL;
1871 goto activate_err; 2213 goto activate_err;
1872 } 2214 }
1873 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD, 2215
1874 adapter->rx_queue.rx_ring[0].next2fill); 2216 for (i = 0; i < adapter->num_rx_queues; i++) {
1875 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD2, 2217 VMXNET3_WRITE_BAR0_REG(adapter,
1876 adapter->rx_queue.rx_ring[1].next2fill); 2218 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2219 adapter->rx_queue[i].rx_ring[0].next2fill);
2220 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2221 (i * VMXNET3_REG_ALIGN)),
2222 adapter->rx_queue[i].rx_ring[1].next2fill);
2223 }
1877 2224
1878 /* Apply the rx filter settins last. */ 2225 /* Apply the rx filter settins last. */
1879 vmxnet3_set_mc(adapter->netdev); 2226 vmxnet3_set_mc(adapter->netdev);
@@ -1883,8 +2230,8 @@ vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1883 * tx queue if the link is up. 2230 * tx queue if the link is up.
1884 */ 2231 */
1885 vmxnet3_check_link(adapter, true); 2232 vmxnet3_check_link(adapter, true);
1886 2233 for (i = 0; i < adapter->num_rx_queues; i++)
1887 napi_enable(&adapter->napi); 2234 napi_enable(&adapter->rx_queue[i].napi);
1888 vmxnet3_enable_all_intrs(adapter); 2235 vmxnet3_enable_all_intrs(adapter);
1889 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 2236 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
1890 return 0; 2237 return 0;
@@ -1896,7 +2243,7 @@ activate_err:
1896irq_err: 2243irq_err:
1897rq_err: 2244rq_err:
1898 /* free up buffers we allocated */ 2245 /* free up buffers we allocated */
1899 vmxnet3_rq_cleanup(&adapter->rx_queue, adapter); 2246 vmxnet3_rq_cleanup_all(adapter);
1900 return err; 2247 return err;
1901} 2248}
1902 2249
@@ -1911,6 +2258,7 @@ vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
1911int 2258int
1912vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 2259vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
1913{ 2260{
2261 int i;
1914 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 2262 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
1915 return 0; 2263 return 0;
1916 2264
@@ -1919,13 +2267,14 @@ vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
1919 VMXNET3_CMD_QUIESCE_DEV); 2267 VMXNET3_CMD_QUIESCE_DEV);
1920 vmxnet3_disable_all_intrs(adapter); 2268 vmxnet3_disable_all_intrs(adapter);
1921 2269
1922 napi_disable(&adapter->napi); 2270 for (i = 0; i < adapter->num_rx_queues; i++)
2271 napi_disable(&adapter->rx_queue[i].napi);
1923 netif_tx_disable(adapter->netdev); 2272 netif_tx_disable(adapter->netdev);
1924 adapter->link_speed = 0; 2273 adapter->link_speed = 0;
1925 netif_carrier_off(adapter->netdev); 2274 netif_carrier_off(adapter->netdev);
1926 2275
1927 vmxnet3_tq_cleanup(&adapter->tx_queue, adapter); 2276 vmxnet3_tq_cleanup_all(adapter);
1928 vmxnet3_rq_cleanup(&adapter->rx_queue, adapter); 2277 vmxnet3_rq_cleanup_all(adapter);
1929 vmxnet3_free_irqs(adapter); 2278 vmxnet3_free_irqs(adapter);
1930 return 0; 2279 return 0;
1931} 2280}
@@ -2047,7 +2396,9 @@ vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2047static void 2396static void
2048vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 2397vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2049{ 2398{
2050 size_t sz; 2399 size_t sz, i, ring0_size, ring1_size, comp_size;
2400 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2401
2051 2402
2052 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 2403 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2053 VMXNET3_MAX_ETH_HDR_SIZE) { 2404 VMXNET3_MAX_ETH_HDR_SIZE) {
@@ -2069,11 +2420,19 @@ vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2069 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 2420 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2070 */ 2421 */
2071 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 2422 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2072 adapter->rx_queue.rx_ring[0].size = (adapter->rx_queue.rx_ring[0].size + 2423 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2073 sz - 1) / sz * sz; 2424 ring0_size = (ring0_size + sz - 1) / sz * sz;
2074 adapter->rx_queue.rx_ring[0].size = min_t(u32, 2425 ring0_size = min_t(u32, rq->rx_ring[0].size, VMXNET3_RX_RING_MAX_SIZE /
2075 adapter->rx_queue.rx_ring[0].size, 2426 sz * sz);
2076 VMXNET3_RX_RING_MAX_SIZE / sz * sz); 2427 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2428 comp_size = ring0_size + ring1_size;
2429
2430 for (i = 0; i < adapter->num_rx_queues; i++) {
2431 rq = &adapter->rx_queue[i];
2432 rq->rx_ring[0].size = ring0_size;
2433 rq->rx_ring[1].size = ring1_size;
2434 rq->comp_ring.size = comp_size;
2435 }
2077} 2436}
2078 2437
2079 2438
@@ -2081,29 +2440,53 @@ int
2081vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 2440vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2082 u32 rx_ring_size, u32 rx_ring2_size) 2441 u32 rx_ring_size, u32 rx_ring2_size)
2083{ 2442{
2084 int err; 2443 int err = 0, i;
2085 2444
2086 adapter->tx_queue.tx_ring.size = tx_ring_size; 2445 for (i = 0; i < adapter->num_tx_queues; i++) {
2087 adapter->tx_queue.data_ring.size = tx_ring_size; 2446 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2088 adapter->tx_queue.comp_ring.size = tx_ring_size; 2447 tq->tx_ring.size = tx_ring_size;
2089 adapter->tx_queue.shared = &adapter->tqd_start->ctrl; 2448 tq->data_ring.size = tx_ring_size;
2090 adapter->tx_queue.stopped = true; 2449 tq->comp_ring.size = tx_ring_size;
2091 err = vmxnet3_tq_create(&adapter->tx_queue, adapter); 2450 tq->shared = &adapter->tqd_start[i].ctrl;
2092 if (err) 2451 tq->stopped = true;
2093 return err; 2452 tq->adapter = adapter;
2453 tq->qid = i;
2454 err = vmxnet3_tq_create(tq, adapter);
2455 /*
2456 * Too late to change num_tx_queues. We cannot do away with
2457 * lesser number of queues than what we asked for
2458 */
2459 if (err)
2460 goto queue_err;
2461 }
2094 2462
2095 adapter->rx_queue.rx_ring[0].size = rx_ring_size; 2463 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2096 adapter->rx_queue.rx_ring[1].size = rx_ring2_size; 2464 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2097 vmxnet3_adjust_rx_ring_size(adapter); 2465 vmxnet3_adjust_rx_ring_size(adapter);
2098 adapter->rx_queue.comp_ring.size = adapter->rx_queue.rx_ring[0].size + 2466 for (i = 0; i < adapter->num_rx_queues; i++) {
2099 adapter->rx_queue.rx_ring[1].size; 2467 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2100 adapter->rx_queue.qid = 0; 2468 /* qid and qid2 for rx queues will be assigned later when num
2101 adapter->rx_queue.qid2 = 1; 2469 * of rx queues is finalized after allocating intrs */
2102 adapter->rx_queue.shared = &adapter->rqd_start->ctrl; 2470 rq->shared = &adapter->rqd_start[i].ctrl;
2103 err = vmxnet3_rq_create(&adapter->rx_queue, adapter); 2471 rq->adapter = adapter;
2104 if (err) 2472 err = vmxnet3_rq_create(rq, adapter);
2105 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 2473 if (err) {
2106 2474 if (i == 0) {
2475 printk(KERN_ERR "Could not allocate any rx"
2476 "queues. Aborting.\n");
2477 goto queue_err;
2478 } else {
2479 printk(KERN_INFO "Number of rx queues changed "
2480 "to : %d.\n", i);
2481 adapter->num_rx_queues = i;
2482 err = 0;
2483 break;
2484 }
2485 }
2486 }
2487 return err;
2488queue_err:
2489 vmxnet3_tq_destroy_all(adapter);
2107 return err; 2490 return err;
2108} 2491}
2109 2492
@@ -2111,11 +2494,12 @@ static int
2111vmxnet3_open(struct net_device *netdev) 2494vmxnet3_open(struct net_device *netdev)
2112{ 2495{
2113 struct vmxnet3_adapter *adapter; 2496 struct vmxnet3_adapter *adapter;
2114 int err; 2497 int err, i;
2115 2498
2116 adapter = netdev_priv(netdev); 2499 adapter = netdev_priv(netdev);
2117 2500
2118 spin_lock_init(&adapter->tx_queue.tx_lock); 2501 for (i = 0; i < adapter->num_tx_queues; i++)
2502 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2119 2503
2120 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE, 2504 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2121 VMXNET3_DEF_RX_RING_SIZE, 2505 VMXNET3_DEF_RX_RING_SIZE,
@@ -2130,8 +2514,8 @@ vmxnet3_open(struct net_device *netdev)
2130 return 0; 2514 return 0;
2131 2515
2132activate_err: 2516activate_err:
2133 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 2517 vmxnet3_rq_destroy_all(adapter);
2134 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 2518 vmxnet3_tq_destroy_all(adapter);
2135queue_err: 2519queue_err:
2136 return err; 2520 return err;
2137} 2521}
@@ -2151,8 +2535,8 @@ vmxnet3_close(struct net_device *netdev)
2151 2535
2152 vmxnet3_quiesce_dev(adapter); 2536 vmxnet3_quiesce_dev(adapter);
2153 2537
2154 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 2538 vmxnet3_rq_destroy_all(adapter);
2155 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 2539 vmxnet3_tq_destroy_all(adapter);
2156 2540
2157 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 2541 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2158 2542
@@ -2164,6 +2548,8 @@ vmxnet3_close(struct net_device *netdev)
2164void 2548void
2165vmxnet3_force_close(struct vmxnet3_adapter *adapter) 2549vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2166{ 2550{
2551 int i;
2552
2167 /* 2553 /*
2168 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 2554 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2169 * vmxnet3_close() will deadlock. 2555 * vmxnet3_close() will deadlock.
@@ -2171,7 +2557,8 @@ vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2171 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 2557 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2172 2558
2173 /* we need to enable NAPI, otherwise dev_close will deadlock */ 2559 /* we need to enable NAPI, otherwise dev_close will deadlock */
2174 napi_enable(&adapter->napi); 2560 for (i = 0; i < adapter->num_rx_queues; i++)
2561 napi_enable(&adapter->rx_queue[i].napi);
2175 dev_close(adapter->netdev); 2562 dev_close(adapter->netdev);
2176} 2563}
2177 2564
@@ -2202,14 +2589,11 @@ vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2202 vmxnet3_reset_dev(adapter); 2589 vmxnet3_reset_dev(adapter);
2203 2590
2204 /* we need to re-create the rx queue based on the new mtu */ 2591 /* we need to re-create the rx queue based on the new mtu */
2205 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 2592 vmxnet3_rq_destroy_all(adapter);
2206 vmxnet3_adjust_rx_ring_size(adapter); 2593 vmxnet3_adjust_rx_ring_size(adapter);
2207 adapter->rx_queue.comp_ring.size = 2594 err = vmxnet3_rq_create_all(adapter);
2208 adapter->rx_queue.rx_ring[0].size +
2209 adapter->rx_queue.rx_ring[1].size;
2210 err = vmxnet3_rq_create(&adapter->rx_queue, adapter);
2211 if (err) { 2595 if (err) {
2212 printk(KERN_ERR "%s: failed to re-create rx queue," 2596 printk(KERN_ERR "%s: failed to re-create rx queues,"
2213 " error %d. Closing it.\n", netdev->name, err); 2597 " error %d. Closing it.\n", netdev->name, err);
2214 goto out; 2598 goto out;
2215 } 2599 }
@@ -2274,6 +2658,55 @@ vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2274 mac[5] = (tmp >> 8) & 0xff; 2658 mac[5] = (tmp >> 8) & 0xff;
2275} 2659}
2276 2660
2661#ifdef CONFIG_PCI_MSI
2662
2663/*
2664 * Enable MSIx vectors.
2665 * Returns :
2666 * 0 on successful enabling of required vectors,
2667 * VMXNET3_LINUX_MIN_MSIX_VECT when only minumum number of vectors required
2668 * could be enabled.
2669 * number of vectors which can be enabled otherwise (this number is smaller
2670 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2671 */
2672
2673static int
2674vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
2675 int vectors)
2676{
2677 int err = 0, vector_threshold;
2678 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
2679
2680 while (vectors >= vector_threshold) {
2681 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2682 vectors);
2683 if (!err) {
2684 adapter->intr.num_intrs = vectors;
2685 return 0;
2686 } else if (err < 0) {
2687 printk(KERN_ERR "Failed to enable MSI-X for %s, error"
2688 " %d\n", adapter->netdev->name, err);
2689 vectors = 0;
2690 } else if (err < vector_threshold) {
2691 break;
2692 } else {
2693 /* If fails to enable required number of MSI-x vectors
2694 * try enabling 3 of them. One each for rx, tx and event
2695 */
2696 vectors = vector_threshold;
2697 printk(KERN_ERR "Failed to enable %d MSI-X for %s, try"
2698 " %d instead\n", vectors, adapter->netdev->name,
2699 vector_threshold);
2700 }
2701 }
2702
2703 printk(KERN_INFO "Number of MSI-X interrupts which can be allocatedi"
2704 " are lower than min threshold required.\n");
2705 return err;
2706}
2707
2708
2709#endif /* CONFIG_PCI_MSI */
2277 2710
2278static void 2711static void
2279vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 2712vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
@@ -2293,16 +2726,47 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2293 2726
2294#ifdef CONFIG_PCI_MSI 2727#ifdef CONFIG_PCI_MSI
2295 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2728 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2296 int err; 2729 int vector, err = 0;
2297 2730
2298 adapter->intr.msix_entries[0].entry = 0; 2731 adapter->intr.num_intrs = (adapter->share_intr ==
2299 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries, 2732 VMXNET3_INTR_TXSHARE) ? 1 :
2300 VMXNET3_LINUX_MAX_MSIX_VECT); 2733 adapter->num_tx_queues;
2301 if (!err) { 2734 adapter->intr.num_intrs += (adapter->share_intr ==
2302 adapter->intr.num_intrs = 1; 2735 VMXNET3_INTR_BUDDYSHARE) ? 0 :
2303 adapter->intr.type = VMXNET3_IT_MSIX; 2736 adapter->num_rx_queues;
2737 adapter->intr.num_intrs += 1; /* for link event */
2738
2739 adapter->intr.num_intrs = (adapter->intr.num_intrs >
2740 VMXNET3_LINUX_MIN_MSIX_VECT
2741 ? adapter->intr.num_intrs :
2742 VMXNET3_LINUX_MIN_MSIX_VECT);
2743
2744 for (vector = 0; vector < adapter->intr.num_intrs; vector++)
2745 adapter->intr.msix_entries[vector].entry = vector;
2746
2747 err = vmxnet3_acquire_msix_vectors(adapter,
2748 adapter->intr.num_intrs);
2749 /* If we cannot allocate one MSIx vector per queue
2750 * then limit the number of rx queues to 1
2751 */
2752 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
2753 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2754 || adapter->num_rx_queues != 2) {
2755 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2756 printk(KERN_ERR "Number of rx queues : 1\n");
2757 adapter->num_rx_queues = 1;
2758 adapter->intr.num_intrs =
2759 VMXNET3_LINUX_MIN_MSIX_VECT;
2760 }
2304 return; 2761 return;
2305 } 2762 }
2763 if (!err)
2764 return;
2765
2766 /* If we cannot allocate MSIx vectors use only one rx queue */
2767 printk(KERN_INFO "Failed to enable MSI-X for %s, error %d."
2768 "#rx queues : 1, try MSI\n", adapter->netdev->name, err);
2769
2306 adapter->intr.type = VMXNET3_IT_MSI; 2770 adapter->intr.type = VMXNET3_IT_MSI;
2307 } 2771 }
2308 2772
@@ -2310,12 +2774,15 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2310 int err; 2774 int err;
2311 err = pci_enable_msi(adapter->pdev); 2775 err = pci_enable_msi(adapter->pdev);
2312 if (!err) { 2776 if (!err) {
2777 adapter->num_rx_queues = 1;
2313 adapter->intr.num_intrs = 1; 2778 adapter->intr.num_intrs = 1;
2314 return; 2779 return;
2315 } 2780 }
2316 } 2781 }
2317#endif /* CONFIG_PCI_MSI */ 2782#endif /* CONFIG_PCI_MSI */
2318 2783
2784 adapter->num_rx_queues = 1;
2785 printk(KERN_INFO "Using INTx interrupt, #Rx queues: 1.\n");
2319 adapter->intr.type = VMXNET3_IT_INTX; 2786 adapter->intr.type = VMXNET3_IT_INTX;
2320 2787
2321 /* INT-X related setting */ 2788 /* INT-X related setting */
@@ -2343,6 +2810,7 @@ vmxnet3_tx_timeout(struct net_device *netdev)
2343 2810
2344 printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name); 2811 printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name);
2345 schedule_work(&adapter->work); 2812 schedule_work(&adapter->work);
2813 netif_wake_queue(adapter->netdev);
2346} 2814}
2347 2815
2348 2816
@@ -2399,8 +2867,29 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2399 struct net_device *netdev; 2867 struct net_device *netdev;
2400 struct vmxnet3_adapter *adapter; 2868 struct vmxnet3_adapter *adapter;
2401 u8 mac[ETH_ALEN]; 2869 u8 mac[ETH_ALEN];
2870 int size;
2871 int num_tx_queues;
2872 int num_rx_queues;
2873
2874#ifdef VMXNET3_RSS
2875 if (enable_mq)
2876 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2877 (int)num_online_cpus());
2878 else
2879#endif
2880 num_rx_queues = 1;
2881
2882 if (enable_mq)
2883 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2884 (int)num_online_cpus());
2885 else
2886 num_tx_queues = 1;
2887
2888 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2889 max(num_tx_queues, num_rx_queues));
2890 printk(KERN_INFO "# of Tx queues : %d, # of Rx queues : %d\n",
2891 num_tx_queues, num_rx_queues);
2402 2892
2403 netdev = alloc_etherdev(sizeof(struct vmxnet3_adapter));
2404 if (!netdev) { 2893 if (!netdev) {
2405 printk(KERN_ERR "Failed to alloc ethernet device for adapter " 2894 printk(KERN_ERR "Failed to alloc ethernet device for adapter "
2406 "%s\n", pci_name(pdev)); 2895 "%s\n", pci_name(pdev));
@@ -2422,9 +2911,12 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2422 goto err_alloc_shared; 2911 goto err_alloc_shared;
2423 } 2912 }
2424 2913
2425 adapter->tqd_start = pci_alloc_consistent(adapter->pdev, 2914 adapter->num_rx_queues = num_rx_queues;
2426 sizeof(struct Vmxnet3_TxQueueDesc) + 2915 adapter->num_tx_queues = num_tx_queues;
2427 sizeof(struct Vmxnet3_RxQueueDesc), 2916
2917 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2918 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2919 adapter->tqd_start = pci_alloc_consistent(adapter->pdev, size,
2428 &adapter->queue_desc_pa); 2920 &adapter->queue_desc_pa);
2429 2921
2430 if (!adapter->tqd_start) { 2922 if (!adapter->tqd_start) {
@@ -2433,8 +2925,8 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2433 err = -ENOMEM; 2925 err = -ENOMEM;
2434 goto err_alloc_queue_desc; 2926 goto err_alloc_queue_desc;
2435 } 2927 }
2436 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start 2928 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
2437 + 1); 2929 adapter->num_tx_queues);
2438 2930
2439 adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL); 2931 adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL);
2440 if (adapter->pm_conf == NULL) { 2932 if (adapter->pm_conf == NULL) {
@@ -2444,6 +2936,17 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2444 goto err_alloc_pm; 2936 goto err_alloc_pm;
2445 } 2937 }
2446 2938
2939#ifdef VMXNET3_RSS
2940
2941 adapter->rss_conf = kmalloc(sizeof(struct UPT1_RSSConf), GFP_KERNEL);
2942 if (adapter->rss_conf == NULL) {
2943 printk(KERN_ERR "Failed to allocate memory for %s\n",
2944 pci_name(pdev));
2945 err = -ENOMEM;
2946 goto err_alloc_rss;
2947 }
2948#endif /* VMXNET3_RSS */
2949
2447 err = vmxnet3_alloc_pci_resources(adapter, &dma64); 2950 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
2448 if (err < 0) 2951 if (err < 0)
2449 goto err_alloc_pci; 2952 goto err_alloc_pci;
@@ -2471,18 +2974,48 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2471 vmxnet3_declare_features(adapter, dma64); 2974 vmxnet3_declare_features(adapter, dma64);
2472 2975
2473 adapter->dev_number = atomic_read(&devices_found); 2976 adapter->dev_number = atomic_read(&devices_found);
2977
2978 adapter->share_intr = irq_share_mode;
2979 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE &&
2980 adapter->num_tx_queues != adapter->num_rx_queues)
2981 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
2982
2474 vmxnet3_alloc_intr_resources(adapter); 2983 vmxnet3_alloc_intr_resources(adapter);
2475 2984
2985#ifdef VMXNET3_RSS
2986 if (adapter->num_rx_queues > 1 &&
2987 adapter->intr.type == VMXNET3_IT_MSIX) {
2988 adapter->rss = true;
2989 printk(KERN_INFO "RSS is enabled.\n");
2990 } else {
2991 adapter->rss = false;
2992 }
2993#endif
2994
2476 vmxnet3_read_mac_addr(adapter, mac); 2995 vmxnet3_read_mac_addr(adapter, mac);
2477 memcpy(netdev->dev_addr, mac, netdev->addr_len); 2996 memcpy(netdev->dev_addr, mac, netdev->addr_len);
2478 2997
2479 netdev->netdev_ops = &vmxnet3_netdev_ops; 2998 netdev->netdev_ops = &vmxnet3_netdev_ops;
2480 netdev->watchdog_timeo = 5 * HZ;
2481 vmxnet3_set_ethtool_ops(netdev); 2999 vmxnet3_set_ethtool_ops(netdev);
3000 netdev->watchdog_timeo = 5 * HZ;
2482 3001
2483 INIT_WORK(&adapter->work, vmxnet3_reset_work); 3002 INIT_WORK(&adapter->work, vmxnet3_reset_work);
2484 3003
2485 netif_napi_add(netdev, &adapter->napi, vmxnet3_poll, 64); 3004 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3005 int i;
3006 for (i = 0; i < adapter->num_rx_queues; i++) {
3007 netif_napi_add(adapter->netdev,
3008 &adapter->rx_queue[i].napi,
3009 vmxnet3_poll_rx_only, 64);
3010 }
3011 } else {
3012 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3013 vmxnet3_poll, 64);
3014 }
3015
3016 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3017 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3018
2486 SET_NETDEV_DEV(netdev, &pdev->dev); 3019 SET_NETDEV_DEV(netdev, &pdev->dev);
2487 err = register_netdev(netdev); 3020 err = register_netdev(netdev);
2488 3021
@@ -2502,11 +3035,14 @@ err_register:
2502err_ver: 3035err_ver:
2503 vmxnet3_free_pci_resources(adapter); 3036 vmxnet3_free_pci_resources(adapter);
2504err_alloc_pci: 3037err_alloc_pci:
3038#ifdef VMXNET3_RSS
3039 kfree(adapter->rss_conf);
3040err_alloc_rss:
3041#endif
2505 kfree(adapter->pm_conf); 3042 kfree(adapter->pm_conf);
2506err_alloc_pm: 3043err_alloc_pm:
2507 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) + 3044 pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
2508 sizeof(struct Vmxnet3_RxQueueDesc), 3045 adapter->queue_desc_pa);
2509 adapter->tqd_start, adapter->queue_desc_pa);
2510err_alloc_queue_desc: 3046err_alloc_queue_desc:
2511 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared), 3047 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2512 adapter->shared, adapter->shared_pa); 3048 adapter->shared, adapter->shared_pa);
@@ -2522,6 +3058,16 @@ vmxnet3_remove_device(struct pci_dev *pdev)
2522{ 3058{
2523 struct net_device *netdev = pci_get_drvdata(pdev); 3059 struct net_device *netdev = pci_get_drvdata(pdev);
2524 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3060 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3061 int size = 0;
3062 int num_rx_queues;
3063
3064#ifdef VMXNET3_RSS
3065 if (enable_mq)
3066 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3067 (int)num_online_cpus());
3068 else
3069#endif
3070 num_rx_queues = 1;
2525 3071
2526 flush_scheduled_work(); 3072 flush_scheduled_work();
2527 3073
@@ -2529,10 +3075,15 @@ vmxnet3_remove_device(struct pci_dev *pdev)
2529 3075
2530 vmxnet3_free_intr_resources(adapter); 3076 vmxnet3_free_intr_resources(adapter);
2531 vmxnet3_free_pci_resources(adapter); 3077 vmxnet3_free_pci_resources(adapter);
3078#ifdef VMXNET3_RSS
3079 kfree(adapter->rss_conf);
3080#endif
2532 kfree(adapter->pm_conf); 3081 kfree(adapter->pm_conf);
2533 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) + 3082
2534 sizeof(struct Vmxnet3_RxQueueDesc), 3083 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2535 adapter->tqd_start, adapter->queue_desc_pa); 3084 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3085 pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
3086 adapter->queue_desc_pa);
2536 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared), 3087 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2537 adapter->shared, adapter->shared_pa); 3088 adapter->shared, adapter->shared_pa);
2538 free_netdev(netdev); 3089 free_netdev(netdev);
@@ -2563,7 +3114,7 @@ vmxnet3_suspend(struct device *device)
2563 vmxnet3_free_intr_resources(adapter); 3114 vmxnet3_free_intr_resources(adapter);
2564 3115
2565 netif_device_detach(netdev); 3116 netif_device_detach(netdev);
2566 netif_stop_queue(netdev); 3117 netif_tx_stop_all_queues(netdev);
2567 3118
2568 /* Create wake-up filters. */ 3119 /* Create wake-up filters. */
2569 pmConf = adapter->pm_conf; 3120 pmConf = adapter->pm_conf;
diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c
index b79070bcc92e..9ddaea636cfa 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethtool.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c
@@ -151,44 +151,42 @@ vmxnet3_get_stats(struct net_device *netdev)
151 struct UPT1_TxStats *devTxStats; 151 struct UPT1_TxStats *devTxStats;
152 struct UPT1_RxStats *devRxStats; 152 struct UPT1_RxStats *devRxStats;
153 struct net_device_stats *net_stats = &netdev->stats; 153 struct net_device_stats *net_stats = &netdev->stats;
154 int i;
154 155
155 adapter = netdev_priv(netdev); 156 adapter = netdev_priv(netdev);
156 157
157 /* Collect the dev stats into the shared area */ 158 /* Collect the dev stats into the shared area */
158 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 159 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
159 160
160 /* Assuming that we have a single queue device */
161 devTxStats = &adapter->tqd_start->stats;
162 devRxStats = &adapter->rqd_start->stats;
163
164 /* Get access to the driver stats per queue */
165 drvTxStats = &adapter->tx_queue.stats;
166 drvRxStats = &adapter->rx_queue.stats;
167
168 memset(net_stats, 0, sizeof(*net_stats)); 161 memset(net_stats, 0, sizeof(*net_stats));
162 for (i = 0; i < adapter->num_tx_queues; i++) {
163 devTxStats = &adapter->tqd_start[i].stats;
164 drvTxStats = &adapter->tx_queue[i].stats;
165 net_stats->tx_packets += devTxStats->ucastPktsTxOK +
166 devTxStats->mcastPktsTxOK +
167 devTxStats->bcastPktsTxOK;
168 net_stats->tx_bytes += devTxStats->ucastBytesTxOK +
169 devTxStats->mcastBytesTxOK +
170 devTxStats->bcastBytesTxOK;
171 net_stats->tx_errors += devTxStats->pktsTxError;
172 net_stats->tx_dropped += drvTxStats->drop_total;
173 }
169 174
170 net_stats->rx_packets = devRxStats->ucastPktsRxOK + 175 for (i = 0; i < adapter->num_rx_queues; i++) {
171 devRxStats->mcastPktsRxOK + 176 devRxStats = &adapter->rqd_start[i].stats;
172 devRxStats->bcastPktsRxOK; 177 drvRxStats = &adapter->rx_queue[i].stats;
173 178 net_stats->rx_packets += devRxStats->ucastPktsRxOK +
174 net_stats->tx_packets = devTxStats->ucastPktsTxOK + 179 devRxStats->mcastPktsRxOK +
175 devTxStats->mcastPktsTxOK + 180 devRxStats->bcastPktsRxOK;
176 devTxStats->bcastPktsTxOK;
177
178 net_stats->rx_bytes = devRxStats->ucastBytesRxOK +
179 devRxStats->mcastBytesRxOK +
180 devRxStats->bcastBytesRxOK;
181
182 net_stats->tx_bytes = devTxStats->ucastBytesTxOK +
183 devTxStats->mcastBytesTxOK +
184 devTxStats->bcastBytesTxOK;
185 181
186 net_stats->rx_errors = devRxStats->pktsRxError; 182 net_stats->rx_bytes += devRxStats->ucastBytesRxOK +
187 net_stats->tx_errors = devTxStats->pktsTxError; 183 devRxStats->mcastBytesRxOK +
188 net_stats->rx_dropped = drvRxStats->drop_total; 184 devRxStats->bcastBytesRxOK;
189 net_stats->tx_dropped = drvTxStats->drop_total;
190 net_stats->multicast = devRxStats->mcastPktsRxOK;
191 185
186 net_stats->rx_errors += devRxStats->pktsRxError;
187 net_stats->rx_dropped += drvRxStats->drop_total;
188 net_stats->multicast += devRxStats->mcastPktsRxOK;
189 }
192 return net_stats; 190 return net_stats;
193} 191}
194 192
@@ -307,24 +305,26 @@ vmxnet3_get_ethtool_stats(struct net_device *netdev,
307 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 305 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
308 u8 *base; 306 u8 *base;
309 int i; 307 int i;
308 int j = 0;
310 309
311 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 310 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
312 311
313 /* this does assume each counter is 64-bit wide */ 312 /* this does assume each counter is 64-bit wide */
313/* TODO change this for multiple queues */
314 314
315 base = (u8 *)&adapter->tqd_start->stats; 315 base = (u8 *)&adapter->tqd_start[j].stats;
316 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++) 316 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++)
317 *buf++ = *(u64 *)(base + vmxnet3_tq_dev_stats[i].offset); 317 *buf++ = *(u64 *)(base + vmxnet3_tq_dev_stats[i].offset);
318 318
319 base = (u8 *)&adapter->tx_queue.stats; 319 base = (u8 *)&adapter->tx_queue[j].stats;
320 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++) 320 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++)
321 *buf++ = *(u64 *)(base + vmxnet3_tq_driver_stats[i].offset); 321 *buf++ = *(u64 *)(base + vmxnet3_tq_driver_stats[i].offset);
322 322
323 base = (u8 *)&adapter->rqd_start->stats; 323 base = (u8 *)&adapter->rqd_start[j].stats;
324 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++) 324 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++)
325 *buf++ = *(u64 *)(base + vmxnet3_rq_dev_stats[i].offset); 325 *buf++ = *(u64 *)(base + vmxnet3_rq_dev_stats[i].offset);
326 326
327 base = (u8 *)&adapter->rx_queue.stats; 327 base = (u8 *)&adapter->rx_queue[j].stats;
328 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++) 328 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++)
329 *buf++ = *(u64 *)(base + vmxnet3_rq_driver_stats[i].offset); 329 *buf++ = *(u64 *)(base + vmxnet3_rq_driver_stats[i].offset);
330 330
@@ -339,6 +339,7 @@ vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
339{ 339{
340 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 340 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
341 u32 *buf = p; 341 u32 *buf = p;
342 int i = 0;
342 343
343 memset(p, 0, vmxnet3_get_regs_len(netdev)); 344 memset(p, 0, vmxnet3_get_regs_len(netdev));
344 345
@@ -347,28 +348,29 @@ vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
347 /* Update vmxnet3_get_regs_len if we want to dump more registers */ 348 /* Update vmxnet3_get_regs_len if we want to dump more registers */
348 349
349 /* make each ring use multiple of 16 bytes */ 350 /* make each ring use multiple of 16 bytes */
350 buf[0] = adapter->tx_queue.tx_ring.next2fill; 351/* TODO change this for multiple queues */
351 buf[1] = adapter->tx_queue.tx_ring.next2comp; 352 buf[0] = adapter->tx_queue[i].tx_ring.next2fill;
352 buf[2] = adapter->tx_queue.tx_ring.gen; 353 buf[1] = adapter->tx_queue[i].tx_ring.next2comp;
354 buf[2] = adapter->tx_queue[i].tx_ring.gen;
353 buf[3] = 0; 355 buf[3] = 0;
354 356
355 buf[4] = adapter->tx_queue.comp_ring.next2proc; 357 buf[4] = adapter->tx_queue[i].comp_ring.next2proc;
356 buf[5] = adapter->tx_queue.comp_ring.gen; 358 buf[5] = adapter->tx_queue[i].comp_ring.gen;
357 buf[6] = adapter->tx_queue.stopped; 359 buf[6] = adapter->tx_queue[i].stopped;
358 buf[7] = 0; 360 buf[7] = 0;
359 361
360 buf[8] = adapter->rx_queue.rx_ring[0].next2fill; 362 buf[8] = adapter->rx_queue[i].rx_ring[0].next2fill;
361 buf[9] = adapter->rx_queue.rx_ring[0].next2comp; 363 buf[9] = adapter->rx_queue[i].rx_ring[0].next2comp;
362 buf[10] = adapter->rx_queue.rx_ring[0].gen; 364 buf[10] = adapter->rx_queue[i].rx_ring[0].gen;
363 buf[11] = 0; 365 buf[11] = 0;
364 366
365 buf[12] = adapter->rx_queue.rx_ring[1].next2fill; 367 buf[12] = adapter->rx_queue[i].rx_ring[1].next2fill;
366 buf[13] = adapter->rx_queue.rx_ring[1].next2comp; 368 buf[13] = adapter->rx_queue[i].rx_ring[1].next2comp;
367 buf[14] = adapter->rx_queue.rx_ring[1].gen; 369 buf[14] = adapter->rx_queue[i].rx_ring[1].gen;
368 buf[15] = 0; 370 buf[15] = 0;
369 371
370 buf[16] = adapter->rx_queue.comp_ring.next2proc; 372 buf[16] = adapter->rx_queue[i].comp_ring.next2proc;
371 buf[17] = adapter->rx_queue.comp_ring.gen; 373 buf[17] = adapter->rx_queue[i].comp_ring.gen;
372 buf[18] = 0; 374 buf[18] = 0;
373 buf[19] = 0; 375 buf[19] = 0;
374} 376}
@@ -435,8 +437,10 @@ vmxnet3_get_ringparam(struct net_device *netdev,
435 param->rx_mini_max_pending = 0; 437 param->rx_mini_max_pending = 0;
436 param->rx_jumbo_max_pending = 0; 438 param->rx_jumbo_max_pending = 0;
437 439
438 param->rx_pending = adapter->rx_queue.rx_ring[0].size; 440 param->rx_pending = adapter->rx_queue[0].rx_ring[0].size *
439 param->tx_pending = adapter->tx_queue.tx_ring.size; 441 adapter->num_rx_queues;
442 param->tx_pending = adapter->tx_queue[0].tx_ring.size *
443 adapter->num_tx_queues;
440 param->rx_mini_pending = 0; 444 param->rx_mini_pending = 0;
441 param->rx_jumbo_pending = 0; 445 param->rx_jumbo_pending = 0;
442} 446}
@@ -480,8 +484,8 @@ vmxnet3_set_ringparam(struct net_device *netdev,
480 sz) != 0) 484 sz) != 0)
481 return -EINVAL; 485 return -EINVAL;
482 486
483 if (new_tx_ring_size == adapter->tx_queue.tx_ring.size && 487 if (new_tx_ring_size == adapter->tx_queue[0].tx_ring.size &&
484 new_rx_ring_size == adapter->rx_queue.rx_ring[0].size) { 488 new_rx_ring_size == adapter->rx_queue[0].rx_ring[0].size) {
485 return 0; 489 return 0;
486 } 490 }
487 491
@@ -498,11 +502,12 @@ vmxnet3_set_ringparam(struct net_device *netdev,
498 502
499 /* recreate the rx queue and the tx queue based on the 503 /* recreate the rx queue and the tx queue based on the
500 * new sizes */ 504 * new sizes */
501 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 505 vmxnet3_tq_destroy_all(adapter);
502 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 506 vmxnet3_rq_destroy_all(adapter);
503 507
504 err = vmxnet3_create_queues(adapter, new_tx_ring_size, 508 err = vmxnet3_create_queues(adapter, new_tx_ring_size,
505 new_rx_ring_size, VMXNET3_DEF_RX_RING_SIZE); 509 new_rx_ring_size, VMXNET3_DEF_RX_RING_SIZE);
510
506 if (err) { 511 if (err) {
507 /* failed, most likely because of OOM, try default 512 /* failed, most likely because of OOM, try default
508 * size */ 513 * size */
@@ -535,6 +540,65 @@ out:
535} 540}
536 541
537 542
543static int
544vmxnet3_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
545 void *rules)
546{
547 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
548 switch (info->cmd) {
549 case ETHTOOL_GRXRINGS:
550 info->data = adapter->num_rx_queues;
551 return 0;
552 }
553 return -EOPNOTSUPP;
554}
555
556
557static int
558vmxnet3_get_rss_indir(struct net_device *netdev,
559 struct ethtool_rxfh_indir *p)
560{
561 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
562 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
563 unsigned int n = min_t(unsigned int, p->size, rssConf->indTableSize);
564
565 p->size = rssConf->indTableSize;
566 while (n--)
567 p->ring_index[n] = rssConf->indTable[n];
568 return 0;
569
570}
571
572static int
573vmxnet3_set_rss_indir(struct net_device *netdev,
574 const struct ethtool_rxfh_indir *p)
575{
576 unsigned int i;
577 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
578 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
579
580 if (p->size != rssConf->indTableSize)
581 return -EINVAL;
582 for (i = 0; i < rssConf->indTableSize; i++) {
583 /*
584 * Return with error code if any of the queue indices
585 * is out of range
586 */
587 if (p->ring_index[i] < 0 ||
588 p->ring_index[i] >= adapter->num_rx_queues)
589 return -EINVAL;
590 }
591
592 for (i = 0; i < rssConf->indTableSize; i++)
593 rssConf->indTable[i] = p->ring_index[i];
594
595 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
596 VMXNET3_CMD_UPDATE_RSSIDT);
597
598 return 0;
599
600}
601
538static struct ethtool_ops vmxnet3_ethtool_ops = { 602static struct ethtool_ops vmxnet3_ethtool_ops = {
539 .get_settings = vmxnet3_get_settings, 603 .get_settings = vmxnet3_get_settings,
540 .get_drvinfo = vmxnet3_get_drvinfo, 604 .get_drvinfo = vmxnet3_get_drvinfo,
@@ -558,6 +622,9 @@ static struct ethtool_ops vmxnet3_ethtool_ops = {
558 .get_ethtool_stats = vmxnet3_get_ethtool_stats, 622 .get_ethtool_stats = vmxnet3_get_ethtool_stats,
559 .get_ringparam = vmxnet3_get_ringparam, 623 .get_ringparam = vmxnet3_get_ringparam,
560 .set_ringparam = vmxnet3_set_ringparam, 624 .set_ringparam = vmxnet3_set_ringparam,
625 .get_rxnfc = vmxnet3_get_rxnfc,
626 .get_rxfh_indir = vmxnet3_get_rss_indir,
627 .set_rxfh_indir = vmxnet3_set_rss_indir,
561}; 628};
562 629
563void vmxnet3_set_ethtool_ops(struct net_device *netdev) 630void vmxnet3_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/vmxnet3/vmxnet3_int.h b/drivers/net/vmxnet3/vmxnet3_int.h
index edf228843afc..7fadeed37f03 100644
--- a/drivers/net/vmxnet3/vmxnet3_int.h
+++ b/drivers/net/vmxnet3/vmxnet3_int.h
@@ -68,11 +68,15 @@
68/* 68/*
69 * Version numbers 69 * Version numbers
70 */ 70 */
71#define VMXNET3_DRIVER_VERSION_STRING "1.0.14.0-k" 71#define VMXNET3_DRIVER_VERSION_STRING "1.0.16.0-k"
72 72
73/* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */ 73/* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */
74#define VMXNET3_DRIVER_VERSION_NUM 0x01000E00 74#define VMXNET3_DRIVER_VERSION_NUM 0x01001000
75 75
76#if defined(CONFIG_PCI_MSI)
77 /* RSS only makes sense if MSI-X is supported. */
78 #define VMXNET3_RSS
79#endif
76 80
77/* 81/*
78 * Capabilities 82 * Capabilities
@@ -218,16 +222,19 @@ struct vmxnet3_tx_ctx {
218}; 222};
219 223
220struct vmxnet3_tx_queue { 224struct vmxnet3_tx_queue {
225 char name[IFNAMSIZ+8]; /* To identify interrupt */
226 struct vmxnet3_adapter *adapter;
221 spinlock_t tx_lock; 227 spinlock_t tx_lock;
222 struct vmxnet3_cmd_ring tx_ring; 228 struct vmxnet3_cmd_ring tx_ring;
223 struct vmxnet3_tx_buf_info *buf_info; 229 struct vmxnet3_tx_buf_info *buf_info;
224 struct vmxnet3_tx_data_ring data_ring; 230 struct vmxnet3_tx_data_ring data_ring;
225 struct vmxnet3_comp_ring comp_ring; 231 struct vmxnet3_comp_ring comp_ring;
226 struct Vmxnet3_TxQueueCtrl *shared; 232 struct Vmxnet3_TxQueueCtrl *shared;
227 struct vmxnet3_tq_driver_stats stats; 233 struct vmxnet3_tq_driver_stats stats;
228 bool stopped; 234 bool stopped;
229 int num_stop; /* # of times the queue is 235 int num_stop; /* # of times the queue is
230 * stopped */ 236 * stopped */
237 int qid;
231} __attribute__((__aligned__(SMP_CACHE_BYTES))); 238} __attribute__((__aligned__(SMP_CACHE_BYTES)));
232 239
233enum vmxnet3_rx_buf_type { 240enum vmxnet3_rx_buf_type {
@@ -259,6 +266,9 @@ struct vmxnet3_rq_driver_stats {
259}; 266};
260 267
261struct vmxnet3_rx_queue { 268struct vmxnet3_rx_queue {
269 char name[IFNAMSIZ + 8]; /* To identify interrupt */
270 struct vmxnet3_adapter *adapter;
271 struct napi_struct napi;
262 struct vmxnet3_cmd_ring rx_ring[2]; 272 struct vmxnet3_cmd_ring rx_ring[2];
263 struct vmxnet3_comp_ring comp_ring; 273 struct vmxnet3_comp_ring comp_ring;
264 struct vmxnet3_rx_ctx rx_ctx; 274 struct vmxnet3_rx_ctx rx_ctx;
@@ -271,7 +281,16 @@ struct vmxnet3_rx_queue {
271 struct vmxnet3_rq_driver_stats stats; 281 struct vmxnet3_rq_driver_stats stats;
272} __attribute__((__aligned__(SMP_CACHE_BYTES))); 282} __attribute__((__aligned__(SMP_CACHE_BYTES)));
273 283
274#define VMXNET3_LINUX_MAX_MSIX_VECT 1 284#define VMXNET3_DEVICE_MAX_TX_QUEUES 8
285#define VMXNET3_DEVICE_MAX_RX_QUEUES 8 /* Keep this value as a power of 2 */
286
287/* Should be less than UPT1_RSS_MAX_IND_TABLE_SIZE */
288#define VMXNET3_RSS_IND_TABLE_SIZE (VMXNET3_DEVICE_MAX_RX_QUEUES * 4)
289
290#define VMXNET3_LINUX_MAX_MSIX_VECT (VMXNET3_DEVICE_MAX_TX_QUEUES + \
291 VMXNET3_DEVICE_MAX_RX_QUEUES + 1)
292#define VMXNET3_LINUX_MIN_MSIX_VECT 3 /* 1 for each : tx, rx and event */
293
275 294
276struct vmxnet3_intr { 295struct vmxnet3_intr {
277 enum vmxnet3_intr_mask_mode mask_mode; 296 enum vmxnet3_intr_mask_mode mask_mode;
@@ -279,27 +298,32 @@ struct vmxnet3_intr {
279 u8 num_intrs; /* # of intr vectors */ 298 u8 num_intrs; /* # of intr vectors */
280 u8 event_intr_idx; /* idx of the intr vector for event */ 299 u8 event_intr_idx; /* idx of the intr vector for event */
281 u8 mod_levels[VMXNET3_LINUX_MAX_MSIX_VECT]; /* moderation level */ 300 u8 mod_levels[VMXNET3_LINUX_MAX_MSIX_VECT]; /* moderation level */
301 char event_msi_vector_name[IFNAMSIZ+11];
282#ifdef CONFIG_PCI_MSI 302#ifdef CONFIG_PCI_MSI
283 struct msix_entry msix_entries[VMXNET3_LINUX_MAX_MSIX_VECT]; 303 struct msix_entry msix_entries[VMXNET3_LINUX_MAX_MSIX_VECT];
284#endif 304#endif
285}; 305};
286 306
307/* Interrupt sharing schemes, share_intr */
308#define VMXNET3_INTR_BUDDYSHARE 0 /* Corresponding tx,rx queues share irq */
309#define VMXNET3_INTR_TXSHARE 1 /* All tx queues share one irq */
310#define VMXNET3_INTR_DONTSHARE 2 /* each queue has its own irq */
311
312
287#define VMXNET3_STATE_BIT_RESETTING 0 313#define VMXNET3_STATE_BIT_RESETTING 0
288#define VMXNET3_STATE_BIT_QUIESCED 1 314#define VMXNET3_STATE_BIT_QUIESCED 1
289struct vmxnet3_adapter { 315struct vmxnet3_adapter {
290 struct vmxnet3_tx_queue tx_queue; 316 struct vmxnet3_tx_queue tx_queue[VMXNET3_DEVICE_MAX_TX_QUEUES];
291 struct vmxnet3_rx_queue rx_queue; 317 struct vmxnet3_rx_queue rx_queue[VMXNET3_DEVICE_MAX_RX_QUEUES];
292 struct napi_struct napi; 318 struct vlan_group *vlan_grp;
293 struct vlan_group *vlan_grp; 319 struct vmxnet3_intr intr;
294 320 struct Vmxnet3_DriverShared *shared;
295 struct vmxnet3_intr intr; 321 struct Vmxnet3_PMConf *pm_conf;
296 322 struct Vmxnet3_TxQueueDesc *tqd_start; /* all tx queue desc */
297 struct Vmxnet3_DriverShared *shared; 323 struct Vmxnet3_RxQueueDesc *rqd_start; /* all rx queue desc */
298 struct Vmxnet3_PMConf *pm_conf; 324 struct net_device *netdev;
299 struct Vmxnet3_TxQueueDesc *tqd_start; /* first tx queue desc */ 325 struct net_device_stats net_stats;
300 struct Vmxnet3_RxQueueDesc *rqd_start; /* first rx queue desc */ 326 struct pci_dev *pdev;
301 struct net_device *netdev;
302 struct pci_dev *pdev;
303 327
304 u8 __iomem *hw_addr0; /* for BAR 0 */ 328 u8 __iomem *hw_addr0; /* for BAR 0 */
305 u8 __iomem *hw_addr1; /* for BAR 1 */ 329 u8 __iomem *hw_addr1; /* for BAR 1 */
@@ -308,6 +332,12 @@ struct vmxnet3_adapter {
308 bool rxcsum; 332 bool rxcsum;
309 bool lro; 333 bool lro;
310 bool jumbo_frame; 334 bool jumbo_frame;
335#ifdef VMXNET3_RSS
336 struct UPT1_RSSConf *rss_conf;
337 bool rss;
338#endif
339 u32 num_rx_queues;
340 u32 num_tx_queues;
311 341
312 /* rx buffer related */ 342 /* rx buffer related */
313 unsigned skb_buf_size; 343 unsigned skb_buf_size;
@@ -327,6 +357,7 @@ struct vmxnet3_adapter {
327 unsigned long state; /* VMXNET3_STATE_BIT_xxx */ 357 unsigned long state; /* VMXNET3_STATE_BIT_xxx */
328 358
329 int dev_number; 359 int dev_number;
360 int share_intr;
330}; 361};
331 362
332#define VMXNET3_WRITE_BAR0_REG(adapter, reg, val) \ 363#define VMXNET3_WRITE_BAR0_REG(adapter, reg, val) \
@@ -366,12 +397,10 @@ void
366vmxnet3_reset_dev(struct vmxnet3_adapter *adapter); 397vmxnet3_reset_dev(struct vmxnet3_adapter *adapter);
367 398
368void 399void
369vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 400vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter);
370 struct vmxnet3_adapter *adapter);
371 401
372void 402void
373vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 403vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter);
374 struct vmxnet3_adapter *adapter);
375 404
376int 405int
377vmxnet3_create_queues(struct vmxnet3_adapter *adapter, 406vmxnet3_create_queues(struct vmxnet3_adapter *adapter,
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 04:52:24 -0500