aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorShreyas Bhatewara <sbhatewara@vmware.com>2010-11-19 05:55:24 -0500
committerDavid S. Miller <davem@davemloft.net>2010-11-19 17:22:57 -0500
commit09c5088e5c5993be217a2c85dca088147ffc9b72 (patch)
tree40c72370d3f9ee2d113fac2c5feb8708ed1d0093 /drivers
parent24912420e923d56461b400d83f250a31bed8a964 (diff)
net-next: Add multiqueue support to vmxnet3 driver
Add multiqueue support to vmxnet3 driver This change adds multiqueue and thus receive side scaling support to vmxnet3 device driver. Number of rx queues is limited to 1 in cases where MSI is not configured or one MSIx vector is not available per rx queue Signed-off-by: Shreyas Bhatewara <sbhatewara@vmware.com> Reviewed-by: Bhavesh Davda <bhavesh@vmware.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers')
-rw-r--r--drivers/net/vmxnet3/vmxnet3_drv.c951
-rw-r--r--drivers/net/vmxnet3/vmxnet3_ethtool.c171
-rw-r--r--drivers/net/vmxnet3/vmxnet3_int.h73
3 files changed, 921 insertions, 274 deletions
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 21314e06e6d7..65860a998321 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -44,6 +44,9 @@ MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
44 44
45static atomic_t devices_found; 45static atomic_t devices_found;
46 46
47#define VMXNET3_MAX_DEVICES 10
48static int enable_mq = 1;
49static int irq_share_mode;
47 50
48/* 51/*
49 * Enable/Disable the given intr 52 * Enable/Disable the given intr
@@ -99,7 +102,7 @@ vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
99static bool 102static bool
100vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 103vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
101{ 104{
102 return netif_queue_stopped(adapter->netdev); 105 return tq->stopped;
103} 106}
104 107
105 108
@@ -107,7 +110,7 @@ static void
107vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 110vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
108{ 111{
109 tq->stopped = false; 112 tq->stopped = false;
110 netif_start_queue(adapter->netdev); 113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
111} 114}
112 115
113 116
@@ -115,7 +118,7 @@ static void
115vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 118vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
116{ 119{
117 tq->stopped = false; 120 tq->stopped = false;
118 netif_wake_queue(adapter->netdev); 121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
119} 122}
120 123
121 124
@@ -124,7 +127,7 @@ vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
124{ 127{
125 tq->stopped = true; 128 tq->stopped = true;
126 tq->num_stop++; 129 tq->num_stop++;
127 netif_stop_queue(adapter->netdev); 130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
128} 131}
129 132
130 133
@@ -135,6 +138,7 @@ static void
135vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 138vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
136{ 139{
137 u32 ret; 140 u32 ret;
141 int i;
138 142
139 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 143 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
140 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 144 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
@@ -145,22 +149,28 @@ vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
145 if (!netif_carrier_ok(adapter->netdev)) 149 if (!netif_carrier_ok(adapter->netdev))
146 netif_carrier_on(adapter->netdev); 150 netif_carrier_on(adapter->netdev);
147 151
148 if (affectTxQueue) 152 if (affectTxQueue) {
149 vmxnet3_tq_start(&adapter->tx_queue, adapter); 153 for (i = 0; i < adapter->num_tx_queues; i++)
154 vmxnet3_tq_start(&adapter->tx_queue[i],
155 adapter);
156 }
150 } else { 157 } else {
151 printk(KERN_INFO "%s: NIC Link is Down\n", 158 printk(KERN_INFO "%s: NIC Link is Down\n",
152 adapter->netdev->name); 159 adapter->netdev->name);
153 if (netif_carrier_ok(adapter->netdev)) 160 if (netif_carrier_ok(adapter->netdev))
154 netif_carrier_off(adapter->netdev); 161 netif_carrier_off(adapter->netdev);
155 162
156 if (affectTxQueue) 163 if (affectTxQueue) {
157 vmxnet3_tq_stop(&adapter->tx_queue, adapter); 164 for (i = 0; i < adapter->num_tx_queues; i++)
165 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
166 }
158 } 167 }
159} 168}
160 169
161static void 170static void
162vmxnet3_process_events(struct vmxnet3_adapter *adapter) 171vmxnet3_process_events(struct vmxnet3_adapter *adapter)
163{ 172{
173 int i;
164 u32 events = le32_to_cpu(adapter->shared->ecr); 174 u32 events = le32_to_cpu(adapter->shared->ecr);
165 if (!events) 175 if (!events)
166 return; 176 return;
@@ -176,16 +186,18 @@ vmxnet3_process_events(struct vmxnet3_adapter *adapter)
176 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 186 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
177 VMXNET3_CMD_GET_QUEUE_STATUS); 187 VMXNET3_CMD_GET_QUEUE_STATUS);
178 188
179 if (adapter->tqd_start->status.stopped) { 189 for (i = 0; i < adapter->num_tx_queues; i++)
180 printk(KERN_ERR "%s: tq error 0x%x\n", 190 if (adapter->tqd_start[i].status.stopped)
181 adapter->netdev->name, 191 dev_err(&adapter->netdev->dev,
182 le32_to_cpu(adapter->tqd_start->status.error)); 192 "%s: tq[%d] error 0x%x\n",
183 } 193 adapter->netdev->name, i, le32_to_cpu(
184 if (adapter->rqd_start->status.stopped) { 194 adapter->tqd_start[i].status.error));
185 printk(KERN_ERR "%s: rq error 0x%x\n", 195 for (i = 0; i < adapter->num_rx_queues; i++)
186 adapter->netdev->name, 196 if (adapter->rqd_start[i].status.stopped)
187 adapter->rqd_start->status.error); 197 dev_err(&adapter->netdev->dev,
188 } 198 "%s: rq[%d] error 0x%x\n",
199 adapter->netdev->name, i,
200 adapter->rqd_start[i].status.error);
189 201
190 schedule_work(&adapter->work); 202 schedule_work(&adapter->work);
191 } 203 }
@@ -410,7 +422,7 @@ vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
410} 422}
411 423
412 424
413void 425static void
414vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 426vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
415 struct vmxnet3_adapter *adapter) 427 struct vmxnet3_adapter *adapter)
416{ 428{
@@ -437,6 +449,17 @@ vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
437} 449}
438 450
439 451
452/* Destroy all tx queues */
453void
454vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
455{
456 int i;
457
458 for (i = 0; i < adapter->num_tx_queues; i++)
459 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
460}
461
462
440static void 463static void
441vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 464vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
442 struct vmxnet3_adapter *adapter) 465 struct vmxnet3_adapter *adapter)
@@ -518,6 +541,14 @@ err:
518 return -ENOMEM; 541 return -ENOMEM;
519} 542}
520 543
544static void
545vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
546{
547 int i;
548
549 for (i = 0; i < adapter->num_tx_queues; i++)
550 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
551}
521 552
522/* 553/*
523 * starting from ring->next2fill, allocate rx buffers for the given ring 554 * starting from ring->next2fill, allocate rx buffers for the given ring
@@ -732,6 +763,17 @@ vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
732} 763}
733 764
734 765
766/* Init all tx queues */
767static void
768vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
769{
770 int i;
771
772 for (i = 0; i < adapter->num_tx_queues; i++)
773 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
774}
775
776
735/* 777/*
736 * parse and copy relevant protocol headers: 778 * parse and copy relevant protocol headers:
737 * For a tso pkt, relevant headers are L2/3/4 including options 779 * For a tso pkt, relevant headers are L2/3/4 including options
@@ -903,6 +945,21 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
903 } 945 }
904 } 946 }
905 947
948 spin_lock_irqsave(&tq->tx_lock, flags);
949
950 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
951 tq->stats.tx_ring_full++;
952 dev_dbg(&adapter->netdev->dev,
953 "tx queue stopped on %s, next2comp %u"
954 " next2fill %u\n", adapter->netdev->name,
955 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
956
957 vmxnet3_tq_stop(tq, adapter);
958 spin_unlock_irqrestore(&tq->tx_lock, flags);
959 return NETDEV_TX_BUSY;
960 }
961
962
906 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter); 963 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
907 if (ret >= 0) { 964 if (ret >= 0) {
908 BUG_ON(ret <= 0 && ctx.copy_size != 0); 965 BUG_ON(ret <= 0 && ctx.copy_size != 0);
@@ -926,20 +983,6 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926 goto drop_pkt; 983 goto drop_pkt;
927 } 984 }
928 985
929 spin_lock_irqsave(&tq->tx_lock, flags);
930
931 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
932 tq->stats.tx_ring_full++;
933 dev_dbg(&adapter->netdev->dev,
934 "tx queue stopped on %s, next2comp %u"
935 " next2fill %u\n", adapter->netdev->name,
936 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
937
938 vmxnet3_tq_stop(tq, adapter);
939 spin_unlock_irqrestore(&tq->tx_lock, flags);
940 return NETDEV_TX_BUSY;
941 }
942
943 /* fill tx descs related to addr & len */ 986 /* fill tx descs related to addr & len */
944 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter); 987 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
945 988
@@ -1000,7 +1043,8 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
1000 if (le32_to_cpu(tq->shared->txNumDeferred) >= 1043 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1001 le32_to_cpu(tq->shared->txThreshold)) { 1044 le32_to_cpu(tq->shared->txThreshold)) {
1002 tq->shared->txNumDeferred = 0; 1045 tq->shared->txNumDeferred = 0;
1003 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_TXPROD, 1046 VMXNET3_WRITE_BAR0_REG(adapter,
1047 VMXNET3_REG_TXPROD + tq->qid * 8,
1004 tq->tx_ring.next2fill); 1048 tq->tx_ring.next2fill);
1005 } 1049 }
1006 1050
@@ -1020,7 +1064,10 @@ vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1020{ 1064{
1021 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1065 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1022 1066
1023 return vmxnet3_tq_xmit(skb, &adapter->tx_queue, adapter, netdev); 1067 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1068 return vmxnet3_tq_xmit(skb,
1069 &adapter->tx_queue[skb->queue_mapping],
1070 adapter, netdev);
1024} 1071}
1025 1072
1026 1073
@@ -1106,9 +1153,9 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1106 break; 1153 break;
1107 } 1154 }
1108 num_rxd++; 1155 num_rxd++;
1109 1156 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1110 idx = rcd->rxdIdx; 1157 idx = rcd->rxdIdx;
1111 ring_idx = rcd->rqID == rq->qid ? 0 : 1; 1158 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1112 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1159 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1113 &rxCmdDesc); 1160 &rxCmdDesc);
1114 rbi = rq->buf_info[ring_idx] + idx; 1161 rbi = rq->buf_info[ring_idx] + idx;
@@ -1260,6 +1307,16 @@ vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1260} 1307}
1261 1308
1262 1309
1310static void
1311vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1312{
1313 int i;
1314
1315 for (i = 0; i < adapter->num_rx_queues; i++)
1316 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1317}
1318
1319
1263void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1320void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1264 struct vmxnet3_adapter *adapter) 1321 struct vmxnet3_adapter *adapter)
1265{ 1322{
@@ -1351,6 +1408,25 @@ vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1351 1408
1352 1409
1353static int 1410static int
1411vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1412{
1413 int i, err = 0;
1414
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
1416 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1417 if (unlikely(err)) {
1418 dev_err(&adapter->netdev->dev, "%s: failed to "
1419 "initialize rx queue%i\n",
1420 adapter->netdev->name, i);
1421 break;
1422 }
1423 }
1424 return err;
1425
1426}
1427
1428
1429static int
1354vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 1430vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1355{ 1431{
1356 int i; 1432 int i;
@@ -1398,33 +1474,177 @@ err:
1398 1474
1399 1475
1400static int 1476static int
1477vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1478{
1479 int i, err = 0;
1480
1481 for (i = 0; i < adapter->num_rx_queues; i++) {
1482 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1483 if (unlikely(err)) {
1484 dev_err(&adapter->netdev->dev,
1485 "%s: failed to create rx queue%i\n",
1486 adapter->netdev->name, i);
1487 goto err_out;
1488 }
1489 }
1490 return err;
1491err_out:
1492 vmxnet3_rq_destroy_all(adapter);
1493 return err;
1494
1495}
1496
1497/* Multiple queue aware polling function for tx and rx */
1498
1499static int
1401vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 1500vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1402{ 1501{
1502 int rcd_done = 0, i;
1403 if (unlikely(adapter->shared->ecr)) 1503 if (unlikely(adapter->shared->ecr))
1404 vmxnet3_process_events(adapter); 1504 vmxnet3_process_events(adapter);
1505 for (i = 0; i < adapter->num_tx_queues; i++)
1506 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1405 1507
1406 vmxnet3_tq_tx_complete(&adapter->tx_queue, adapter); 1508 for (i = 0; i < adapter->num_rx_queues; i++)
1407 return vmxnet3_rq_rx_complete(&adapter->rx_queue, adapter, budget); 1509 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1510 adapter, budget);
1511 return rcd_done;
1408} 1512}
1409 1513
1410 1514
1411static int 1515static int
1412vmxnet3_poll(struct napi_struct *napi, int budget) 1516vmxnet3_poll(struct napi_struct *napi, int budget)
1413{ 1517{
1414 struct vmxnet3_adapter *adapter = container_of(napi, 1518 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1415 struct vmxnet3_adapter, napi); 1519 struct vmxnet3_rx_queue, napi);
1520 int rxd_done;
1521
1522 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1523
1524 if (rxd_done < budget) {
1525 napi_complete(napi);
1526 vmxnet3_enable_all_intrs(rx_queue->adapter);
1527 }
1528 return rxd_done;
1529}
1530
1531/*
1532 * NAPI polling function for MSI-X mode with multiple Rx queues
1533 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1534 */
1535
1536static int
1537vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1538{
1539 struct vmxnet3_rx_queue *rq = container_of(napi,
1540 struct vmxnet3_rx_queue, napi);
1541 struct vmxnet3_adapter *adapter = rq->adapter;
1416 int rxd_done; 1542 int rxd_done;
1417 1543
1418 rxd_done = vmxnet3_do_poll(adapter, budget); 1544 /* When sharing interrupt with corresponding tx queue, process
1545 * tx completions in that queue as well
1546 */
1547 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1548 struct vmxnet3_tx_queue *tq =
1549 &adapter->tx_queue[rq - adapter->rx_queue];
1550 vmxnet3_tq_tx_complete(tq, adapter);
1551 }
1552
1553 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1419 1554
1420 if (rxd_done < budget) { 1555 if (rxd_done < budget) {
1421 napi_complete(napi); 1556 napi_complete(napi);
1422 vmxnet3_enable_intr(adapter, 0); 1557 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1423 } 1558 }
1424 return rxd_done; 1559 return rxd_done;
1425} 1560}
1426 1561
1427 1562
1563#ifdef CONFIG_PCI_MSI
1564
1565/*
1566 * Handle completion interrupts on tx queues
1567 * Returns whether or not the intr is handled
1568 */
1569
1570static irqreturn_t
1571vmxnet3_msix_tx(int irq, void *data)
1572{
1573 struct vmxnet3_tx_queue *tq = data;
1574 struct vmxnet3_adapter *adapter = tq->adapter;
1575
1576 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1577 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1578
1579 /* Handle the case where only one irq is allocate for all tx queues */
1580 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1581 int i;
1582 for (i = 0; i < adapter->num_tx_queues; i++) {
1583 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1584 vmxnet3_tq_tx_complete(txq, adapter);
1585 }
1586 } else {
1587 vmxnet3_tq_tx_complete(tq, adapter);
1588 }
1589 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1590
1591 return IRQ_HANDLED;
1592}
1593
1594
1595/*
1596 * Handle completion interrupts on rx queues. Returns whether or not the
1597 * intr is handled
1598 */
1599
1600static irqreturn_t
1601vmxnet3_msix_rx(int irq, void *data)
1602{
1603 struct vmxnet3_rx_queue *rq = data;
1604 struct vmxnet3_adapter *adapter = rq->adapter;
1605
1606 /* disable intr if needed */
1607 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1608 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1609 napi_schedule(&rq->napi);
1610
1611 return IRQ_HANDLED;
1612}
1613
1614/*
1615 *----------------------------------------------------------------------------
1616 *
1617 * vmxnet3_msix_event --
1618 *
1619 * vmxnet3 msix event intr handler
1620 *
1621 * Result:
1622 * whether or not the intr is handled
1623 *
1624 *----------------------------------------------------------------------------
1625 */
1626
1627static irqreturn_t
1628vmxnet3_msix_event(int irq, void *data)
1629{
1630 struct net_device *dev = data;
1631 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1632
1633 /* disable intr if needed */
1634 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1635 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1636
1637 if (adapter->shared->ecr)
1638 vmxnet3_process_events(adapter);
1639
1640 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1641
1642 return IRQ_HANDLED;
1643}
1644
1645#endif /* CONFIG_PCI_MSI */
1646
1647
1428/* Interrupt handler for vmxnet3 */ 1648/* Interrupt handler for vmxnet3 */
1429static irqreturn_t 1649static irqreturn_t
1430vmxnet3_intr(int irq, void *dev_id) 1650vmxnet3_intr(int irq, void *dev_id)
@@ -1432,7 +1652,7 @@ vmxnet3_intr(int irq, void *dev_id)
1432 struct net_device *dev = dev_id; 1652 struct net_device *dev = dev_id;
1433 struct vmxnet3_adapter *adapter = netdev_priv(dev); 1653 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1434 1654
1435 if (unlikely(adapter->intr.type == VMXNET3_IT_INTX)) { 1655 if (adapter->intr.type == VMXNET3_IT_INTX) {
1436 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 1656 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1437 if (unlikely(icr == 0)) 1657 if (unlikely(icr == 0))
1438 /* not ours */ 1658 /* not ours */
@@ -1442,77 +1662,144 @@ vmxnet3_intr(int irq, void *dev_id)
1442 1662
1443 /* disable intr if needed */ 1663 /* disable intr if needed */
1444 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1664 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1445 vmxnet3_disable_intr(adapter, 0); 1665 vmxnet3_disable_all_intrs(adapter);
1446 1666
1447 napi_schedule(&adapter->napi); 1667 napi_schedule(&adapter->rx_queue[0].napi);
1448 1668
1449 return IRQ_HANDLED; 1669 return IRQ_HANDLED;
1450} 1670}
1451 1671
1452#ifdef CONFIG_NET_POLL_CONTROLLER 1672#ifdef CONFIG_NET_POLL_CONTROLLER
1453 1673
1454
1455/* netpoll callback. */ 1674/* netpoll callback. */
1456static void 1675static void
1457vmxnet3_netpoll(struct net_device *netdev) 1676vmxnet3_netpoll(struct net_device *netdev)
1458{ 1677{
1459 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1678 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1460 int irq;
1461 1679
1462#ifdef CONFIG_PCI_MSI 1680 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1463 if (adapter->intr.type == VMXNET3_IT_MSIX) 1681 vmxnet3_disable_all_intrs(adapter);
1464 irq = adapter->intr.msix_entries[0].vector; 1682
1465 else 1683 vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
1466#endif 1684 vmxnet3_enable_all_intrs(adapter);
1467 irq = adapter->pdev->irq;
1468 1685
1469 disable_irq(irq);
1470 vmxnet3_intr(irq, netdev);
1471 enable_irq(irq);
1472} 1686}
1473#endif 1687#endif /* CONFIG_NET_POLL_CONTROLLER */
1474 1688
1475static int 1689static int
1476vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 1690vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1477{ 1691{
1478 int err; 1692 struct vmxnet3_intr *intr = &adapter->intr;
1693 int err = 0, i;
1694 int vector = 0;
1479 1695
1480#ifdef CONFIG_PCI_MSI 1696#ifdef CONFIG_PCI_MSI
1481 if (adapter->intr.type == VMXNET3_IT_MSIX) { 1697 if (adapter->intr.type == VMXNET3_IT_MSIX) {
1482 /* we only use 1 MSI-X vector */ 1698 for (i = 0; i < adapter->num_tx_queues; i++) {
1483 err = request_irq(adapter->intr.msix_entries[0].vector, 1699 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1484 vmxnet3_intr, 0, adapter->netdev->name, 1700 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1485 adapter->netdev); 1701 adapter->netdev->name, vector);
1486 } else if (adapter->intr.type == VMXNET3_IT_MSI) { 1702 err = request_irq(
1703 intr->msix_entries[vector].vector,
1704 vmxnet3_msix_tx, 0,
1705 adapter->tx_queue[i].name,
1706 &adapter->tx_queue[i]);
1707 } else {
1708 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1709 adapter->netdev->name, vector);
1710 }
1711 if (err) {
1712 dev_err(&adapter->netdev->dev,
1713 "Failed to request irq for MSIX, %s, "
1714 "error %d\n",
1715 adapter->tx_queue[i].name, err);
1716 return err;
1717 }
1718
1719 /* Handle the case where only 1 MSIx was allocated for
1720 * all tx queues */
1721 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1722 for (; i < adapter->num_tx_queues; i++)
1723 adapter->tx_queue[i].comp_ring.intr_idx
1724 = vector;
1725 vector++;
1726 break;
1727 } else {
1728 adapter->tx_queue[i].comp_ring.intr_idx
1729 = vector++;
1730 }
1731 }
1732 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1733 vector = 0;
1734
1735 for (i = 0; i < adapter->num_rx_queues; i++) {
1736 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1737 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1738 adapter->netdev->name, vector);
1739 else
1740 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1741 adapter->netdev->name, vector);
1742 err = request_irq(intr->msix_entries[vector].vector,
1743 vmxnet3_msix_rx, 0,
1744 adapter->rx_queue[i].name,
1745 &(adapter->rx_queue[i]));
1746 if (err) {
1747 printk(KERN_ERR "Failed to request irq for MSIX"
1748 ", %s, error %d\n",
1749 adapter->rx_queue[i].name, err);
1750 return err;
1751 }
1752
1753 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1754 }
1755
1756 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1757 adapter->netdev->name, vector);
1758 err = request_irq(intr->msix_entries[vector].vector,
1759 vmxnet3_msix_event, 0,
1760 intr->event_msi_vector_name, adapter->netdev);
1761 intr->event_intr_idx = vector;
1762
1763 } else if (intr->type == VMXNET3_IT_MSI) {
1764 adapter->num_rx_queues = 1;
1487 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 1765 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1488 adapter->netdev->name, adapter->netdev); 1766 adapter->netdev->name, adapter->netdev);
1489 } else 1767 } else {
1490#endif 1768#endif
1491 { 1769 adapter->num_rx_queues = 1;
1492 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 1770 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1493 IRQF_SHARED, adapter->netdev->name, 1771 IRQF_SHARED, adapter->netdev->name,
1494 adapter->netdev); 1772 adapter->netdev);
1773#ifdef CONFIG_PCI_MSI
1495 } 1774 }
1496 1775#endif
1497 if (err) 1776 intr->num_intrs = vector + 1;
1777 if (err) {
1498 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error" 1778 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error"
1499 ":%d\n", adapter->netdev->name, adapter->intr.type, err); 1779 ":%d\n", adapter->netdev->name, intr->type, err);
1780 } else {
1781 /* Number of rx queues will not change after this */
1782 for (i = 0; i < adapter->num_rx_queues; i++) {
1783 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1784 rq->qid = i;
1785 rq->qid2 = i + adapter->num_rx_queues;
1786 }
1500 1787
1501 1788
1502 if (!err) {
1503 int i;
1504 /* init our intr settings */
1505 for (i = 0; i < adapter->intr.num_intrs; i++)
1506 adapter->intr.mod_levels[i] = UPT1_IML_ADAPTIVE;
1507 1789
1508 /* next setup intr index for all intr sources */ 1790 /* init our intr settings */
1509 adapter->tx_queue.comp_ring.intr_idx = 0; 1791 for (i = 0; i < intr->num_intrs; i++)
1510 adapter->rx_queue.comp_ring.intr_idx = 0; 1792 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1511 adapter->intr.event_intr_idx = 0; 1793 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1794 adapter->intr.event_intr_idx = 0;
1795 for (i = 0; i < adapter->num_tx_queues; i++)
1796 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1797 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1798 }
1512 1799
1513 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors " 1800 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors "
1514 "allocated\n", adapter->netdev->name, adapter->intr.type, 1801 "allocated\n", adapter->netdev->name, intr->type,
1515 adapter->intr.mask_mode, adapter->intr.num_intrs); 1802 intr->mask_mode, intr->num_intrs);
1516 } 1803 }
1517 1804
1518 return err; 1805 return err;
@@ -1522,18 +1809,32 @@ vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1522static void 1809static void
1523vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 1810vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1524{ 1811{
1525 BUG_ON(adapter->intr.type == VMXNET3_IT_AUTO || 1812 struct vmxnet3_intr *intr = &adapter->intr;
1526 adapter->intr.num_intrs <= 0); 1813 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1527 1814
1528 switch (adapter->intr.type) { 1815 switch (intr->type) {
1529#ifdef CONFIG_PCI_MSI 1816#ifdef CONFIG_PCI_MSI
1530 case VMXNET3_IT_MSIX: 1817 case VMXNET3_IT_MSIX:
1531 { 1818 {
1532 int i; 1819 int i, vector = 0;
1533 1820
1534 for (i = 0; i < adapter->intr.num_intrs; i++) 1821 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1535 free_irq(adapter->intr.msix_entries[i].vector, 1822 for (i = 0; i < adapter->num_tx_queues; i++) {
1536 adapter->netdev); 1823 free_irq(intr->msix_entries[vector++].vector,
1824 &(adapter->tx_queue[i]));
1825 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1826 break;
1827 }
1828 }
1829
1830 for (i = 0; i < adapter->num_rx_queues; i++) {
1831 free_irq(intr->msix_entries[vector++].vector,
1832 &(adapter->rx_queue[i]));
1833 }
1834
1835 free_irq(intr->msix_entries[vector].vector,
1836 adapter->netdev);
1837 BUG_ON(vector >= intr->num_intrs);
1537 break; 1838 break;
1538 } 1839 }
1539#endif 1840#endif
@@ -1727,6 +2028,15 @@ vmxnet3_set_mc(struct net_device *netdev)
1727 kfree(new_table); 2028 kfree(new_table);
1728} 2029}
1729 2030
2031void
2032vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2033{
2034 int i;
2035
2036 for (i = 0; i < adapter->num_rx_queues; i++)
2037 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2038}
2039
1730 2040
1731/* 2041/*
1732 * Set up driver_shared based on settings in adapter. 2042 * Set up driver_shared based on settings in adapter.
@@ -1774,40 +2084,72 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
1774 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2084 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
1775 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2085 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
1776 devRead->misc.queueDescLen = cpu_to_le32( 2086 devRead->misc.queueDescLen = cpu_to_le32(
1777 sizeof(struct Vmxnet3_TxQueueDesc) + 2087 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
1778 sizeof(struct Vmxnet3_RxQueueDesc)); 2088 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
1779 2089
1780 /* tx queue settings */ 2090 /* tx queue settings */
1781 BUG_ON(adapter->tx_queue.tx_ring.base == NULL); 2091 devRead->misc.numTxQueues = adapter->num_tx_queues;
1782 2092 for (i = 0; i < adapter->num_tx_queues; i++) {
1783 devRead->misc.numTxQueues = 1; 2093 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
1784 tqc = &adapter->tqd_start->conf; 2094 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
1785 tqc->txRingBasePA = cpu_to_le64(adapter->tx_queue.tx_ring.basePA); 2095 tqc = &adapter->tqd_start[i].conf;
1786 tqc->dataRingBasePA = cpu_to_le64(adapter->tx_queue.data_ring.basePA); 2096 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
1787 tqc->compRingBasePA = cpu_to_le64(adapter->tx_queue.comp_ring.basePA); 2097 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
1788 tqc->ddPA = cpu_to_le64(virt_to_phys( 2098 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
1789 adapter->tx_queue.buf_info)); 2099 tqc->ddPA = cpu_to_le64(virt_to_phys(tq->buf_info));
1790 tqc->txRingSize = cpu_to_le32(adapter->tx_queue.tx_ring.size); 2100 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
1791 tqc->dataRingSize = cpu_to_le32(adapter->tx_queue.data_ring.size); 2101 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
1792 tqc->compRingSize = cpu_to_le32(adapter->tx_queue.comp_ring.size); 2102 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
1793 tqc->ddLen = cpu_to_le32(sizeof(struct vmxnet3_tx_buf_info) * 2103 tqc->ddLen = cpu_to_le32(
1794 tqc->txRingSize); 2104 sizeof(struct vmxnet3_tx_buf_info) *
1795 tqc->intrIdx = adapter->tx_queue.comp_ring.intr_idx; 2105 tqc->txRingSize);
2106 tqc->intrIdx = tq->comp_ring.intr_idx;
2107 }
1796 2108
1797 /* rx queue settings */ 2109 /* rx queue settings */
1798 devRead->misc.numRxQueues = 1; 2110 devRead->misc.numRxQueues = adapter->num_rx_queues;
1799 rqc = &adapter->rqd_start->conf; 2111 for (i = 0; i < adapter->num_rx_queues; i++) {
1800 rqc->rxRingBasePA[0] = cpu_to_le64(adapter->rx_queue.rx_ring[0].basePA); 2112 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1801 rqc->rxRingBasePA[1] = cpu_to_le64(adapter->rx_queue.rx_ring[1].basePA); 2113 rqc = &adapter->rqd_start[i].conf;
1802 rqc->compRingBasePA = cpu_to_le64(adapter->rx_queue.comp_ring.basePA); 2114 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
1803 rqc->ddPA = cpu_to_le64(virt_to_phys( 2115 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
1804 adapter->rx_queue.buf_info)); 2116 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
1805 rqc->rxRingSize[0] = cpu_to_le32(adapter->rx_queue.rx_ring[0].size); 2117 rqc->ddPA = cpu_to_le64(virt_to_phys(
1806 rqc->rxRingSize[1] = cpu_to_le32(adapter->rx_queue.rx_ring[1].size); 2118 rq->buf_info));
1807 rqc->compRingSize = cpu_to_le32(adapter->rx_queue.comp_ring.size); 2119 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
1808 rqc->ddLen = cpu_to_le32(sizeof(struct vmxnet3_rx_buf_info) * 2120 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
1809 (rqc->rxRingSize[0] + rqc->rxRingSize[1])); 2121 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
1810 rqc->intrIdx = adapter->rx_queue.comp_ring.intr_idx; 2122 rqc->ddLen = cpu_to_le32(
2123 sizeof(struct vmxnet3_rx_buf_info) *
2124 (rqc->rxRingSize[0] +
2125 rqc->rxRingSize[1]));
2126 rqc->intrIdx = rq->comp_ring.intr_idx;
2127 }
2128
2129#ifdef VMXNET3_RSS
2130 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2131
2132 if (adapter->rss) {
2133 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2134 devRead->misc.uptFeatures |= UPT1_F_RSS;
2135 devRead->misc.numRxQueues = adapter->num_rx_queues;
2136 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2137 UPT1_RSS_HASH_TYPE_IPV4 |
2138 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2139 UPT1_RSS_HASH_TYPE_IPV6;
2140 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2141 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2142 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2143 get_random_bytes(&rssConf->hashKey[0], rssConf->hashKeySize);
2144 for (i = 0; i < rssConf->indTableSize; i++)
2145 rssConf->indTable[i] = i % adapter->num_rx_queues;
2146
2147 devRead->rssConfDesc.confVer = 1;
2148 devRead->rssConfDesc.confLen = sizeof(*rssConf);
2149 devRead->rssConfDesc.confPA = virt_to_phys(rssConf);
2150 }
2151
2152#endif /* VMXNET3_RSS */
1811 2153
1812 /* intr settings */ 2154 /* intr settings */
1813 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2155 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
@@ -1829,18 +2171,18 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
1829int 2171int
1830vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 2172vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1831{ 2173{
1832 int err; 2174 int err, i;
1833 u32 ret; 2175 u32 ret;
1834 2176
1835 dev_dbg(&adapter->netdev->dev, 2177 dev_dbg(&adapter->netdev->dev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
1836 "%s: skb_buf_size %d, rx_buf_per_pkt %d, ring sizes" 2178 " ring sizes %u %u %u\n", adapter->netdev->name,
1837 " %u %u %u\n", adapter->netdev->name, adapter->skb_buf_size, 2179 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
1838 adapter->rx_buf_per_pkt, adapter->tx_queue.tx_ring.size, 2180 adapter->tx_queue[0].tx_ring.size,
1839 adapter->rx_queue.rx_ring[0].size, 2181 adapter->rx_queue[0].rx_ring[0].size,
1840 adapter->rx_queue.rx_ring[1].size); 2182 adapter->rx_queue[0].rx_ring[1].size);
1841 2183
1842 vmxnet3_tq_init(&adapter->tx_queue, adapter); 2184 vmxnet3_tq_init_all(adapter);
1843 err = vmxnet3_rq_init(&adapter->rx_queue, adapter); 2185 err = vmxnet3_rq_init_all(adapter);
1844 if (err) { 2186 if (err) {
1845 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n", 2187 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n",
1846 adapter->netdev->name, err); 2188 adapter->netdev->name, err);
@@ -1870,10 +2212,15 @@ vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1870 err = -EINVAL; 2212 err = -EINVAL;
1871 goto activate_err; 2213 goto activate_err;
1872 } 2214 }
1873 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD, 2215
1874 adapter->rx_queue.rx_ring[0].next2fill); 2216 for (i = 0; i < adapter->num_rx_queues; i++) {
1875 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD2, 2217 VMXNET3_WRITE_BAR0_REG(adapter,
1876 adapter->rx_queue.rx_ring[1].next2fill); 2218 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2219 adapter->rx_queue[i].rx_ring[0].next2fill);
2220 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2221 (i * VMXNET3_REG_ALIGN)),
2222 adapter->rx_queue[i].rx_ring[1].next2fill);
2223 }
1877 2224
1878 /* Apply the rx filter settins last. */ 2225 /* Apply the rx filter settins last. */
1879 vmxnet3_set_mc(adapter->netdev); 2226 vmxnet3_set_mc(adapter->netdev);
@@ -1883,8 +2230,8 @@ vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1883 * tx queue if the link is up. 2230 * tx queue if the link is up.
1884 */ 2231 */
1885 vmxnet3_check_link(adapter, true); 2232 vmxnet3_check_link(adapter, true);
1886 2233 for (i = 0; i < adapter->num_rx_queues; i++)
1887 napi_enable(&adapter->napi); 2234 napi_enable(&adapter->rx_queue[i].napi);
1888 vmxnet3_enable_all_intrs(adapter); 2235 vmxnet3_enable_all_intrs(adapter);
1889 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 2236 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
1890 return 0; 2237 return 0;
@@ -1896,7 +2243,7 @@ activate_err:
1896irq_err: 2243irq_err:
1897rq_err: 2244rq_err:
1898 /* free up buffers we allocated */ 2245 /* free up buffers we allocated */
1899 vmxnet3_rq_cleanup(&adapter->rx_queue, adapter); 2246 vmxnet3_rq_cleanup_all(adapter);
1900 return err; 2247 return err;
1901} 2248}
1902 2249
@@ -1911,6 +2258,7 @@ vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
1911int 2258int
1912vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 2259vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
1913{ 2260{
2261 int i;
1914 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 2262 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
1915 return 0; 2263 return 0;
1916 2264
@@ -1919,13 +2267,14 @@ vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
1919 VMXNET3_CMD_QUIESCE_DEV); 2267 VMXNET3_CMD_QUIESCE_DEV);
1920 vmxnet3_disable_all_intrs(adapter); 2268 vmxnet3_disable_all_intrs(adapter);
1921 2269
1922 napi_disable(&adapter->napi); 2270 for (i = 0; i < adapter->num_rx_queues; i++)
2271 napi_disable(&adapter->rx_queue[i].napi);
1923 netif_tx_disable(adapter->netdev); 2272 netif_tx_disable(adapter->netdev);
1924 adapter->link_speed = 0; 2273 adapter->link_speed = 0;
1925 netif_carrier_off(adapter->netdev); 2274 netif_carrier_off(adapter->netdev);
1926 2275
1927 vmxnet3_tq_cleanup(&adapter->tx_queue, adapter); 2276 vmxnet3_tq_cleanup_all(adapter);
1928 vmxnet3_rq_cleanup(&adapter->rx_queue, adapter); 2277 vmxnet3_rq_cleanup_all(adapter);
1929 vmxnet3_free_irqs(adapter); 2278 vmxnet3_free_irqs(adapter);
1930 return 0; 2279 return 0;
1931} 2280}
@@ -2047,7 +2396,9 @@ vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2047static void 2396static void
2048vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 2397vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2049{ 2398{
2050 size_t sz; 2399 size_t sz, i, ring0_size, ring1_size, comp_size;
2400 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2401
2051 2402
2052 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 2403 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2053 VMXNET3_MAX_ETH_HDR_SIZE) { 2404 VMXNET3_MAX_ETH_HDR_SIZE) {
@@ -2069,11 +2420,19 @@ vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2069 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 2420 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2070 */ 2421 */
2071 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 2422 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2072 adapter->rx_queue.rx_ring[0].size = (adapter->rx_queue.rx_ring[0].size + 2423 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2073 sz - 1) / sz * sz; 2424 ring0_size = (ring0_size + sz - 1) / sz * sz;
2074 adapter->rx_queue.rx_ring[0].size = min_t(u32, 2425 ring0_size = min_t(u32, rq->rx_ring[0].size, VMXNET3_RX_RING_MAX_SIZE /
2075 adapter->rx_queue.rx_ring[0].size, 2426 sz * sz);
2076 VMXNET3_RX_RING_MAX_SIZE / sz * sz); 2427 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2428 comp_size = ring0_size + ring1_size;
2429
2430 for (i = 0; i < adapter->num_rx_queues; i++) {
2431 rq = &adapter->rx_queue[i];
2432 rq->rx_ring[0].size = ring0_size;
2433 rq->rx_ring[1].size = ring1_size;
2434 rq->comp_ring.size = comp_size;
2435 }
2077} 2436}
2078 2437
2079 2438
@@ -2081,29 +2440,53 @@ int
2081vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 2440vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2082 u32 rx_ring_size, u32 rx_ring2_size) 2441 u32 rx_ring_size, u32 rx_ring2_size)
2083{ 2442{
2084 int err; 2443 int err = 0, i;
2085 2444
2086 adapter->tx_queue.tx_ring.size = tx_ring_size; 2445 for (i = 0; i < adapter->num_tx_queues; i++) {
2087 adapter->tx_queue.data_ring.size = tx_ring_size; 2446 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2088 adapter->tx_queue.comp_ring.size = tx_ring_size; 2447 tq->tx_ring.size = tx_ring_size;
2089 adapter->tx_queue.shared = &adapter->tqd_start->ctrl; 2448 tq->data_ring.size = tx_ring_size;
2090 adapter->tx_queue.stopped = true; 2449 tq->comp_ring.size = tx_ring_size;
2091 err = vmxnet3_tq_create(&adapter->tx_queue, adapter); 2450 tq->shared = &adapter->tqd_start[i].ctrl;
2092 if (err) 2451 tq->stopped = true;
2093 return err; 2452 tq->adapter = adapter;
2453 tq->qid = i;
2454 err = vmxnet3_tq_create(tq, adapter);
2455 /*
2456 * Too late to change num_tx_queues. We cannot do away with
2457 * lesser number of queues than what we asked for
2458 */
2459 if (err)
2460 goto queue_err;
2461 }
2094 2462
2095 adapter->rx_queue.rx_ring[0].size = rx_ring_size; 2463 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2096 adapter->rx_queue.rx_ring[1].size = rx_ring2_size; 2464 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2097 vmxnet3_adjust_rx_ring_size(adapter); 2465 vmxnet3_adjust_rx_ring_size(adapter);
2098 adapter->rx_queue.comp_ring.size = adapter->rx_queue.rx_ring[0].size + 2466 for (i = 0; i < adapter->num_rx_queues; i++) {
2099 adapter->rx_queue.rx_ring[1].size; 2467 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2100 adapter->rx_queue.qid = 0; 2468 /* qid and qid2 for rx queues will be assigned later when num
2101 adapter->rx_queue.qid2 = 1; 2469 * of rx queues is finalized after allocating intrs */
2102 adapter->rx_queue.shared = &adapter->rqd_start->ctrl; 2470 rq->shared = &adapter->rqd_start[i].ctrl;
2103 err = vmxnet3_rq_create(&adapter->rx_queue, adapter); 2471 rq->adapter = adapter;
2104 if (err) 2472 err = vmxnet3_rq_create(rq, adapter);
2105 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 2473 if (err) {
2106 2474 if (i == 0) {
2475 printk(KERN_ERR "Could not allocate any rx"
2476 "queues. Aborting.\n");
2477 goto queue_err;
2478 } else {
2479 printk(KERN_INFO "Number of rx queues changed "
2480 "to : %d.\n", i);
2481 adapter->num_rx_queues = i;
2482 err = 0;
2483 break;
2484 }
2485 }
2486 }
2487 return err;
2488queue_err:
2489 vmxnet3_tq_destroy_all(adapter);
2107 return err; 2490 return err;
2108} 2491}
2109 2492
@@ -2111,11 +2494,12 @@ static int
2111vmxnet3_open(struct net_device *netdev) 2494vmxnet3_open(struct net_device *netdev)
2112{ 2495{
2113 struct vmxnet3_adapter *adapter; 2496 struct vmxnet3_adapter *adapter;
2114 int err; 2497 int err, i;
2115 2498
2116 adapter = netdev_priv(netdev); 2499 adapter = netdev_priv(netdev);
2117 2500
2118 spin_lock_init(&adapter->tx_queue.tx_lock); 2501 for (i = 0; i < adapter->num_tx_queues; i++)
2502 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2119 2503
2120 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE, 2504 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2121 VMXNET3_DEF_RX_RING_SIZE, 2505 VMXNET3_DEF_RX_RING_SIZE,
@@ -2130,8 +2514,8 @@ vmxnet3_open(struct net_device *netdev)
2130 return 0; 2514 return 0;
2131 2515
2132activate_err: 2516activate_err:
2133 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 2517 vmxnet3_rq_destroy_all(adapter);
2134 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 2518 vmxnet3_tq_destroy_all(adapter);
2135queue_err: 2519queue_err:
2136 return err; 2520 return err;
2137} 2521}
@@ -2151,8 +2535,8 @@ vmxnet3_close(struct net_device *netdev)
2151 2535
2152 vmxnet3_quiesce_dev(adapter); 2536 vmxnet3_quiesce_dev(adapter);
2153 2537
2154 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 2538 vmxnet3_rq_destroy_all(adapter);
2155 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 2539 vmxnet3_tq_destroy_all(adapter);
2156 2540
2157 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 2541 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2158 2542
@@ -2164,6 +2548,8 @@ vmxnet3_close(struct net_device *netdev)
2164void 2548void
2165vmxnet3_force_close(struct vmxnet3_adapter *adapter) 2549vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2166{ 2550{
2551 int i;
2552
2167 /* 2553 /*
2168 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 2554 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2169 * vmxnet3_close() will deadlock. 2555 * vmxnet3_close() will deadlock.
@@ -2171,7 +2557,8 @@ vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2171 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 2557 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2172 2558
2173 /* we need to enable NAPI, otherwise dev_close will deadlock */ 2559 /* we need to enable NAPI, otherwise dev_close will deadlock */
2174 napi_enable(&adapter->napi); 2560 for (i = 0; i < adapter->num_rx_queues; i++)
2561 napi_enable(&adapter->rx_queue[i].napi);
2175 dev_close(adapter->netdev); 2562 dev_close(adapter->netdev);
2176} 2563}
2177 2564
@@ -2202,14 +2589,11 @@ vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2202 vmxnet3_reset_dev(adapter); 2589 vmxnet3_reset_dev(adapter);
2203 2590
2204 /* we need to re-create the rx queue based on the new mtu */ 2591 /* we need to re-create the rx queue based on the new mtu */
2205 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 2592 vmxnet3_rq_destroy_all(adapter);
2206 vmxnet3_adjust_rx_ring_size(adapter); 2593 vmxnet3_adjust_rx_ring_size(adapter);
2207 adapter->rx_queue.comp_ring.size = 2594 err = vmxnet3_rq_create_all(adapter);
2208 adapter->rx_queue.rx_ring[0].size +
2209 adapter->rx_queue.rx_ring[1].size;
2210 err = vmxnet3_rq_create(&adapter->rx_queue, adapter);
2211 if (err) { 2595 if (err) {
2212 printk(KERN_ERR "%s: failed to re-create rx queue," 2596 printk(KERN_ERR "%s: failed to re-create rx queues,"
2213 " error %d. Closing it.\n", netdev->name, err); 2597 " error %d. Closing it.\n", netdev->name, err);
2214 goto out; 2598 goto out;
2215 } 2599 }
@@ -2274,6 +2658,55 @@ vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2274 mac[5] = (tmp >> 8) & 0xff; 2658 mac[5] = (tmp >> 8) & 0xff;
2275} 2659}
2276 2660
2661#ifdef CONFIG_PCI_MSI
2662
2663/*
2664 * Enable MSIx vectors.
2665 * Returns :
2666 * 0 on successful enabling of required vectors,
2667 * VMXNET3_LINUX_MIN_MSIX_VECT when only minumum number of vectors required
2668 * could be enabled.
2669 * number of vectors which can be enabled otherwise (this number is smaller
2670 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2671 */
2672
2673static int
2674vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
2675 int vectors)
2676{
2677 int err = 0, vector_threshold;
2678 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
2679
2680 while (vectors >= vector_threshold) {
2681 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2682 vectors);
2683 if (!err) {
2684 adapter->intr.num_intrs = vectors;
2685 return 0;
2686 } else if (err < 0) {
2687 printk(KERN_ERR "Failed to enable MSI-X for %s, error"
2688 " %d\n", adapter->netdev->name, err);
2689 vectors = 0;
2690 } else if (err < vector_threshold) {
2691 break;
2692 } else {
2693 /* If fails to enable required number of MSI-x vectors
2694 * try enabling 3 of them. One each for rx, tx and event
2695 */
2696 vectors = vector_threshold;
2697 printk(KERN_ERR "Failed to enable %d MSI-X for %s, try"
2698 " %d instead\n", vectors, adapter->netdev->name,
2699 vector_threshold);
2700 }
2701 }
2702
2703 printk(KERN_INFO "Number of MSI-X interrupts which can be allocatedi"
2704 " are lower than min threshold required.\n");
2705 return err;
2706}
2707
2708
2709#endif /* CONFIG_PCI_MSI */
2277 2710
2278static void 2711static void
2279vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 2712vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
@@ -2293,16 +2726,47 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2293 2726
2294#ifdef CONFIG_PCI_MSI 2727#ifdef CONFIG_PCI_MSI
2295 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2728 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2296 int err; 2729 int vector, err = 0;
2297 2730
2298 adapter->intr.msix_entries[0].entry = 0; 2731 adapter->intr.num_intrs = (adapter->share_intr ==
2299 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries, 2732 VMXNET3_INTR_TXSHARE) ? 1 :
2300 VMXNET3_LINUX_MAX_MSIX_VECT); 2733 adapter->num_tx_queues;
2301 if (!err) { 2734 adapter->intr.num_intrs += (adapter->share_intr ==
2302 adapter->intr.num_intrs = 1; 2735 VMXNET3_INTR_BUDDYSHARE) ? 0 :
2303 adapter->intr.type = VMXNET3_IT_MSIX; 2736 adapter->num_rx_queues;
2737 adapter->intr.num_intrs += 1; /* for link event */
2738
2739 adapter->intr.num_intrs = (adapter->intr.num_intrs >
2740 VMXNET3_LINUX_MIN_MSIX_VECT
2741 ? adapter->intr.num_intrs :
2742 VMXNET3_LINUX_MIN_MSIX_VECT);
2743
2744 for (vector = 0; vector < adapter->intr.num_intrs; vector++)
2745 adapter->intr.msix_entries[vector].entry = vector;
2746
2747 err = vmxnet3_acquire_msix_vectors(adapter,
2748 adapter->intr.num_intrs);
2749 /* If we cannot allocate one MSIx vector per queue
2750 * then limit the number of rx queues to 1
2751 */
2752 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
2753 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2754 || adapter->num_rx_queues != 2) {
2755 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2756 printk(KERN_ERR "Number of rx queues : 1\n");
2757 adapter->num_rx_queues = 1;
2758 adapter->intr.num_intrs =
2759 VMXNET3_LINUX_MIN_MSIX_VECT;
2760 }
2304 return; 2761 return;
2305 } 2762 }
2763 if (!err)
2764 return;
2765
2766 /* If we cannot allocate MSIx vectors use only one rx queue */
2767 printk(KERN_INFO "Failed to enable MSI-X for %s, error %d."
2768 "#rx queues : 1, try MSI\n", adapter->netdev->name, err);
2769
2306 adapter->intr.type = VMXNET3_IT_MSI; 2770 adapter->intr.type = VMXNET3_IT_MSI;
2307 } 2771 }
2308 2772
@@ -2310,12 +2774,15 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2310 int err; 2774 int err;
2311 err = pci_enable_msi(adapter->pdev); 2775 err = pci_enable_msi(adapter->pdev);
2312 if (!err) { 2776 if (!err) {
2777 adapter->num_rx_queues = 1;
2313 adapter->intr.num_intrs = 1; 2778 adapter->intr.num_intrs = 1;
2314 return; 2779 return;
2315 } 2780 }
2316 } 2781 }
2317#endif /* CONFIG_PCI_MSI */ 2782#endif /* CONFIG_PCI_MSI */
2318 2783
2784 adapter->num_rx_queues = 1;
2785 printk(KERN_INFO "Using INTx interrupt, #Rx queues: 1.\n");
2319 adapter->intr.type = VMXNET3_IT_INTX; 2786 adapter->intr.type = VMXNET3_IT_INTX;
2320 2787
2321 /* INT-X related setting */ 2788 /* INT-X related setting */
@@ -2343,6 +2810,7 @@ vmxnet3_tx_timeout(struct net_device *netdev)
2343 2810
2344 printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name); 2811 printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name);
2345 schedule_work(&adapter->work); 2812 schedule_work(&adapter->work);
2813 netif_wake_queue(adapter->netdev);
2346} 2814}
2347 2815
2348 2816
@@ -2399,8 +2867,29 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2399 struct net_device *netdev; 2867 struct net_device *netdev;
2400 struct vmxnet3_adapter *adapter; 2868 struct vmxnet3_adapter *adapter;
2401 u8 mac[ETH_ALEN]; 2869 u8 mac[ETH_ALEN];
2870 int size;
2871 int num_tx_queues;
2872 int num_rx_queues;
2873
2874#ifdef VMXNET3_RSS
2875 if (enable_mq)
2876 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2877 (int)num_online_cpus());
2878 else
2879#endif
2880 num_rx_queues = 1;
2881
2882 if (enable_mq)
2883 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2884 (int)num_online_cpus());
2885 else
2886 num_tx_queues = 1;
2887
2888 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2889 max(num_tx_queues, num_rx_queues));
2890 printk(KERN_INFO "# of Tx queues : %d, # of Rx queues : %d\n",
2891 num_tx_queues, num_rx_queues);
2402 2892
2403 netdev = alloc_etherdev(sizeof(struct vmxnet3_adapter));
2404 if (!netdev) { 2893 if (!netdev) {
2405 printk(KERN_ERR "Failed to alloc ethernet device for adapter " 2894 printk(KERN_ERR "Failed to alloc ethernet device for adapter "
2406 "%s\n", pci_name(pdev)); 2895 "%s\n", pci_name(pdev));
@@ -2422,9 +2911,12 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2422 goto err_alloc_shared; 2911 goto err_alloc_shared;
2423 } 2912 }
2424 2913
2425 adapter->tqd_start = pci_alloc_consistent(adapter->pdev, 2914 adapter->num_rx_queues = num_rx_queues;
2426 sizeof(struct Vmxnet3_TxQueueDesc) + 2915 adapter->num_tx_queues = num_tx_queues;
2427 sizeof(struct Vmxnet3_RxQueueDesc), 2916
2917 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2918 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2919 adapter->tqd_start = pci_alloc_consistent(adapter->pdev, size,
2428 &adapter->queue_desc_pa); 2920 &adapter->queue_desc_pa);
2429 2921
2430 if (!adapter->tqd_start) { 2922 if (!adapter->tqd_start) {
@@ -2433,8 +2925,8 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2433 err = -ENOMEM; 2925 err = -ENOMEM;
2434 goto err_alloc_queue_desc; 2926 goto err_alloc_queue_desc;
2435 } 2927 }
2436 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start 2928 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
2437 + 1); 2929 adapter->num_tx_queues);
2438 2930
2439 adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL); 2931 adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL);
2440 if (adapter->pm_conf == NULL) { 2932 if (adapter->pm_conf == NULL) {
@@ -2444,6 +2936,17 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2444 goto err_alloc_pm; 2936 goto err_alloc_pm;
2445 } 2937 }
2446 2938
2939#ifdef VMXNET3_RSS
2940
2941 adapter->rss_conf = kmalloc(sizeof(struct UPT1_RSSConf), GFP_KERNEL);
2942 if (adapter->rss_conf == NULL) {
2943 printk(KERN_ERR "Failed to allocate memory for %s\n",
2944 pci_name(pdev));
2945 err = -ENOMEM;
2946 goto err_alloc_rss;
2947 }
2948#endif /* VMXNET3_RSS */
2949
2447 err = vmxnet3_alloc_pci_resources(adapter, &dma64); 2950 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
2448 if (err < 0) 2951 if (err < 0)
2449 goto err_alloc_pci; 2952 goto err_alloc_pci;
@@ -2471,18 +2974,48 @@ vmxnet3_probe_device(struct pci_dev *pdev,
2471 vmxnet3_declare_features(adapter, dma64); 2974 vmxnet3_declare_features(adapter, dma64);
2472 2975
2473 adapter->dev_number = atomic_read(&devices_found); 2976 adapter->dev_number = atomic_read(&devices_found);
2977
2978 adapter->share_intr = irq_share_mode;
2979 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE &&
2980 adapter->num_tx_queues != adapter->num_rx_queues)
2981 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
2982
2474 vmxnet3_alloc_intr_resources(adapter); 2983 vmxnet3_alloc_intr_resources(adapter);
2475 2984
2985#ifdef VMXNET3_RSS
2986 if (adapter->num_rx_queues > 1 &&
2987 adapter->intr.type == VMXNET3_IT_MSIX) {
2988 adapter->rss = true;
2989 printk(KERN_INFO "RSS is enabled.\n");
2990 } else {
2991 adapter->rss = false;
2992 }
2993#endif
2994
2476 vmxnet3_read_mac_addr(adapter, mac); 2995 vmxnet3_read_mac_addr(adapter, mac);
2477 memcpy(netdev->dev_addr, mac, netdev->addr_len); 2996 memcpy(netdev->dev_addr, mac, netdev->addr_len);
2478 2997
2479 netdev->netdev_ops = &vmxnet3_netdev_ops; 2998 netdev->netdev_ops = &vmxnet3_netdev_ops;
2480 netdev->watchdog_timeo = 5 * HZ;
2481 vmxnet3_set_ethtool_ops(netdev); 2999 vmxnet3_set_ethtool_ops(netdev);
3000 netdev->watchdog_timeo = 5 * HZ;
2482 3001
2483 INIT_WORK(&adapter->work, vmxnet3_reset_work); 3002 INIT_WORK(&adapter->work, vmxnet3_reset_work);
2484 3003
2485 netif_napi_add(netdev, &adapter->napi, vmxnet3_poll, 64); 3004 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3005 int i;
3006 for (i = 0; i < adapter->num_rx_queues; i++) {
3007 netif_napi_add(adapter->netdev,
3008 &adapter->rx_queue[i].napi,
3009 vmxnet3_poll_rx_only, 64);
3010 }
3011 } else {
3012 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3013 vmxnet3_poll, 64);
3014 }
3015
3016 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3017 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3018
2486 SET_NETDEV_DEV(netdev, &pdev->dev); 3019 SET_NETDEV_DEV(netdev, &pdev->dev);
2487 err = register_netdev(netdev); 3020 err = register_netdev(netdev);
2488 3021
@@ -2502,11 +3035,14 @@ err_register:
2502err_ver: 3035err_ver:
2503 vmxnet3_free_pci_resources(adapter); 3036 vmxnet3_free_pci_resources(adapter);
2504err_alloc_pci: 3037err_alloc_pci:
3038#ifdef VMXNET3_RSS
3039 kfree(adapter->rss_conf);
3040err_alloc_rss:
3041#endif
2505 kfree(adapter->pm_conf); 3042 kfree(adapter->pm_conf);
2506err_alloc_pm: 3043err_alloc_pm:
2507 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) + 3044 pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
2508 sizeof(struct Vmxnet3_RxQueueDesc), 3045 adapter->queue_desc_pa);
2509 adapter->tqd_start, adapter->queue_desc_pa);
2510err_alloc_queue_desc: 3046err_alloc_queue_desc:
2511 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared), 3047 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2512 adapter->shared, adapter->shared_pa); 3048 adapter->shared, adapter->shared_pa);
@@ -2522,6 +3058,16 @@ vmxnet3_remove_device(struct pci_dev *pdev)
2522{ 3058{
2523 struct net_device *netdev = pci_get_drvdata(pdev); 3059 struct net_device *netdev = pci_get_drvdata(pdev);
2524 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3060 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3061 int size = 0;
3062 int num_rx_queues;
3063
3064#ifdef VMXNET3_RSS
3065 if (enable_mq)
3066 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3067 (int)num_online_cpus());
3068 else
3069#endif
3070 num_rx_queues = 1;
2525 3071
2526 flush_scheduled_work(); 3072 flush_scheduled_work();
2527 3073
@@ -2529,10 +3075,15 @@ vmxnet3_remove_device(struct pci_dev *pdev)
2529 3075
2530 vmxnet3_free_intr_resources(adapter); 3076 vmxnet3_free_intr_resources(adapter);
2531 vmxnet3_free_pci_resources(adapter); 3077 vmxnet3_free_pci_resources(adapter);
3078#ifdef VMXNET3_RSS
3079 kfree(adapter->rss_conf);
3080#endif
2532 kfree(adapter->pm_conf); 3081 kfree(adapter->pm_conf);
2533 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) + 3082
2534 sizeof(struct Vmxnet3_RxQueueDesc), 3083 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2535 adapter->tqd_start, adapter->queue_desc_pa); 3084 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3085 pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
3086 adapter->queue_desc_pa);
2536 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared), 3087 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2537 adapter->shared, adapter->shared_pa); 3088 adapter->shared, adapter->shared_pa);
2538 free_netdev(netdev); 3089 free_netdev(netdev);
@@ -2563,7 +3114,7 @@ vmxnet3_suspend(struct device *device)
2563 vmxnet3_free_intr_resources(adapter); 3114 vmxnet3_free_intr_resources(adapter);
2564 3115
2565 netif_device_detach(netdev); 3116 netif_device_detach(netdev);
2566 netif_stop_queue(netdev); 3117 netif_tx_stop_all_queues(netdev);
2567 3118
2568 /* Create wake-up filters. */ 3119 /* Create wake-up filters. */
2569 pmConf = adapter->pm_conf; 3120 pmConf = adapter->pm_conf;
diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c
index b79070bcc92e..9ddaea636cfa 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethtool.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c
@@ -151,44 +151,42 @@ vmxnet3_get_stats(struct net_device *netdev)
151 struct UPT1_TxStats *devTxStats; 151 struct UPT1_TxStats *devTxStats;
152 struct UPT1_RxStats *devRxStats; 152 struct UPT1_RxStats *devRxStats;
153 struct net_device_stats *net_stats = &netdev->stats; 153 struct net_device_stats *net_stats = &netdev->stats;
154 int i;
154 155
155 adapter = netdev_priv(netdev); 156 adapter = netdev_priv(netdev);
156 157
157 /* Collect the dev stats into the shared area */ 158 /* Collect the dev stats into the shared area */
158 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 159 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
159 160
160 /* Assuming that we have a single queue device */
161 devTxStats = &adapter->tqd_start->stats;
162 devRxStats = &adapter->rqd_start->stats;
163
164 /* Get access to the driver stats per queue */
165 drvTxStats = &adapter->tx_queue.stats;
166 drvRxStats = &adapter->rx_queue.stats;
167
168 memset(net_stats, 0, sizeof(*net_stats)); 161 memset(net_stats, 0, sizeof(*net_stats));
162 for (i = 0; i < adapter->num_tx_queues; i++) {
163 devTxStats = &adapter->tqd_start[i].stats;
164 drvTxStats = &adapter->tx_queue[i].stats;
165 net_stats->tx_packets += devTxStats->ucastPktsTxOK +
166 devTxStats->mcastPktsTxOK +
167 devTxStats->bcastPktsTxOK;
168 net_stats->tx_bytes += devTxStats->ucastBytesTxOK +
169 devTxStats->mcastBytesTxOK +
170 devTxStats->bcastBytesTxOK;
171 net_stats->tx_errors += devTxStats->pktsTxError;
172 net_stats->tx_dropped += drvTxStats->drop_total;
173 }
169 174
170 net_stats->rx_packets = devRxStats->ucastPktsRxOK + 175 for (i = 0; i < adapter->num_rx_queues; i++) {
171 devRxStats->mcastPktsRxOK + 176 devRxStats = &adapter->rqd_start[i].stats;
172 devRxStats->bcastPktsRxOK; 177 drvRxStats = &adapter->rx_queue[i].stats;
173 178 net_stats->rx_packets += devRxStats->ucastPktsRxOK +
174 net_stats->tx_packets = devTxStats->ucastPktsTxOK + 179 devRxStats->mcastPktsRxOK +
175 devTxStats->mcastPktsTxOK + 180 devRxStats->bcastPktsRxOK;
176 devTxStats->bcastPktsTxOK;
177
178 net_stats->rx_bytes = devRxStats->ucastBytesRxOK +
179 devRxStats->mcastBytesRxOK +
180 devRxStats->bcastBytesRxOK;
181
182 net_stats->tx_bytes = devTxStats->ucastBytesTxOK +
183 devTxStats->mcastBytesTxOK +
184 devTxStats->bcastBytesTxOK;
185 181
186 net_stats->rx_errors = devRxStats->pktsRxError; 182 net_stats->rx_bytes += devRxStats->ucastBytesRxOK +
187 net_stats->tx_errors = devTxStats->pktsTxError; 183 devRxStats->mcastBytesRxOK +
188 net_stats->rx_dropped = drvRxStats->drop_total; 184 devRxStats->bcastBytesRxOK;
189 net_stats->tx_dropped = drvTxStats->drop_total;
190 net_stats->multicast = devRxStats->mcastPktsRxOK;
191 185
186 net_stats->rx_errors += devRxStats->pktsRxError;
187 net_stats->rx_dropped += drvRxStats->drop_total;
188 net_stats->multicast += devRxStats->mcastPktsRxOK;
189 }
192 return net_stats; 190 return net_stats;
193} 191}
194 192
@@ -307,24 +305,26 @@ vmxnet3_get_ethtool_stats(struct net_device *netdev,
307 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 305 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
308 u8 *base; 306 u8 *base;
309 int i; 307 int i;
308 int j = 0;
310 309
311 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 310 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
312 311
313 /* this does assume each counter is 64-bit wide */ 312 /* this does assume each counter is 64-bit wide */
313/* TODO change this for multiple queues */
314 314
315 base = (u8 *)&adapter->tqd_start->stats; 315 base = (u8 *)&adapter->tqd_start[j].stats;
316 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++) 316 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++)
317 *buf++ = *(u64 *)(base + vmxnet3_tq_dev_stats[i].offset); 317 *buf++ = *(u64 *)(base + vmxnet3_tq_dev_stats[i].offset);
318 318
319 base = (u8 *)&adapter->tx_queue.stats; 319 base = (u8 *)&adapter->tx_queue[j].stats;
320 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++) 320 for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++)
321 *buf++ = *(u64 *)(base + vmxnet3_tq_driver_stats[i].offset); 321 *buf++ = *(u64 *)(base + vmxnet3_tq_driver_stats[i].offset);
322 322
323 base = (u8 *)&adapter->rqd_start->stats; 323 base = (u8 *)&adapter->rqd_start[j].stats;
324 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++) 324 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++)
325 *buf++ = *(u64 *)(base + vmxnet3_rq_dev_stats[i].offset); 325 *buf++ = *(u64 *)(base + vmxnet3_rq_dev_stats[i].offset);
326 326
327 base = (u8 *)&adapter->rx_queue.stats; 327 base = (u8 *)&adapter->rx_queue[j].stats;
328 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++) 328 for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++)
329 *buf++ = *(u64 *)(base + vmxnet3_rq_driver_stats[i].offset); 329 *buf++ = *(u64 *)(base + vmxnet3_rq_driver_stats[i].offset);
330 330
@@ -339,6 +339,7 @@ vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
339{ 339{
340 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 340 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
341 u32 *buf = p; 341 u32 *buf = p;
342 int i = 0;
342 343
343 memset(p, 0, vmxnet3_get_regs_len(netdev)); 344 memset(p, 0, vmxnet3_get_regs_len(netdev));
344 345
@@ -347,28 +348,29 @@ vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
347 /* Update vmxnet3_get_regs_len if we want to dump more registers */ 348 /* Update vmxnet3_get_regs_len if we want to dump more registers */
348 349
349 /* make each ring use multiple of 16 bytes */ 350 /* make each ring use multiple of 16 bytes */
350 buf[0] = adapter->tx_queue.tx_ring.next2fill; 351/* TODO change this for multiple queues */
351 buf[1] = adapter->tx_queue.tx_ring.next2comp; 352 buf[0] = adapter->tx_queue[i].tx_ring.next2fill;
352 buf[2] = adapter->tx_queue.tx_ring.gen; 353 buf[1] = adapter->tx_queue[i].tx_ring.next2comp;
354 buf[2] = adapter->tx_queue[i].tx_ring.gen;
353 buf[3] = 0; 355 buf[3] = 0;
354 356
355 buf[4] = adapter->tx_queue.comp_ring.next2proc; 357 buf[4] = adapter->tx_queue[i].comp_ring.next2proc;
356 buf[5] = adapter->tx_queue.comp_ring.gen; 358 buf[5] = adapter->tx_queue[i].comp_ring.gen;
357 buf[6] = adapter->tx_queue.stopped; 359 buf[6] = adapter->tx_queue[i].stopped;
358 buf[7] = 0; 360 buf[7] = 0;
359 361
360 buf[8] = adapter->rx_queue.rx_ring[0].next2fill; 362 buf[8] = adapter->rx_queue[i].rx_ring[0].next2fill;
361 buf[9] = adapter->rx_queue.rx_ring[0].next2comp; 363 buf[9] = adapter->rx_queue[i].rx_ring[0].next2comp;
362 buf[10] = adapter->rx_queue.rx_ring[0].gen; 364 buf[10] = adapter->rx_queue[i].rx_ring[0].gen;
363 buf[11] = 0; 365 buf[11] = 0;
364 366
365 buf[12] = adapter->rx_queue.rx_ring[1].next2fill; 367 buf[12] = adapter->rx_queue[i].rx_ring[1].next2fill;
366 buf[13] = adapter->rx_queue.rx_ring[1].next2comp; 368 buf[13] = adapter->rx_queue[i].rx_ring[1].next2comp;
367 buf[14] = adapter->rx_queue.rx_ring[1].gen; 369 buf[14] = adapter->rx_queue[i].rx_ring[1].gen;
368 buf[15] = 0; 370 buf[15] = 0;
369 371
370 buf[16] = adapter->rx_queue.comp_ring.next2proc; 372 buf[16] = adapter->rx_queue[i].comp_ring.next2proc;
371 buf[17] = adapter->rx_queue.comp_ring.gen; 373 buf[17] = adapter->rx_queue[i].comp_ring.gen;
372 buf[18] = 0; 374 buf[18] = 0;
373 buf[19] = 0; 375 buf[19] = 0;
374} 376}
@@ -435,8 +437,10 @@ vmxnet3_get_ringparam(struct net_device *netdev,
435 param->rx_mini_max_pending = 0; 437 param->rx_mini_max_pending = 0;
436 param->rx_jumbo_max_pending = 0; 438 param->rx_jumbo_max_pending = 0;
437 439
438 param->rx_pending = adapter->rx_queue.rx_ring[0].size; 440 param->rx_pending = adapter->rx_queue[0].rx_ring[0].size *
439 param->tx_pending = adapter->tx_queue.tx_ring.size; 441 adapter->num_rx_queues;
442 param->tx_pending = adapter->tx_queue[0].tx_ring.size *
443 adapter->num_tx_queues;
440 param->rx_mini_pending = 0; 444 param->rx_mini_pending = 0;
441 param->rx_jumbo_pending = 0; 445 param->rx_jumbo_pending = 0;
442} 446}
@@ -480,8 +484,8 @@ vmxnet3_set_ringparam(struct net_device *netdev,
480 sz) != 0) 484 sz) != 0)
481 return -EINVAL; 485 return -EINVAL;
482 486
483 if (new_tx_ring_size == adapter->tx_queue.tx_ring.size && 487 if (new_tx_ring_size == adapter->tx_queue[0].tx_ring.size &&
484 new_rx_ring_size == adapter->rx_queue.rx_ring[0].size) { 488 new_rx_ring_size == adapter->rx_queue[0].rx_ring[0].size) {
485 return 0; 489 return 0;
486 } 490 }
487 491
@@ -498,11 +502,12 @@ vmxnet3_set_ringparam(struct net_device *netdev,
498 502
499 /* recreate the rx queue and the tx queue based on the 503 /* recreate the rx queue and the tx queue based on the
500 * new sizes */ 504 * new sizes */
501 vmxnet3_tq_destroy(&adapter->tx_queue, adapter); 505 vmxnet3_tq_destroy_all(adapter);
502 vmxnet3_rq_destroy(&adapter->rx_queue, adapter); 506 vmxnet3_rq_destroy_all(adapter);
503 507
504 err = vmxnet3_create_queues(adapter, new_tx_ring_size, 508 err = vmxnet3_create_queues(adapter, new_tx_ring_size,
505 new_rx_ring_size, VMXNET3_DEF_RX_RING_SIZE); 509 new_rx_ring_size, VMXNET3_DEF_RX_RING_SIZE);
510
506 if (err) { 511 if (err) {
507 /* failed, most likely because of OOM, try default 512 /* failed, most likely because of OOM, try default
508 * size */ 513 * size */
@@ -535,6 +540,65 @@ out:
535} 540}
536 541
537 542
543static int
544vmxnet3_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
545 void *rules)
546{
547 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
548 switch (info->cmd) {
549 case ETHTOOL_GRXRINGS:
550 info->data = adapter->num_rx_queues;
551 return 0;
552 }
553 return -EOPNOTSUPP;
554}
555
556
557static int
558vmxnet3_get_rss_indir(struct net_device *netdev,
559 struct ethtool_rxfh_indir *p)
560{
561 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
562 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
563 unsigned int n = min_t(unsigned int, p->size, rssConf->indTableSize);
564
565 p->size = rssConf->indTableSize;
566 while (n--)
567 p->ring_index[n] = rssConf->indTable[n];
568 return 0;
569
570}
571
572static int
573vmxnet3_set_rss_indir(struct net_device *netdev,
574 const struct ethtool_rxfh_indir *p)
575{
576 unsigned int i;
577 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
578 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
579
580 if (p->size != rssConf->indTableSize)
581 return -EINVAL;
582 for (i = 0; i < rssConf->indTableSize; i++) {
583 /*
584 * Return with error code if any of the queue indices
585 * is out of range
586 */
587 if (p->ring_index[i] < 0 ||
588 p->ring_index[i] >= adapter->num_rx_queues)
589 return -EINVAL;
590 }
591
592 for (i = 0; i < rssConf->indTableSize; i++)
593 rssConf->indTable[i] = p->ring_index[i];
594
595 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
596 VMXNET3_CMD_UPDATE_RSSIDT);
597
598 return 0;
599
600}
601
538static struct ethtool_ops vmxnet3_ethtool_ops = { 602static struct ethtool_ops vmxnet3_ethtool_ops = {
539 .get_settings = vmxnet3_get_settings, 603 .get_settings = vmxnet3_get_settings,
540 .get_drvinfo = vmxnet3_get_drvinfo, 604 .get_drvinfo = vmxnet3_get_drvinfo,
@@ -558,6 +622,9 @@ static struct ethtool_ops vmxnet3_ethtool_ops = {
558 .get_ethtool_stats = vmxnet3_get_ethtool_stats, 622 .get_ethtool_stats = vmxnet3_get_ethtool_stats,
559 .get_ringparam = vmxnet3_get_ringparam, 623 .get_ringparam = vmxnet3_get_ringparam,
560 .set_ringparam = vmxnet3_set_ringparam, 624 .set_ringparam = vmxnet3_set_ringparam,
625 .get_rxnfc = vmxnet3_get_rxnfc,
626 .get_rxfh_indir = vmxnet3_get_rss_indir,
627 .set_rxfh_indir = vmxnet3_set_rss_indir,
561}; 628};
562 629
563void vmxnet3_set_ethtool_ops(struct net_device *netdev) 630void vmxnet3_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/vmxnet3/vmxnet3_int.h b/drivers/net/vmxnet3/vmxnet3_int.h
index edf228843afc..7fadeed37f03 100644
--- a/drivers/net/vmxnet3/vmxnet3_int.h
+++ b/drivers/net/vmxnet3/vmxnet3_int.h
@@ -68,11 +68,15 @@
68/* 68/*
69 * Version numbers 69 * Version numbers
70 */ 70 */
71#define VMXNET3_DRIVER_VERSION_STRING "1.0.14.0-k" 71#define VMXNET3_DRIVER_VERSION_STRING "1.0.16.0-k"
72 72
73/* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */ 73/* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */
74#define VMXNET3_DRIVER_VERSION_NUM 0x01000E00 74#define VMXNET3_DRIVER_VERSION_NUM 0x01001000
75 75
76#if defined(CONFIG_PCI_MSI)
77 /* RSS only makes sense if MSI-X is supported. */
78 #define VMXNET3_RSS
79#endif
76 80
77/* 81/*
78 * Capabilities 82 * Capabilities
@@ -218,16 +222,19 @@ struct vmxnet3_tx_ctx {
218}; 222};
219 223
220struct vmxnet3_tx_queue { 224struct vmxnet3_tx_queue {
225 char name[IFNAMSIZ+8]; /* To identify interrupt */
226 struct vmxnet3_adapter *adapter;
221 spinlock_t tx_lock; 227 spinlock_t tx_lock;
222 struct vmxnet3_cmd_ring tx_ring; 228 struct vmxnet3_cmd_ring tx_ring;
223 struct vmxnet3_tx_buf_info *buf_info; 229 struct vmxnet3_tx_buf_info *buf_info;
224 struct vmxnet3_tx_data_ring data_ring; 230 struct vmxnet3_tx_data_ring data_ring;
225 struct vmxnet3_comp_ring comp_ring; 231 struct vmxnet3_comp_ring comp_ring;
226 struct Vmxnet3_TxQueueCtrl *shared; 232 struct Vmxnet3_TxQueueCtrl *shared;
227 struct vmxnet3_tq_driver_stats stats; 233 struct vmxnet3_tq_driver_stats stats;
228 bool stopped; 234 bool stopped;
229 int num_stop; /* # of times the queue is 235 int num_stop; /* # of times the queue is
230 * stopped */ 236 * stopped */
237 int qid;
231} __attribute__((__aligned__(SMP_CACHE_BYTES))); 238} __attribute__((__aligned__(SMP_CACHE_BYTES)));
232 239
233enum vmxnet3_rx_buf_type { 240enum vmxnet3_rx_buf_type {
@@ -259,6 +266,9 @@ struct vmxnet3_rq_driver_stats {
259}; 266};
260 267
261struct vmxnet3_rx_queue { 268struct vmxnet3_rx_queue {
269 char name[IFNAMSIZ + 8]; /* To identify interrupt */
270 struct vmxnet3_adapter *adapter;
271 struct napi_struct napi;
262 struct vmxnet3_cmd_ring rx_ring[2]; 272 struct vmxnet3_cmd_ring rx_ring[2];
263 struct vmxnet3_comp_ring comp_ring; 273 struct vmxnet3_comp_ring comp_ring;
264 struct vmxnet3_rx_ctx rx_ctx; 274 struct vmxnet3_rx_ctx rx_ctx;
@@ -271,7 +281,16 @@ struct vmxnet3_rx_queue {
271 struct vmxnet3_rq_driver_stats stats; 281 struct vmxnet3_rq_driver_stats stats;
272} __attribute__((__aligned__(SMP_CACHE_BYTES))); 282} __attribute__((__aligned__(SMP_CACHE_BYTES)));
273 283
274#define VMXNET3_LINUX_MAX_MSIX_VECT 1 284#define VMXNET3_DEVICE_MAX_TX_QUEUES 8
285#define VMXNET3_DEVICE_MAX_RX_QUEUES 8 /* Keep this value as a power of 2 */
286
287/* Should be less than UPT1_RSS_MAX_IND_TABLE_SIZE */
288#define VMXNET3_RSS_IND_TABLE_SIZE (VMXNET3_DEVICE_MAX_RX_QUEUES * 4)
289
290#define VMXNET3_LINUX_MAX_MSIX_VECT (VMXNET3_DEVICE_MAX_TX_QUEUES + \
291 VMXNET3_DEVICE_MAX_RX_QUEUES + 1)
292#define VMXNET3_LINUX_MIN_MSIX_VECT 3 /* 1 for each : tx, rx and event */
293
275 294
276struct vmxnet3_intr { 295struct vmxnet3_intr {
277 enum vmxnet3_intr_mask_mode mask_mode; 296 enum vmxnet3_intr_mask_mode mask_mode;
@@ -279,27 +298,32 @@ struct vmxnet3_intr {
279 u8 num_intrs; /* # of intr vectors */ 298 u8 num_intrs; /* # of intr vectors */
280 u8 event_intr_idx; /* idx of the intr vector for event */ 299 u8 event_intr_idx; /* idx of the intr vector for event */
281 u8 mod_levels[VMXNET3_LINUX_MAX_MSIX_VECT]; /* moderation level */ 300 u8 mod_levels[VMXNET3_LINUX_MAX_MSIX_VECT]; /* moderation level */
301 char event_msi_vector_name[IFNAMSIZ+11];
282#ifdef CONFIG_PCI_MSI 302#ifdef CONFIG_PCI_MSI
283 struct msix_entry msix_entries[VMXNET3_LINUX_MAX_MSIX_VECT]; 303 struct msix_entry msix_entries[VMXNET3_LINUX_MAX_MSIX_VECT];
284#endif 304#endif
285}; 305};
286 306
307/* Interrupt sharing schemes, share_intr */
308#define VMXNET3_INTR_BUDDYSHARE 0 /* Corresponding tx,rx queues share irq */
309#define VMXNET3_INTR_TXSHARE 1 /* All tx queues share one irq */
310#define VMXNET3_INTR_DONTSHARE 2 /* each queue has its own irq */
311
312
287#define VMXNET3_STATE_BIT_RESETTING 0 313#define VMXNET3_STATE_BIT_RESETTING 0
288#define VMXNET3_STATE_BIT_QUIESCED 1 314#define VMXNET3_STATE_BIT_QUIESCED 1
289struct vmxnet3_adapter { 315struct vmxnet3_adapter {
290 struct vmxnet3_tx_queue tx_queue; 316 struct vmxnet3_tx_queue tx_queue[VMXNET3_DEVICE_MAX_TX_QUEUES];
291 struct vmxnet3_rx_queue rx_queue; 317 struct vmxnet3_rx_queue rx_queue[VMXNET3_DEVICE_MAX_RX_QUEUES];
292 struct napi_struct napi; 318 struct vlan_group *vlan_grp;
293 struct vlan_group *vlan_grp; 319 struct vmxnet3_intr intr;
294 320 struct Vmxnet3_DriverShared *shared;
295 struct vmxnet3_intr intr; 321 struct Vmxnet3_PMConf *pm_conf;
296 322 struct Vmxnet3_TxQueueDesc *tqd_start; /* all tx queue desc */
297 struct Vmxnet3_DriverShared *shared; 323 struct Vmxnet3_RxQueueDesc *rqd_start; /* all rx queue desc */
298 struct Vmxnet3_PMConf *pm_conf; 324 struct net_device *netdev;
299 struct Vmxnet3_TxQueueDesc *tqd_start; /* first tx queue desc */ 325 struct net_device_stats net_stats;
300 struct Vmxnet3_RxQueueDesc *rqd_start; /* first rx queue desc */ 326 struct pci_dev *pdev;
301 struct net_device *netdev;
302 struct pci_dev *pdev;
303 327
304 u8 __iomem *hw_addr0; /* for BAR 0 */ 328 u8 __iomem *hw_addr0; /* for BAR 0 */
305 u8 __iomem *hw_addr1; /* for BAR 1 */ 329 u8 __iomem *hw_addr1; /* for BAR 1 */
@@ -308,6 +332,12 @@ struct vmxnet3_adapter {
308 bool rxcsum; 332 bool rxcsum;
309 bool lro; 333 bool lro;
310 bool jumbo_frame; 334 bool jumbo_frame;
335#ifdef VMXNET3_RSS
336 struct UPT1_RSSConf *rss_conf;
337 bool rss;
338#endif
339 u32 num_rx_queues;
340 u32 num_tx_queues;
311 341
312 /* rx buffer related */ 342 /* rx buffer related */
313 unsigned skb_buf_size; 343 unsigned skb_buf_size;
@@ -327,6 +357,7 @@ struct vmxnet3_adapter {
327 unsigned long state; /* VMXNET3_STATE_BIT_xxx */ 357 unsigned long state; /* VMXNET3_STATE_BIT_xxx */
328 358
329 int dev_number; 359 int dev_number;
360 int share_intr;
330}; 361};
331 362
332#define VMXNET3_WRITE_BAR0_REG(adapter, reg, val) \ 363#define VMXNET3_WRITE_BAR0_REG(adapter, reg, val) \
@@ -366,12 +397,10 @@ void
366vmxnet3_reset_dev(struct vmxnet3_adapter *adapter); 397vmxnet3_reset_dev(struct vmxnet3_adapter *adapter);
367 398
368void 399void
369vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 400vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter);
370 struct vmxnet3_adapter *adapter);
371 401
372void 402void
373vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 403vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter);
374 struct vmxnet3_adapter *adapter);
375 404
376int 405int
377vmxnet3_create_queues(struct vmxnet3_adapter *adapter, 406vmxnet3_create_queues(struct vmxnet3_adapter *adapter,
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-22 19:07:02 -0500