diff options
author | Stephen Hemminger <shemminger@linux-foundation.org> | 2007-10-03 19:41:36 -0400 |
---|---|---|
committer | David S. Miller <davem@sunset.davemloft.net> | 2007-10-10 19:47:45 -0400 |
commit | bea3348eef27e6044b6161fd04c3152215f96411 (patch) | |
tree | f0990b263e5ce42505d290a4c346fe990bcd4c33 /drivers/net/cxgb3 | |
parent | dde4e47e8fe333a5649a3fa0e7db1fa7c08d6158 (diff) |
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers/net/cxgb3')
-rw-r--r-- | drivers/net/cxgb3/adapter.h | 22 | ||||
-rw-r--r-- | drivers/net/cxgb3/cxgb3_main.c | 96 | ||||
-rw-r--r-- | drivers/net/cxgb3/sge.c | 170 |
3 files changed, 102 insertions, 186 deletions
diff --git a/drivers/net/cxgb3/adapter.h b/drivers/net/cxgb3/adapter.h index 20e887de2545..044261703381 100644 --- a/drivers/net/cxgb3/adapter.h +++ b/drivers/net/cxgb3/adapter.h | |||
@@ -49,11 +49,13 @@ | |||
49 | typedef irqreturn_t(*intr_handler_t) (int, void *); | 49 | typedef irqreturn_t(*intr_handler_t) (int, void *); |
50 | 50 | ||
51 | struct vlan_group; | 51 | struct vlan_group; |
52 | |||
53 | struct adapter; | 52 | struct adapter; |
53 | struct sge_qset; | ||
54 | |||
54 | struct port_info { | 55 | struct port_info { |
55 | struct adapter *adapter; | 56 | struct adapter *adapter; |
56 | struct vlan_group *vlan_grp; | 57 | struct vlan_group *vlan_grp; |
58 | struct sge_qset *qs; | ||
57 | const struct port_type_info *port_type; | 59 | const struct port_type_info *port_type; |
58 | u8 port_id; | 60 | u8 port_id; |
59 | u8 rx_csum_offload; | 61 | u8 rx_csum_offload; |
@@ -173,10 +175,12 @@ enum { /* per port SGE statistics */ | |||
173 | }; | 175 | }; |
174 | 176 | ||
175 | struct sge_qset { /* an SGE queue set */ | 177 | struct sge_qset { /* an SGE queue set */ |
178 | struct adapter *adap; | ||
179 | struct napi_struct napi; | ||
176 | struct sge_rspq rspq; | 180 | struct sge_rspq rspq; |
177 | struct sge_fl fl[SGE_RXQ_PER_SET]; | 181 | struct sge_fl fl[SGE_RXQ_PER_SET]; |
178 | struct sge_txq txq[SGE_TXQ_PER_SET]; | 182 | struct sge_txq txq[SGE_TXQ_PER_SET]; |
179 | struct net_device *netdev; /* associated net device */ | 183 | struct net_device *netdev; |
180 | unsigned long txq_stopped; /* which Tx queues are stopped */ | 184 | unsigned long txq_stopped; /* which Tx queues are stopped */ |
181 | struct timer_list tx_reclaim_timer; /* reclaims TX buffers */ | 185 | struct timer_list tx_reclaim_timer; /* reclaims TX buffers */ |
182 | unsigned long port_stats[SGE_PSTAT_MAX]; | 186 | unsigned long port_stats[SGE_PSTAT_MAX]; |
@@ -221,12 +225,6 @@ struct adapter { | |||
221 | struct delayed_work adap_check_task; | 225 | struct delayed_work adap_check_task; |
222 | struct work_struct ext_intr_handler_task; | 226 | struct work_struct ext_intr_handler_task; |
223 | 227 | ||
224 | /* | ||
225 | * Dummy netdevices are needed when using multiple receive queues with | ||
226 | * NAPI as each netdevice can service only one queue. | ||
227 | */ | ||
228 | struct net_device *dummy_netdev[SGE_QSETS - 1]; | ||
229 | |||
230 | struct dentry *debugfs_root; | 228 | struct dentry *debugfs_root; |
231 | 229 | ||
232 | struct mutex mdio_lock; | 230 | struct mutex mdio_lock; |
@@ -253,12 +251,6 @@ static inline struct port_info *adap2pinfo(struct adapter *adap, int idx) | |||
253 | return netdev_priv(adap->port[idx]); | 251 | return netdev_priv(adap->port[idx]); |
254 | } | 252 | } |
255 | 253 | ||
256 | /* | ||
257 | * We use the spare atalk_ptr to map a net device to its SGE queue set. | ||
258 | * This is a macro so it can be used as l-value. | ||
259 | */ | ||
260 | #define dev2qset(netdev) ((netdev)->atalk_ptr) | ||
261 | |||
262 | #define OFFLOAD_DEVMAP_BIT 15 | 254 | #define OFFLOAD_DEVMAP_BIT 15 |
263 | 255 | ||
264 | #define tdev2adap(d) container_of(d, struct adapter, tdev) | 256 | #define tdev2adap(d) container_of(d, struct adapter, tdev) |
@@ -284,7 +276,7 @@ int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb); | |||
284 | void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p); | 276 | void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p); |
285 | int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, | 277 | int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, |
286 | int irq_vec_idx, const struct qset_params *p, | 278 | int irq_vec_idx, const struct qset_params *p, |
287 | int ntxq, struct net_device *netdev); | 279 | int ntxq, struct net_device *dev); |
288 | int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, | 280 | int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, |
289 | unsigned char *data); | 281 | unsigned char *data); |
290 | irqreturn_t t3_sge_intr_msix(int irq, void *cookie); | 282 | irqreturn_t t3_sge_intr_msix(int irq, void *cookie); |
diff --git a/drivers/net/cxgb3/cxgb3_main.c b/drivers/net/cxgb3/cxgb3_main.c index 5ab319cfe5de..5db7d4e27ec0 100644 --- a/drivers/net/cxgb3/cxgb3_main.c +++ b/drivers/net/cxgb3/cxgb3_main.c | |||
@@ -339,49 +339,17 @@ static void setup_rss(struct adapter *adap) | |||
339 | V_RRCPLCPUSIZE(6), cpus, rspq_map); | 339 | V_RRCPLCPUSIZE(6), cpus, rspq_map); |
340 | } | 340 | } |
341 | 341 | ||
342 | /* | 342 | static void init_napi(struct adapter *adap) |
343 | * If we have multiple receive queues per port serviced by NAPI we need one | ||
344 | * netdevice per queue as NAPI operates on netdevices. We already have one | ||
345 | * netdevice, namely the one associated with the interface, so we use dummy | ||
346 | * ones for any additional queues. Note that these netdevices exist purely | ||
347 | * so that NAPI has something to work with, they do not represent network | ||
348 | * ports and are not registered. | ||
349 | */ | ||
350 | static int init_dummy_netdevs(struct adapter *adap) | ||
351 | { | 343 | { |
352 | int i, j, dummy_idx = 0; | 344 | int i; |
353 | struct net_device *nd; | ||
354 | |||
355 | for_each_port(adap, i) { | ||
356 | struct net_device *dev = adap->port[i]; | ||
357 | const struct port_info *pi = netdev_priv(dev); | ||
358 | |||
359 | for (j = 0; j < pi->nqsets - 1; j++) { | ||
360 | if (!adap->dummy_netdev[dummy_idx]) { | ||
361 | struct port_info *p; | ||
362 | |||
363 | nd = alloc_netdev(sizeof(*p), "", ether_setup); | ||
364 | if (!nd) | ||
365 | goto free_all; | ||
366 | 345 | ||
367 | p = netdev_priv(nd); | 346 | for (i = 0; i < SGE_QSETS; i++) { |
368 | p->adapter = adap; | 347 | struct sge_qset *qs = &adap->sge.qs[i]; |
369 | nd->weight = 64; | ||
370 | set_bit(__LINK_STATE_START, &nd->state); | ||
371 | adap->dummy_netdev[dummy_idx] = nd; | ||
372 | } | ||
373 | strcpy(adap->dummy_netdev[dummy_idx]->name, dev->name); | ||
374 | dummy_idx++; | ||
375 | } | ||
376 | } | ||
377 | return 0; | ||
378 | 348 | ||
379 | free_all: | 349 | if (qs->adap) |
380 | while (--dummy_idx >= 0) { | 350 | netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll, |
381 | free_netdev(adap->dummy_netdev[dummy_idx]); | 351 | 64); |
382 | adap->dummy_netdev[dummy_idx] = NULL; | ||
383 | } | 352 | } |
384 | return -ENOMEM; | ||
385 | } | 353 | } |
386 | 354 | ||
387 | /* | 355 | /* |
@@ -392,20 +360,18 @@ free_all: | |||
392 | static void quiesce_rx(struct adapter *adap) | 360 | static void quiesce_rx(struct adapter *adap) |
393 | { | 361 | { |
394 | int i; | 362 | int i; |
395 | struct net_device *dev; | ||
396 | 363 | ||
397 | for_each_port(adap, i) { | 364 | for (i = 0; i < SGE_QSETS; i++) |
398 | dev = adap->port[i]; | 365 | if (adap->sge.qs[i].adap) |
399 | while (test_bit(__LINK_STATE_RX_SCHED, &dev->state)) | 366 | napi_disable(&adap->sge.qs[i].napi); |
400 | msleep(1); | 367 | } |
401 | } | ||
402 | 368 | ||
403 | for (i = 0; i < ARRAY_SIZE(adap->dummy_netdev); i++) { | 369 | static void enable_all_napi(struct adapter *adap) |
404 | dev = adap->dummy_netdev[i]; | 370 | { |
405 | if (dev) | 371 | int i; |
406 | while (test_bit(__LINK_STATE_RX_SCHED, &dev->state)) | 372 | for (i = 0; i < SGE_QSETS; i++) |
407 | msleep(1); | 373 | if (adap->sge.qs[i].adap) |
408 | } | 374 | napi_enable(&adap->sge.qs[i].napi); |
409 | } | 375 | } |
410 | 376 | ||
411 | /** | 377 | /** |
@@ -418,7 +384,7 @@ static void quiesce_rx(struct adapter *adap) | |||
418 | */ | 384 | */ |
419 | static int setup_sge_qsets(struct adapter *adap) | 385 | static int setup_sge_qsets(struct adapter *adap) |
420 | { | 386 | { |
421 | int i, j, err, irq_idx = 0, qset_idx = 0, dummy_dev_idx = 0; | 387 | int i, j, err, irq_idx = 0, qset_idx = 0; |
422 | unsigned int ntxq = SGE_TXQ_PER_SET; | 388 | unsigned int ntxq = SGE_TXQ_PER_SET; |
423 | 389 | ||
424 | if (adap->params.rev > 0 && !(adap->flags & USING_MSI)) | 390 | if (adap->params.rev > 0 && !(adap->flags & USING_MSI)) |
@@ -426,15 +392,14 @@ static int setup_sge_qsets(struct adapter *adap) | |||
426 | 392 | ||
427 | for_each_port(adap, i) { | 393 | for_each_port(adap, i) { |
428 | struct net_device *dev = adap->port[i]; | 394 | struct net_device *dev = adap->port[i]; |
429 | const struct port_info *pi = netdev_priv(dev); | 395 | struct port_info *pi = netdev_priv(dev); |
430 | 396 | ||
397 | pi->qs = &adap->sge.qs[pi->first_qset]; | ||
431 | for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { | 398 | for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { |
432 | err = t3_sge_alloc_qset(adap, qset_idx, 1, | 399 | err = t3_sge_alloc_qset(adap, qset_idx, 1, |
433 | (adap->flags & USING_MSIX) ? qset_idx + 1 : | 400 | (adap->flags & USING_MSIX) ? qset_idx + 1 : |
434 | irq_idx, | 401 | irq_idx, |
435 | &adap->params.sge.qset[qset_idx], ntxq, | 402 | &adap->params.sge.qset[qset_idx], ntxq, dev); |
436 | j == 0 ? dev : | ||
437 | adap-> dummy_netdev[dummy_dev_idx++]); | ||
438 | if (err) { | 403 | if (err) { |
439 | t3_free_sge_resources(adap); | 404 | t3_free_sge_resources(adap); |
440 | return err; | 405 | return err; |
@@ -845,21 +810,18 @@ static int cxgb_up(struct adapter *adap) | |||
845 | goto out; | 810 | goto out; |
846 | } | 811 | } |
847 | 812 | ||
848 | err = init_dummy_netdevs(adap); | ||
849 | if (err) | ||
850 | goto out; | ||
851 | |||
852 | err = t3_init_hw(adap, 0); | 813 | err = t3_init_hw(adap, 0); |
853 | if (err) | 814 | if (err) |
854 | goto out; | 815 | goto out; |
855 | 816 | ||
856 | t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); | 817 | t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); |
857 | 818 | ||
858 | err = setup_sge_qsets(adap); | 819 | err = setup_sge_qsets(adap); |
859 | if (err) | 820 | if (err) |
860 | goto out; | 821 | goto out; |
861 | 822 | ||
862 | setup_rss(adap); | 823 | setup_rss(adap); |
824 | init_napi(adap); | ||
863 | adap->flags |= FULL_INIT_DONE; | 825 | adap->flags |= FULL_INIT_DONE; |
864 | } | 826 | } |
865 | 827 | ||
@@ -886,6 +848,7 @@ static int cxgb_up(struct adapter *adap) | |||
886 | adap->name, adap))) | 848 | adap->name, adap))) |
887 | goto irq_err; | 849 | goto irq_err; |
888 | 850 | ||
851 | enable_all_napi(adap); | ||
889 | t3_sge_start(adap); | 852 | t3_sge_start(adap); |
890 | t3_intr_enable(adap); | 853 | t3_intr_enable(adap); |
891 | 854 | ||
@@ -1012,8 +975,10 @@ static int cxgb_open(struct net_device *dev) | |||
1012 | int other_ports = adapter->open_device_map & PORT_MASK; | 975 | int other_ports = adapter->open_device_map & PORT_MASK; |
1013 | int err; | 976 | int err; |
1014 | 977 | ||
1015 | if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0) | 978 | if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0) { |
979 | quiesce_rx(adapter); | ||
1016 | return err; | 980 | return err; |
981 | } | ||
1017 | 982 | ||
1018 | set_bit(pi->port_id, &adapter->open_device_map); | 983 | set_bit(pi->port_id, &adapter->open_device_map); |
1019 | if (is_offload(adapter) && !ofld_disable) { | 984 | if (is_offload(adapter) && !ofld_disable) { |
@@ -2524,7 +2489,6 @@ static int __devinit init_one(struct pci_dev *pdev, | |||
2524 | #ifdef CONFIG_NET_POLL_CONTROLLER | 2489 | #ifdef CONFIG_NET_POLL_CONTROLLER |
2525 | netdev->poll_controller = cxgb_netpoll; | 2490 | netdev->poll_controller = cxgb_netpoll; |
2526 | #endif | 2491 | #endif |
2527 | netdev->weight = 64; | ||
2528 | 2492 | ||
2529 | SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops); | 2493 | SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops); |
2530 | } | 2494 | } |
@@ -2625,12 +2589,6 @@ static void __devexit remove_one(struct pci_dev *pdev) | |||
2625 | t3_free_sge_resources(adapter); | 2589 | t3_free_sge_resources(adapter); |
2626 | cxgb_disable_msi(adapter); | 2590 | cxgb_disable_msi(adapter); |
2627 | 2591 | ||
2628 | for (i = 0; i < ARRAY_SIZE(adapter->dummy_netdev); i++) | ||
2629 | if (adapter->dummy_netdev[i]) { | ||
2630 | free_netdev(adapter->dummy_netdev[i]); | ||
2631 | adapter->dummy_netdev[i] = NULL; | ||
2632 | } | ||
2633 | |||
2634 | for_each_port(adapter, i) | 2592 | for_each_port(adapter, i) |
2635 | if (adapter->port[i]) | 2593 | if (adapter->port[i]) |
2636 | free_netdev(adapter->port[i]); | 2594 | free_netdev(adapter->port[i]); |
diff --git a/drivers/net/cxgb3/sge.c b/drivers/net/cxgb3/sge.c index 58a5f60521ed..069c1aca8a6b 100644 --- a/drivers/net/cxgb3/sge.c +++ b/drivers/net/cxgb3/sge.c | |||
@@ -591,9 +591,6 @@ void t3_free_qset(struct adapter *adapter, struct sge_qset *q) | |||
591 | q->rspq.desc, q->rspq.phys_addr); | 591 | q->rspq.desc, q->rspq.phys_addr); |
592 | } | 592 | } |
593 | 593 | ||
594 | if (q->netdev) | ||
595 | q->netdev->atalk_ptr = NULL; | ||
596 | |||
597 | memset(q, 0, sizeof(*q)); | 594 | memset(q, 0, sizeof(*q)); |
598 | } | 595 | } |
599 | 596 | ||
@@ -1074,7 +1071,7 @@ int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev) | |||
1074 | unsigned int ndesc, pidx, credits, gen, compl; | 1071 | unsigned int ndesc, pidx, credits, gen, compl; |
1075 | const struct port_info *pi = netdev_priv(dev); | 1072 | const struct port_info *pi = netdev_priv(dev); |
1076 | struct adapter *adap = pi->adapter; | 1073 | struct adapter *adap = pi->adapter; |
1077 | struct sge_qset *qs = dev2qset(dev); | 1074 | struct sge_qset *qs = pi->qs; |
1078 | struct sge_txq *q = &qs->txq[TXQ_ETH]; | 1075 | struct sge_txq *q = &qs->txq[TXQ_ETH]; |
1079 | 1076 | ||
1080 | /* | 1077 | /* |
@@ -1326,13 +1323,12 @@ static void restart_ctrlq(unsigned long data) | |||
1326 | struct sk_buff *skb; | 1323 | struct sk_buff *skb; |
1327 | struct sge_qset *qs = (struct sge_qset *)data; | 1324 | struct sge_qset *qs = (struct sge_qset *)data; |
1328 | struct sge_txq *q = &qs->txq[TXQ_CTRL]; | 1325 | struct sge_txq *q = &qs->txq[TXQ_CTRL]; |
1329 | const struct port_info *pi = netdev_priv(qs->netdev); | ||
1330 | struct adapter *adap = pi->adapter; | ||
1331 | 1326 | ||
1332 | spin_lock(&q->lock); | 1327 | spin_lock(&q->lock); |
1333 | again:reclaim_completed_tx_imm(q); | 1328 | again:reclaim_completed_tx_imm(q); |
1334 | 1329 | ||
1335 | while (q->in_use < q->size && (skb = __skb_dequeue(&q->sendq)) != NULL) { | 1330 | while (q->in_use < q->size && |
1331 | (skb = __skb_dequeue(&q->sendq)) != NULL) { | ||
1336 | 1332 | ||
1337 | write_imm(&q->desc[q->pidx], skb, skb->len, q->gen); | 1333 | write_imm(&q->desc[q->pidx], skb, skb->len, q->gen); |
1338 | 1334 | ||
@@ -1354,7 +1350,7 @@ static void restart_ctrlq(unsigned long data) | |||
1354 | } | 1350 | } |
1355 | 1351 | ||
1356 | spin_unlock(&q->lock); | 1352 | spin_unlock(&q->lock); |
1357 | t3_write_reg(adap, A_SG_KDOORBELL, | 1353 | t3_write_reg(qs->adap, A_SG_KDOORBELL, |
1358 | F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); | 1354 | F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); |
1359 | } | 1355 | } |
1360 | 1356 | ||
@@ -1638,8 +1634,7 @@ static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb) | |||
1638 | else { | 1634 | else { |
1639 | struct sge_qset *qs = rspq_to_qset(q); | 1635 | struct sge_qset *qs = rspq_to_qset(q); |
1640 | 1636 | ||
1641 | if (__netif_rx_schedule_prep(qs->netdev)) | 1637 | napi_schedule(&qs->napi); |
1642 | __netif_rx_schedule(qs->netdev); | ||
1643 | q->rx_head = skb; | 1638 | q->rx_head = skb; |
1644 | } | 1639 | } |
1645 | q->rx_tail = skb; | 1640 | q->rx_tail = skb; |
@@ -1675,34 +1670,30 @@ static inline void deliver_partial_bundle(struct t3cdev *tdev, | |||
1675 | * receive handler. Batches need to be of modest size as we do prefetches | 1670 | * receive handler. Batches need to be of modest size as we do prefetches |
1676 | * on the packets in each. | 1671 | * on the packets in each. |
1677 | */ | 1672 | */ |
1678 | static int ofld_poll(struct net_device *dev, int *budget) | 1673 | static int ofld_poll(struct napi_struct *napi, int budget) |
1679 | { | 1674 | { |
1680 | const struct port_info *pi = netdev_priv(dev); | 1675 | struct sge_qset *qs = container_of(napi, struct sge_qset, napi); |
1681 | struct adapter *adapter = pi->adapter; | ||
1682 | struct sge_qset *qs = dev2qset(dev); | ||
1683 | struct sge_rspq *q = &qs->rspq; | 1676 | struct sge_rspq *q = &qs->rspq; |
1684 | int work_done, limit = min(*budget, dev->quota), avail = limit; | 1677 | struct adapter *adapter = qs->adap; |
1678 | int work_done = 0; | ||
1685 | 1679 | ||
1686 | while (avail) { | 1680 | while (work_done < budget) { |
1687 | struct sk_buff *head, *tail, *skbs[RX_BUNDLE_SIZE]; | 1681 | struct sk_buff *head, *tail, *skbs[RX_BUNDLE_SIZE]; |
1688 | int ngathered; | 1682 | int ngathered; |
1689 | 1683 | ||
1690 | spin_lock_irq(&q->lock); | 1684 | spin_lock_irq(&q->lock); |
1691 | head = q->rx_head; | 1685 | head = q->rx_head; |
1692 | if (!head) { | 1686 | if (!head) { |
1693 | work_done = limit - avail; | 1687 | napi_complete(napi); |
1694 | *budget -= work_done; | ||
1695 | dev->quota -= work_done; | ||
1696 | __netif_rx_complete(dev); | ||
1697 | spin_unlock_irq(&q->lock); | 1688 | spin_unlock_irq(&q->lock); |
1698 | return 0; | 1689 | return work_done; |
1699 | } | 1690 | } |
1700 | 1691 | ||
1701 | tail = q->rx_tail; | 1692 | tail = q->rx_tail; |
1702 | q->rx_head = q->rx_tail = NULL; | 1693 | q->rx_head = q->rx_tail = NULL; |
1703 | spin_unlock_irq(&q->lock); | 1694 | spin_unlock_irq(&q->lock); |
1704 | 1695 | ||
1705 | for (ngathered = 0; avail && head; avail--) { | 1696 | for (ngathered = 0; work_done < budget && head; work_done++) { |
1706 | prefetch(head->data); | 1697 | prefetch(head->data); |
1707 | skbs[ngathered] = head; | 1698 | skbs[ngathered] = head; |
1708 | head = head->next; | 1699 | head = head->next; |
@@ -1724,10 +1715,8 @@ static int ofld_poll(struct net_device *dev, int *budget) | |||
1724 | } | 1715 | } |
1725 | deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered); | 1716 | deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered); |
1726 | } | 1717 | } |
1727 | work_done = limit - avail; | 1718 | |
1728 | *budget -= work_done; | 1719 | return work_done; |
1729 | dev->quota -= work_done; | ||
1730 | return 1; | ||
1731 | } | 1720 | } |
1732 | 1721 | ||
1733 | /** | 1722 | /** |
@@ -2071,50 +2060,47 @@ static inline int is_pure_response(const struct rsp_desc *r) | |||
2071 | 2060 | ||
2072 | /** | 2061 | /** |
2073 | * napi_rx_handler - the NAPI handler for Rx processing | 2062 | * napi_rx_handler - the NAPI handler for Rx processing |
2074 | * @dev: the net device | 2063 | * @napi: the napi instance |
2075 | * @budget: how many packets we can process in this round | 2064 | * @budget: how many packets we can process in this round |
2076 | * | 2065 | * |
2077 | * Handler for new data events when using NAPI. | 2066 | * Handler for new data events when using NAPI. |
2078 | */ | 2067 | */ |
2079 | static int napi_rx_handler(struct net_device *dev, int *budget) | 2068 | static int napi_rx_handler(struct napi_struct *napi, int budget) |
2080 | { | 2069 | { |
2081 | const struct port_info *pi = netdev_priv(dev); | 2070 | struct sge_qset *qs = container_of(napi, struct sge_qset, napi); |
2082 | struct adapter *adap = pi->adapter; | 2071 | struct adapter *adap = qs->adap; |
2083 | struct sge_qset *qs = dev2qset(dev); | 2072 | int work_done = process_responses(adap, qs, budget); |
2084 | int effective_budget = min(*budget, dev->quota); | ||
2085 | |||
2086 | int work_done = process_responses(adap, qs, effective_budget); | ||
2087 | *budget -= work_done; | ||
2088 | dev->quota -= work_done; | ||
2089 | 2073 | ||
2090 | if (work_done >= effective_budget) | 2074 | if (likely(work_done < budget)) { |
2091 | return 1; | 2075 | napi_complete(napi); |
2092 | |||
2093 | netif_rx_complete(dev); | ||
2094 | 2076 | ||
2095 | /* | 2077 | /* |
2096 | * Because we don't atomically flush the following write it is | 2078 | * Because we don't atomically flush the following |
2097 | * possible that in very rare cases it can reach the device in a way | 2079 | * write it is possible that in very rare cases it can |
2098 | * that races with a new response being written plus an error interrupt | 2080 | * reach the device in a way that races with a new |
2099 | * causing the NAPI interrupt handler below to return unhandled status | 2081 | * response being written plus an error interrupt |
2100 | * to the OS. To protect against this would require flushing the write | 2082 | * causing the NAPI interrupt handler below to return |
2101 | * and doing both the write and the flush with interrupts off. Way too | 2083 | * unhandled status to the OS. To protect against |
2102 | * expensive and unjustifiable given the rarity of the race. | 2084 | * this would require flushing the write and doing |
2103 | * | 2085 | * both the write and the flush with interrupts off. |
2104 | * The race cannot happen at all with MSI-X. | 2086 | * Way too expensive and unjustifiable given the |
2105 | */ | 2087 | * rarity of the race. |
2106 | t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | | 2088 | * |
2107 | V_NEWTIMER(qs->rspq.next_holdoff) | | 2089 | * The race cannot happen at all with MSI-X. |
2108 | V_NEWINDEX(qs->rspq.cidx)); | 2090 | */ |
2109 | return 0; | 2091 | t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | |
2092 | V_NEWTIMER(qs->rspq.next_holdoff) | | ||
2093 | V_NEWINDEX(qs->rspq.cidx)); | ||
2094 | } | ||
2095 | return work_done; | ||
2110 | } | 2096 | } |
2111 | 2097 | ||
2112 | /* | 2098 | /* |
2113 | * Returns true if the device is already scheduled for polling. | 2099 | * Returns true if the device is already scheduled for polling. |
2114 | */ | 2100 | */ |
2115 | static inline int napi_is_scheduled(struct net_device *dev) | 2101 | static inline int napi_is_scheduled(struct napi_struct *napi) |
2116 | { | 2102 | { |
2117 | return test_bit(__LINK_STATE_RX_SCHED, &dev->state); | 2103 | return test_bit(NAPI_STATE_SCHED, &napi->state); |
2118 | } | 2104 | } |
2119 | 2105 | ||
2120 | /** | 2106 | /** |
@@ -2197,8 +2183,7 @@ static inline int handle_responses(struct adapter *adap, struct sge_rspq *q) | |||
2197 | V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx)); | 2183 | V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx)); |
2198 | return 0; | 2184 | return 0; |
2199 | } | 2185 | } |
2200 | if (likely(__netif_rx_schedule_prep(qs->netdev))) | 2186 | napi_schedule(&qs->napi); |
2201 | __netif_rx_schedule(qs->netdev); | ||
2202 | return 1; | 2187 | return 1; |
2203 | } | 2188 | } |
2204 | 2189 | ||
@@ -2209,8 +2194,7 @@ static inline int handle_responses(struct adapter *adap, struct sge_rspq *q) | |||
2209 | irqreturn_t t3_sge_intr_msix(int irq, void *cookie) | 2194 | irqreturn_t t3_sge_intr_msix(int irq, void *cookie) |
2210 | { | 2195 | { |
2211 | struct sge_qset *qs = cookie; | 2196 | struct sge_qset *qs = cookie; |
2212 | const struct port_info *pi = netdev_priv(qs->netdev); | 2197 | struct adapter *adap = qs->adap; |
2213 | struct adapter *adap = pi->adapter; | ||
2214 | struct sge_rspq *q = &qs->rspq; | 2198 | struct sge_rspq *q = &qs->rspq; |
2215 | 2199 | ||
2216 | spin_lock(&q->lock); | 2200 | spin_lock(&q->lock); |
@@ -2229,13 +2213,11 @@ irqreturn_t t3_sge_intr_msix(int irq, void *cookie) | |||
2229 | irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie) | 2213 | irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie) |
2230 | { | 2214 | { |
2231 | struct sge_qset *qs = cookie; | 2215 | struct sge_qset *qs = cookie; |
2232 | const struct port_info *pi = netdev_priv(qs->netdev); | ||
2233 | struct adapter *adap = pi->adapter; | ||
2234 | struct sge_rspq *q = &qs->rspq; | 2216 | struct sge_rspq *q = &qs->rspq; |
2235 | 2217 | ||
2236 | spin_lock(&q->lock); | 2218 | spin_lock(&q->lock); |
2237 | 2219 | ||
2238 | if (handle_responses(adap, q) < 0) | 2220 | if (handle_responses(qs->adap, q) < 0) |
2239 | q->unhandled_irqs++; | 2221 | q->unhandled_irqs++; |
2240 | spin_unlock(&q->lock); | 2222 | spin_unlock(&q->lock); |
2241 | return IRQ_HANDLED; | 2223 | return IRQ_HANDLED; |
@@ -2278,11 +2260,13 @@ static irqreturn_t t3_intr_msi(int irq, void *cookie) | |||
2278 | return IRQ_HANDLED; | 2260 | return IRQ_HANDLED; |
2279 | } | 2261 | } |
2280 | 2262 | ||
2281 | static int rspq_check_napi(struct net_device *dev, struct sge_rspq *q) | 2263 | static int rspq_check_napi(struct sge_qset *qs) |
2282 | { | 2264 | { |
2283 | if (!napi_is_scheduled(dev) && is_new_response(&q->desc[q->cidx], q)) { | 2265 | struct sge_rspq *q = &qs->rspq; |
2284 | if (likely(__netif_rx_schedule_prep(dev))) | 2266 | |
2285 | __netif_rx_schedule(dev); | 2267 | if (!napi_is_scheduled(&qs->napi) && |
2268 | is_new_response(&q->desc[q->cidx], q)) { | ||
2269 | napi_schedule(&qs->napi); | ||
2286 | return 1; | 2270 | return 1; |
2287 | } | 2271 | } |
2288 | return 0; | 2272 | return 0; |
@@ -2303,10 +2287,9 @@ irqreturn_t t3_intr_msi_napi(int irq, void *cookie) | |||
2303 | 2287 | ||
2304 | spin_lock(&q->lock); | 2288 | spin_lock(&q->lock); |
2305 | 2289 | ||
2306 | new_packets = rspq_check_napi(adap->sge.qs[0].netdev, q); | 2290 | new_packets = rspq_check_napi(&adap->sge.qs[0]); |
2307 | if (adap->params.nports == 2) | 2291 | if (adap->params.nports == 2) |
2308 | new_packets += rspq_check_napi(adap->sge.qs[1].netdev, | 2292 | new_packets += rspq_check_napi(&adap->sge.qs[1]); |
2309 | &adap->sge.qs[1].rspq); | ||
2310 | if (!new_packets && t3_slow_intr_handler(adap) == 0) | 2293 | if (!new_packets && t3_slow_intr_handler(adap) == 0) |
2311 | q->unhandled_irqs++; | 2294 | q->unhandled_irqs++; |
2312 | 2295 | ||
@@ -2409,9 +2392,9 @@ static irqreturn_t t3b_intr(int irq, void *cookie) | |||
2409 | static irqreturn_t t3b_intr_napi(int irq, void *cookie) | 2392 | static irqreturn_t t3b_intr_napi(int irq, void *cookie) |
2410 | { | 2393 | { |
2411 | u32 map; | 2394 | u32 map; |
2412 | struct net_device *dev; | ||
2413 | struct adapter *adap = cookie; | 2395 | struct adapter *adap = cookie; |
2414 | struct sge_rspq *q0 = &adap->sge.qs[0].rspq; | 2396 | struct sge_qset *qs0 = &adap->sge.qs[0]; |
2397 | struct sge_rspq *q0 = &qs0->rspq; | ||
2415 | 2398 | ||
2416 | t3_write_reg(adap, A_PL_CLI, 0); | 2399 | t3_write_reg(adap, A_PL_CLI, 0); |
2417 | map = t3_read_reg(adap, A_SG_DATA_INTR); | 2400 | map = t3_read_reg(adap, A_SG_DATA_INTR); |
@@ -2424,18 +2407,11 @@ static irqreturn_t t3b_intr_napi(int irq, void *cookie) | |||
2424 | if (unlikely(map & F_ERRINTR)) | 2407 | if (unlikely(map & F_ERRINTR)) |
2425 | t3_slow_intr_handler(adap); | 2408 | t3_slow_intr_handler(adap); |
2426 | 2409 | ||
2427 | if (likely(map & 1)) { | 2410 | if (likely(map & 1)) |
2428 | dev = adap->sge.qs[0].netdev; | 2411 | napi_schedule(&qs0->napi); |
2429 | |||
2430 | if (likely(__netif_rx_schedule_prep(dev))) | ||
2431 | __netif_rx_schedule(dev); | ||
2432 | } | ||
2433 | if (map & 2) { | ||
2434 | dev = adap->sge.qs[1].netdev; | ||
2435 | 2412 | ||
2436 | if (likely(__netif_rx_schedule_prep(dev))) | 2413 | if (map & 2) |
2437 | __netif_rx_schedule(dev); | 2414 | napi_schedule(&adap->sge.qs[1].napi); |
2438 | } | ||
2439 | 2415 | ||
2440 | spin_unlock(&q0->lock); | 2416 | spin_unlock(&q0->lock); |
2441 | return IRQ_HANDLED; | 2417 | return IRQ_HANDLED; |
@@ -2514,8 +2490,7 @@ static void sge_timer_cb(unsigned long data) | |||
2514 | { | 2490 | { |
2515 | spinlock_t *lock; | 2491 | spinlock_t *lock; |
2516 | struct sge_qset *qs = (struct sge_qset *)data; | 2492 | struct sge_qset *qs = (struct sge_qset *)data; |
2517 | const struct port_info *pi = netdev_priv(qs->netdev); | 2493 | struct adapter *adap = qs->adap; |
2518 | struct adapter *adap = pi->adapter; | ||
2519 | 2494 | ||
2520 | if (spin_trylock(&qs->txq[TXQ_ETH].lock)) { | 2495 | if (spin_trylock(&qs->txq[TXQ_ETH].lock)) { |
2521 | reclaim_completed_tx(adap, &qs->txq[TXQ_ETH]); | 2496 | reclaim_completed_tx(adap, &qs->txq[TXQ_ETH]); |
@@ -2526,9 +2501,9 @@ static void sge_timer_cb(unsigned long data) | |||
2526 | spin_unlock(&qs->txq[TXQ_OFLD].lock); | 2501 | spin_unlock(&qs->txq[TXQ_OFLD].lock); |
2527 | } | 2502 | } |
2528 | lock = (adap->flags & USING_MSIX) ? &qs->rspq.lock : | 2503 | lock = (adap->flags & USING_MSIX) ? &qs->rspq.lock : |
2529 | &adap->sge.qs[0].rspq.lock; | 2504 | &adap->sge.qs[0].rspq.lock; |
2530 | if (spin_trylock_irq(lock)) { | 2505 | if (spin_trylock_irq(lock)) { |
2531 | if (!napi_is_scheduled(qs->netdev)) { | 2506 | if (!napi_is_scheduled(&qs->napi)) { |
2532 | u32 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS); | 2507 | u32 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS); |
2533 | 2508 | ||
2534 | if (qs->fl[0].credits < qs->fl[0].size) | 2509 | if (qs->fl[0].credits < qs->fl[0].size) |
@@ -2562,12 +2537,9 @@ static void sge_timer_cb(unsigned long data) | |||
2562 | */ | 2537 | */ |
2563 | void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) | 2538 | void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) |
2564 | { | 2539 | { |
2565 | if (!qs->netdev) | ||
2566 | return; | ||
2567 | |||
2568 | qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */ | 2540 | qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */ |
2569 | qs->rspq.polling = p->polling; | 2541 | qs->rspq.polling = p->polling; |
2570 | qs->netdev->poll = p->polling ? napi_rx_handler : ofld_poll; | 2542 | qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll; |
2571 | } | 2543 | } |
2572 | 2544 | ||
2573 | /** | 2545 | /** |
@@ -2587,7 +2559,7 @@ void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) | |||
2587 | */ | 2559 | */ |
2588 | int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, | 2560 | int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, |
2589 | int irq_vec_idx, const struct qset_params *p, | 2561 | int irq_vec_idx, const struct qset_params *p, |
2590 | int ntxq, struct net_device *netdev) | 2562 | int ntxq, struct net_device *dev) |
2591 | { | 2563 | { |
2592 | int i, ret = -ENOMEM; | 2564 | int i, ret = -ENOMEM; |
2593 | struct sge_qset *q = &adapter->sge.qs[id]; | 2565 | struct sge_qset *q = &adapter->sge.qs[id]; |
@@ -2708,16 +2680,10 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, | |||
2708 | } | 2680 | } |
2709 | 2681 | ||
2710 | spin_unlock(&adapter->sge.reg_lock); | 2682 | spin_unlock(&adapter->sge.reg_lock); |
2711 | q->netdev = netdev; | ||
2712 | t3_update_qset_coalesce(q, p); | ||
2713 | 2683 | ||
2714 | /* | 2684 | q->adap = adapter; |
2715 | * We use atalk_ptr as a backpointer to a qset. In case a device is | 2685 | q->netdev = dev; |
2716 | * associated with multiple queue sets only the first one sets | 2686 | t3_update_qset_coalesce(q, p); |
2717 | * atalk_ptr. | ||
2718 | */ | ||
2719 | if (netdev->atalk_ptr == NULL) | ||
2720 | netdev->atalk_ptr = q; | ||
2721 | 2687 | ||
2722 | refill_fl(adapter, &q->fl[0], q->fl[0].size, GFP_KERNEL); | 2688 | refill_fl(adapter, &q->fl[0], q->fl[0].size, GFP_KERNEL); |
2723 | refill_fl(adapter, &q->fl[1], q->fl[1].size, GFP_KERNEL); | 2689 | refill_fl(adapter, &q->fl[1], q->fl[1].size, GFP_KERNEL); |