aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/cxgb4/cxgb4_main.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/cxgb4/cxgb4_main.c')
-rw-r--r--drivers/net/cxgb4/cxgb4_main.c3388
1 files changed, 3388 insertions, 0 deletions
diff --git a/drivers/net/cxgb4/cxgb4_main.c b/drivers/net/cxgb4/cxgb4_main.c
new file mode 100644
index 000000000000..a7e30a23d322
--- /dev/null
+++ b/drivers/net/cxgb4/cxgb4_main.c
@@ -0,0 +1,3388 @@
1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
44#include <linux/if_vlan.h>
45#include <linux/init.h>
46#include <linux/log2.h>
47#include <linux/mdio.h>
48#include <linux/module.h>
49#include <linux/moduleparam.h>
50#include <linux/mutex.h>
51#include <linux/netdevice.h>
52#include <linux/pci.h>
53#include <linux/aer.h>
54#include <linux/rtnetlink.h>
55#include <linux/sched.h>
56#include <linux/seq_file.h>
57#include <linux/sockios.h>
58#include <linux/vmalloc.h>
59#include <linux/workqueue.h>
60#include <net/neighbour.h>
61#include <net/netevent.h>
62#include <asm/uaccess.h>
63
64#include "cxgb4.h"
65#include "t4_regs.h"
66#include "t4_msg.h"
67#include "t4fw_api.h"
68#include "l2t.h"
69
70#define DRV_VERSION "1.0.0-ko"
71#define DRV_DESC "Chelsio T4 Network Driver"
72
73/*
74 * Max interrupt hold-off timer value in us. Queues fall back to this value
75 * under extreme memory pressure so it's largish to give the system time to
76 * recover.
77 */
78#define MAX_SGE_TIMERVAL 200U
79
80enum {
81 MEMWIN0_APERTURE = 65536,
82 MEMWIN0_BASE = 0x30000,
83 MEMWIN1_APERTURE = 32768,
84 MEMWIN1_BASE = 0x28000,
85 MEMWIN2_APERTURE = 2048,
86 MEMWIN2_BASE = 0x1b800,
87};
88
89enum {
90 MAX_TXQ_ENTRIES = 16384,
91 MAX_CTRL_TXQ_ENTRIES = 1024,
92 MAX_RSPQ_ENTRIES = 16384,
93 MAX_RX_BUFFERS = 16384,
94 MIN_TXQ_ENTRIES = 32,
95 MIN_CTRL_TXQ_ENTRIES = 32,
96 MIN_RSPQ_ENTRIES = 128,
97 MIN_FL_ENTRIES = 16
98};
99
100#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
101 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
102 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
103
104#define CH_DEVICE(devid) { PCI_VDEVICE(CHELSIO, devid), 0 }
105
106static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
107 CH_DEVICE(0xa000), /* PE10K */
108 { 0, }
109};
110
111#define FW_FNAME "cxgb4/t4fw.bin"
112
113MODULE_DESCRIPTION(DRV_DESC);
114MODULE_AUTHOR("Chelsio Communications");
115MODULE_LICENSE("Dual BSD/GPL");
116MODULE_VERSION(DRV_VERSION);
117MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
118MODULE_FIRMWARE(FW_FNAME);
119
120static int dflt_msg_enable = DFLT_MSG_ENABLE;
121
122module_param(dflt_msg_enable, int, 0644);
123MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
124
125/*
126 * The driver uses the best interrupt scheme available on a platform in the
127 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
128 * of these schemes the driver may consider as follows:
129 *
130 * msi = 2: choose from among all three options
131 * msi = 1: only consider MSI and INTx interrupts
132 * msi = 0: force INTx interrupts
133 */
134static int msi = 2;
135
136module_param(msi, int, 0644);
137MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
138
139/*
140 * Queue interrupt hold-off timer values. Queues default to the first of these
141 * upon creation.
142 */
143static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
144
145module_param_array(intr_holdoff, uint, NULL, 0644);
146MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
147 "0..4 in microseconds");
148
149static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
150
151module_param_array(intr_cnt, uint, NULL, 0644);
152MODULE_PARM_DESC(intr_cnt,
153 "thresholds 1..3 for queue interrupt packet counters");
154
155static int vf_acls;
156
157#ifdef CONFIG_PCI_IOV
158module_param(vf_acls, bool, 0644);
159MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
160
161static unsigned int num_vf[4];
162
163module_param_array(num_vf, uint, NULL, 0644);
164MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
165#endif
166
167static struct dentry *cxgb4_debugfs_root;
168
169static LIST_HEAD(adapter_list);
170static DEFINE_MUTEX(uld_mutex);
171static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
172static const char *uld_str[] = { "RDMA", "iSCSI" };
173
174static void link_report(struct net_device *dev)
175{
176 if (!netif_carrier_ok(dev))
177 netdev_info(dev, "link down\n");
178 else {
179 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
180
181 const char *s = "10Mbps";
182 const struct port_info *p = netdev_priv(dev);
183
184 switch (p->link_cfg.speed) {
185 case SPEED_10000:
186 s = "10Gbps";
187 break;
188 case SPEED_1000:
189 s = "1000Mbps";
190 break;
191 case SPEED_100:
192 s = "100Mbps";
193 break;
194 }
195
196 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
197 fc[p->link_cfg.fc]);
198 }
199}
200
201void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
202{
203 struct net_device *dev = adapter->port[port_id];
204
205 /* Skip changes from disabled ports. */
206 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
207 if (link_stat)
208 netif_carrier_on(dev);
209 else
210 netif_carrier_off(dev);
211
212 link_report(dev);
213 }
214}
215
216void t4_os_portmod_changed(const struct adapter *adap, int port_id)
217{
218 static const char *mod_str[] = {
219 NULL, "LR", "SR", "ER", "passive DA", "active DA"
220 };
221
222 const struct net_device *dev = adap->port[port_id];
223 const struct port_info *pi = netdev_priv(dev);
224
225 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
226 netdev_info(dev, "port module unplugged\n");
227 else
228 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
229}
230
231/*
232 * Configure the exact and hash address filters to handle a port's multicast
233 * and secondary unicast MAC addresses.
234 */
235static int set_addr_filters(const struct net_device *dev, bool sleep)
236{
237 u64 mhash = 0;
238 u64 uhash = 0;
239 bool free = true;
240 u16 filt_idx[7];
241 const u8 *addr[7];
242 int ret, naddr = 0;
243 const struct dev_addr_list *d;
244 const struct netdev_hw_addr *ha;
245 int uc_cnt = netdev_uc_count(dev);
246 const struct port_info *pi = netdev_priv(dev);
247
248 /* first do the secondary unicast addresses */
249 netdev_for_each_uc_addr(ha, dev) {
250 addr[naddr++] = ha->addr;
251 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
252 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
253 naddr, addr, filt_idx, &uhash, sleep);
254 if (ret < 0)
255 return ret;
256
257 free = false;
258 naddr = 0;
259 }
260 }
261
262 /* next set up the multicast addresses */
263 netdev_for_each_mc_addr(d, dev) {
264 addr[naddr++] = d->dmi_addr;
265 if (naddr >= ARRAY_SIZE(addr) || d->next == NULL) {
266 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
267 naddr, addr, filt_idx, &mhash, sleep);
268 if (ret < 0)
269 return ret;
270
271 free = false;
272 naddr = 0;
273 }
274 }
275
276 return t4_set_addr_hash(pi->adapter, 0, pi->viid, uhash != 0,
277 uhash | mhash, sleep);
278}
279
280/*
281 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
282 * If @mtu is -1 it is left unchanged.
283 */
284static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
285{
286 int ret;
287 struct port_info *pi = netdev_priv(dev);
288
289 ret = set_addr_filters(dev, sleep_ok);
290 if (ret == 0)
291 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, mtu,
292 (dev->flags & IFF_PROMISC) ? 1 : 0,
293 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1,
294 sleep_ok);
295 return ret;
296}
297
298/**
299 * link_start - enable a port
300 * @dev: the port to enable
301 *
302 * Performs the MAC and PHY actions needed to enable a port.
303 */
304static int link_start(struct net_device *dev)
305{
306 int ret;
307 struct port_info *pi = netdev_priv(dev);
308
309 /*
310 * We do not set address filters and promiscuity here, the stack does
311 * that step explicitly.
312 */
313 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, dev->mtu, -1, -1, -1,
314 true);
315 if (ret == 0) {
316 ret = t4_change_mac(pi->adapter, 0, pi->viid,
317 pi->xact_addr_filt, dev->dev_addr, true,
318 false);
319 if (ret >= 0) {
320 pi->xact_addr_filt = ret;
321 ret = 0;
322 }
323 }
324 if (ret == 0)
325 ret = t4_link_start(pi->adapter, 0, pi->tx_chan, &pi->link_cfg);
326 if (ret == 0)
327 ret = t4_enable_vi(pi->adapter, 0, pi->viid, true, true);
328 return ret;
329}
330
331/*
332 * Response queue handler for the FW event queue.
333 */
334static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
335 const struct pkt_gl *gl)
336{
337 u8 opcode = ((const struct rss_header *)rsp)->opcode;
338
339 rsp++; /* skip RSS header */
340 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
341 const struct cpl_sge_egr_update *p = (void *)rsp;
342 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
343 struct sge_txq *txq = q->adap->sge.egr_map[qid];
344
345 txq->restarts++;
346 if ((u8 *)txq < (u8 *)q->adap->sge.ethrxq) {
347 struct sge_eth_txq *eq;
348
349 eq = container_of(txq, struct sge_eth_txq, q);
350 netif_tx_wake_queue(eq->txq);
351 } else {
352 struct sge_ofld_txq *oq;
353
354 oq = container_of(txq, struct sge_ofld_txq, q);
355 tasklet_schedule(&oq->qresume_tsk);
356 }
357 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
358 const struct cpl_fw6_msg *p = (void *)rsp;
359
360 if (p->type == 0)
361 t4_handle_fw_rpl(q->adap, p->data);
362 } else if (opcode == CPL_L2T_WRITE_RPL) {
363 const struct cpl_l2t_write_rpl *p = (void *)rsp;
364
365 do_l2t_write_rpl(q->adap, p);
366 } else
367 dev_err(q->adap->pdev_dev,
368 "unexpected CPL %#x on FW event queue\n", opcode);
369 return 0;
370}
371
372/**
373 * uldrx_handler - response queue handler for ULD queues
374 * @q: the response queue that received the packet
375 * @rsp: the response queue descriptor holding the offload message
376 * @gl: the gather list of packet fragments
377 *
378 * Deliver an ingress offload packet to a ULD. All processing is done by
379 * the ULD, we just maintain statistics.
380 */
381static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
382 const struct pkt_gl *gl)
383{
384 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
385
386 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
387 rxq->stats.nomem++;
388 return -1;
389 }
390 if (gl == NULL)
391 rxq->stats.imm++;
392 else if (gl == CXGB4_MSG_AN)
393 rxq->stats.an++;
394 else
395 rxq->stats.pkts++;
396 return 0;
397}
398
399static void disable_msi(struct adapter *adapter)
400{
401 if (adapter->flags & USING_MSIX) {
402 pci_disable_msix(adapter->pdev);
403 adapter->flags &= ~USING_MSIX;
404 } else if (adapter->flags & USING_MSI) {
405 pci_disable_msi(adapter->pdev);
406 adapter->flags &= ~USING_MSI;
407 }
408}
409
410/*
411 * Interrupt handler for non-data events used with MSI-X.
412 */
413static irqreturn_t t4_nondata_intr(int irq, void *cookie)
414{
415 struct adapter *adap = cookie;
416
417 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
418 if (v & PFSW) {
419 adap->swintr = 1;
420 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
421 }
422 t4_slow_intr_handler(adap);
423 return IRQ_HANDLED;
424}
425
426/*
427 * Name the MSI-X interrupts.
428 */
429static void name_msix_vecs(struct adapter *adap)
430{
431 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
432
433 /* non-data interrupts */
434 snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
435 adap->msix_info[0].desc[n] = 0;
436
437 /* FW events */
438 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
439 adap->msix_info[1].desc[n] = 0;
440
441 /* Ethernet queues */
442 for_each_port(adap, j) {
443 struct net_device *d = adap->port[j];
444 const struct port_info *pi = netdev_priv(d);
445
446 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
447 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
448 d->name, i);
449 adap->msix_info[msi_idx].desc[n] = 0;
450 }
451 }
452
453 /* offload queues */
454 for_each_ofldrxq(&adap->sge, i) {
455 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
456 adap->name, i);
457 adap->msix_info[msi_idx++].desc[n] = 0;
458 }
459 for_each_rdmarxq(&adap->sge, i) {
460 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
461 adap->name, i);
462 adap->msix_info[msi_idx++].desc[n] = 0;
463 }
464}
465
466static int request_msix_queue_irqs(struct adapter *adap)
467{
468 struct sge *s = &adap->sge;
469 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
470
471 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
472 adap->msix_info[1].desc, &s->fw_evtq);
473 if (err)
474 return err;
475
476 for_each_ethrxq(s, ethqidx) {
477 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
478 adap->msix_info[msi].desc,
479 &s->ethrxq[ethqidx].rspq);
480 if (err)
481 goto unwind;
482 msi++;
483 }
484 for_each_ofldrxq(s, ofldqidx) {
485 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
486 adap->msix_info[msi].desc,
487 &s->ofldrxq[ofldqidx].rspq);
488 if (err)
489 goto unwind;
490 msi++;
491 }
492 for_each_rdmarxq(s, rdmaqidx) {
493 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
494 adap->msix_info[msi].desc,
495 &s->rdmarxq[rdmaqidx].rspq);
496 if (err)
497 goto unwind;
498 msi++;
499 }
500 return 0;
501
502unwind:
503 while (--rdmaqidx >= 0)
504 free_irq(adap->msix_info[--msi].vec,
505 &s->rdmarxq[rdmaqidx].rspq);
506 while (--ofldqidx >= 0)
507 free_irq(adap->msix_info[--msi].vec,
508 &s->ofldrxq[ofldqidx].rspq);
509 while (--ethqidx >= 0)
510 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
511 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
512 return err;
513}
514
515static void free_msix_queue_irqs(struct adapter *adap)
516{
517 int i, msi = 2;
518 struct sge *s = &adap->sge;
519
520 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
521 for_each_ethrxq(s, i)
522 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
523 for_each_ofldrxq(s, i)
524 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
525 for_each_rdmarxq(s, i)
526 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
527}
528
529/**
530 * setup_rss - configure RSS
531 * @adap: the adapter
532 *
533 * Sets up RSS to distribute packets to multiple receive queues. We
534 * configure the RSS CPU lookup table to distribute to the number of HW
535 * receive queues, and the response queue lookup table to narrow that
536 * down to the response queues actually configured for each port.
537 * We always configure the RSS mapping for all ports since the mapping
538 * table has plenty of entries.
539 */
540static int setup_rss(struct adapter *adap)
541{
542 int i, j, err;
543 u16 rss[MAX_ETH_QSETS];
544
545 for_each_port(adap, i) {
546 const struct port_info *pi = adap2pinfo(adap, i);
547 const struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
548
549 for (j = 0; j < pi->nqsets; j++)
550 rss[j] = q[j].rspq.abs_id;
551
552 err = t4_config_rss_range(adap, 0, pi->viid, 0, pi->rss_size,
553 rss, pi->nqsets);
554 if (err)
555 return err;
556 }
557 return 0;
558}
559
560/*
561 * Wait until all NAPI handlers are descheduled.
562 */
563static void quiesce_rx(struct adapter *adap)
564{
565 int i;
566
567 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
568 struct sge_rspq *q = adap->sge.ingr_map[i];
569
570 if (q && q->handler)
571 napi_disable(&q->napi);
572 }
573}
574
575/*
576 * Enable NAPI scheduling and interrupt generation for all Rx queues.
577 */
578static void enable_rx(struct adapter *adap)
579{
580 int i;
581
582 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
583 struct sge_rspq *q = adap->sge.ingr_map[i];
584
585 if (!q)
586 continue;
587 if (q->handler)
588 napi_enable(&q->napi);
589 /* 0-increment GTS to start the timer and enable interrupts */
590 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
591 SEINTARM(q->intr_params) |
592 INGRESSQID(q->cntxt_id));
593 }
594}
595
596/**
597 * setup_sge_queues - configure SGE Tx/Rx/response queues
598 * @adap: the adapter
599 *
600 * Determines how many sets of SGE queues to use and initializes them.
601 * We support multiple queue sets per port if we have MSI-X, otherwise
602 * just one queue set per port.
603 */
604static int setup_sge_queues(struct adapter *adap)
605{
606 int err, msi_idx, i, j;
607 struct sge *s = &adap->sge;
608
609 bitmap_zero(s->starving_fl, MAX_EGRQ);
610 bitmap_zero(s->txq_maperr, MAX_EGRQ);
611
612 if (adap->flags & USING_MSIX)
613 msi_idx = 1; /* vector 0 is for non-queue interrupts */
614 else {
615 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
616 NULL, NULL);
617 if (err)
618 return err;
619 msi_idx = -((int)s->intrq.abs_id + 1);
620 }
621
622 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
623 msi_idx, NULL, fwevtq_handler);
624 if (err) {
625freeout: t4_free_sge_resources(adap);
626 return err;
627 }
628
629 for_each_port(adap, i) {
630 struct net_device *dev = adap->port[i];
631 struct port_info *pi = netdev_priv(dev);
632 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
633 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
634
635 for (j = 0; j < pi->nqsets; j++, q++) {
636 if (msi_idx > 0)
637 msi_idx++;
638 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
639 msi_idx, &q->fl,
640 t4_ethrx_handler);
641 if (err)
642 goto freeout;
643 q->rspq.idx = j;
644 memset(&q->stats, 0, sizeof(q->stats));
645 }
646 for (j = 0; j < pi->nqsets; j++, t++) {
647 err = t4_sge_alloc_eth_txq(adap, t, dev,
648 netdev_get_tx_queue(dev, j),
649 s->fw_evtq.cntxt_id);
650 if (err)
651 goto freeout;
652 }
653 }
654
655 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
656 for_each_ofldrxq(s, i) {
657 struct sge_ofld_rxq *q = &s->ofldrxq[i];
658 struct net_device *dev = adap->port[i / j];
659
660 if (msi_idx > 0)
661 msi_idx++;
662 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
663 &q->fl, uldrx_handler);
664 if (err)
665 goto freeout;
666 memset(&q->stats, 0, sizeof(q->stats));
667 s->ofld_rxq[i] = q->rspq.abs_id;
668 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
669 s->fw_evtq.cntxt_id);
670 if (err)
671 goto freeout;
672 }
673
674 for_each_rdmarxq(s, i) {
675 struct sge_ofld_rxq *q = &s->rdmarxq[i];
676
677 if (msi_idx > 0)
678 msi_idx++;
679 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
680 msi_idx, &q->fl, uldrx_handler);
681 if (err)
682 goto freeout;
683 memset(&q->stats, 0, sizeof(q->stats));
684 s->rdma_rxq[i] = q->rspq.abs_id;
685 }
686
687 for_each_port(adap, i) {
688 /*
689 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
690 * have RDMA queues, and that's the right value.
691 */
692 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
693 s->fw_evtq.cntxt_id,
694 s->rdmarxq[i].rspq.cntxt_id);
695 if (err)
696 goto freeout;
697 }
698
699 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
700 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
701 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
702 return 0;
703}
704
705/*
706 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
707 * started but failed, and a negative errno if flash load couldn't start.
708 */
709static int upgrade_fw(struct adapter *adap)
710{
711 int ret;
712 u32 vers;
713 const struct fw_hdr *hdr;
714 const struct firmware *fw;
715 struct device *dev = adap->pdev_dev;
716
717 ret = request_firmware(&fw, FW_FNAME, dev);
718 if (ret < 0) {
719 dev_err(dev, "unable to load firmware image " FW_FNAME
720 ", error %d\n", ret);
721 return ret;
722 }
723
724 hdr = (const struct fw_hdr *)fw->data;
725 vers = ntohl(hdr->fw_ver);
726 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
727 ret = -EINVAL; /* wrong major version, won't do */
728 goto out;
729 }
730
731 /*
732 * If the flash FW is unusable or we found something newer, load it.
733 */
734 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
735 vers > adap->params.fw_vers) {
736 ret = -t4_load_fw(adap, fw->data, fw->size);
737 if (!ret)
738 dev_info(dev, "firmware upgraded to version %pI4 from "
739 FW_FNAME "\n", &hdr->fw_ver);
740 }
741out: release_firmware(fw);
742 return ret;
743}
744
745/*
746 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
747 * The allocated memory is cleared.
748 */
749void *t4_alloc_mem(size_t size)
750{
751 void *p = kmalloc(size, GFP_KERNEL);
752
753 if (!p)
754 p = vmalloc(size);
755 if (p)
756 memset(p, 0, size);
757 return p;
758}
759
760/*
761 * Free memory allocated through alloc_mem().
762 */
763void t4_free_mem(void *addr)
764{
765 if (is_vmalloc_addr(addr))
766 vfree(addr);
767 else
768 kfree(addr);
769}
770
771static inline int is_offload(const struct adapter *adap)
772{
773 return adap->params.offload;
774}
775
776/*
777 * Implementation of ethtool operations.
778 */
779
780static u32 get_msglevel(struct net_device *dev)
781{
782 return netdev2adap(dev)->msg_enable;
783}
784
785static void set_msglevel(struct net_device *dev, u32 val)
786{
787 netdev2adap(dev)->msg_enable = val;
788}
789
790static char stats_strings[][ETH_GSTRING_LEN] = {
791 "TxOctetsOK ",
792 "TxFramesOK ",
793 "TxBroadcastFrames ",
794 "TxMulticastFrames ",
795 "TxUnicastFrames ",
796 "TxErrorFrames ",
797
798 "TxFrames64 ",
799 "TxFrames65To127 ",
800 "TxFrames128To255 ",
801 "TxFrames256To511 ",
802 "TxFrames512To1023 ",
803 "TxFrames1024To1518 ",
804 "TxFrames1519ToMax ",
805
806 "TxFramesDropped ",
807 "TxPauseFrames ",
808 "TxPPP0Frames ",
809 "TxPPP1Frames ",
810 "TxPPP2Frames ",
811 "TxPPP3Frames ",
812 "TxPPP4Frames ",
813 "TxPPP5Frames ",
814 "TxPPP6Frames ",
815 "TxPPP7Frames ",
816
817 "RxOctetsOK ",
818 "RxFramesOK ",
819 "RxBroadcastFrames ",
820 "RxMulticastFrames ",
821 "RxUnicastFrames ",
822
823 "RxFramesTooLong ",
824 "RxJabberErrors ",
825 "RxFCSErrors ",
826 "RxLengthErrors ",
827 "RxSymbolErrors ",
828 "RxRuntFrames ",
829
830 "RxFrames64 ",
831 "RxFrames65To127 ",
832 "RxFrames128To255 ",
833 "RxFrames256To511 ",
834 "RxFrames512To1023 ",
835 "RxFrames1024To1518 ",
836 "RxFrames1519ToMax ",
837
838 "RxPauseFrames ",
839 "RxPPP0Frames ",
840 "RxPPP1Frames ",
841 "RxPPP2Frames ",
842 "RxPPP3Frames ",
843 "RxPPP4Frames ",
844 "RxPPP5Frames ",
845 "RxPPP6Frames ",
846 "RxPPP7Frames ",
847
848 "RxBG0FramesDropped ",
849 "RxBG1FramesDropped ",
850 "RxBG2FramesDropped ",
851 "RxBG3FramesDropped ",
852 "RxBG0FramesTrunc ",
853 "RxBG1FramesTrunc ",
854 "RxBG2FramesTrunc ",
855 "RxBG3FramesTrunc ",
856
857 "TSO ",
858 "TxCsumOffload ",
859 "RxCsumGood ",
860 "VLANextractions ",
861 "VLANinsertions ",
862};
863
864static int get_sset_count(struct net_device *dev, int sset)
865{
866 switch (sset) {
867 case ETH_SS_STATS:
868 return ARRAY_SIZE(stats_strings);
869 default:
870 return -EOPNOTSUPP;
871 }
872}
873
874#define T4_REGMAP_SIZE (160 * 1024)
875
876static int get_regs_len(struct net_device *dev)
877{
878 return T4_REGMAP_SIZE;
879}
880
881static int get_eeprom_len(struct net_device *dev)
882{
883 return EEPROMSIZE;
884}
885
886static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
887{
888 struct adapter *adapter = netdev2adap(dev);
889
890 strcpy(info->driver, KBUILD_MODNAME);
891 strcpy(info->version, DRV_VERSION);
892 strcpy(info->bus_info, pci_name(adapter->pdev));
893
894 if (!adapter->params.fw_vers)
895 strcpy(info->fw_version, "N/A");
896 else
897 snprintf(info->fw_version, sizeof(info->fw_version),
898 "%u.%u.%u.%u, TP %u.%u.%u.%u",
899 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
900 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
901 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
902 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
903 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
904 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
905 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
906 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
907}
908
909static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
910{
911 if (stringset == ETH_SS_STATS)
912 memcpy(data, stats_strings, sizeof(stats_strings));
913}
914
915/*
916 * port stats maintained per queue of the port. They should be in the same
917 * order as in stats_strings above.
918 */
919struct queue_port_stats {
920 u64 tso;
921 u64 tx_csum;
922 u64 rx_csum;
923 u64 vlan_ex;
924 u64 vlan_ins;
925};
926
927static void collect_sge_port_stats(const struct adapter *adap,
928 const struct port_info *p, struct queue_port_stats *s)
929{
930 int i;
931 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
932 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
933
934 memset(s, 0, sizeof(*s));
935 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
936 s->tso += tx->tso;
937 s->tx_csum += tx->tx_cso;
938 s->rx_csum += rx->stats.rx_cso;
939 s->vlan_ex += rx->stats.vlan_ex;
940 s->vlan_ins += tx->vlan_ins;
941 }
942}
943
944static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
945 u64 *data)
946{
947 struct port_info *pi = netdev_priv(dev);
948 struct adapter *adapter = pi->adapter;
949
950 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
951
952 data += sizeof(struct port_stats) / sizeof(u64);
953 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
954}
955
956/*
957 * Return a version number to identify the type of adapter. The scheme is:
958 * - bits 0..9: chip version
959 * - bits 10..15: chip revision
960 */
961static inline unsigned int mk_adap_vers(const struct adapter *ap)
962{
963 return 4 | (ap->params.rev << 10);
964}
965
966static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
967 unsigned int end)
968{
969 u32 *p = buf + start;
970
971 for ( ; start <= end; start += sizeof(u32))
972 *p++ = t4_read_reg(ap, start);
973}
974
975static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
976 void *buf)
977{
978 static const unsigned int reg_ranges[] = {
979 0x1008, 0x1108,
980 0x1180, 0x11b4,
981 0x11fc, 0x123c,
982 0x1300, 0x173c,
983 0x1800, 0x18fc,
984 0x3000, 0x30d8,
985 0x30e0, 0x5924,
986 0x5960, 0x59d4,
987 0x5a00, 0x5af8,
988 0x6000, 0x6098,
989 0x6100, 0x6150,
990 0x6200, 0x6208,
991 0x6240, 0x6248,
992 0x6280, 0x6338,
993 0x6370, 0x638c,
994 0x6400, 0x643c,
995 0x6500, 0x6524,
996 0x6a00, 0x6a38,
997 0x6a60, 0x6a78,
998 0x6b00, 0x6b84,
999 0x6bf0, 0x6c84,
1000 0x6cf0, 0x6d84,
1001 0x6df0, 0x6e84,
1002 0x6ef0, 0x6f84,
1003 0x6ff0, 0x7084,
1004 0x70f0, 0x7184,
1005 0x71f0, 0x7284,
1006 0x72f0, 0x7384,
1007 0x73f0, 0x7450,
1008 0x7500, 0x7530,
1009 0x7600, 0x761c,
1010 0x7680, 0x76cc,
1011 0x7700, 0x7798,
1012 0x77c0, 0x77fc,
1013 0x7900, 0x79fc,
1014 0x7b00, 0x7c38,
1015 0x7d00, 0x7efc,
1016 0x8dc0, 0x8e1c,
1017 0x8e30, 0x8e78,
1018 0x8ea0, 0x8f6c,
1019 0x8fc0, 0x9074,
1020 0x90fc, 0x90fc,
1021 0x9400, 0x9458,
1022 0x9600, 0x96bc,
1023 0x9800, 0x9808,
1024 0x9820, 0x983c,
1025 0x9850, 0x9864,
1026 0x9c00, 0x9c6c,
1027 0x9c80, 0x9cec,
1028 0x9d00, 0x9d6c,
1029 0x9d80, 0x9dec,
1030 0x9e00, 0x9e6c,
1031 0x9e80, 0x9eec,
1032 0x9f00, 0x9f6c,
1033 0x9f80, 0x9fec,
1034 0xd004, 0xd03c,
1035 0xdfc0, 0xdfe0,
1036 0xe000, 0xea7c,
1037 0xf000, 0x11190,
1038 0x19040, 0x19124,
1039 0x19150, 0x191b0,
1040 0x191d0, 0x191e8,
1041 0x19238, 0x1924c,
1042 0x193f8, 0x19474,
1043 0x19490, 0x194f8,
1044 0x19800, 0x19f30,
1045 0x1a000, 0x1a06c,
1046 0x1a0b0, 0x1a120,
1047 0x1a128, 0x1a138,
1048 0x1a190, 0x1a1c4,
1049 0x1a1fc, 0x1a1fc,
1050 0x1e040, 0x1e04c,
1051 0x1e240, 0x1e28c,
1052 0x1e2c0, 0x1e2c0,
1053 0x1e2e0, 0x1e2e0,
1054 0x1e300, 0x1e384,
1055 0x1e3c0, 0x1e3c8,
1056 0x1e440, 0x1e44c,
1057 0x1e640, 0x1e68c,
1058 0x1e6c0, 0x1e6c0,
1059 0x1e6e0, 0x1e6e0,
1060 0x1e700, 0x1e784,
1061 0x1e7c0, 0x1e7c8,
1062 0x1e840, 0x1e84c,
1063 0x1ea40, 0x1ea8c,
1064 0x1eac0, 0x1eac0,
1065 0x1eae0, 0x1eae0,
1066 0x1eb00, 0x1eb84,
1067 0x1ebc0, 0x1ebc8,
1068 0x1ec40, 0x1ec4c,
1069 0x1ee40, 0x1ee8c,
1070 0x1eec0, 0x1eec0,
1071 0x1eee0, 0x1eee0,
1072 0x1ef00, 0x1ef84,
1073 0x1efc0, 0x1efc8,
1074 0x1f040, 0x1f04c,
1075 0x1f240, 0x1f28c,
1076 0x1f2c0, 0x1f2c0,
1077 0x1f2e0, 0x1f2e0,
1078 0x1f300, 0x1f384,
1079 0x1f3c0, 0x1f3c8,
1080 0x1f440, 0x1f44c,
1081 0x1f640, 0x1f68c,
1082 0x1f6c0, 0x1f6c0,
1083 0x1f6e0, 0x1f6e0,
1084 0x1f700, 0x1f784,
1085 0x1f7c0, 0x1f7c8,
1086 0x1f840, 0x1f84c,
1087 0x1fa40, 0x1fa8c,
1088 0x1fac0, 0x1fac0,
1089 0x1fae0, 0x1fae0,
1090 0x1fb00, 0x1fb84,
1091 0x1fbc0, 0x1fbc8,
1092 0x1fc40, 0x1fc4c,
1093 0x1fe40, 0x1fe8c,
1094 0x1fec0, 0x1fec0,
1095 0x1fee0, 0x1fee0,
1096 0x1ff00, 0x1ff84,
1097 0x1ffc0, 0x1ffc8,
1098 0x20000, 0x2002c,
1099 0x20100, 0x2013c,
1100 0x20190, 0x201c8,
1101 0x20200, 0x20318,
1102 0x20400, 0x20528,
1103 0x20540, 0x20614,
1104 0x21000, 0x21040,
1105 0x2104c, 0x21060,
1106 0x210c0, 0x210ec,
1107 0x21200, 0x21268,
1108 0x21270, 0x21284,
1109 0x212fc, 0x21388,
1110 0x21400, 0x21404,
1111 0x21500, 0x21518,
1112 0x2152c, 0x2153c,
1113 0x21550, 0x21554,
1114 0x21600, 0x21600,
1115 0x21608, 0x21628,
1116 0x21630, 0x2163c,
1117 0x21700, 0x2171c,
1118 0x21780, 0x2178c,
1119 0x21800, 0x21c38,
1120 0x21c80, 0x21d7c,
1121 0x21e00, 0x21e04,
1122 0x22000, 0x2202c,
1123 0x22100, 0x2213c,
1124 0x22190, 0x221c8,
1125 0x22200, 0x22318,
1126 0x22400, 0x22528,
1127 0x22540, 0x22614,
1128 0x23000, 0x23040,
1129 0x2304c, 0x23060,
1130 0x230c0, 0x230ec,
1131 0x23200, 0x23268,
1132 0x23270, 0x23284,
1133 0x232fc, 0x23388,
1134 0x23400, 0x23404,
1135 0x23500, 0x23518,
1136 0x2352c, 0x2353c,
1137 0x23550, 0x23554,
1138 0x23600, 0x23600,
1139 0x23608, 0x23628,
1140 0x23630, 0x2363c,
1141 0x23700, 0x2371c,
1142 0x23780, 0x2378c,
1143 0x23800, 0x23c38,
1144 0x23c80, 0x23d7c,
1145 0x23e00, 0x23e04,
1146 0x24000, 0x2402c,
1147 0x24100, 0x2413c,
1148 0x24190, 0x241c8,
1149 0x24200, 0x24318,
1150 0x24400, 0x24528,
1151 0x24540, 0x24614,
1152 0x25000, 0x25040,
1153 0x2504c, 0x25060,
1154 0x250c0, 0x250ec,
1155 0x25200, 0x25268,
1156 0x25270, 0x25284,
1157 0x252fc, 0x25388,
1158 0x25400, 0x25404,
1159 0x25500, 0x25518,
1160 0x2552c, 0x2553c,
1161 0x25550, 0x25554,
1162 0x25600, 0x25600,
1163 0x25608, 0x25628,
1164 0x25630, 0x2563c,
1165 0x25700, 0x2571c,
1166 0x25780, 0x2578c,
1167 0x25800, 0x25c38,
1168 0x25c80, 0x25d7c,
1169 0x25e00, 0x25e04,
1170 0x26000, 0x2602c,
1171 0x26100, 0x2613c,
1172 0x26190, 0x261c8,
1173 0x26200, 0x26318,
1174 0x26400, 0x26528,
1175 0x26540, 0x26614,
1176 0x27000, 0x27040,
1177 0x2704c, 0x27060,
1178 0x270c0, 0x270ec,
1179 0x27200, 0x27268,
1180 0x27270, 0x27284,
1181 0x272fc, 0x27388,
1182 0x27400, 0x27404,
1183 0x27500, 0x27518,
1184 0x2752c, 0x2753c,
1185 0x27550, 0x27554,
1186 0x27600, 0x27600,
1187 0x27608, 0x27628,
1188 0x27630, 0x2763c,
1189 0x27700, 0x2771c,
1190 0x27780, 0x2778c,
1191 0x27800, 0x27c38,
1192 0x27c80, 0x27d7c,
1193 0x27e00, 0x27e04
1194 };
1195
1196 int i;
1197 struct adapter *ap = netdev2adap(dev);
1198
1199 regs->version = mk_adap_vers(ap);
1200
1201 memset(buf, 0, T4_REGMAP_SIZE);
1202 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1203 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1204}
1205
1206static int restart_autoneg(struct net_device *dev)
1207{
1208 struct port_info *p = netdev_priv(dev);
1209
1210 if (!netif_running(dev))
1211 return -EAGAIN;
1212 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1213 return -EINVAL;
1214 t4_restart_aneg(p->adapter, 0, p->tx_chan);
1215 return 0;
1216}
1217
1218static int identify_port(struct net_device *dev, u32 data)
1219{
1220 if (data == 0)
1221 data = 2; /* default to 2 seconds */
1222
1223 return t4_identify_port(netdev2adap(dev), 0, netdev2pinfo(dev)->viid,
1224 data * 5);
1225}
1226
1227static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1228{
1229 unsigned int v = 0;
1230
1231 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XAUI) {
1232 v |= SUPPORTED_TP;
1233 if (caps & FW_PORT_CAP_SPEED_100M)
1234 v |= SUPPORTED_100baseT_Full;
1235 if (caps & FW_PORT_CAP_SPEED_1G)
1236 v |= SUPPORTED_1000baseT_Full;
1237 if (caps & FW_PORT_CAP_SPEED_10G)
1238 v |= SUPPORTED_10000baseT_Full;
1239 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1240 v |= SUPPORTED_Backplane;
1241 if (caps & FW_PORT_CAP_SPEED_1G)
1242 v |= SUPPORTED_1000baseKX_Full;
1243 if (caps & FW_PORT_CAP_SPEED_10G)
1244 v |= SUPPORTED_10000baseKX4_Full;
1245 } else if (type == FW_PORT_TYPE_KR)
1246 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1247 else if (type == FW_PORT_TYPE_FIBER)
1248 v |= SUPPORTED_FIBRE;
1249
1250 if (caps & FW_PORT_CAP_ANEG)
1251 v |= SUPPORTED_Autoneg;
1252 return v;
1253}
1254
1255static unsigned int to_fw_linkcaps(unsigned int caps)
1256{
1257 unsigned int v = 0;
1258
1259 if (caps & ADVERTISED_100baseT_Full)
1260 v |= FW_PORT_CAP_SPEED_100M;
1261 if (caps & ADVERTISED_1000baseT_Full)
1262 v |= FW_PORT_CAP_SPEED_1G;
1263 if (caps & ADVERTISED_10000baseT_Full)
1264 v |= FW_PORT_CAP_SPEED_10G;
1265 return v;
1266}
1267
1268static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1269{
1270 const struct port_info *p = netdev_priv(dev);
1271
1272 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1273 p->port_type == FW_PORT_TYPE_BT_XAUI)
1274 cmd->port = PORT_TP;
1275 else if (p->port_type == FW_PORT_TYPE_FIBER)
1276 cmd->port = PORT_FIBRE;
1277 else if (p->port_type == FW_PORT_TYPE_TWINAX)
1278 cmd->port = PORT_DA;
1279 else
1280 cmd->port = PORT_OTHER;
1281
1282 if (p->mdio_addr >= 0) {
1283 cmd->phy_address = p->mdio_addr;
1284 cmd->transceiver = XCVR_EXTERNAL;
1285 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1286 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1287 } else {
1288 cmd->phy_address = 0; /* not really, but no better option */
1289 cmd->transceiver = XCVR_INTERNAL;
1290 cmd->mdio_support = 0;
1291 }
1292
1293 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1294 cmd->advertising = from_fw_linkcaps(p->port_type,
1295 p->link_cfg.advertising);
1296 cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1297 cmd->duplex = DUPLEX_FULL;
1298 cmd->autoneg = p->link_cfg.autoneg;
1299 cmd->maxtxpkt = 0;
1300 cmd->maxrxpkt = 0;
1301 return 0;
1302}
1303
1304static unsigned int speed_to_caps(int speed)
1305{
1306 if (speed == SPEED_100)
1307 return FW_PORT_CAP_SPEED_100M;
1308 if (speed == SPEED_1000)
1309 return FW_PORT_CAP_SPEED_1G;
1310 if (speed == SPEED_10000)
1311 return FW_PORT_CAP_SPEED_10G;
1312 return 0;
1313}
1314
1315static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1316{
1317 unsigned int cap;
1318 struct port_info *p = netdev_priv(dev);
1319 struct link_config *lc = &p->link_cfg;
1320
1321 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1322 return -EINVAL;
1323
1324 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1325 /*
1326 * PHY offers a single speed. See if that's what's
1327 * being requested.
1328 */
1329 if (cmd->autoneg == AUTONEG_DISABLE &&
1330 (lc->supported & speed_to_caps(cmd->speed)))
1331 return 0;
1332 return -EINVAL;
1333 }
1334
1335 if (cmd->autoneg == AUTONEG_DISABLE) {
1336 cap = speed_to_caps(cmd->speed);
1337
1338 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1339 cmd->speed == SPEED_10000)
1340 return -EINVAL;
1341 lc->requested_speed = cap;
1342 lc->advertising = 0;
1343 } else {
1344 cap = to_fw_linkcaps(cmd->advertising);
1345 if (!(lc->supported & cap))
1346 return -EINVAL;
1347 lc->requested_speed = 0;
1348 lc->advertising = cap | FW_PORT_CAP_ANEG;
1349 }
1350 lc->autoneg = cmd->autoneg;
1351
1352 if (netif_running(dev))
1353 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1354 return 0;
1355}
1356
1357static void get_pauseparam(struct net_device *dev,
1358 struct ethtool_pauseparam *epause)
1359{
1360 struct port_info *p = netdev_priv(dev);
1361
1362 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1363 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1364 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1365}
1366
1367static int set_pauseparam(struct net_device *dev,
1368 struct ethtool_pauseparam *epause)
1369{
1370 struct port_info *p = netdev_priv(dev);
1371 struct link_config *lc = &p->link_cfg;
1372
1373 if (epause->autoneg == AUTONEG_DISABLE)
1374 lc->requested_fc = 0;
1375 else if (lc->supported & FW_PORT_CAP_ANEG)
1376 lc->requested_fc = PAUSE_AUTONEG;
1377 else
1378 return -EINVAL;
1379
1380 if (epause->rx_pause)
1381 lc->requested_fc |= PAUSE_RX;
1382 if (epause->tx_pause)
1383 lc->requested_fc |= PAUSE_TX;
1384 if (netif_running(dev))
1385 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1386 return 0;
1387}
1388
1389static u32 get_rx_csum(struct net_device *dev)
1390{
1391 struct port_info *p = netdev_priv(dev);
1392
1393 return p->rx_offload & RX_CSO;
1394}
1395
1396static int set_rx_csum(struct net_device *dev, u32 data)
1397{
1398 struct port_info *p = netdev_priv(dev);
1399
1400 if (data)
1401 p->rx_offload |= RX_CSO;
1402 else
1403 p->rx_offload &= ~RX_CSO;
1404 return 0;
1405}
1406
1407static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1408{
1409 const struct port_info *pi = netdev_priv(dev);
1410 const struct sge *s = &pi->adapter->sge;
1411
1412 e->rx_max_pending = MAX_RX_BUFFERS;
1413 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1414 e->rx_jumbo_max_pending = 0;
1415 e->tx_max_pending = MAX_TXQ_ENTRIES;
1416
1417 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1418 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1419 e->rx_jumbo_pending = 0;
1420 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1421}
1422
1423static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1424{
1425 int i;
1426 const struct port_info *pi = netdev_priv(dev);
1427 struct adapter *adapter = pi->adapter;
1428 struct sge *s = &adapter->sge;
1429
1430 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1431 e->tx_pending > MAX_TXQ_ENTRIES ||
1432 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1433 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1434 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1435 return -EINVAL;
1436
1437 if (adapter->flags & FULL_INIT_DONE)
1438 return -EBUSY;
1439
1440 for (i = 0; i < pi->nqsets; ++i) {
1441 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1442 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1443 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1444 }
1445 return 0;
1446}
1447
1448static int closest_timer(const struct sge *s, int time)
1449{
1450 int i, delta, match = 0, min_delta = INT_MAX;
1451
1452 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1453 delta = time - s->timer_val[i];
1454 if (delta < 0)
1455 delta = -delta;
1456 if (delta < min_delta) {
1457 min_delta = delta;
1458 match = i;
1459 }
1460 }
1461 return match;
1462}
1463
1464static int closest_thres(const struct sge *s, int thres)
1465{
1466 int i, delta, match = 0, min_delta = INT_MAX;
1467
1468 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1469 delta = thres - s->counter_val[i];
1470 if (delta < 0)
1471 delta = -delta;
1472 if (delta < min_delta) {
1473 min_delta = delta;
1474 match = i;
1475 }
1476 }
1477 return match;
1478}
1479
1480/*
1481 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1482 */
1483static unsigned int qtimer_val(const struct adapter *adap,
1484 const struct sge_rspq *q)
1485{
1486 unsigned int idx = q->intr_params >> 1;
1487
1488 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1489}
1490
1491/**
1492 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1493 * @adap: the adapter
1494 * @q: the Rx queue
1495 * @us: the hold-off time in us, or 0 to disable timer
1496 * @cnt: the hold-off packet count, or 0 to disable counter
1497 *
1498 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1499 * one of the two needs to be enabled for the queue to generate interrupts.
1500 */
1501static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1502 unsigned int us, unsigned int cnt)
1503{
1504 if ((us | cnt) == 0)
1505 cnt = 1;
1506
1507 if (cnt) {
1508 int err;
1509 u32 v, new_idx;
1510
1511 new_idx = closest_thres(&adap->sge, cnt);
1512 if (q->desc && q->pktcnt_idx != new_idx) {
1513 /* the queue has already been created, update it */
1514 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1515 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1516 FW_PARAMS_PARAM_YZ(q->cntxt_id);
1517 err = t4_set_params(adap, 0, 0, 0, 1, &v, &new_idx);
1518 if (err)
1519 return err;
1520 }
1521 q->pktcnt_idx = new_idx;
1522 }
1523
1524 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1525 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1526 return 0;
1527}
1528
1529static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1530{
1531 const struct port_info *pi = netdev_priv(dev);
1532 struct adapter *adap = pi->adapter;
1533
1534 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1535 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1536}
1537
1538static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1539{
1540 const struct port_info *pi = netdev_priv(dev);
1541 const struct adapter *adap = pi->adapter;
1542 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1543
1544 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1545 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1546 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1547 return 0;
1548}
1549
1550/*
1551 * Translate a physical EEPROM address to virtual. The first 1K is accessed
1552 * through virtual addresses starting at 31K, the rest is accessed through
1553 * virtual addresses starting at 0. This mapping is correct only for PF0.
1554 */
1555static int eeprom_ptov(unsigned int phys_addr)
1556{
1557 if (phys_addr < 1024)
1558 return phys_addr + (31 << 10);
1559 if (phys_addr < EEPROMSIZE)
1560 return phys_addr - 1024;
1561 return -EINVAL;
1562}
1563
1564/*
1565 * The next two routines implement eeprom read/write from physical addresses.
1566 * The physical->virtual translation is correct only for PF0.
1567 */
1568static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1569{
1570 int vaddr = eeprom_ptov(phys_addr);
1571
1572 if (vaddr >= 0)
1573 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1574 return vaddr < 0 ? vaddr : 0;
1575}
1576
1577static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1578{
1579 int vaddr = eeprom_ptov(phys_addr);
1580
1581 if (vaddr >= 0)
1582 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1583 return vaddr < 0 ? vaddr : 0;
1584}
1585
1586#define EEPROM_MAGIC 0x38E2F10C
1587
1588static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1589 u8 *data)
1590{
1591 int i, err = 0;
1592 struct adapter *adapter = netdev2adap(dev);
1593
1594 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1595 if (!buf)
1596 return -ENOMEM;
1597
1598 e->magic = EEPROM_MAGIC;
1599 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1600 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1601
1602 if (!err)
1603 memcpy(data, buf + e->offset, e->len);
1604 kfree(buf);
1605 return err;
1606}
1607
1608static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1609 u8 *data)
1610{
1611 u8 *buf;
1612 int err = 0;
1613 u32 aligned_offset, aligned_len, *p;
1614 struct adapter *adapter = netdev2adap(dev);
1615
1616 if (eeprom->magic != EEPROM_MAGIC)
1617 return -EINVAL;
1618
1619 aligned_offset = eeprom->offset & ~3;
1620 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1621
1622 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1623 /*
1624 * RMW possibly needed for first or last words.
1625 */
1626 buf = kmalloc(aligned_len, GFP_KERNEL);
1627 if (!buf)
1628 return -ENOMEM;
1629 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1630 if (!err && aligned_len > 4)
1631 err = eeprom_rd_phys(adapter,
1632 aligned_offset + aligned_len - 4,
1633 (u32 *)&buf[aligned_len - 4]);
1634 if (err)
1635 goto out;
1636 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1637 } else
1638 buf = data;
1639
1640 err = t4_seeprom_wp(adapter, false);
1641 if (err)
1642 goto out;
1643
1644 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1645 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1646 aligned_offset += 4;
1647 }
1648
1649 if (!err)
1650 err = t4_seeprom_wp(adapter, true);
1651out:
1652 if (buf != data)
1653 kfree(buf);
1654 return err;
1655}
1656
1657static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1658{
1659 int ret;
1660 const struct firmware *fw;
1661 struct adapter *adap = netdev2adap(netdev);
1662
1663 ef->data[sizeof(ef->data) - 1] = '\0';
1664 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1665 if (ret < 0)
1666 return ret;
1667
1668 ret = t4_load_fw(adap, fw->data, fw->size);
1669 release_firmware(fw);
1670 if (!ret)
1671 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1672 return ret;
1673}
1674
1675#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1676#define BCAST_CRC 0xa0ccc1a6
1677
1678static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1679{
1680 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1681 wol->wolopts = netdev2adap(dev)->wol;
1682 memset(&wol->sopass, 0, sizeof(wol->sopass));
1683}
1684
1685static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1686{
1687 int err = 0;
1688 struct port_info *pi = netdev_priv(dev);
1689
1690 if (wol->wolopts & ~WOL_SUPPORTED)
1691 return -EINVAL;
1692 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1693 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1694 if (wol->wolopts & WAKE_BCAST) {
1695 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1696 ~0ULL, 0, false);
1697 if (!err)
1698 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1699 ~6ULL, ~0ULL, BCAST_CRC, true);
1700 } else
1701 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1702 return err;
1703}
1704
1705static int set_tso(struct net_device *dev, u32 value)
1706{
1707 if (value)
1708 dev->features |= NETIF_F_TSO | NETIF_F_TSO6;
1709 else
1710 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
1711 return 0;
1712}
1713
1714static struct ethtool_ops cxgb_ethtool_ops = {
1715 .get_settings = get_settings,
1716 .set_settings = set_settings,
1717 .get_drvinfo = get_drvinfo,
1718 .get_msglevel = get_msglevel,
1719 .set_msglevel = set_msglevel,
1720 .get_ringparam = get_sge_param,
1721 .set_ringparam = set_sge_param,
1722 .get_coalesce = get_coalesce,
1723 .set_coalesce = set_coalesce,
1724 .get_eeprom_len = get_eeprom_len,
1725 .get_eeprom = get_eeprom,
1726 .set_eeprom = set_eeprom,
1727 .get_pauseparam = get_pauseparam,
1728 .set_pauseparam = set_pauseparam,
1729 .get_rx_csum = get_rx_csum,
1730 .set_rx_csum = set_rx_csum,
1731 .set_tx_csum = ethtool_op_set_tx_ipv6_csum,
1732 .set_sg = ethtool_op_set_sg,
1733 .get_link = ethtool_op_get_link,
1734 .get_strings = get_strings,
1735 .phys_id = identify_port,
1736 .nway_reset = restart_autoneg,
1737 .get_sset_count = get_sset_count,
1738 .get_ethtool_stats = get_stats,
1739 .get_regs_len = get_regs_len,
1740 .get_regs = get_regs,
1741 .get_wol = get_wol,
1742 .set_wol = set_wol,
1743 .set_tso = set_tso,
1744 .flash_device = set_flash,
1745};
1746
1747/*
1748 * debugfs support
1749 */
1750
1751static int mem_open(struct inode *inode, struct file *file)
1752{
1753 file->private_data = inode->i_private;
1754 return 0;
1755}
1756
1757static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
1758 loff_t *ppos)
1759{
1760 loff_t pos = *ppos;
1761 loff_t avail = file->f_path.dentry->d_inode->i_size;
1762 unsigned int mem = (uintptr_t)file->private_data & 3;
1763 struct adapter *adap = file->private_data - mem;
1764
1765 if (pos < 0)
1766 return -EINVAL;
1767 if (pos >= avail)
1768 return 0;
1769 if (count > avail - pos)
1770 count = avail - pos;
1771
1772 while (count) {
1773 size_t len;
1774 int ret, ofst;
1775 __be32 data[16];
1776
1777 if (mem == MEM_MC)
1778 ret = t4_mc_read(adap, pos, data, NULL);
1779 else
1780 ret = t4_edc_read(adap, mem, pos, data, NULL);
1781 if (ret)
1782 return ret;
1783
1784 ofst = pos % sizeof(data);
1785 len = min(count, sizeof(data) - ofst);
1786 if (copy_to_user(buf, (u8 *)data + ofst, len))
1787 return -EFAULT;
1788
1789 buf += len;
1790 pos += len;
1791 count -= len;
1792 }
1793 count = pos - *ppos;
1794 *ppos = pos;
1795 return count;
1796}
1797
1798static const struct file_operations mem_debugfs_fops = {
1799 .owner = THIS_MODULE,
1800 .open = mem_open,
1801 .read = mem_read,
1802};
1803
1804static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
1805 unsigned int idx, unsigned int size_mb)
1806{
1807 struct dentry *de;
1808
1809 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
1810 (void *)adap + idx, &mem_debugfs_fops);
1811 if (de && de->d_inode)
1812 de->d_inode->i_size = size_mb << 20;
1813}
1814
1815static int __devinit setup_debugfs(struct adapter *adap)
1816{
1817 int i;
1818
1819 if (IS_ERR_OR_NULL(adap->debugfs_root))
1820 return -1;
1821
1822 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
1823 if (i & EDRAM0_ENABLE)
1824 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
1825 if (i & EDRAM1_ENABLE)
1826 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
1827 if (i & EXT_MEM_ENABLE)
1828 add_debugfs_mem(adap, "mc", MEM_MC,
1829 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
1830 if (adap->l2t)
1831 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
1832 &t4_l2t_fops);
1833 return 0;
1834}
1835
1836/*
1837 * upper-layer driver support
1838 */
1839
1840/*
1841 * Allocate an active-open TID and set it to the supplied value.
1842 */
1843int cxgb4_alloc_atid(struct tid_info *t, void *data)
1844{
1845 int atid = -1;
1846
1847 spin_lock_bh(&t->atid_lock);
1848 if (t->afree) {
1849 union aopen_entry *p = t->afree;
1850
1851 atid = p - t->atid_tab;
1852 t->afree = p->next;
1853 p->data = data;
1854 t->atids_in_use++;
1855 }
1856 spin_unlock_bh(&t->atid_lock);
1857 return atid;
1858}
1859EXPORT_SYMBOL(cxgb4_alloc_atid);
1860
1861/*
1862 * Release an active-open TID.
1863 */
1864void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
1865{
1866 union aopen_entry *p = &t->atid_tab[atid];
1867
1868 spin_lock_bh(&t->atid_lock);
1869 p->next = t->afree;
1870 t->afree = p;
1871 t->atids_in_use--;
1872 spin_unlock_bh(&t->atid_lock);
1873}
1874EXPORT_SYMBOL(cxgb4_free_atid);
1875
1876/*
1877 * Allocate a server TID and set it to the supplied value.
1878 */
1879int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
1880{
1881 int stid;
1882
1883 spin_lock_bh(&t->stid_lock);
1884 if (family == PF_INET) {
1885 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
1886 if (stid < t->nstids)
1887 __set_bit(stid, t->stid_bmap);
1888 else
1889 stid = -1;
1890 } else {
1891 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
1892 if (stid < 0)
1893 stid = -1;
1894 }
1895 if (stid >= 0) {
1896 t->stid_tab[stid].data = data;
1897 stid += t->stid_base;
1898 t->stids_in_use++;
1899 }
1900 spin_unlock_bh(&t->stid_lock);
1901 return stid;
1902}
1903EXPORT_SYMBOL(cxgb4_alloc_stid);
1904
1905/*
1906 * Release a server TID.
1907 */
1908void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
1909{
1910 stid -= t->stid_base;
1911 spin_lock_bh(&t->stid_lock);
1912 if (family == PF_INET)
1913 __clear_bit(stid, t->stid_bmap);
1914 else
1915 bitmap_release_region(t->stid_bmap, stid, 2);
1916 t->stid_tab[stid].data = NULL;
1917 t->stids_in_use--;
1918 spin_unlock_bh(&t->stid_lock);
1919}
1920EXPORT_SYMBOL(cxgb4_free_stid);
1921
1922/*
1923 * Populate a TID_RELEASE WR. Caller must properly size the skb.
1924 */
1925static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
1926 unsigned int tid)
1927{
1928 struct cpl_tid_release *req;
1929
1930 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1931 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
1932 INIT_TP_WR(req, tid);
1933 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
1934}
1935
1936/*
1937 * Queue a TID release request and if necessary schedule a work queue to
1938 * process it.
1939 */
1940void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
1941 unsigned int tid)
1942{
1943 void **p = &t->tid_tab[tid];
1944 struct adapter *adap = container_of(t, struct adapter, tids);
1945
1946 spin_lock_bh(&adap->tid_release_lock);
1947 *p = adap->tid_release_head;
1948 /* Low 2 bits encode the Tx channel number */
1949 adap->tid_release_head = (void **)((uintptr_t)p | chan);
1950 if (!adap->tid_release_task_busy) {
1951 adap->tid_release_task_busy = true;
1952 schedule_work(&adap->tid_release_task);
1953 }
1954 spin_unlock_bh(&adap->tid_release_lock);
1955}
1956EXPORT_SYMBOL(cxgb4_queue_tid_release);
1957
1958/*
1959 * Process the list of pending TID release requests.
1960 */
1961static void process_tid_release_list(struct work_struct *work)
1962{
1963 struct sk_buff *skb;
1964 struct adapter *adap;
1965
1966 adap = container_of(work, struct adapter, tid_release_task);
1967
1968 spin_lock_bh(&adap->tid_release_lock);
1969 while (adap->tid_release_head) {
1970 void **p = adap->tid_release_head;
1971 unsigned int chan = (uintptr_t)p & 3;
1972 p = (void *)p - chan;
1973
1974 adap->tid_release_head = *p;
1975 *p = NULL;
1976 spin_unlock_bh(&adap->tid_release_lock);
1977
1978 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
1979 GFP_KERNEL)))
1980 schedule_timeout_uninterruptible(1);
1981
1982 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
1983 t4_ofld_send(adap, skb);
1984 spin_lock_bh(&adap->tid_release_lock);
1985 }
1986 adap->tid_release_task_busy = false;
1987 spin_unlock_bh(&adap->tid_release_lock);
1988}
1989
1990/*
1991 * Release a TID and inform HW. If we are unable to allocate the release
1992 * message we defer to a work queue.
1993 */
1994void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
1995{
1996 void *old;
1997 struct sk_buff *skb;
1998 struct adapter *adap = container_of(t, struct adapter, tids);
1999
2000 old = t->tid_tab[tid];
2001 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2002 if (likely(skb)) {
2003 t->tid_tab[tid] = NULL;
2004 mk_tid_release(skb, chan, tid);
2005 t4_ofld_send(adap, skb);
2006 } else
2007 cxgb4_queue_tid_release(t, chan, tid);
2008 if (old)
2009 atomic_dec(&t->tids_in_use);
2010}
2011EXPORT_SYMBOL(cxgb4_remove_tid);
2012
2013/*
2014 * Allocate and initialize the TID tables. Returns 0 on success.
2015 */
2016static int tid_init(struct tid_info *t)
2017{
2018 size_t size;
2019 unsigned int natids = t->natids;
2020
2021 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2022 t->nstids * sizeof(*t->stid_tab) +
2023 BITS_TO_LONGS(t->nstids) * sizeof(long);
2024 t->tid_tab = t4_alloc_mem(size);
2025 if (!t->tid_tab)
2026 return -ENOMEM;
2027
2028 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2029 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2030 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2031 spin_lock_init(&t->stid_lock);
2032 spin_lock_init(&t->atid_lock);
2033
2034 t->stids_in_use = 0;
2035 t->afree = NULL;
2036 t->atids_in_use = 0;
2037 atomic_set(&t->tids_in_use, 0);
2038
2039 /* Setup the free list for atid_tab and clear the stid bitmap. */
2040 if (natids) {
2041 while (--natids)
2042 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2043 t->afree = t->atid_tab;
2044 }
2045 bitmap_zero(t->stid_bmap, t->nstids);
2046 return 0;
2047}
2048
2049/**
2050 * cxgb4_create_server - create an IP server
2051 * @dev: the device
2052 * @stid: the server TID
2053 * @sip: local IP address to bind server to
2054 * @sport: the server's TCP port
2055 * @queue: queue to direct messages from this server to
2056 *
2057 * Create an IP server for the given port and address.
2058 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2059 */
2060int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2061 __be32 sip, __be16 sport, unsigned int queue)
2062{
2063 unsigned int chan;
2064 struct sk_buff *skb;
2065 struct adapter *adap;
2066 struct cpl_pass_open_req *req;
2067
2068 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2069 if (!skb)
2070 return -ENOMEM;
2071
2072 adap = netdev2adap(dev);
2073 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2074 INIT_TP_WR(req, 0);
2075 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2076 req->local_port = sport;
2077 req->peer_port = htons(0);
2078 req->local_ip = sip;
2079 req->peer_ip = htonl(0);
2080 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2081 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2082 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2083 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2084 return t4_mgmt_tx(adap, skb);
2085}
2086EXPORT_SYMBOL(cxgb4_create_server);
2087
2088/**
2089 * cxgb4_create_server6 - create an IPv6 server
2090 * @dev: the device
2091 * @stid: the server TID
2092 * @sip: local IPv6 address to bind server to
2093 * @sport: the server's TCP port
2094 * @queue: queue to direct messages from this server to
2095 *
2096 * Create an IPv6 server for the given port and address.
2097 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2098 */
2099int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2100 const struct in6_addr *sip, __be16 sport,
2101 unsigned int queue)
2102{
2103 unsigned int chan;
2104 struct sk_buff *skb;
2105 struct adapter *adap;
2106 struct cpl_pass_open_req6 *req;
2107
2108 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2109 if (!skb)
2110 return -ENOMEM;
2111
2112 adap = netdev2adap(dev);
2113 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2114 INIT_TP_WR(req, 0);
2115 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2116 req->local_port = sport;
2117 req->peer_port = htons(0);
2118 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2119 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2120 req->peer_ip_hi = cpu_to_be64(0);
2121 req->peer_ip_lo = cpu_to_be64(0);
2122 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2123 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2124 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2125 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2126 return t4_mgmt_tx(adap, skb);
2127}
2128EXPORT_SYMBOL(cxgb4_create_server6);
2129
2130/**
2131 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2132 * @mtus: the HW MTU table
2133 * @mtu: the target MTU
2134 * @idx: index of selected entry in the MTU table
2135 *
2136 * Returns the index and the value in the HW MTU table that is closest to
2137 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2138 * table, in which case that smallest available value is selected.
2139 */
2140unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2141 unsigned int *idx)
2142{
2143 unsigned int i = 0;
2144
2145 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2146 ++i;
2147 if (idx)
2148 *idx = i;
2149 return mtus[i];
2150}
2151EXPORT_SYMBOL(cxgb4_best_mtu);
2152
2153/**
2154 * cxgb4_port_chan - get the HW channel of a port
2155 * @dev: the net device for the port
2156 *
2157 * Return the HW Tx channel of the given port.
2158 */
2159unsigned int cxgb4_port_chan(const struct net_device *dev)
2160{
2161 return netdev2pinfo(dev)->tx_chan;
2162}
2163EXPORT_SYMBOL(cxgb4_port_chan);
2164
2165/**
2166 * cxgb4_port_viid - get the VI id of a port
2167 * @dev: the net device for the port
2168 *
2169 * Return the VI id of the given port.
2170 */
2171unsigned int cxgb4_port_viid(const struct net_device *dev)
2172{
2173 return netdev2pinfo(dev)->viid;
2174}
2175EXPORT_SYMBOL(cxgb4_port_viid);
2176
2177/**
2178 * cxgb4_port_idx - get the index of a port
2179 * @dev: the net device for the port
2180 *
2181 * Return the index of the given port.
2182 */
2183unsigned int cxgb4_port_idx(const struct net_device *dev)
2184{
2185 return netdev2pinfo(dev)->port_id;
2186}
2187EXPORT_SYMBOL(cxgb4_port_idx);
2188
2189/**
2190 * cxgb4_netdev_by_hwid - return the net device of a HW port
2191 * @pdev: identifies the adapter
2192 * @id: the HW port id
2193 *
2194 * Return the net device associated with the interface with the given HW
2195 * id.
2196 */
2197struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2198{
2199 const struct adapter *adap = pci_get_drvdata(pdev);
2200
2201 if (!adap || id >= NCHAN)
2202 return NULL;
2203 id = adap->chan_map[id];
2204 return id < MAX_NPORTS ? adap->port[id] : NULL;
2205}
2206EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2207
2208void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2209 struct tp_tcp_stats *v6)
2210{
2211 struct adapter *adap = pci_get_drvdata(pdev);
2212
2213 spin_lock(&adap->stats_lock);
2214 t4_tp_get_tcp_stats(adap, v4, v6);
2215 spin_unlock(&adap->stats_lock);
2216}
2217EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2218
2219void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2220 const unsigned int *pgsz_order)
2221{
2222 struct adapter *adap = netdev2adap(dev);
2223
2224 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2225 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2226 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2227 HPZ3(pgsz_order[3]));
2228}
2229EXPORT_SYMBOL(cxgb4_iscsi_init);
2230
2231static struct pci_driver cxgb4_driver;
2232
2233static void check_neigh_update(struct neighbour *neigh)
2234{
2235 const struct device *parent;
2236 const struct net_device *netdev = neigh->dev;
2237
2238 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2239 netdev = vlan_dev_real_dev(netdev);
2240 parent = netdev->dev.parent;
2241 if (parent && parent->driver == &cxgb4_driver.driver)
2242 t4_l2t_update(dev_get_drvdata(parent), neigh);
2243}
2244
2245static int netevent_cb(struct notifier_block *nb, unsigned long event,
2246 void *data)
2247{
2248 switch (event) {
2249 case NETEVENT_NEIGH_UPDATE:
2250 check_neigh_update(data);
2251 break;
2252 case NETEVENT_PMTU_UPDATE:
2253 case NETEVENT_REDIRECT:
2254 default:
2255 break;
2256 }
2257 return 0;
2258}
2259
2260static bool netevent_registered;
2261static struct notifier_block cxgb4_netevent_nb = {
2262 .notifier_call = netevent_cb
2263};
2264
2265static void uld_attach(struct adapter *adap, unsigned int uld)
2266{
2267 void *handle;
2268 struct cxgb4_lld_info lli;
2269
2270 lli.pdev = adap->pdev;
2271 lli.l2t = adap->l2t;
2272 lli.tids = &adap->tids;
2273 lli.ports = adap->port;
2274 lli.vr = &adap->vres;
2275 lli.mtus = adap->params.mtus;
2276 if (uld == CXGB4_ULD_RDMA) {
2277 lli.rxq_ids = adap->sge.rdma_rxq;
2278 lli.nrxq = adap->sge.rdmaqs;
2279 } else if (uld == CXGB4_ULD_ISCSI) {
2280 lli.rxq_ids = adap->sge.ofld_rxq;
2281 lli.nrxq = adap->sge.ofldqsets;
2282 }
2283 lli.ntxq = adap->sge.ofldqsets;
2284 lli.nchan = adap->params.nports;
2285 lli.nports = adap->params.nports;
2286 lli.wr_cred = adap->params.ofldq_wr_cred;
2287 lli.adapter_type = adap->params.rev;
2288 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2289 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2290 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF));
2291 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2292 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF));
2293 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2294 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2295 lli.fw_vers = adap->params.fw_vers;
2296
2297 handle = ulds[uld].add(&lli);
2298 if (IS_ERR(handle)) {
2299 dev_warn(adap->pdev_dev,
2300 "could not attach to the %s driver, error %ld\n",
2301 uld_str[uld], PTR_ERR(handle));
2302 return;
2303 }
2304
2305 adap->uld_handle[uld] = handle;
2306
2307 if (!netevent_registered) {
2308 register_netevent_notifier(&cxgb4_netevent_nb);
2309 netevent_registered = true;
2310 }
2311}
2312
2313static void attach_ulds(struct adapter *adap)
2314{
2315 unsigned int i;
2316
2317 mutex_lock(&uld_mutex);
2318 list_add_tail(&adap->list_node, &adapter_list);
2319 for (i = 0; i < CXGB4_ULD_MAX; i++)
2320 if (ulds[i].add)
2321 uld_attach(adap, i);
2322 mutex_unlock(&uld_mutex);
2323}
2324
2325static void detach_ulds(struct adapter *adap)
2326{
2327 unsigned int i;
2328
2329 mutex_lock(&uld_mutex);
2330 list_del(&adap->list_node);
2331 for (i = 0; i < CXGB4_ULD_MAX; i++)
2332 if (adap->uld_handle[i]) {
2333 ulds[i].state_change(adap->uld_handle[i],
2334 CXGB4_STATE_DETACH);
2335 adap->uld_handle[i] = NULL;
2336 }
2337 if (netevent_registered && list_empty(&adapter_list)) {
2338 unregister_netevent_notifier(&cxgb4_netevent_nb);
2339 netevent_registered = false;
2340 }
2341 mutex_unlock(&uld_mutex);
2342}
2343
2344static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2345{
2346 unsigned int i;
2347
2348 mutex_lock(&uld_mutex);
2349 for (i = 0; i < CXGB4_ULD_MAX; i++)
2350 if (adap->uld_handle[i])
2351 ulds[i].state_change(adap->uld_handle[i], new_state);
2352 mutex_unlock(&uld_mutex);
2353}
2354
2355/**
2356 * cxgb4_register_uld - register an upper-layer driver
2357 * @type: the ULD type
2358 * @p: the ULD methods
2359 *
2360 * Registers an upper-layer driver with this driver and notifies the ULD
2361 * about any presently available devices that support its type. Returns
2362 * %-EBUSY if a ULD of the same type is already registered.
2363 */
2364int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2365{
2366 int ret = 0;
2367 struct adapter *adap;
2368
2369 if (type >= CXGB4_ULD_MAX)
2370 return -EINVAL;
2371 mutex_lock(&uld_mutex);
2372 if (ulds[type].add) {
2373 ret = -EBUSY;
2374 goto out;
2375 }
2376 ulds[type] = *p;
2377 list_for_each_entry(adap, &adapter_list, list_node)
2378 uld_attach(adap, type);
2379out: mutex_unlock(&uld_mutex);
2380 return ret;
2381}
2382EXPORT_SYMBOL(cxgb4_register_uld);
2383
2384/**
2385 * cxgb4_unregister_uld - unregister an upper-layer driver
2386 * @type: the ULD type
2387 *
2388 * Unregisters an existing upper-layer driver.
2389 */
2390int cxgb4_unregister_uld(enum cxgb4_uld type)
2391{
2392 struct adapter *adap;
2393
2394 if (type >= CXGB4_ULD_MAX)
2395 return -EINVAL;
2396 mutex_lock(&uld_mutex);
2397 list_for_each_entry(adap, &adapter_list, list_node)
2398 adap->uld_handle[type] = NULL;
2399 ulds[type].add = NULL;
2400 mutex_unlock(&uld_mutex);
2401 return 0;
2402}
2403EXPORT_SYMBOL(cxgb4_unregister_uld);
2404
2405/**
2406 * cxgb_up - enable the adapter
2407 * @adap: adapter being enabled
2408 *
2409 * Called when the first port is enabled, this function performs the
2410 * actions necessary to make an adapter operational, such as completing
2411 * the initialization of HW modules, and enabling interrupts.
2412 *
2413 * Must be called with the rtnl lock held.
2414 */
2415static int cxgb_up(struct adapter *adap)
2416{
2417 int err = 0;
2418
2419 if (!(adap->flags & FULL_INIT_DONE)) {
2420 err = setup_sge_queues(adap);
2421 if (err)
2422 goto out;
2423 err = setup_rss(adap);
2424 if (err) {
2425 t4_free_sge_resources(adap);
2426 goto out;
2427 }
2428 if (adap->flags & USING_MSIX)
2429 name_msix_vecs(adap);
2430 adap->flags |= FULL_INIT_DONE;
2431 }
2432
2433 if (adap->flags & USING_MSIX) {
2434 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2435 adap->msix_info[0].desc, adap);
2436 if (err)
2437 goto irq_err;
2438
2439 err = request_msix_queue_irqs(adap);
2440 if (err) {
2441 free_irq(adap->msix_info[0].vec, adap);
2442 goto irq_err;
2443 }
2444 } else {
2445 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2446 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2447 adap->name, adap);
2448 if (err)
2449 goto irq_err;
2450 }
2451 enable_rx(adap);
2452 t4_sge_start(adap);
2453 t4_intr_enable(adap);
2454 notify_ulds(adap, CXGB4_STATE_UP);
2455 out:
2456 return err;
2457 irq_err:
2458 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2459 goto out;
2460}
2461
2462static void cxgb_down(struct adapter *adapter)
2463{
2464 t4_intr_disable(adapter);
2465 cancel_work_sync(&adapter->tid_release_task);
2466 adapter->tid_release_task_busy = false;
2467
2468 if (adapter->flags & USING_MSIX) {
2469 free_msix_queue_irqs(adapter);
2470 free_irq(adapter->msix_info[0].vec, adapter);
2471 } else
2472 free_irq(adapter->pdev->irq, adapter);
2473 quiesce_rx(adapter);
2474}
2475
2476/*
2477 * net_device operations
2478 */
2479static int cxgb_open(struct net_device *dev)
2480{
2481 int err;
2482 struct port_info *pi = netdev_priv(dev);
2483 struct adapter *adapter = pi->adapter;
2484
2485 if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
2486 return err;
2487
2488 dev->real_num_tx_queues = pi->nqsets;
2489 set_bit(pi->tx_chan, &adapter->open_device_map);
2490 link_start(dev);
2491 netif_tx_start_all_queues(dev);
2492 return 0;
2493}
2494
2495static int cxgb_close(struct net_device *dev)
2496{
2497 int ret;
2498 struct port_info *pi = netdev_priv(dev);
2499 struct adapter *adapter = pi->adapter;
2500
2501 netif_tx_stop_all_queues(dev);
2502 netif_carrier_off(dev);
2503 ret = t4_enable_vi(adapter, 0, pi->viid, false, false);
2504
2505 clear_bit(pi->tx_chan, &adapter->open_device_map);
2506
2507 if (!adapter->open_device_map)
2508 cxgb_down(adapter);
2509 return 0;
2510}
2511
2512static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
2513{
2514 struct port_stats stats;
2515 struct port_info *p = netdev_priv(dev);
2516 struct adapter *adapter = p->adapter;
2517 struct net_device_stats *ns = &dev->stats;
2518
2519 spin_lock(&adapter->stats_lock);
2520 t4_get_port_stats(adapter, p->tx_chan, &stats);
2521 spin_unlock(&adapter->stats_lock);
2522
2523 ns->tx_bytes = stats.tx_octets;
2524 ns->tx_packets = stats.tx_frames;
2525 ns->rx_bytes = stats.rx_octets;
2526 ns->rx_packets = stats.rx_frames;
2527 ns->multicast = stats.rx_mcast_frames;
2528
2529 /* detailed rx_errors */
2530 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2531 stats.rx_runt;
2532 ns->rx_over_errors = 0;
2533 ns->rx_crc_errors = stats.rx_fcs_err;
2534 ns->rx_frame_errors = stats.rx_symbol_err;
2535 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2536 stats.rx_ovflow2 + stats.rx_ovflow3 +
2537 stats.rx_trunc0 + stats.rx_trunc1 +
2538 stats.rx_trunc2 + stats.rx_trunc3;
2539 ns->rx_missed_errors = 0;
2540
2541 /* detailed tx_errors */
2542 ns->tx_aborted_errors = 0;
2543 ns->tx_carrier_errors = 0;
2544 ns->tx_fifo_errors = 0;
2545 ns->tx_heartbeat_errors = 0;
2546 ns->tx_window_errors = 0;
2547
2548 ns->tx_errors = stats.tx_error_frames;
2549 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2550 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2551 return ns;
2552}
2553
2554static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2555{
2556 int ret = 0, prtad, devad;
2557 struct port_info *pi = netdev_priv(dev);
2558 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2559
2560 switch (cmd) {
2561 case SIOCGMIIPHY:
2562 if (pi->mdio_addr < 0)
2563 return -EOPNOTSUPP;
2564 data->phy_id = pi->mdio_addr;
2565 break;
2566 case SIOCGMIIREG:
2567 case SIOCSMIIREG:
2568 if (mdio_phy_id_is_c45(data->phy_id)) {
2569 prtad = mdio_phy_id_prtad(data->phy_id);
2570 devad = mdio_phy_id_devad(data->phy_id);
2571 } else if (data->phy_id < 32) {
2572 prtad = data->phy_id;
2573 devad = 0;
2574 data->reg_num &= 0x1f;
2575 } else
2576 return -EINVAL;
2577
2578 if (cmd == SIOCGMIIREG)
2579 ret = t4_mdio_rd(pi->adapter, 0, prtad, devad,
2580 data->reg_num, &data->val_out);
2581 else
2582 ret = t4_mdio_wr(pi->adapter, 0, prtad, devad,
2583 data->reg_num, data->val_in);
2584 break;
2585 default:
2586 return -EOPNOTSUPP;
2587 }
2588 return ret;
2589}
2590
2591static void cxgb_set_rxmode(struct net_device *dev)
2592{
2593 /* unfortunately we can't return errors to the stack */
2594 set_rxmode(dev, -1, false);
2595}
2596
2597static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2598{
2599 int ret;
2600 struct port_info *pi = netdev_priv(dev);
2601
2602 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2603 return -EINVAL;
2604 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, new_mtu, -1, -1, -1,
2605 true);
2606 if (!ret)
2607 dev->mtu = new_mtu;
2608 return ret;
2609}
2610
2611static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2612{
2613 int ret;
2614 struct sockaddr *addr = p;
2615 struct port_info *pi = netdev_priv(dev);
2616
2617 if (!is_valid_ether_addr(addr->sa_data))
2618 return -EINVAL;
2619
2620 ret = t4_change_mac(pi->adapter, 0, pi->viid, pi->xact_addr_filt,
2621 addr->sa_data, true, true);
2622 if (ret < 0)
2623 return ret;
2624
2625 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2626 pi->xact_addr_filt = ret;
2627 return 0;
2628}
2629
2630static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2631{
2632 struct port_info *pi = netdev_priv(dev);
2633
2634 pi->vlan_grp = grp;
2635 t4_set_vlan_accel(pi->adapter, 1 << pi->tx_chan, grp != NULL);
2636}
2637
2638#ifdef CONFIG_NET_POLL_CONTROLLER
2639static void cxgb_netpoll(struct net_device *dev)
2640{
2641 struct port_info *pi = netdev_priv(dev);
2642 struct adapter *adap = pi->adapter;
2643
2644 if (adap->flags & USING_MSIX) {
2645 int i;
2646 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2647
2648 for (i = pi->nqsets; i; i--, rx++)
2649 t4_sge_intr_msix(0, &rx->rspq);
2650 } else
2651 t4_intr_handler(adap)(0, adap);
2652}
2653#endif
2654
2655static const struct net_device_ops cxgb4_netdev_ops = {
2656 .ndo_open = cxgb_open,
2657 .ndo_stop = cxgb_close,
2658 .ndo_start_xmit = t4_eth_xmit,
2659 .ndo_get_stats = cxgb_get_stats,
2660 .ndo_set_rx_mode = cxgb_set_rxmode,
2661 .ndo_set_mac_address = cxgb_set_mac_addr,
2662 .ndo_validate_addr = eth_validate_addr,
2663 .ndo_do_ioctl = cxgb_ioctl,
2664 .ndo_change_mtu = cxgb_change_mtu,
2665 .ndo_vlan_rx_register = vlan_rx_register,
2666#ifdef CONFIG_NET_POLL_CONTROLLER
2667 .ndo_poll_controller = cxgb_netpoll,
2668#endif
2669};
2670
2671void t4_fatal_err(struct adapter *adap)
2672{
2673 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2674 t4_intr_disable(adap);
2675 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2676}
2677
2678static void setup_memwin(struct adapter *adap)
2679{
2680 u32 bar0;
2681
2682 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2683 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2684 (bar0 + MEMWIN0_BASE) | BIR(0) |
2685 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2686 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2687 (bar0 + MEMWIN1_BASE) | BIR(0) |
2688 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2689 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2690 (bar0 + MEMWIN2_BASE) | BIR(0) |
2691 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2692}
2693
2694/*
2695 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2696 */
2697#define MAX_ATIDS 8192U
2698
2699/*
2700 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2701 */
2702static int adap_init0(struct adapter *adap)
2703{
2704 int ret;
2705 u32 v, port_vec;
2706 enum dev_state state;
2707 u32 params[7], val[7];
2708 struct fw_caps_config_cmd c;
2709
2710 ret = t4_check_fw_version(adap);
2711 if (ret == -EINVAL || ret > 0) {
2712 if (upgrade_fw(adap) >= 0) /* recache FW version */
2713 ret = t4_check_fw_version(adap);
2714 }
2715 if (ret < 0)
2716 return ret;
2717
2718 /* contact FW, request master */
2719 ret = t4_fw_hello(adap, 0, 0, MASTER_MUST, &state);
2720 if (ret < 0) {
2721 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2722 ret);
2723 return ret;
2724 }
2725
2726 /* reset device */
2727 ret = t4_fw_reset(adap, 0, PIORSTMODE | PIORST);
2728 if (ret < 0)
2729 goto bye;
2730
2731 /* get device capabilities */
2732 memset(&c, 0, sizeof(c));
2733 c.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2734 FW_CMD_REQUEST | FW_CMD_READ);
2735 c.retval_len16 = htonl(FW_LEN16(c));
2736 ret = t4_wr_mbox(adap, 0, &c, sizeof(c), &c);
2737 if (ret < 0)
2738 goto bye;
2739
2740 /* select capabilities we'll be using */
2741 if (c.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2742 if (!vf_acls)
2743 c.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2744 else
2745 c.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2746 } else if (vf_acls) {
2747 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2748 goto bye;
2749 }
2750 c.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2751 FW_CMD_REQUEST | FW_CMD_WRITE);
2752 ret = t4_wr_mbox(adap, 0, &c, sizeof(c), NULL);
2753 if (ret < 0)
2754 goto bye;
2755
2756 ret = t4_config_glbl_rss(adap, 0,
2757 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2758 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2759 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2760 if (ret < 0)
2761 goto bye;
2762
2763 ret = t4_cfg_pfvf(adap, 0, 0, 0, 64, 64, 64, 0, 0, 4, 0xf, 0xf, 16,
2764 FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2765 if (ret < 0)
2766 goto bye;
2767
2768 for (v = 0; v < SGE_NTIMERS - 1; v++)
2769 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
2770 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
2771 adap->sge.counter_val[0] = 1;
2772 for (v = 1; v < SGE_NCOUNTERS; v++)
2773 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
2774 THRESHOLD_3_MASK);
2775 t4_sge_init(adap);
2776
2777 /* get basic stuff going */
2778 ret = t4_early_init(adap, 0);
2779 if (ret < 0)
2780 goto bye;
2781
2782#define FW_PARAM_DEV(param) \
2783 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2784 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2785
2786#define FW_PARAM_PFVF(param) \
2787 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2788 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2789
2790 params[0] = FW_PARAM_DEV(PORTVEC);
2791 params[1] = FW_PARAM_PFVF(L2T_START);
2792 params[2] = FW_PARAM_PFVF(L2T_END);
2793 params[3] = FW_PARAM_PFVF(FILTER_START);
2794 params[4] = FW_PARAM_PFVF(FILTER_END);
2795 ret = t4_query_params(adap, 0, 0, 0, 5, params, val);
2796 if (ret < 0)
2797 goto bye;
2798 port_vec = val[0];
2799 adap->tids.ftid_base = val[3];
2800 adap->tids.nftids = val[4] - val[3] + 1;
2801
2802 if (c.ofldcaps) {
2803 /* query offload-related parameters */
2804 params[0] = FW_PARAM_DEV(NTID);
2805 params[1] = FW_PARAM_PFVF(SERVER_START);
2806 params[2] = FW_PARAM_PFVF(SERVER_END);
2807 params[3] = FW_PARAM_PFVF(TDDP_START);
2808 params[4] = FW_PARAM_PFVF(TDDP_END);
2809 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2810 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
2811 if (ret < 0)
2812 goto bye;
2813 adap->tids.ntids = val[0];
2814 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
2815 adap->tids.stid_base = val[1];
2816 adap->tids.nstids = val[2] - val[1] + 1;
2817 adap->vres.ddp.start = val[3];
2818 adap->vres.ddp.size = val[4] - val[3] + 1;
2819 adap->params.ofldq_wr_cred = val[5];
2820 adap->params.offload = 1;
2821 }
2822 if (c.rdmacaps) {
2823 params[0] = FW_PARAM_PFVF(STAG_START);
2824 params[1] = FW_PARAM_PFVF(STAG_END);
2825 params[2] = FW_PARAM_PFVF(RQ_START);
2826 params[3] = FW_PARAM_PFVF(RQ_END);
2827 params[4] = FW_PARAM_PFVF(PBL_START);
2828 params[5] = FW_PARAM_PFVF(PBL_END);
2829 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
2830 if (ret < 0)
2831 goto bye;
2832 adap->vres.stag.start = val[0];
2833 adap->vres.stag.size = val[1] - val[0] + 1;
2834 adap->vres.rq.start = val[2];
2835 adap->vres.rq.size = val[3] - val[2] + 1;
2836 adap->vres.pbl.start = val[4];
2837 adap->vres.pbl.size = val[5] - val[4] + 1;
2838 }
2839 if (c.iscsicaps) {
2840 params[0] = FW_PARAM_PFVF(ISCSI_START);
2841 params[1] = FW_PARAM_PFVF(ISCSI_END);
2842 ret = t4_query_params(adap, 0, 0, 0, 2, params, val);
2843 if (ret < 0)
2844 goto bye;
2845 adap->vres.iscsi.start = val[0];
2846 adap->vres.iscsi.size = val[1] - val[0] + 1;
2847 }
2848#undef FW_PARAM_PFVF
2849#undef FW_PARAM_DEV
2850
2851 adap->params.nports = hweight32(port_vec);
2852 adap->params.portvec = port_vec;
2853 adap->flags |= FW_OK;
2854
2855 /* These are finalized by FW initialization, load their values now */
2856 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
2857 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
2858 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
2859 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
2860 adap->params.b_wnd);
2861
2862 /* tweak some settings */
2863 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2864 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2865 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2866 v = t4_read_reg(adap, TP_PIO_DATA);
2867 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2868 setup_memwin(adap);
2869 return 0;
2870
2871 /*
2872 * If a command timed out or failed with EIO FW does not operate within
2873 * its spec or something catastrophic happened to HW/FW, stop issuing
2874 * commands.
2875 */
2876bye: if (ret != -ETIMEDOUT && ret != -EIO)
2877 t4_fw_bye(adap, 0);
2878 return ret;
2879}
2880
2881static inline bool is_10g_port(const struct link_config *lc)
2882{
2883 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
2884}
2885
2886static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
2887 unsigned int size, unsigned int iqe_size)
2888{
2889 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
2890 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
2891 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
2892 q->iqe_len = iqe_size;
2893 q->size = size;
2894}
2895
2896/*
2897 * Perform default configuration of DMA queues depending on the number and type
2898 * of ports we found and the number of available CPUs. Most settings can be
2899 * modified by the admin prior to actual use.
2900 */
2901static void __devinit cfg_queues(struct adapter *adap)
2902{
2903 struct sge *s = &adap->sge;
2904 int i, q10g = 0, n10g = 0, qidx = 0;
2905
2906 for_each_port(adap, i)
2907 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
2908
2909 /*
2910 * We default to 1 queue per non-10G port and up to # of cores queues
2911 * per 10G port.
2912 */
2913 if (n10g)
2914 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
2915 if (q10g > num_online_cpus())
2916 q10g = num_online_cpus();
2917
2918 for_each_port(adap, i) {
2919 struct port_info *pi = adap2pinfo(adap, i);
2920
2921 pi->first_qset = qidx;
2922 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
2923 qidx += pi->nqsets;
2924 }
2925
2926 s->ethqsets = qidx;
2927 s->max_ethqsets = qidx; /* MSI-X may lower it later */
2928
2929 if (is_offload(adap)) {
2930 /*
2931 * For offload we use 1 queue/channel if all ports are up to 1G,
2932 * otherwise we divide all available queues amongst the channels
2933 * capped by the number of available cores.
2934 */
2935 if (n10g) {
2936 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
2937 num_online_cpus());
2938 s->ofldqsets = roundup(i, adap->params.nports);
2939 } else
2940 s->ofldqsets = adap->params.nports;
2941 /* For RDMA one Rx queue per channel suffices */
2942 s->rdmaqs = adap->params.nports;
2943 }
2944
2945 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
2946 struct sge_eth_rxq *r = &s->ethrxq[i];
2947
2948 init_rspq(&r->rspq, 0, 0, 1024, 64);
2949 r->fl.size = 72;
2950 }
2951
2952 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
2953 s->ethtxq[i].q.size = 1024;
2954
2955 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
2956 s->ctrlq[i].q.size = 512;
2957
2958 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
2959 s->ofldtxq[i].q.size = 1024;
2960
2961 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
2962 struct sge_ofld_rxq *r = &s->ofldrxq[i];
2963
2964 init_rspq(&r->rspq, 0, 0, 1024, 64);
2965 r->rspq.uld = CXGB4_ULD_ISCSI;
2966 r->fl.size = 72;
2967 }
2968
2969 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
2970 struct sge_ofld_rxq *r = &s->rdmarxq[i];
2971
2972 init_rspq(&r->rspq, 0, 0, 511, 64);
2973 r->rspq.uld = CXGB4_ULD_RDMA;
2974 r->fl.size = 72;
2975 }
2976
2977 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
2978 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
2979}
2980
2981/*
2982 * Reduce the number of Ethernet queues across all ports to at most n.
2983 * n provides at least one queue per port.
2984 */
2985static void __devinit reduce_ethqs(struct adapter *adap, int n)
2986{
2987 int i;
2988 struct port_info *pi;
2989
2990 while (n < adap->sge.ethqsets)
2991 for_each_port(adap, i) {
2992 pi = adap2pinfo(adap, i);
2993 if (pi->nqsets > 1) {
2994 pi->nqsets--;
2995 adap->sge.ethqsets--;
2996 if (adap->sge.ethqsets <= n)
2997 break;
2998 }
2999 }
3000
3001 n = 0;
3002 for_each_port(adap, i) {
3003 pi = adap2pinfo(adap, i);
3004 pi->first_qset = n;
3005 n += pi->nqsets;
3006 }
3007}
3008
3009/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3010#define EXTRA_VECS 2
3011
3012static int __devinit enable_msix(struct adapter *adap)
3013{
3014 int ofld_need = 0;
3015 int i, err, want, need;
3016 struct sge *s = &adap->sge;
3017 unsigned int nchan = adap->params.nports;
3018 struct msix_entry entries[MAX_INGQ + 1];
3019
3020 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3021 entries[i].entry = i;
3022
3023 want = s->max_ethqsets + EXTRA_VECS;
3024 if (is_offload(adap)) {
3025 want += s->rdmaqs + s->ofldqsets;
3026 /* need nchan for each possible ULD */
3027 ofld_need = 2 * nchan;
3028 }
3029 need = adap->params.nports + EXTRA_VECS + ofld_need;
3030
3031 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3032 want = err;
3033
3034 if (!err) {
3035 /*
3036 * Distribute available vectors to the various queue groups.
3037 * Every group gets its minimum requirement and NIC gets top
3038 * priority for leftovers.
3039 */
3040 i = want - EXTRA_VECS - ofld_need;
3041 if (i < s->max_ethqsets) {
3042 s->max_ethqsets = i;
3043 if (i < s->ethqsets)
3044 reduce_ethqs(adap, i);
3045 }
3046 if (is_offload(adap)) {
3047 i = want - EXTRA_VECS - s->max_ethqsets;
3048 i -= ofld_need - nchan;
3049 s->ofldqsets = (i / nchan) * nchan; /* round down */
3050 }
3051 for (i = 0; i < want; ++i)
3052 adap->msix_info[i].vec = entries[i].vector;
3053 } else if (err > 0)
3054 dev_info(adap->pdev_dev,
3055 "only %d MSI-X vectors left, not using MSI-X\n", err);
3056 return err;
3057}
3058
3059#undef EXTRA_VECS
3060
3061static void __devinit print_port_info(struct adapter *adap)
3062{
3063 static const char *base[] = {
3064 "R", "KX4", "T", "KX", "T", "KR", "CX4"
3065 };
3066
3067 int i;
3068 char buf[80];
3069
3070 for_each_port(adap, i) {
3071 struct net_device *dev = adap->port[i];
3072 const struct port_info *pi = netdev_priv(dev);
3073 char *bufp = buf;
3074
3075 if (!test_bit(i, &adap->registered_device_map))
3076 continue;
3077
3078 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3079 bufp += sprintf(bufp, "100/");
3080 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3081 bufp += sprintf(bufp, "1000/");
3082 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3083 bufp += sprintf(bufp, "10G/");
3084 if (bufp != buf)
3085 --bufp;
3086 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3087
3088 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s\n",
3089 adap->params.vpd.id, adap->params.rev,
3090 buf, is_offload(adap) ? "R" : "",
3091 adap->params.pci.width,
3092 (adap->flags & USING_MSIX) ? " MSI-X" :
3093 (adap->flags & USING_MSI) ? " MSI" : "");
3094 if (adap->name == dev->name)
3095 netdev_info(dev, "S/N: %s, E/C: %s\n",
3096 adap->params.vpd.sn, adap->params.vpd.ec);
3097 }
3098}
3099
3100#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | NETIF_F_TSO6 |\
3101 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3102
3103static int __devinit init_one(struct pci_dev *pdev,
3104 const struct pci_device_id *ent)
3105{
3106 int func, i, err;
3107 struct port_info *pi;
3108 unsigned int highdma = 0;
3109 struct adapter *adapter = NULL;
3110
3111 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3112
3113 err = pci_request_regions(pdev, KBUILD_MODNAME);
3114 if (err) {
3115 /* Just info, some other driver may have claimed the device. */
3116 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3117 return err;
3118 }
3119
3120 /* We control everything through PF 0 */
3121 func = PCI_FUNC(pdev->devfn);
3122 if (func > 0)
3123 goto sriov;
3124
3125 err = pci_enable_device(pdev);
3126 if (err) {
3127 dev_err(&pdev->dev, "cannot enable PCI device\n");
3128 goto out_release_regions;
3129 }
3130
3131 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3132 highdma = NETIF_F_HIGHDMA;
3133 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3134 if (err) {
3135 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3136 "coherent allocations\n");
3137 goto out_disable_device;
3138 }
3139 } else {
3140 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3141 if (err) {
3142 dev_err(&pdev->dev, "no usable DMA configuration\n");
3143 goto out_disable_device;
3144 }
3145 }
3146
3147 pci_enable_pcie_error_reporting(pdev);
3148 pci_set_master(pdev);
3149 pci_save_state(pdev);
3150
3151 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3152 if (!adapter) {
3153 err = -ENOMEM;
3154 goto out_disable_device;
3155 }
3156
3157 adapter->regs = pci_ioremap_bar(pdev, 0);
3158 if (!adapter->regs) {
3159 dev_err(&pdev->dev, "cannot map device registers\n");
3160 err = -ENOMEM;
3161 goto out_free_adapter;
3162 }
3163
3164 adapter->pdev = pdev;
3165 adapter->pdev_dev = &pdev->dev;
3166 adapter->name = pci_name(pdev);
3167 adapter->msg_enable = dflt_msg_enable;
3168 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3169
3170 spin_lock_init(&adapter->stats_lock);
3171 spin_lock_init(&adapter->tid_release_lock);
3172
3173 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3174
3175 err = t4_prep_adapter(adapter);
3176 if (err)
3177 goto out_unmap_bar;
3178 err = adap_init0(adapter);
3179 if (err)
3180 goto out_unmap_bar;
3181
3182 for_each_port(adapter, i) {
3183 struct net_device *netdev;
3184
3185 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3186 MAX_ETH_QSETS);
3187 if (!netdev) {
3188 err = -ENOMEM;
3189 goto out_free_dev;
3190 }
3191
3192 SET_NETDEV_DEV(netdev, &pdev->dev);
3193
3194 adapter->port[i] = netdev;
3195 pi = netdev_priv(netdev);
3196 pi->adapter = adapter;
3197 pi->xact_addr_filt = -1;
3198 pi->rx_offload = RX_CSO;
3199 pi->port_id = i;
3200 netif_carrier_off(netdev);
3201 netif_tx_stop_all_queues(netdev);
3202 netdev->irq = pdev->irq;
3203
3204 netdev->features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
3205 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3206 netdev->features |= NETIF_F_GRO | highdma;
3207 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3208 netdev->vlan_features = netdev->features & VLAN_FEAT;
3209
3210 netdev->netdev_ops = &cxgb4_netdev_ops;
3211 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3212 }
3213
3214 pci_set_drvdata(pdev, adapter);
3215
3216 if (adapter->flags & FW_OK) {
3217 err = t4_port_init(adapter, 0, 0, 0);
3218 if (err)
3219 goto out_free_dev;
3220 }
3221
3222 /*
3223 * Configure queues and allocate tables now, they can be needed as
3224 * soon as the first register_netdev completes.
3225 */
3226 cfg_queues(adapter);
3227
3228 adapter->l2t = t4_init_l2t();
3229 if (!adapter->l2t) {
3230 /* We tolerate a lack of L2T, giving up some functionality */
3231 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3232 adapter->params.offload = 0;
3233 }
3234
3235 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3236 dev_warn(&pdev->dev, "could not allocate TID table, "
3237 "continuing\n");
3238 adapter->params.offload = 0;
3239 }
3240
3241 /*
3242 * The card is now ready to go. If any errors occur during device
3243 * registration we do not fail the whole card but rather proceed only
3244 * with the ports we manage to register successfully. However we must
3245 * register at least one net device.
3246 */
3247 for_each_port(adapter, i) {
3248 err = register_netdev(adapter->port[i]);
3249 if (err)
3250 dev_warn(&pdev->dev,
3251 "cannot register net device %s, skipping\n",
3252 adapter->port[i]->name);
3253 else {
3254 /*
3255 * Change the name we use for messages to the name of
3256 * the first successfully registered interface.
3257 */
3258 if (!adapter->registered_device_map)
3259 adapter->name = adapter->port[i]->name;
3260
3261 __set_bit(i, &adapter->registered_device_map);
3262 adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3263 }
3264 }
3265 if (!adapter->registered_device_map) {
3266 dev_err(&pdev->dev, "could not register any net devices\n");
3267 goto out_free_dev;
3268 }
3269
3270 if (cxgb4_debugfs_root) {
3271 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3272 cxgb4_debugfs_root);
3273 setup_debugfs(adapter);
3274 }
3275
3276 /* See what interrupts we'll be using */
3277 if (msi > 1 && enable_msix(adapter) == 0)
3278 adapter->flags |= USING_MSIX;
3279 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3280 adapter->flags |= USING_MSI;
3281
3282 if (is_offload(adapter))
3283 attach_ulds(adapter);
3284
3285 print_port_info(adapter);
3286
3287sriov:
3288#ifdef CONFIG_PCI_IOV
3289 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3290 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3291 dev_info(&pdev->dev,
3292 "instantiated %u virtual functions\n",
3293 num_vf[func]);
3294#endif
3295 return 0;
3296
3297 out_free_dev:
3298 t4_free_mem(adapter->tids.tid_tab);
3299 t4_free_mem(adapter->l2t);
3300 for_each_port(adapter, i)
3301 if (adapter->port[i])
3302 free_netdev(adapter->port[i]);
3303 if (adapter->flags & FW_OK)
3304 t4_fw_bye(adapter, 0);
3305 out_unmap_bar:
3306 iounmap(adapter->regs);
3307 out_free_adapter:
3308 kfree(adapter);
3309 out_disable_device:
3310 pci_disable_pcie_error_reporting(pdev);
3311 pci_disable_device(pdev);
3312 out_release_regions:
3313 pci_release_regions(pdev);
3314 pci_set_drvdata(pdev, NULL);
3315 return err;
3316}
3317
3318static void __devexit remove_one(struct pci_dev *pdev)
3319{
3320 struct adapter *adapter = pci_get_drvdata(pdev);
3321
3322 pci_disable_sriov(pdev);
3323
3324 if (adapter) {
3325 int i;
3326
3327 if (is_offload(adapter))
3328 detach_ulds(adapter);
3329
3330 for_each_port(adapter, i)
3331 if (test_bit(i, &adapter->registered_device_map))
3332 unregister_netdev(adapter->port[i]);
3333
3334 if (adapter->debugfs_root)
3335 debugfs_remove_recursive(adapter->debugfs_root);
3336
3337 t4_sge_stop(adapter);
3338 t4_free_sge_resources(adapter);
3339 t4_free_mem(adapter->l2t);
3340 t4_free_mem(adapter->tids.tid_tab);
3341 disable_msi(adapter);
3342
3343 for_each_port(adapter, i)
3344 if (adapter->port[i])
3345 free_netdev(adapter->port[i]);
3346
3347 if (adapter->flags & FW_OK)
3348 t4_fw_bye(adapter, 0);
3349 iounmap(adapter->regs);
3350 kfree(adapter);
3351 pci_disable_pcie_error_reporting(pdev);
3352 pci_disable_device(pdev);
3353 pci_release_regions(pdev);
3354 pci_set_drvdata(pdev, NULL);
3355 } else if (PCI_FUNC(pdev->devfn) > 0)
3356 pci_release_regions(pdev);
3357}
3358
3359static struct pci_driver cxgb4_driver = {
3360 .name = KBUILD_MODNAME,
3361 .id_table = cxgb4_pci_tbl,
3362 .probe = init_one,
3363 .remove = __devexit_p(remove_one),
3364};
3365
3366static int __init cxgb4_init_module(void)
3367{
3368 int ret;
3369
3370 /* Debugfs support is optional, just warn if this fails */
3371 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3372 if (!cxgb4_debugfs_root)
3373 pr_warning("could not create debugfs entry, continuing\n");
3374
3375 ret = pci_register_driver(&cxgb4_driver);
3376 if (ret < 0)
3377 debugfs_remove(cxgb4_debugfs_root);
3378 return ret;
3379}
3380
3381static void __exit cxgb4_cleanup_module(void)
3382{
3383 pci_unregister_driver(&cxgb4_driver);
3384 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3385}
3386
3387module_init(cxgb4_init_module);
3388module_exit(cxgb4_cleanup_module);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-24 12:18:29 -0500