aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/igb/igb_main.c
diff options
context:
space:
mode:
authorAuke Kok <auke-jan.h.kok@intel.com>2008-01-24 05:22:38 -0500
committerDavid S. Miller <davem@davemloft.net>2008-01-28 18:10:33 -0500
commit9d5c824399dea881779d78a6c147288bf2dccb6b (patch)
tree8c76b20c3cf1d81a63973e97578cea6a8a82a354 /drivers/net/igb/igb_main.c
parentb491edd5817f1618f4e06d67638739591a714bdb (diff)
igb: PCI-Express 82575 Gigabit Ethernet driver
We are pleased to announce a new Gigabit Ethernet product and its driver to the linux community. This product is the Intel(R) 82575 Gigabit Ethernet adapter family. Physical adapters will be available to the public soon. These adapters come in 2- and 4-port versions (copper PHY) currently. Other variants will be available later. The 82575 chipset supports significantly different features that warrant a new driver. The descriptor format is (just like the ixgbe driver) different. The device can use multiple MSI-X vectors and multiple queues for both send and receive. This allows us to optimize some of the driver code specifically as well compared to the e1000-supported devices. This version of the igb driver no lnger uses fake netdevices and incorporates napi_struct members for each ring to do the multi- queue polling. multi-queue is enabled by default and the driver supports NAPI mode only. All the namespace collisions should be gone in this version too. The register macro's have been condensed to improve readability. Signed-off-by: Auke Kok <auke-jan.h.kok@intel.com> Signed-off-by: Jeff Garzik <jgarzik@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers/net/igb/igb_main.c')
-rw-r--r--drivers/net/igb/igb_main.c4138
1 files changed, 4138 insertions, 0 deletions
diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c
new file mode 100644
index 000000000000..f3c144d5d72f
--- /dev/null
+++ b/drivers/net/igb/igb_main.c
@@ -0,0 +1,4138 @@
1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
34#include <linux/tcp.h>
35#include <linux/ipv6.h>
36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
42#include <linux/delay.h>
43#include <linux/interrupt.h>
44#include <linux/if_ether.h>
45
46#include "igb.h"
47
48#define DRV_VERSION "1.0.8-k2"
49char igb_driver_name[] = "igb";
50char igb_driver_version[] = DRV_VERSION;
51static const char igb_driver_string[] =
52 "Intel(R) Gigabit Ethernet Network Driver";
53static const char igb_copyright[] = "Copyright (c) 2007 Intel Corporation.";
54
55
56static const struct e1000_info *igb_info_tbl[] = {
57 [board_82575] = &e1000_82575_info,
58};
59
60static struct pci_device_id igb_pci_tbl[] = {
61 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
62 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
63 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
64 /* required last entry */
65 {0, }
66};
67
68MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
69
70void igb_reset(struct igb_adapter *);
71static int igb_setup_all_tx_resources(struct igb_adapter *);
72static int igb_setup_all_rx_resources(struct igb_adapter *);
73static void igb_free_all_tx_resources(struct igb_adapter *);
74static void igb_free_all_rx_resources(struct igb_adapter *);
75static void igb_free_tx_resources(struct igb_adapter *, struct igb_ring *);
76static void igb_free_rx_resources(struct igb_adapter *, struct igb_ring *);
77void igb_update_stats(struct igb_adapter *);
78static int igb_probe(struct pci_dev *, const struct pci_device_id *);
79static void __devexit igb_remove(struct pci_dev *pdev);
80static int igb_sw_init(struct igb_adapter *);
81static int igb_open(struct net_device *);
82static int igb_close(struct net_device *);
83static void igb_configure_tx(struct igb_adapter *);
84static void igb_configure_rx(struct igb_adapter *);
85static void igb_setup_rctl(struct igb_adapter *);
86static void igb_clean_all_tx_rings(struct igb_adapter *);
87static void igb_clean_all_rx_rings(struct igb_adapter *);
88static void igb_clean_tx_ring(struct igb_adapter *, struct igb_ring *);
89static void igb_clean_rx_ring(struct igb_adapter *, struct igb_ring *);
90static void igb_set_multi(struct net_device *);
91static void igb_update_phy_info(unsigned long);
92static void igb_watchdog(unsigned long);
93static void igb_watchdog_task(struct work_struct *);
94static int igb_xmit_frame_ring_adv(struct sk_buff *, struct net_device *,
95 struct igb_ring *);
96static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
97static struct net_device_stats *igb_get_stats(struct net_device *);
98static int igb_change_mtu(struct net_device *, int);
99static int igb_set_mac(struct net_device *, void *);
100static irqreturn_t igb_intr(int irq, void *);
101static irqreturn_t igb_intr_msi(int irq, void *);
102static irqreturn_t igb_msix_other(int irq, void *);
103static irqreturn_t igb_msix_rx(int irq, void *);
104static irqreturn_t igb_msix_tx(int irq, void *);
105static int igb_clean_rx_ring_msix(struct napi_struct *, int);
106static bool igb_clean_tx_irq(struct igb_adapter *, struct igb_ring *);
107static int igb_clean(struct napi_struct *, int);
108static bool igb_clean_rx_irq_adv(struct igb_adapter *,
109 struct igb_ring *, int *, int);
110static void igb_alloc_rx_buffers_adv(struct igb_adapter *,
111 struct igb_ring *, int);
112static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
113static void igb_tx_timeout(struct net_device *);
114static void igb_reset_task(struct work_struct *);
115static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
116static void igb_vlan_rx_add_vid(struct net_device *, u16);
117static void igb_vlan_rx_kill_vid(struct net_device *, u16);
118static void igb_restore_vlan(struct igb_adapter *);
119
120static int igb_suspend(struct pci_dev *, pm_message_t);
121#ifdef CONFIG_PM
122static int igb_resume(struct pci_dev *);
123#endif
124static void igb_shutdown(struct pci_dev *);
125
126#ifdef CONFIG_NET_POLL_CONTROLLER
127/* for netdump / net console */
128static void igb_netpoll(struct net_device *);
129#endif
130
131static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
132 pci_channel_state_t);
133static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
134static void igb_io_resume(struct pci_dev *);
135
136static struct pci_error_handlers igb_err_handler = {
137 .error_detected = igb_io_error_detected,
138 .slot_reset = igb_io_slot_reset,
139 .resume = igb_io_resume,
140};
141
142
143static struct pci_driver igb_driver = {
144 .name = igb_driver_name,
145 .id_table = igb_pci_tbl,
146 .probe = igb_probe,
147 .remove = __devexit_p(igb_remove),
148#ifdef CONFIG_PM
149 /* Power Managment Hooks */
150 .suspend = igb_suspend,
151 .resume = igb_resume,
152#endif
153 .shutdown = igb_shutdown,
154 .err_handler = &igb_err_handler
155};
156
157MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
158MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
159MODULE_LICENSE("GPL");
160MODULE_VERSION(DRV_VERSION);
161
162#ifdef DEBUG
163/**
164 * igb_get_hw_dev_name - return device name string
165 * used by hardware layer to print debugging information
166 **/
167char *igb_get_hw_dev_name(struct e1000_hw *hw)
168{
169 struct igb_adapter *adapter = hw->back;
170 return adapter->netdev->name;
171}
172#endif
173
174/**
175 * igb_init_module - Driver Registration Routine
176 *
177 * igb_init_module is the first routine called when the driver is
178 * loaded. All it does is register with the PCI subsystem.
179 **/
180static int __init igb_init_module(void)
181{
182 int ret;
183 printk(KERN_INFO "%s - version %s\n",
184 igb_driver_string, igb_driver_version);
185
186 printk(KERN_INFO "%s\n", igb_copyright);
187
188 ret = pci_register_driver(&igb_driver);
189 return ret;
190}
191
192module_init(igb_init_module);
193
194/**
195 * igb_exit_module - Driver Exit Cleanup Routine
196 *
197 * igb_exit_module is called just before the driver is removed
198 * from memory.
199 **/
200static void __exit igb_exit_module(void)
201{
202 pci_unregister_driver(&igb_driver);
203}
204
205module_exit(igb_exit_module);
206
207/**
208 * igb_alloc_queues - Allocate memory for all rings
209 * @adapter: board private structure to initialize
210 *
211 * We allocate one ring per queue at run-time since we don't know the
212 * number of queues at compile-time.
213 **/
214static int igb_alloc_queues(struct igb_adapter *adapter)
215{
216 int i;
217
218 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
219 sizeof(struct igb_ring), GFP_KERNEL);
220 if (!adapter->tx_ring)
221 return -ENOMEM;
222
223 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
224 sizeof(struct igb_ring), GFP_KERNEL);
225 if (!adapter->rx_ring) {
226 kfree(adapter->tx_ring);
227 return -ENOMEM;
228 }
229
230 for (i = 0; i < adapter->num_rx_queues; i++) {
231 struct igb_ring *ring = &(adapter->rx_ring[i]);
232 ring->adapter = adapter;
233 ring->itr_register = E1000_ITR;
234
235 if (!ring->napi.poll)
236 netif_napi_add(adapter->netdev, &ring->napi, igb_clean,
237 adapter->napi.weight /
238 adapter->num_rx_queues);
239 }
240 return 0;
241}
242
243#define IGB_N0_QUEUE -1
244static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue,
245 int tx_queue, int msix_vector)
246{
247 u32 msixbm = 0;
248 struct e1000_hw *hw = &adapter->hw;
249 /* The 82575 assigns vectors using a bitmask, which matches the
250 bitmask for the EICR/EIMS/EIMC registers. To assign one
251 or more queues to a vector, we write the appropriate bits
252 into the MSIXBM register for that vector. */
253 if (rx_queue > IGB_N0_QUEUE) {
254 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
255 adapter->rx_ring[rx_queue].eims_value = msixbm;
256 }
257 if (tx_queue > IGB_N0_QUEUE) {
258 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
259 adapter->tx_ring[tx_queue].eims_value =
260 E1000_EICR_TX_QUEUE0 << tx_queue;
261 }
262 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
263}
264
265/**
266 * igb_configure_msix - Configure MSI-X hardware
267 *
268 * igb_configure_msix sets up the hardware to properly
269 * generate MSI-X interrupts.
270 **/
271static void igb_configure_msix(struct igb_adapter *adapter)
272{
273 u32 tmp;
274 int i, vector = 0;
275 struct e1000_hw *hw = &adapter->hw;
276
277 adapter->eims_enable_mask = 0;
278
279 for (i = 0; i < adapter->num_tx_queues; i++) {
280 struct igb_ring *tx_ring = &adapter->tx_ring[i];
281 igb_assign_vector(adapter, IGB_N0_QUEUE, i, vector++);
282 adapter->eims_enable_mask |= tx_ring->eims_value;
283 if (tx_ring->itr_val)
284 writel(1000000000 / (tx_ring->itr_val * 256),
285 hw->hw_addr + tx_ring->itr_register);
286 else
287 writel(1, hw->hw_addr + tx_ring->itr_register);
288 }
289
290 for (i = 0; i < adapter->num_rx_queues; i++) {
291 struct igb_ring *rx_ring = &adapter->rx_ring[i];
292 igb_assign_vector(adapter, i, IGB_N0_QUEUE, vector++);
293 adapter->eims_enable_mask |= rx_ring->eims_value;
294 if (rx_ring->itr_val)
295 writel(1000000000 / (rx_ring->itr_val * 256),
296 hw->hw_addr + rx_ring->itr_register);
297 else
298 writel(1, hw->hw_addr + rx_ring->itr_register);
299 }
300
301
302 /* set vector for other causes, i.e. link changes */
303 array_wr32(E1000_MSIXBM(0), vector++,
304 E1000_EIMS_OTHER);
305
306 /* disable IAM for ICR interrupt bits */
307 wr32(E1000_IAM, 0);
308
309 tmp = rd32(E1000_CTRL_EXT);
310 /* enable MSI-X PBA support*/
311 tmp |= E1000_CTRL_EXT_PBA_CLR;
312
313 /* Auto-Mask interrupts upon ICR read. */
314 tmp |= E1000_CTRL_EXT_EIAME;
315 tmp |= E1000_CTRL_EXT_IRCA;
316
317 wr32(E1000_CTRL_EXT, tmp);
318 adapter->eims_enable_mask |= E1000_EIMS_OTHER;
319
320 wrfl();
321}
322
323/**
324 * igb_request_msix - Initialize MSI-X interrupts
325 *
326 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
327 * kernel.
328 **/
329static int igb_request_msix(struct igb_adapter *adapter)
330{
331 struct net_device *netdev = adapter->netdev;
332 int i, err = 0, vector = 0;
333
334 vector = 0;
335
336 for (i = 0; i < adapter->num_tx_queues; i++) {
337 struct igb_ring *ring = &(adapter->tx_ring[i]);
338 sprintf(ring->name, "%s-tx%d", netdev->name, i);
339 err = request_irq(adapter->msix_entries[vector].vector,
340 &igb_msix_tx, 0, ring->name,
341 &(adapter->tx_ring[i]));
342 if (err)
343 goto out;
344 ring->itr_register = E1000_EITR(0) + (vector << 2);
345 ring->itr_val = adapter->itr;
346 vector++;
347 }
348 for (i = 0; i < adapter->num_rx_queues; i++) {
349 struct igb_ring *ring = &(adapter->rx_ring[i]);
350 if (strlen(netdev->name) < (IFNAMSIZ - 5))
351 sprintf(ring->name, "%s-rx%d", netdev->name, i);
352 else
353 memcpy(ring->name, netdev->name, IFNAMSIZ);
354 err = request_irq(adapter->msix_entries[vector].vector,
355 &igb_msix_rx, 0, ring->name,
356 &(adapter->rx_ring[i]));
357 if (err)
358 goto out;
359 ring->itr_register = E1000_EITR(0) + (vector << 2);
360 ring->itr_val = adapter->itr;
361 vector++;
362 }
363
364 err = request_irq(adapter->msix_entries[vector].vector,
365 &igb_msix_other, 0, netdev->name, netdev);
366 if (err)
367 goto out;
368
369 adapter->napi.poll = igb_clean_rx_ring_msix;
370 for (i = 0; i < adapter->num_rx_queues; i++)
371 adapter->rx_ring[i].napi.poll = adapter->napi.poll;
372 igb_configure_msix(adapter);
373 return 0;
374out:
375 return err;
376}
377
378static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
379{
380 if (adapter->msix_entries) {
381 pci_disable_msix(adapter->pdev);
382 kfree(adapter->msix_entries);
383 adapter->msix_entries = NULL;
384 } else if (adapter->msi_enabled)
385 pci_disable_msi(adapter->pdev);
386 return;
387}
388
389
390/**
391 * igb_set_interrupt_capability - set MSI or MSI-X if supported
392 *
393 * Attempt to configure interrupts using the best available
394 * capabilities of the hardware and kernel.
395 **/
396static void igb_set_interrupt_capability(struct igb_adapter *adapter)
397{
398 int err;
399 int numvecs, i;
400
401 numvecs = adapter->num_tx_queues + adapter->num_rx_queues + 1;
402 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
403 GFP_KERNEL);
404 if (!adapter->msix_entries)
405 goto msi_only;
406
407 for (i = 0; i < numvecs; i++)
408 adapter->msix_entries[i].entry = i;
409
410 err = pci_enable_msix(adapter->pdev,
411 adapter->msix_entries,
412 numvecs);
413 if (err == 0)
414 return;
415
416 igb_reset_interrupt_capability(adapter);
417
418 /* If we can't do MSI-X, try MSI */
419msi_only:
420 adapter->num_rx_queues = 1;
421 if (!pci_enable_msi(adapter->pdev))
422 adapter->msi_enabled = 1;
423 return;
424}
425
426/**
427 * igb_request_irq - initialize interrupts
428 *
429 * Attempts to configure interrupts using the best available
430 * capabilities of the hardware and kernel.
431 **/
432static int igb_request_irq(struct igb_adapter *adapter)
433{
434 struct net_device *netdev = adapter->netdev;
435 struct e1000_hw *hw = &adapter->hw;
436 int err = 0;
437
438 if (adapter->msix_entries) {
439 err = igb_request_msix(adapter);
440 if (!err) {
441 struct e1000_hw *hw = &adapter->hw;
442 /* enable IAM, auto-mask,
443 * DO NOT USE EIAME or IAME in legacy mode */
444 wr32(E1000_IAM, IMS_ENABLE_MASK);
445 goto request_done;
446 }
447 /* fall back to MSI */
448 igb_reset_interrupt_capability(adapter);
449 if (!pci_enable_msi(adapter->pdev))
450 adapter->msi_enabled = 1;
451 igb_free_all_tx_resources(adapter);
452 igb_free_all_rx_resources(adapter);
453 adapter->num_rx_queues = 1;
454 igb_alloc_queues(adapter);
455 }
456 if (adapter->msi_enabled) {
457 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
458 netdev->name, netdev);
459 if (!err)
460 goto request_done;
461 /* fall back to legacy interrupts */
462 igb_reset_interrupt_capability(adapter);
463 adapter->msi_enabled = 0;
464 }
465
466 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
467 netdev->name, netdev);
468
469 if (err) {
470 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
471 err);
472 goto request_done;
473 }
474
475 /* enable IAM, auto-mask */
476 wr32(E1000_IAM, IMS_ENABLE_MASK);
477
478request_done:
479 return err;
480}
481
482static void igb_free_irq(struct igb_adapter *adapter)
483{
484 struct net_device *netdev = adapter->netdev;
485
486 if (adapter->msix_entries) {
487 int vector = 0, i;
488
489 for (i = 0; i < adapter->num_tx_queues; i++)
490 free_irq(adapter->msix_entries[vector++].vector,
491 &(adapter->tx_ring[i]));
492 for (i = 0; i < adapter->num_rx_queues; i++)
493 free_irq(adapter->msix_entries[vector++].vector,
494 &(adapter->rx_ring[i]));
495
496 free_irq(adapter->msix_entries[vector++].vector, netdev);
497 return;
498 }
499
500 free_irq(adapter->pdev->irq, netdev);
501}
502
503/**
504 * igb_irq_disable - Mask off interrupt generation on the NIC
505 * @adapter: board private structure
506 **/
507static void igb_irq_disable(struct igb_adapter *adapter)
508{
509 struct e1000_hw *hw = &adapter->hw;
510
511 if (adapter->msix_entries) {
512 wr32(E1000_EIMC, ~0);
513 wr32(E1000_EIAC, 0);
514 }
515 wr32(E1000_IMC, ~0);
516 wrfl();
517 synchronize_irq(adapter->pdev->irq);
518}
519
520/**
521 * igb_irq_enable - Enable default interrupt generation settings
522 * @adapter: board private structure
523 **/
524static void igb_irq_enable(struct igb_adapter *adapter)
525{
526 struct e1000_hw *hw = &adapter->hw;
527
528 if (adapter->msix_entries) {
529 wr32(E1000_EIMS,
530 adapter->eims_enable_mask);
531 wr32(E1000_EIAC,
532 adapter->eims_enable_mask);
533 wr32(E1000_IMS, E1000_IMS_LSC);
534 } else
535 wr32(E1000_IMS, IMS_ENABLE_MASK);
536}
537
538static void igb_update_mng_vlan(struct igb_adapter *adapter)
539{
540 struct net_device *netdev = adapter->netdev;
541 u16 vid = adapter->hw.mng_cookie.vlan_id;
542 u16 old_vid = adapter->mng_vlan_id;
543 if (adapter->vlgrp) {
544 if (!vlan_group_get_device(adapter->vlgrp, vid)) {
545 if (adapter->hw.mng_cookie.status &
546 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
547 igb_vlan_rx_add_vid(netdev, vid);
548 adapter->mng_vlan_id = vid;
549 } else
550 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
551
552 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
553 (vid != old_vid) &&
554 !vlan_group_get_device(adapter->vlgrp, old_vid))
555 igb_vlan_rx_kill_vid(netdev, old_vid);
556 } else
557 adapter->mng_vlan_id = vid;
558 }
559}
560
561/**
562 * igb_release_hw_control - release control of the h/w to f/w
563 * @adapter: address of board private structure
564 *
565 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
566 * For ASF and Pass Through versions of f/w this means that the
567 * driver is no longer loaded.
568 *
569 **/
570static void igb_release_hw_control(struct igb_adapter *adapter)
571{
572 struct e1000_hw *hw = &adapter->hw;
573 u32 ctrl_ext;
574
575 /* Let firmware take over control of h/w */
576 ctrl_ext = rd32(E1000_CTRL_EXT);
577 wr32(E1000_CTRL_EXT,
578 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
579}
580
581
582/**
583 * igb_get_hw_control - get control of the h/w from f/w
584 * @adapter: address of board private structure
585 *
586 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
587 * For ASF and Pass Through versions of f/w this means that
588 * the driver is loaded.
589 *
590 **/
591static void igb_get_hw_control(struct igb_adapter *adapter)
592{
593 struct e1000_hw *hw = &adapter->hw;
594 u32 ctrl_ext;
595
596 /* Let firmware know the driver has taken over */
597 ctrl_ext = rd32(E1000_CTRL_EXT);
598 wr32(E1000_CTRL_EXT,
599 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
600}
601
602static void igb_init_manageability(struct igb_adapter *adapter)
603{
604 struct e1000_hw *hw = &adapter->hw;
605
606 if (adapter->en_mng_pt) {
607 u32 manc2h = rd32(E1000_MANC2H);
608 u32 manc = rd32(E1000_MANC);
609
610 /* disable hardware interception of ARP */
611 manc &= ~(E1000_MANC_ARP_EN);
612
613 /* enable receiving management packets to the host */
614 /* this will probably generate destination unreachable messages
615 * from the host OS, but the packets will be handled on SMBUS */
616 manc |= E1000_MANC_EN_MNG2HOST;
617#define E1000_MNG2HOST_PORT_623 (1 << 5)
618#define E1000_MNG2HOST_PORT_664 (1 << 6)
619 manc2h |= E1000_MNG2HOST_PORT_623;
620 manc2h |= E1000_MNG2HOST_PORT_664;
621 wr32(E1000_MANC2H, manc2h);
622
623 wr32(E1000_MANC, manc);
624 }
625}
626
627static void igb_release_manageability(struct igb_adapter *adapter)
628{
629 struct e1000_hw *hw = &adapter->hw;
630
631 if (adapter->en_mng_pt) {
632 u32 manc = rd32(E1000_MANC);
633
634 /* re-enable hardware interception of ARP */
635 manc |= E1000_MANC_ARP_EN;
636 manc &= ~E1000_MANC_EN_MNG2HOST;
637
638 /* don't explicitly have to mess with MANC2H since
639 * MANC has an enable disable that gates MANC2H */
640
641 /* XXX stop the hardware watchdog ? */
642 wr32(E1000_MANC, manc);
643 }
644}
645
646/**
647 * igb_configure - configure the hardware for RX and TX
648 * @adapter: private board structure
649 **/
650static void igb_configure(struct igb_adapter *adapter)
651{
652 struct net_device *netdev = adapter->netdev;
653 int i;
654
655 igb_get_hw_control(adapter);
656 igb_set_multi(netdev);
657
658 igb_restore_vlan(adapter);
659 igb_init_manageability(adapter);
660
661 igb_configure_tx(adapter);
662 igb_setup_rctl(adapter);
663 igb_configure_rx(adapter);
664 /* call IGB_DESC_UNUSED which always leaves
665 * at least 1 descriptor unused to make sure
666 * next_to_use != next_to_clean */
667 for (i = 0; i < adapter->num_rx_queues; i++) {
668 struct igb_ring *ring = &adapter->rx_ring[i];
669 igb_alloc_rx_buffers_adv(adapter, ring, IGB_DESC_UNUSED(ring));
670 }
671
672
673 adapter->tx_queue_len = netdev->tx_queue_len;
674}
675
676
677/**
678 * igb_up - Open the interface and prepare it to handle traffic
679 * @adapter: board private structure
680 **/
681
682int igb_up(struct igb_adapter *adapter)
683{
684 struct e1000_hw *hw = &adapter->hw;
685 int i;
686
687 /* hardware has been reset, we need to reload some things */
688 igb_configure(adapter);
689
690 clear_bit(__IGB_DOWN, &adapter->state);
691
692 napi_enable(&adapter->napi);
693
694 if (adapter->msix_entries) {
695 for (i = 0; i < adapter->num_rx_queues; i++)
696 napi_enable(&adapter->rx_ring[i].napi);
697 igb_configure_msix(adapter);
698 }
699
700 /* Clear any pending interrupts. */
701 rd32(E1000_ICR);
702 igb_irq_enable(adapter);
703
704 /* Fire a link change interrupt to start the watchdog. */
705 wr32(E1000_ICS, E1000_ICS_LSC);
706 return 0;
707}
708
709void igb_down(struct igb_adapter *adapter)
710{
711 struct e1000_hw *hw = &adapter->hw;
712 struct net_device *netdev = adapter->netdev;
713 u32 tctl, rctl;
714 int i;
715
716 /* signal that we're down so the interrupt handler does not
717 * reschedule our watchdog timer */
718 set_bit(__IGB_DOWN, &adapter->state);
719
720 /* disable receives in the hardware */
721 rctl = rd32(E1000_RCTL);
722 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
723 /* flush and sleep below */
724
725 netif_stop_queue(netdev);
726
727 /* disable transmits in the hardware */
728 tctl = rd32(E1000_TCTL);
729 tctl &= ~E1000_TCTL_EN;
730 wr32(E1000_TCTL, tctl);
731 /* flush both disables and wait for them to finish */
732 wrfl();
733 msleep(10);
734
735 napi_disable(&adapter->napi);
736
737 if (adapter->msix_entries)
738 for (i = 0; i < adapter->num_rx_queues; i++)
739 napi_disable(&adapter->rx_ring[i].napi);
740 igb_irq_disable(adapter);
741
742 del_timer_sync(&adapter->watchdog_timer);
743 del_timer_sync(&adapter->phy_info_timer);
744
745 netdev->tx_queue_len = adapter->tx_queue_len;
746 netif_carrier_off(netdev);
747 adapter->link_speed = 0;
748 adapter->link_duplex = 0;
749
750 igb_reset(adapter);
751 igb_clean_all_tx_rings(adapter);
752 igb_clean_all_rx_rings(adapter);
753}
754
755void igb_reinit_locked(struct igb_adapter *adapter)
756{
757 WARN_ON(in_interrupt());
758 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
759 msleep(1);
760 igb_down(adapter);
761 igb_up(adapter);
762 clear_bit(__IGB_RESETTING, &adapter->state);
763}
764
765void igb_reset(struct igb_adapter *adapter)
766{
767 struct e1000_hw *hw = &adapter->hw;
768 struct e1000_fc_info *fc = &adapter->hw.fc;
769 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
770 u16 hwm;
771
772 /* Repartition Pba for greater than 9k mtu
773 * To take effect CTRL.RST is required.
774 */
775 pba = E1000_PBA_34K;
776
777 if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
778 /* adjust PBA for jumbo frames */
779 wr32(E1000_PBA, pba);
780
781 /* To maintain wire speed transmits, the Tx FIFO should be
782 * large enough to accommodate two full transmit packets,
783 * rounded up to the next 1KB and expressed in KB. Likewise,
784 * the Rx FIFO should be large enough to accommodate at least
785 * one full receive packet and is similarly rounded up and
786 * expressed in KB. */
787 pba = rd32(E1000_PBA);
788 /* upper 16 bits has Tx packet buffer allocation size in KB */
789 tx_space = pba >> 16;
790 /* lower 16 bits has Rx packet buffer allocation size in KB */
791 pba &= 0xffff;
792 /* the tx fifo also stores 16 bytes of information about the tx
793 * but don't include ethernet FCS because hardware appends it */
794 min_tx_space = (adapter->max_frame_size +
795 sizeof(struct e1000_tx_desc) -
796 ETH_FCS_LEN) * 2;
797 min_tx_space = ALIGN(min_tx_space, 1024);
798 min_tx_space >>= 10;
799 /* software strips receive CRC, so leave room for it */
800 min_rx_space = adapter->max_frame_size;
801 min_rx_space = ALIGN(min_rx_space, 1024);
802 min_rx_space >>= 10;
803
804 /* If current Tx allocation is less than the min Tx FIFO size,
805 * and the min Tx FIFO size is less than the current Rx FIFO
806 * allocation, take space away from current Rx allocation */
807 if (tx_space < min_tx_space &&
808 ((min_tx_space - tx_space) < pba)) {
809 pba = pba - (min_tx_space - tx_space);
810
811 /* if short on rx space, rx wins and must trump tx
812 * adjustment */
813 if (pba < min_rx_space)
814 pba = min_rx_space;
815 }
816 }
817 wr32(E1000_PBA, pba);
818
819 /* flow control settings */
820 /* The high water mark must be low enough to fit one full frame
821 * (or the size used for early receive) above it in the Rx FIFO.
822 * Set it to the lower of:
823 * - 90% of the Rx FIFO size, or
824 * - the full Rx FIFO size minus one full frame */
825 hwm = min(((pba << 10) * 9 / 10),
826 ((pba << 10) - adapter->max_frame_size));
827
828 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
829 fc->low_water = fc->high_water - 8;
830 fc->pause_time = 0xFFFF;
831 fc->send_xon = 1;
832 fc->type = fc->original_type;
833
834 /* Allow time for pending master requests to run */
835 adapter->hw.mac.ops.reset_hw(&adapter->hw);
836 wr32(E1000_WUC, 0);
837
838 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
839 dev_err(&adapter->pdev->dev, "Hardware Error\n");
840
841 igb_update_mng_vlan(adapter);
842
843 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
844 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
845
846 igb_reset_adaptive(&adapter->hw);
847 adapter->hw.phy.ops.get_phy_info(&adapter->hw);
848 igb_release_manageability(adapter);
849}
850
851/**
852 * igb_probe - Device Initialization Routine
853 * @pdev: PCI device information struct
854 * @ent: entry in igb_pci_tbl
855 *
856 * Returns 0 on success, negative on failure
857 *
858 * igb_probe initializes an adapter identified by a pci_dev structure.
859 * The OS initialization, configuring of the adapter private structure,
860 * and a hardware reset occur.
861 **/
862static int __devinit igb_probe(struct pci_dev *pdev,
863 const struct pci_device_id *ent)
864{
865 struct net_device *netdev;
866 struct igb_adapter *adapter;
867 struct e1000_hw *hw;
868 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
869 unsigned long mmio_start, mmio_len;
870 static int cards_found;
871 int i, err, pci_using_dac;
872 u16 eeprom_data = 0;
873 u16 eeprom_apme_mask = IGB_EEPROM_APME;
874 u32 part_num;
875
876 err = pci_enable_device(pdev);
877 if (err)
878 return err;
879
880 pci_using_dac = 0;
881 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
882 if (!err) {
883 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
884 if (!err)
885 pci_using_dac = 1;
886 } else {
887 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
888 if (err) {
889 err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
890 if (err) {
891 dev_err(&pdev->dev, "No usable DMA "
892 "configuration, aborting\n");
893 goto err_dma;
894 }
895 }
896 }
897
898 err = pci_request_regions(pdev, igb_driver_name);
899 if (err)
900 goto err_pci_reg;
901
902 pci_set_master(pdev);
903
904 err = -ENOMEM;
905 netdev = alloc_etherdev(sizeof(struct igb_adapter));
906 if (!netdev)
907 goto err_alloc_etherdev;
908
909 SET_NETDEV_DEV(netdev, &pdev->dev);
910
911 pci_set_drvdata(pdev, netdev);
912 adapter = netdev_priv(netdev);
913 adapter->netdev = netdev;
914 adapter->pdev = pdev;
915 hw = &adapter->hw;
916 hw->back = adapter;
917 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
918
919 mmio_start = pci_resource_start(pdev, 0);
920 mmio_len = pci_resource_len(pdev, 0);
921
922 err = -EIO;
923 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
924 if (!adapter->hw.hw_addr)
925 goto err_ioremap;
926
927 netdev->open = &igb_open;
928 netdev->stop = &igb_close;
929 netdev->get_stats = &igb_get_stats;
930 netdev->set_multicast_list = &igb_set_multi;
931 netdev->set_mac_address = &igb_set_mac;
932 netdev->change_mtu = &igb_change_mtu;
933 netdev->do_ioctl = &igb_ioctl;
934 igb_set_ethtool_ops(netdev);
935 netdev->tx_timeout = &igb_tx_timeout;
936 netdev->watchdog_timeo = 5 * HZ;
937 netif_napi_add(netdev, &adapter->napi, igb_clean, 64);
938 netdev->vlan_rx_register = igb_vlan_rx_register;
939 netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
940 netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
941#ifdef CONFIG_NET_POLL_CONTROLLER
942 netdev->poll_controller = igb_netpoll;
943#endif
944 netdev->hard_start_xmit = &igb_xmit_frame_adv;
945
946 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
947
948 netdev->mem_start = mmio_start;
949 netdev->mem_end = mmio_start + mmio_len;
950
951 adapter->bd_number = cards_found;
952
953 /* PCI config space info */
954 hw->vendor_id = pdev->vendor;
955 hw->device_id = pdev->device;
956 hw->revision_id = pdev->revision;
957 hw->subsystem_vendor_id = pdev->subsystem_vendor;
958 hw->subsystem_device_id = pdev->subsystem_device;
959
960 /* setup the private structure */
961 hw->back = adapter;
962 /* Copy the default MAC, PHY and NVM function pointers */
963 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
964 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
965 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
966 /* Initialize skew-specific constants */
967 err = ei->get_invariants(hw);
968 if (err)
969 goto err_hw_init;
970
971 err = igb_sw_init(adapter);
972 if (err)
973 goto err_sw_init;
974
975 igb_get_bus_info_pcie(hw);
976
977 hw->phy.autoneg_wait_to_complete = false;
978 hw->mac.adaptive_ifs = true;
979
980 /* Copper options */
981 if (hw->phy.media_type == e1000_media_type_copper) {
982 hw->phy.mdix = AUTO_ALL_MODES;
983 hw->phy.disable_polarity_correction = false;
984 hw->phy.ms_type = e1000_ms_hw_default;
985 }
986
987 if (igb_check_reset_block(hw))
988 dev_info(&pdev->dev,
989 "PHY reset is blocked due to SOL/IDER session.\n");
990
991 netdev->features = NETIF_F_SG |
992 NETIF_F_HW_CSUM |
993 NETIF_F_HW_VLAN_TX |
994 NETIF_F_HW_VLAN_RX |
995 NETIF_F_HW_VLAN_FILTER;
996
997 netdev->features |= NETIF_F_TSO;
998
999 netdev->features |= NETIF_F_TSO6;
1000 if (pci_using_dac)
1001 netdev->features |= NETIF_F_HIGHDMA;
1002
1003 netdev->features |= NETIF_F_LLTX;
1004 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1005
1006 /* before reading the NVM, reset the controller to put the device in a
1007 * known good starting state */
1008 hw->mac.ops.reset_hw(hw);
1009
1010 /* make sure the NVM is good */
1011 if (igb_validate_nvm_checksum(hw) < 0) {
1012 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1013 err = -EIO;
1014 goto err_eeprom;
1015 }
1016
1017 /* copy the MAC address out of the NVM */
1018 if (hw->mac.ops.read_mac_addr(hw))
1019 dev_err(&pdev->dev, "NVM Read Error\n");
1020
1021 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1022 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1023
1024 if (!is_valid_ether_addr(netdev->perm_addr)) {
1025 dev_err(&pdev->dev, "Invalid MAC Address\n");
1026 err = -EIO;
1027 goto err_eeprom;
1028 }
1029
1030 init_timer(&adapter->watchdog_timer);
1031 adapter->watchdog_timer.function = &igb_watchdog;
1032 adapter->watchdog_timer.data = (unsigned long) adapter;
1033
1034 init_timer(&adapter->phy_info_timer);
1035 adapter->phy_info_timer.function = &igb_update_phy_info;
1036 adapter->phy_info_timer.data = (unsigned long) adapter;
1037
1038 INIT_WORK(&adapter->reset_task, igb_reset_task);
1039 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1040
1041 /* Initialize link & ring properties that are user-changeable */
1042 adapter->tx_ring->count = 256;
1043 for (i = 0; i < adapter->num_tx_queues; i++)
1044 adapter->tx_ring[i].count = adapter->tx_ring->count;
1045 adapter->rx_ring->count = 256;
1046 for (i = 0; i < adapter->num_rx_queues; i++)
1047 adapter->rx_ring[i].count = adapter->rx_ring->count;
1048
1049 adapter->fc_autoneg = true;
1050 hw->mac.autoneg = true;
1051 hw->phy.autoneg_advertised = 0x2f;
1052
1053 hw->fc.original_type = e1000_fc_default;
1054 hw->fc.type = e1000_fc_default;
1055
1056 adapter->itr_setting = 3;
1057 adapter->itr = IGB_START_ITR;
1058
1059 igb_validate_mdi_setting(hw);
1060
1061 adapter->rx_csum = 1;
1062
1063 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1064 * enable the ACPI Magic Packet filter
1065 */
1066
1067 if (hw->bus.func == 0 ||
1068 hw->device_id == E1000_DEV_ID_82575EB_COPPER)
1069 hw->nvm.ops.read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1,
1070 &eeprom_data);
1071
1072 if (eeprom_data & eeprom_apme_mask)
1073 adapter->eeprom_wol |= E1000_WUFC_MAG;
1074
1075 /* now that we have the eeprom settings, apply the special cases where
1076 * the eeprom may be wrong or the board simply won't support wake on
1077 * lan on a particular port */
1078 switch (pdev->device) {
1079 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1080 adapter->eeprom_wol = 0;
1081 break;
1082 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1083 /* Wake events only supported on port A for dual fiber
1084 * regardless of eeprom setting */
1085 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1086 adapter->eeprom_wol = 0;
1087 break;
1088 }
1089
1090 /* initialize the wol settings based on the eeprom settings */
1091 adapter->wol = adapter->eeprom_wol;
1092
1093 /* reset the hardware with the new settings */
1094 igb_reset(adapter);
1095
1096 /* let the f/w know that the h/w is now under the control of the
1097 * driver. */
1098 igb_get_hw_control(adapter);
1099
1100 /* tell the stack to leave us alone until igb_open() is called */
1101 netif_carrier_off(netdev);
1102 netif_stop_queue(netdev);
1103
1104 strcpy(netdev->name, "eth%d");
1105 err = register_netdev(netdev);
1106 if (err)
1107 goto err_register;
1108
1109 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1110 /* print bus type/speed/width info */
1111 dev_info(&pdev->dev,
1112 "%s: (PCIe:%s:%s) %02x:%02x:%02x:%02x:%02x:%02x\n",
1113 netdev->name,
1114 ((hw->bus.speed == e1000_bus_speed_2500)
1115 ? "2.5Gb/s" : "unknown"),
1116 ((hw->bus.width == e1000_bus_width_pcie_x4)
1117 ? "Width x4" : (hw->bus.width == e1000_bus_width_pcie_x1)
1118 ? "Width x1" : "unknown"),
1119 netdev->dev_addr[0], netdev->dev_addr[1], netdev->dev_addr[2],
1120 netdev->dev_addr[3], netdev->dev_addr[4], netdev->dev_addr[5]);
1121
1122 igb_read_part_num(hw, &part_num);
1123 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1124 (part_num >> 8), (part_num & 0xff));
1125
1126 dev_info(&pdev->dev,
1127 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1128 adapter->msix_entries ? "MSI-X" :
1129 adapter->msi_enabled ? "MSI" : "legacy",
1130 adapter->num_rx_queues, adapter->num_tx_queues);
1131
1132 cards_found++;
1133 return 0;
1134
1135err_register:
1136 igb_release_hw_control(adapter);
1137err_eeprom:
1138 if (!igb_check_reset_block(hw))
1139 hw->phy.ops.reset_phy(hw);
1140
1141 if (hw->flash_address)
1142 iounmap(hw->flash_address);
1143
1144 igb_remove_device(hw);
1145 kfree(adapter->tx_ring);
1146 kfree(adapter->rx_ring);
1147err_sw_init:
1148err_hw_init:
1149 iounmap(hw->hw_addr);
1150err_ioremap:
1151 free_netdev(netdev);
1152err_alloc_etherdev:
1153 pci_release_regions(pdev);
1154err_pci_reg:
1155err_dma:
1156 pci_disable_device(pdev);
1157 return err;
1158}
1159
1160/**
1161 * igb_remove - Device Removal Routine
1162 * @pdev: PCI device information struct
1163 *
1164 * igb_remove is called by the PCI subsystem to alert the driver
1165 * that it should release a PCI device. The could be caused by a
1166 * Hot-Plug event, or because the driver is going to be removed from
1167 * memory.
1168 **/
1169static void __devexit igb_remove(struct pci_dev *pdev)
1170{
1171 struct net_device *netdev = pci_get_drvdata(pdev);
1172 struct igb_adapter *adapter = netdev_priv(netdev);
1173
1174 /* flush_scheduled work may reschedule our watchdog task, so
1175 * explicitly disable watchdog tasks from being rescheduled */
1176 set_bit(__IGB_DOWN, &adapter->state);
1177 del_timer_sync(&adapter->watchdog_timer);
1178 del_timer_sync(&adapter->phy_info_timer);
1179
1180 flush_scheduled_work();
1181
1182
1183 igb_release_manageability(adapter);
1184
1185 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1186 * would have already happened in close and is redundant. */
1187 igb_release_hw_control(adapter);
1188
1189 unregister_netdev(netdev);
1190
1191 if (!igb_check_reset_block(&adapter->hw))
1192 adapter->hw.phy.ops.reset_phy(&adapter->hw);
1193
1194 igb_remove_device(&adapter->hw);
1195 igb_reset_interrupt_capability(adapter);
1196
1197 kfree(adapter->tx_ring);
1198 kfree(adapter->rx_ring);
1199
1200 iounmap(adapter->hw.hw_addr);
1201 if (adapter->hw.flash_address)
1202 iounmap(adapter->hw.flash_address);
1203 pci_release_regions(pdev);
1204
1205 free_netdev(netdev);
1206
1207 pci_disable_device(pdev);
1208}
1209
1210/**
1211 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1212 * @adapter: board private structure to initialize
1213 *
1214 * igb_sw_init initializes the Adapter private data structure.
1215 * Fields are initialized based on PCI device information and
1216 * OS network device settings (MTU size).
1217 **/
1218static int __devinit igb_sw_init(struct igb_adapter *adapter)
1219{
1220 struct e1000_hw *hw = &adapter->hw;
1221 struct net_device *netdev = adapter->netdev;
1222 struct pci_dev *pdev = adapter->pdev;
1223
1224 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1225
1226 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1227 adapter->rx_ps_hdr_size = 0; /* disable packet split */
1228 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1229 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1230
1231 /* Number of supported queues. */
1232 /* Having more queues than CPUs doesn't make sense. */
1233 adapter->num_tx_queues = 1;
1234 adapter->num_rx_queues = min(IGB_MAX_RX_QUEUES, num_online_cpus());
1235
1236 igb_set_interrupt_capability(adapter);
1237
1238 if (igb_alloc_queues(adapter)) {
1239 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1240 return -ENOMEM;
1241 }
1242
1243 /* Explicitly disable IRQ since the NIC can be in any state. */
1244 igb_irq_disable(adapter);
1245
1246 set_bit(__IGB_DOWN, &adapter->state);
1247 return 0;
1248}
1249
1250/**
1251 * igb_open - Called when a network interface is made active
1252 * @netdev: network interface device structure
1253 *
1254 * Returns 0 on success, negative value on failure
1255 *
1256 * The open entry point is called when a network interface is made
1257 * active by the system (IFF_UP). At this point all resources needed
1258 * for transmit and receive operations are allocated, the interrupt
1259 * handler is registered with the OS, the watchdog timer is started,
1260 * and the stack is notified that the interface is ready.
1261 **/
1262static int igb_open(struct net_device *netdev)
1263{
1264 struct igb_adapter *adapter = netdev_priv(netdev);
1265 struct e1000_hw *hw = &adapter->hw;
1266 int err;
1267 int i;
1268
1269 /* disallow open during test */
1270 if (test_bit(__IGB_TESTING, &adapter->state))
1271 return -EBUSY;
1272
1273 /* allocate transmit descriptors */
1274 err = igb_setup_all_tx_resources(adapter);
1275 if (err)
1276 goto err_setup_tx;
1277
1278 /* allocate receive descriptors */
1279 err = igb_setup_all_rx_resources(adapter);
1280 if (err)
1281 goto err_setup_rx;
1282
1283 /* e1000_power_up_phy(adapter); */
1284
1285 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1286 if ((adapter->hw.mng_cookie.status &
1287 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
1288 igb_update_mng_vlan(adapter);
1289
1290 /* before we allocate an interrupt, we must be ready to handle it.
1291 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1292 * as soon as we call pci_request_irq, so we have to setup our
1293 * clean_rx handler before we do so. */
1294 igb_configure(adapter);
1295
1296 err = igb_request_irq(adapter);
1297 if (err)
1298 goto err_req_irq;
1299
1300 /* From here on the code is the same as igb_up() */
1301 clear_bit(__IGB_DOWN, &adapter->state);
1302
1303 napi_enable(&adapter->napi);
1304 if (adapter->msix_entries)
1305 for (i = 0; i < adapter->num_rx_queues; i++)
1306 napi_enable(&adapter->rx_ring[i].napi);
1307
1308 igb_irq_enable(adapter);
1309
1310 /* Clear any pending interrupts. */
1311 rd32(E1000_ICR);
1312 /* Fire a link status change interrupt to start the watchdog. */
1313 wr32(E1000_ICS, E1000_ICS_LSC);
1314
1315 return 0;
1316
1317err_req_irq:
1318 igb_release_hw_control(adapter);
1319 /* e1000_power_down_phy(adapter); */
1320 igb_free_all_rx_resources(adapter);
1321err_setup_rx:
1322 igb_free_all_tx_resources(adapter);
1323err_setup_tx:
1324 igb_reset(adapter);
1325
1326 return err;
1327}
1328
1329/**
1330 * igb_close - Disables a network interface
1331 * @netdev: network interface device structure
1332 *
1333 * Returns 0, this is not allowed to fail
1334 *
1335 * The close entry point is called when an interface is de-activated
1336 * by the OS. The hardware is still under the driver's control, but
1337 * needs to be disabled. A global MAC reset is issued to stop the
1338 * hardware, and all transmit and receive resources are freed.
1339 **/
1340static int igb_close(struct net_device *netdev)
1341{
1342 struct igb_adapter *adapter = netdev_priv(netdev);
1343
1344 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1345 igb_down(adapter);
1346
1347 igb_free_irq(adapter);
1348
1349 igb_free_all_tx_resources(adapter);
1350 igb_free_all_rx_resources(adapter);
1351
1352 /* kill manageability vlan ID if supported, but not if a vlan with
1353 * the same ID is registered on the host OS (let 8021q kill it) */
1354 if ((adapter->hw.mng_cookie.status &
1355 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
1356 !(adapter->vlgrp &&
1357 vlan_group_get_device(adapter->vlgrp, adapter->mng_vlan_id)))
1358 igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
1359
1360 return 0;
1361}
1362
1363/**
1364 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
1365 * @adapter: board private structure
1366 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1367 *
1368 * Return 0 on success, negative on failure
1369 **/
1370
1371int igb_setup_tx_resources(struct igb_adapter *adapter,
1372 struct igb_ring *tx_ring)
1373{
1374 struct pci_dev *pdev = adapter->pdev;
1375 int size;
1376
1377 size = sizeof(struct igb_buffer) * tx_ring->count;
1378 tx_ring->buffer_info = vmalloc(size);
1379 if (!tx_ring->buffer_info)
1380 goto err;
1381 memset(tx_ring->buffer_info, 0, size);
1382
1383 /* round up to nearest 4K */
1384 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc)
1385 + sizeof(u32);
1386 tx_ring->size = ALIGN(tx_ring->size, 4096);
1387
1388 tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
1389 &tx_ring->dma);
1390
1391 if (!tx_ring->desc)
1392 goto err;
1393
1394 tx_ring->adapter = adapter;
1395 tx_ring->next_to_use = 0;
1396 tx_ring->next_to_clean = 0;
1397 spin_lock_init(&tx_ring->tx_clean_lock);
1398 spin_lock_init(&tx_ring->tx_lock);
1399 return 0;
1400
1401err:
1402 vfree(tx_ring->buffer_info);
1403 dev_err(&adapter->pdev->dev,
1404 "Unable to allocate memory for the transmit descriptor ring\n");
1405 return -ENOMEM;
1406}
1407
1408/**
1409 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1410 * (Descriptors) for all queues
1411 * @adapter: board private structure
1412 *
1413 * Return 0 on success, negative on failure
1414 **/
1415static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1416{
1417 int i, err = 0;
1418
1419 for (i = 0; i < adapter->num_tx_queues; i++) {
1420 err = igb_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1421 if (err) {
1422 dev_err(&adapter->pdev->dev,
1423 "Allocation for Tx Queue %u failed\n", i);
1424 for (i--; i >= 0; i--)
1425 igb_free_tx_resources(adapter,
1426 &adapter->tx_ring[i]);
1427 break;
1428 }
1429 }
1430
1431 return err;
1432}
1433
1434/**
1435 * igb_configure_tx - Configure transmit Unit after Reset
1436 * @adapter: board private structure
1437 *
1438 * Configure the Tx unit of the MAC after a reset.
1439 **/
1440static void igb_configure_tx(struct igb_adapter *adapter)
1441{
1442 u64 tdba, tdwba;
1443 struct e1000_hw *hw = &adapter->hw;
1444 u32 tctl;
1445 u32 txdctl, txctrl;
1446 int i;
1447
1448 for (i = 0; i < adapter->num_tx_queues; i++) {
1449 struct igb_ring *ring = &(adapter->tx_ring[i]);
1450
1451 wr32(E1000_TDLEN(i),
1452 ring->count * sizeof(struct e1000_tx_desc));
1453 tdba = ring->dma;
1454 wr32(E1000_TDBAL(i),
1455 tdba & 0x00000000ffffffffULL);
1456 wr32(E1000_TDBAH(i), tdba >> 32);
1457
1458 tdwba = ring->dma + ring->count * sizeof(struct e1000_tx_desc);
1459 tdwba |= 1; /* enable head wb */
1460 wr32(E1000_TDWBAL(i),
1461 tdwba & 0x00000000ffffffffULL);
1462 wr32(E1000_TDWBAH(i), tdwba >> 32);
1463
1464 ring->head = E1000_TDH(i);
1465 ring->tail = E1000_TDT(i);
1466 writel(0, hw->hw_addr + ring->tail);
1467 writel(0, hw->hw_addr + ring->head);
1468 txdctl = rd32(E1000_TXDCTL(i));
1469 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1470 wr32(E1000_TXDCTL(i), txdctl);
1471
1472 /* Turn off Relaxed Ordering on head write-backs. The
1473 * writebacks MUST be delivered in order or it will
1474 * completely screw up our bookeeping.
1475 */
1476 txctrl = rd32(E1000_DCA_TXCTRL(i));
1477 txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1478 wr32(E1000_DCA_TXCTRL(i), txctrl);
1479 }
1480
1481
1482
1483 /* Use the default values for the Tx Inter Packet Gap (IPG) timer */
1484
1485 /* Program the Transmit Control Register */
1486
1487 tctl = rd32(E1000_TCTL);
1488 tctl &= ~E1000_TCTL_CT;
1489 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1490 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1491
1492 igb_config_collision_dist(hw);
1493
1494 /* Setup Transmit Descriptor Settings for eop descriptor */
1495 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS;
1496
1497 /* Enable transmits */
1498 tctl |= E1000_TCTL_EN;
1499
1500 wr32(E1000_TCTL, tctl);
1501}
1502
1503/**
1504 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
1505 * @adapter: board private structure
1506 * @rx_ring: rx descriptor ring (for a specific queue) to setup
1507 *
1508 * Returns 0 on success, negative on failure
1509 **/
1510
1511int igb_setup_rx_resources(struct igb_adapter *adapter,
1512 struct igb_ring *rx_ring)
1513{
1514 struct pci_dev *pdev = adapter->pdev;
1515 int size, desc_len;
1516
1517 size = sizeof(struct igb_buffer) * rx_ring->count;
1518 rx_ring->buffer_info = vmalloc(size);
1519 if (!rx_ring->buffer_info)
1520 goto err;
1521 memset(rx_ring->buffer_info, 0, size);
1522
1523 desc_len = sizeof(union e1000_adv_rx_desc);
1524
1525 /* Round up to nearest 4K */
1526 rx_ring->size = rx_ring->count * desc_len;
1527 rx_ring->size = ALIGN(rx_ring->size, 4096);
1528
1529 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
1530 &rx_ring->dma);
1531
1532 if (!rx_ring->desc)
1533 goto err;
1534
1535 rx_ring->next_to_clean = 0;
1536 rx_ring->next_to_use = 0;
1537 rx_ring->pending_skb = NULL;
1538
1539 rx_ring->adapter = adapter;
1540 /* FIXME: do we want to setup ring->napi->poll here? */
1541 rx_ring->napi.poll = adapter->napi.poll;
1542
1543 return 0;
1544
1545err:
1546 vfree(rx_ring->buffer_info);
1547 dev_err(&adapter->pdev->dev, "Unable to allocate memory for "
1548 "the receive descriptor ring\n");
1549 return -ENOMEM;
1550}
1551
1552/**
1553 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
1554 * (Descriptors) for all queues
1555 * @adapter: board private structure
1556 *
1557 * Return 0 on success, negative on failure
1558 **/
1559static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
1560{
1561 int i, err = 0;
1562
1563 for (i = 0; i < adapter->num_rx_queues; i++) {
1564 err = igb_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1565 if (err) {
1566 dev_err(&adapter->pdev->dev,
1567 "Allocation for Rx Queue %u failed\n", i);
1568 for (i--; i >= 0; i--)
1569 igb_free_rx_resources(adapter,
1570 &adapter->rx_ring[i]);
1571 break;
1572 }
1573 }
1574
1575 return err;
1576}
1577
1578/**
1579 * igb_setup_rctl - configure the receive control registers
1580 * @adapter: Board private structure
1581 **/
1582static void igb_setup_rctl(struct igb_adapter *adapter)
1583{
1584 struct e1000_hw *hw = &adapter->hw;
1585 u32 rctl;
1586 u32 srrctl = 0;
1587 int i;
1588
1589 rctl = rd32(E1000_RCTL);
1590
1591 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1592
1593 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
1594 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
1595 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
1596
1597 /* disable the stripping of CRC because it breaks
1598 * BMC firmware connected over SMBUS
1599 rctl |= E1000_RCTL_SECRC;
1600 */
1601
1602 rctl &= ~E1000_RCTL_SBP;
1603
1604 if (adapter->netdev->mtu <= ETH_DATA_LEN)
1605 rctl &= ~E1000_RCTL_LPE;
1606 else
1607 rctl |= E1000_RCTL_LPE;
1608 if (adapter->rx_buffer_len <= IGB_RXBUFFER_2048) {
1609 /* Setup buffer sizes */
1610 rctl &= ~E1000_RCTL_SZ_4096;
1611 rctl |= E1000_RCTL_BSEX;
1612 switch (adapter->rx_buffer_len) {
1613 case IGB_RXBUFFER_256:
1614 rctl |= E1000_RCTL_SZ_256;
1615 rctl &= ~E1000_RCTL_BSEX;
1616 break;
1617 case IGB_RXBUFFER_512:
1618 rctl |= E1000_RCTL_SZ_512;
1619 rctl &= ~E1000_RCTL_BSEX;
1620 break;
1621 case IGB_RXBUFFER_1024:
1622 rctl |= E1000_RCTL_SZ_1024;
1623 rctl &= ~E1000_RCTL_BSEX;
1624 break;
1625 case IGB_RXBUFFER_2048:
1626 default:
1627 rctl |= E1000_RCTL_SZ_2048;
1628 rctl &= ~E1000_RCTL_BSEX;
1629 break;
1630 case IGB_RXBUFFER_4096:
1631 rctl |= E1000_RCTL_SZ_4096;
1632 break;
1633 case IGB_RXBUFFER_8192:
1634 rctl |= E1000_RCTL_SZ_8192;
1635 break;
1636 case IGB_RXBUFFER_16384:
1637 rctl |= E1000_RCTL_SZ_16384;
1638 break;
1639 }
1640 } else {
1641 rctl &= ~E1000_RCTL_BSEX;
1642 srrctl = adapter->rx_buffer_len >> E1000_SRRCTL_BSIZEPKT_SHIFT;
1643 }
1644
1645 /* 82575 and greater support packet-split where the protocol
1646 * header is placed in skb->data and the packet data is
1647 * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
1648 * In the case of a non-split, skb->data is linearly filled,
1649 * followed by the page buffers. Therefore, skb->data is
1650 * sized to hold the largest protocol header.
1651 */
1652 /* allocations using alloc_page take too long for regular MTU
1653 * so only enable packet split for jumbo frames */
1654 if (rctl & E1000_RCTL_LPE) {
1655 adapter->rx_ps_hdr_size = IGB_RXBUFFER_128;
1656 srrctl = adapter->rx_ps_hdr_size <<
1657 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1658 /* buffer size is ALWAYS one page */
1659 srrctl |= PAGE_SIZE >> E1000_SRRCTL_BSIZEPKT_SHIFT;
1660 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1661 } else {
1662 adapter->rx_ps_hdr_size = 0;
1663 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1664 }
1665
1666 for (i = 0; i < adapter->num_rx_queues; i++)
1667 wr32(E1000_SRRCTL(i), srrctl);
1668
1669 wr32(E1000_RCTL, rctl);
1670}
1671
1672/**
1673 * igb_configure_rx - Configure receive Unit after Reset
1674 * @adapter: board private structure
1675 *
1676 * Configure the Rx unit of the MAC after a reset.
1677 **/
1678static void igb_configure_rx(struct igb_adapter *adapter)
1679{
1680 u64 rdba;
1681 struct e1000_hw *hw = &adapter->hw;
1682 u32 rctl, rxcsum;
1683 u32 rxdctl;
1684 int i;
1685
1686 /* disable receives while setting up the descriptors */
1687 rctl = rd32(E1000_RCTL);
1688 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1689 wrfl();
1690 mdelay(10);
1691
1692 if (adapter->itr_setting > 3)
1693 wr32(E1000_ITR,
1694 1000000000 / (adapter->itr * 256));
1695
1696 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1697 * the Base and Length of the Rx Descriptor Ring */
1698 for (i = 0; i < adapter->num_rx_queues; i++) {
1699 struct igb_ring *ring = &(adapter->rx_ring[i]);
1700 rdba = ring->dma;
1701 wr32(E1000_RDBAL(i),
1702 rdba & 0x00000000ffffffffULL);
1703 wr32(E1000_RDBAH(i), rdba >> 32);
1704 wr32(E1000_RDLEN(i),
1705 ring->count * sizeof(union e1000_adv_rx_desc));
1706
1707 ring->head = E1000_RDH(i);
1708 ring->tail = E1000_RDT(i);
1709 writel(0, hw->hw_addr + ring->tail);
1710 writel(0, hw->hw_addr + ring->head);
1711
1712 rxdctl = rd32(E1000_RXDCTL(i));
1713 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1714 rxdctl &= 0xFFF00000;
1715 rxdctl |= IGB_RX_PTHRESH;
1716 rxdctl |= IGB_RX_HTHRESH << 8;
1717 rxdctl |= IGB_RX_WTHRESH << 16;
1718 wr32(E1000_RXDCTL(i), rxdctl);
1719 }
1720
1721 if (adapter->num_rx_queues > 1) {
1722 u32 random[10];
1723 u32 mrqc;
1724 u32 j, shift;
1725 union e1000_reta {
1726 u32 dword;
1727 u8 bytes[4];
1728 } reta;
1729
1730 get_random_bytes(&random[0], 40);
1731
1732 shift = 6;
1733 for (j = 0; j < (32 * 4); j++) {
1734 reta.bytes[j & 3] =
1735 (j % adapter->num_rx_queues) << shift;
1736 if ((j & 3) == 3)
1737 writel(reta.dword,
1738 hw->hw_addr + E1000_RETA(0) + (j & ~3));
1739 }
1740 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
1741
1742 /* Fill out hash function seeds */
1743 for (j = 0; j < 10; j++)
1744 array_wr32(E1000_RSSRK(0), j, random[j]);
1745
1746 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
1747 E1000_MRQC_RSS_FIELD_IPV4_TCP);
1748 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
1749 E1000_MRQC_RSS_FIELD_IPV6_TCP);
1750 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
1751 E1000_MRQC_RSS_FIELD_IPV6_UDP);
1752 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
1753 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
1754
1755
1756 wr32(E1000_MRQC, mrqc);
1757
1758 /* Multiqueue and raw packet checksumming are mutually
1759 * exclusive. Note that this not the same as TCP/IP
1760 * checksumming, which works fine. */
1761 rxcsum = rd32(E1000_RXCSUM);
1762 rxcsum |= E1000_RXCSUM_PCSD;
1763 wr32(E1000_RXCSUM, rxcsum);
1764 } else {
1765 /* Enable Receive Checksum Offload for TCP and UDP */
1766 rxcsum = rd32(E1000_RXCSUM);
1767 if (adapter->rx_csum) {
1768 rxcsum |= E1000_RXCSUM_TUOFL;
1769
1770 /* Enable IPv4 payload checksum for UDP fragments
1771 * Must be used in conjunction with packet-split. */
1772 if (adapter->rx_ps_hdr_size)
1773 rxcsum |= E1000_RXCSUM_IPPCSE;
1774 } else {
1775 rxcsum &= ~E1000_RXCSUM_TUOFL;
1776 /* don't need to clear IPPCSE as it defaults to 0 */
1777 }
1778 wr32(E1000_RXCSUM, rxcsum);
1779 }
1780
1781 if (adapter->vlgrp)
1782 wr32(E1000_RLPML,
1783 adapter->max_frame_size + VLAN_TAG_SIZE);
1784 else
1785 wr32(E1000_RLPML, adapter->max_frame_size);
1786
1787 /* Enable Receives */
1788 wr32(E1000_RCTL, rctl);
1789}
1790
1791/**
1792 * igb_free_tx_resources - Free Tx Resources per Queue
1793 * @adapter: board private structure
1794 * @tx_ring: Tx descriptor ring for a specific queue
1795 *
1796 * Free all transmit software resources
1797 **/
1798static void igb_free_tx_resources(struct igb_adapter *adapter,
1799 struct igb_ring *tx_ring)
1800{
1801 struct pci_dev *pdev = adapter->pdev;
1802
1803 igb_clean_tx_ring(adapter, tx_ring);
1804
1805 vfree(tx_ring->buffer_info);
1806 tx_ring->buffer_info = NULL;
1807
1808 pci_free_consistent(pdev, tx_ring->size, tx_ring->desc, tx_ring->dma);
1809
1810 tx_ring->desc = NULL;
1811}
1812
1813/**
1814 * igb_free_all_tx_resources - Free Tx Resources for All Queues
1815 * @adapter: board private structure
1816 *
1817 * Free all transmit software resources
1818 **/
1819static void igb_free_all_tx_resources(struct igb_adapter *adapter)
1820{
1821 int i;
1822
1823 for (i = 0; i < adapter->num_tx_queues; i++)
1824 igb_free_tx_resources(adapter, &adapter->tx_ring[i]);
1825}
1826
1827static void igb_unmap_and_free_tx_resource(struct igb_adapter *adapter,
1828 struct igb_buffer *buffer_info)
1829{
1830 if (buffer_info->dma) {
1831 pci_unmap_page(adapter->pdev,
1832 buffer_info->dma,
1833 buffer_info->length,
1834 PCI_DMA_TODEVICE);
1835 buffer_info->dma = 0;
1836 }
1837 if (buffer_info->skb) {
1838 dev_kfree_skb_any(buffer_info->skb);
1839 buffer_info->skb = NULL;
1840 }
1841 buffer_info->time_stamp = 0;
1842 /* buffer_info must be completely set up in the transmit path */
1843}
1844
1845/**
1846 * igb_clean_tx_ring - Free Tx Buffers
1847 * @adapter: board private structure
1848 * @tx_ring: ring to be cleaned
1849 **/
1850static void igb_clean_tx_ring(struct igb_adapter *adapter,
1851 struct igb_ring *tx_ring)
1852{
1853 struct igb_buffer *buffer_info;
1854 unsigned long size;
1855 unsigned int i;
1856
1857 if (!tx_ring->buffer_info)
1858 return;
1859 /* Free all the Tx ring sk_buffs */
1860
1861 for (i = 0; i < tx_ring->count; i++) {
1862 buffer_info = &tx_ring->buffer_info[i];
1863 igb_unmap_and_free_tx_resource(adapter, buffer_info);
1864 }
1865
1866 size = sizeof(struct igb_buffer) * tx_ring->count;
1867 memset(tx_ring->buffer_info, 0, size);
1868
1869 /* Zero out the descriptor ring */
1870
1871 memset(tx_ring->desc, 0, tx_ring->size);
1872
1873 tx_ring->next_to_use = 0;
1874 tx_ring->next_to_clean = 0;
1875
1876 writel(0, adapter->hw.hw_addr + tx_ring->head);
1877 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1878}
1879
1880/**
1881 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
1882 * @adapter: board private structure
1883 **/
1884static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
1885{
1886 int i;
1887
1888 for (i = 0; i < adapter->num_tx_queues; i++)
1889 igb_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1890}
1891
1892/**
1893 * igb_free_rx_resources - Free Rx Resources
1894 * @adapter: board private structure
1895 * @rx_ring: ring to clean the resources from
1896 *
1897 * Free all receive software resources
1898 **/
1899static void igb_free_rx_resources(struct igb_adapter *adapter,
1900 struct igb_ring *rx_ring)
1901{
1902 struct pci_dev *pdev = adapter->pdev;
1903
1904 igb_clean_rx_ring(adapter, rx_ring);
1905
1906 vfree(rx_ring->buffer_info);
1907 rx_ring->buffer_info = NULL;
1908
1909 pci_free_consistent(pdev, rx_ring->size, rx_ring->desc, rx_ring->dma);
1910
1911 rx_ring->desc = NULL;
1912}
1913
1914/**
1915 * igb_free_all_rx_resources - Free Rx Resources for All Queues
1916 * @adapter: board private structure
1917 *
1918 * Free all receive software resources
1919 **/
1920static void igb_free_all_rx_resources(struct igb_adapter *adapter)
1921{
1922 int i;
1923
1924 for (i = 0; i < adapter->num_rx_queues; i++)
1925 igb_free_rx_resources(adapter, &adapter->rx_ring[i]);
1926}
1927
1928/**
1929 * igb_clean_rx_ring - Free Rx Buffers per Queue
1930 * @adapter: board private structure
1931 * @rx_ring: ring to free buffers from
1932 **/
1933static void igb_clean_rx_ring(struct igb_adapter *adapter,
1934 struct igb_ring *rx_ring)
1935{
1936 struct igb_buffer *buffer_info;
1937 struct pci_dev *pdev = adapter->pdev;
1938 unsigned long size;
1939 unsigned int i;
1940
1941 if (!rx_ring->buffer_info)
1942 return;
1943 /* Free all the Rx ring sk_buffs */
1944 for (i = 0; i < rx_ring->count; i++) {
1945 buffer_info = &rx_ring->buffer_info[i];
1946 if (buffer_info->dma) {
1947 if (adapter->rx_ps_hdr_size)
1948 pci_unmap_single(pdev, buffer_info->dma,
1949 adapter->rx_ps_hdr_size,
1950 PCI_DMA_FROMDEVICE);
1951 else
1952 pci_unmap_single(pdev, buffer_info->dma,
1953 adapter->rx_buffer_len,
1954 PCI_DMA_FROMDEVICE);
1955 buffer_info->dma = 0;
1956 }
1957
1958 if (buffer_info->skb) {
1959 dev_kfree_skb(buffer_info->skb);
1960 buffer_info->skb = NULL;
1961 }
1962 if (buffer_info->page) {
1963 pci_unmap_page(pdev, buffer_info->page_dma,
1964 PAGE_SIZE, PCI_DMA_FROMDEVICE);
1965 put_page(buffer_info->page);
1966 buffer_info->page = NULL;
1967 buffer_info->page_dma = 0;
1968 }
1969 }
1970
1971 /* there also may be some cached data from a chained receive */
1972 if (rx_ring->pending_skb) {
1973 dev_kfree_skb(rx_ring->pending_skb);
1974 rx_ring->pending_skb = NULL;
1975 }
1976
1977 size = sizeof(struct igb_buffer) * rx_ring->count;
1978 memset(rx_ring->buffer_info, 0, size);
1979
1980 /* Zero out the descriptor ring */
1981 memset(rx_ring->desc, 0, rx_ring->size);
1982
1983 rx_ring->next_to_clean = 0;
1984 rx_ring->next_to_use = 0;
1985
1986 writel(0, adapter->hw.hw_addr + rx_ring->head);
1987 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1988}
1989
1990/**
1991 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
1992 * @adapter: board private structure
1993 **/
1994static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
1995{
1996 int i;
1997
1998 for (i = 0; i < adapter->num_rx_queues; i++)
1999 igb_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2000}
2001
2002/**
2003 * igb_set_mac - Change the Ethernet Address of the NIC
2004 * @netdev: network interface device structure
2005 * @p: pointer to an address structure
2006 *
2007 * Returns 0 on success, negative on failure
2008 **/
2009static int igb_set_mac(struct net_device *netdev, void *p)
2010{
2011 struct igb_adapter *adapter = netdev_priv(netdev);
2012 struct sockaddr *addr = p;
2013
2014 if (!is_valid_ether_addr(addr->sa_data))
2015 return -EADDRNOTAVAIL;
2016
2017 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2018 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
2019
2020 adapter->hw.mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2021
2022 return 0;
2023}
2024
2025/**
2026 * igb_set_multi - Multicast and Promiscuous mode set
2027 * @netdev: network interface device structure
2028 *
2029 * The set_multi entry point is called whenever the multicast address
2030 * list or the network interface flags are updated. This routine is
2031 * responsible for configuring the hardware for proper multicast,
2032 * promiscuous mode, and all-multi behavior.
2033 **/
2034static void igb_set_multi(struct net_device *netdev)
2035{
2036 struct igb_adapter *adapter = netdev_priv(netdev);
2037 struct e1000_hw *hw = &adapter->hw;
2038 struct e1000_mac_info *mac = &hw->mac;
2039 struct dev_mc_list *mc_ptr;
2040 u8 *mta_list;
2041 u32 rctl;
2042 int i;
2043
2044 /* Check for Promiscuous and All Multicast modes */
2045
2046 rctl = rd32(E1000_RCTL);
2047
2048 if (netdev->flags & IFF_PROMISC)
2049 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2050 else if (netdev->flags & IFF_ALLMULTI) {
2051 rctl |= E1000_RCTL_MPE;
2052 rctl &= ~E1000_RCTL_UPE;
2053 } else
2054 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
2055
2056 wr32(E1000_RCTL, rctl);
2057
2058 if (!netdev->mc_count) {
2059 /* nothing to program, so clear mc list */
2060 igb_update_mc_addr_list(hw, NULL, 0, 1,
2061 mac->rar_entry_count);
2062 return;
2063 }
2064
2065 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2066 if (!mta_list)
2067 return;
2068
2069 /* The shared function expects a packed array of only addresses. */
2070 mc_ptr = netdev->mc_list;
2071
2072 for (i = 0; i < netdev->mc_count; i++) {
2073 if (!mc_ptr)
2074 break;
2075 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2076 mc_ptr = mc_ptr->next;
2077 }
2078 igb_update_mc_addr_list(hw, mta_list, i, 1, mac->rar_entry_count);
2079 kfree(mta_list);
2080}
2081
2082/* Need to wait a few seconds after link up to get diagnostic information from
2083 * the phy */
2084static void igb_update_phy_info(unsigned long data)
2085{
2086 struct igb_adapter *adapter = (struct igb_adapter *) data;
2087 adapter->hw.phy.ops.get_phy_info(&adapter->hw);
2088}
2089
2090/**
2091 * igb_watchdog - Timer Call-back
2092 * @data: pointer to adapter cast into an unsigned long
2093 **/
2094static void igb_watchdog(unsigned long data)
2095{
2096 struct igb_adapter *adapter = (struct igb_adapter *)data;
2097 /* Do the rest outside of interrupt context */
2098 schedule_work(&adapter->watchdog_task);
2099}
2100
2101static void igb_watchdog_task(struct work_struct *work)
2102{
2103 struct igb_adapter *adapter = container_of(work,
2104 struct igb_adapter, watchdog_task);
2105 struct e1000_hw *hw = &adapter->hw;
2106
2107 struct net_device *netdev = adapter->netdev;
2108 struct igb_ring *tx_ring = adapter->tx_ring;
2109 struct e1000_mac_info *mac = &adapter->hw.mac;
2110 u32 link;
2111 s32 ret_val;
2112
2113 if ((netif_carrier_ok(netdev)) &&
2114 (rd32(E1000_STATUS) & E1000_STATUS_LU))
2115 goto link_up;
2116
2117 ret_val = hw->mac.ops.check_for_link(&adapter->hw);
2118 if ((ret_val == E1000_ERR_PHY) &&
2119 (hw->phy.type == e1000_phy_igp_3) &&
2120 (rd32(E1000_CTRL) &
2121 E1000_PHY_CTRL_GBE_DISABLE))
2122 dev_info(&adapter->pdev->dev,
2123 "Gigabit has been disabled, downgrading speed\n");
2124
2125 if ((hw->phy.media_type == e1000_media_type_internal_serdes) &&
2126 !(rd32(E1000_TXCW) & E1000_TXCW_ANE))
2127 link = mac->serdes_has_link;
2128 else
2129 link = rd32(E1000_STATUS) &
2130 E1000_STATUS_LU;
2131
2132 if (link) {
2133 if (!netif_carrier_ok(netdev)) {
2134 u32 ctrl;
2135 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2136 &adapter->link_speed,
2137 &adapter->link_duplex);
2138
2139 ctrl = rd32(E1000_CTRL);
2140 dev_info(&adapter->pdev->dev,
2141 "NIC Link is Up %d Mbps %s, "
2142 "Flow Control: %s\n",
2143 adapter->link_speed,
2144 adapter->link_duplex == FULL_DUPLEX ?
2145 "Full Duplex" : "Half Duplex",
2146 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2147 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2148 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2149 E1000_CTRL_TFCE) ? "TX" : "None")));
2150
2151 /* tweak tx_queue_len according to speed/duplex and
2152 * adjust the timeout factor */
2153 netdev->tx_queue_len = adapter->tx_queue_len;
2154 adapter->tx_timeout_factor = 1;
2155 switch (adapter->link_speed) {
2156 case SPEED_10:
2157 netdev->tx_queue_len = 10;
2158 adapter->tx_timeout_factor = 14;
2159 break;
2160 case SPEED_100:
2161 netdev->tx_queue_len = 100;
2162 /* maybe add some timeout factor ? */
2163 break;
2164 }
2165
2166 netif_carrier_on(netdev);
2167 netif_wake_queue(netdev);
2168
2169 if (!test_bit(__IGB_DOWN, &adapter->state))
2170 mod_timer(&adapter->phy_info_timer,
2171 round_jiffies(jiffies + 2 * HZ));
2172 }
2173 } else {
2174 if (netif_carrier_ok(netdev)) {
2175 adapter->link_speed = 0;
2176 adapter->link_duplex = 0;
2177 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
2178 netif_carrier_off(netdev);
2179 netif_stop_queue(netdev);
2180 if (!test_bit(__IGB_DOWN, &adapter->state))
2181 mod_timer(&adapter->phy_info_timer,
2182 round_jiffies(jiffies + 2 * HZ));
2183 }
2184 }
2185
2186link_up:
2187 igb_update_stats(adapter);
2188
2189 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2190 adapter->tpt_old = adapter->stats.tpt;
2191 mac->collision_delta = adapter->stats.colc - adapter->colc_old;
2192 adapter->colc_old = adapter->stats.colc;
2193
2194 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
2195 adapter->gorc_old = adapter->stats.gorc;
2196 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
2197 adapter->gotc_old = adapter->stats.gotc;
2198
2199 igb_update_adaptive(&adapter->hw);
2200
2201 if (!netif_carrier_ok(netdev)) {
2202 if (IGB_DESC_UNUSED(tx_ring) + 1 < tx_ring->count) {
2203 /* We've lost link, so the controller stops DMA,
2204 * but we've got queued Tx work that's never going
2205 * to get done, so reset controller to flush Tx.
2206 * (Do the reset outside of interrupt context). */
2207 adapter->tx_timeout_count++;
2208 schedule_work(&adapter->reset_task);
2209 }
2210 }
2211
2212 /* Cause software interrupt to ensure rx ring is cleaned */
2213 wr32(E1000_ICS, E1000_ICS_RXDMT0);
2214
2215 /* Force detection of hung controller every watchdog period */
2216 tx_ring->detect_tx_hung = true;
2217
2218 /* Reset the timer */
2219 if (!test_bit(__IGB_DOWN, &adapter->state))
2220 mod_timer(&adapter->watchdog_timer,
2221 round_jiffies(jiffies + 2 * HZ));
2222}
2223
2224enum latency_range {
2225 lowest_latency = 0,
2226 low_latency = 1,
2227 bulk_latency = 2,
2228 latency_invalid = 255
2229};
2230
2231
2232static void igb_lower_rx_eitr(struct igb_adapter *adapter,
2233 struct igb_ring *rx_ring)
2234{
2235 struct e1000_hw *hw = &adapter->hw;
2236 int new_val;
2237
2238 new_val = rx_ring->itr_val / 2;
2239 if (new_val < IGB_MIN_DYN_ITR)
2240 new_val = IGB_MIN_DYN_ITR;
2241
2242 if (new_val != rx_ring->itr_val) {
2243 rx_ring->itr_val = new_val;
2244 wr32(rx_ring->itr_register,
2245 1000000000 / (new_val * 256));
2246 }
2247}
2248
2249static void igb_raise_rx_eitr(struct igb_adapter *adapter,
2250 struct igb_ring *rx_ring)
2251{
2252 struct e1000_hw *hw = &adapter->hw;
2253 int new_val;
2254
2255 new_val = rx_ring->itr_val * 2;
2256 if (new_val > IGB_MAX_DYN_ITR)
2257 new_val = IGB_MAX_DYN_ITR;
2258
2259 if (new_val != rx_ring->itr_val) {
2260 rx_ring->itr_val = new_val;
2261 wr32(rx_ring->itr_register,
2262 1000000000 / (new_val * 256));
2263 }
2264}
2265
2266/**
2267 * igb_update_itr - update the dynamic ITR value based on statistics
2268 * Stores a new ITR value based on packets and byte
2269 * counts during the last interrupt. The advantage of per interrupt
2270 * computation is faster updates and more accurate ITR for the current
2271 * traffic pattern. Constants in this function were computed
2272 * based on theoretical maximum wire speed and thresholds were set based
2273 * on testing data as well as attempting to minimize response time
2274 * while increasing bulk throughput.
2275 * this functionality is controlled by the InterruptThrottleRate module
2276 * parameter (see igb_param.c)
2277 * NOTE: These calculations are only valid when operating in a single-
2278 * queue environment.
2279 * @adapter: pointer to adapter
2280 * @itr_setting: current adapter->itr
2281 * @packets: the number of packets during this measurement interval
2282 * @bytes: the number of bytes during this measurement interval
2283 **/
2284static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
2285 int packets, int bytes)
2286{
2287 unsigned int retval = itr_setting;
2288
2289 if (packets == 0)
2290 goto update_itr_done;
2291
2292 switch (itr_setting) {
2293 case lowest_latency:
2294 /* handle TSO and jumbo frames */
2295 if (bytes/packets > 8000)
2296 retval = bulk_latency;
2297 else if ((packets < 5) && (bytes > 512))
2298 retval = low_latency;
2299 break;
2300 case low_latency: /* 50 usec aka 20000 ints/s */
2301 if (bytes > 10000) {
2302 /* this if handles the TSO accounting */
2303 if (bytes/packets > 8000) {
2304 retval = bulk_latency;
2305 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
2306 retval = bulk_latency;
2307 } else if ((packets > 35)) {
2308 retval = lowest_latency;
2309 }
2310 } else if (bytes/packets > 2000) {
2311 retval = bulk_latency;
2312 } else if (packets <= 2 && bytes < 512) {
2313 retval = lowest_latency;
2314 }
2315 break;
2316 case bulk_latency: /* 250 usec aka 4000 ints/s */
2317 if (bytes > 25000) {
2318 if (packets > 35)
2319 retval = low_latency;
2320 } else if (bytes < 6000) {
2321 retval = low_latency;
2322 }
2323 break;
2324 }
2325
2326update_itr_done:
2327 return retval;
2328}
2329
2330static void igb_set_itr(struct igb_adapter *adapter, u16 itr_register,
2331 int rx_only)
2332{
2333 u16 current_itr;
2334 u32 new_itr = adapter->itr;
2335
2336 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2337 if (adapter->link_speed != SPEED_1000) {
2338 current_itr = 0;
2339 new_itr = 4000;
2340 goto set_itr_now;
2341 }
2342
2343 adapter->rx_itr = igb_update_itr(adapter,
2344 adapter->rx_itr,
2345 adapter->rx_ring->total_packets,
2346 adapter->rx_ring->total_bytes);
2347 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2348 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2349 adapter->rx_itr = low_latency;
2350
2351 if (!rx_only) {
2352 adapter->tx_itr = igb_update_itr(adapter,
2353 adapter->tx_itr,
2354 adapter->tx_ring->total_packets,
2355 adapter->tx_ring->total_bytes);
2356 /* conservative mode (itr 3) eliminates the
2357 * lowest_latency setting */
2358 if (adapter->itr_setting == 3 &&
2359 adapter->tx_itr == lowest_latency)
2360 adapter->tx_itr = low_latency;
2361
2362 current_itr = max(adapter->rx_itr, adapter->tx_itr);
2363 } else {
2364 current_itr = adapter->rx_itr;
2365 }
2366
2367 switch (current_itr) {
2368 /* counts and packets in update_itr are dependent on these numbers */
2369 case lowest_latency:
2370 new_itr = 70000;
2371 break;
2372 case low_latency:
2373 new_itr = 20000; /* aka hwitr = ~200 */
2374 break;
2375 case bulk_latency:
2376 new_itr = 4000;
2377 break;
2378 default:
2379 break;
2380 }
2381
2382set_itr_now:
2383 if (new_itr != adapter->itr) {
2384 /* this attempts to bias the interrupt rate towards Bulk
2385 * by adding intermediate steps when interrupt rate is
2386 * increasing */
2387 new_itr = new_itr > adapter->itr ?
2388 min(adapter->itr + (new_itr >> 2), new_itr) :
2389 new_itr;
2390 /* Don't write the value here; it resets the adapter's
2391 * internal timer, and causes us to delay far longer than
2392 * we should between interrupts. Instead, we write the ITR
2393 * value at the beginning of the next interrupt so the timing
2394 * ends up being correct.
2395 */
2396 adapter->itr = new_itr;
2397 adapter->set_itr = 1;
2398 }
2399
2400 return;
2401}
2402
2403
2404#define IGB_TX_FLAGS_CSUM 0x00000001
2405#define IGB_TX_FLAGS_VLAN 0x00000002
2406#define IGB_TX_FLAGS_TSO 0x00000004
2407#define IGB_TX_FLAGS_IPV4 0x00000008
2408#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
2409#define IGB_TX_FLAGS_VLAN_SHIFT 16
2410
2411static inline int igb_tso_adv(struct igb_adapter *adapter,
2412 struct igb_ring *tx_ring,
2413 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2414{
2415 struct e1000_adv_tx_context_desc *context_desc;
2416 unsigned int i;
2417 int err;
2418 struct igb_buffer *buffer_info;
2419 u32 info = 0, tu_cmd = 0;
2420 u32 mss_l4len_idx, l4len;
2421 *hdr_len = 0;
2422
2423 if (skb_header_cloned(skb)) {
2424 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2425 if (err)
2426 return err;
2427 }
2428
2429 l4len = tcp_hdrlen(skb);
2430 *hdr_len += l4len;
2431
2432 if (skb->protocol == htons(ETH_P_IP)) {
2433 struct iphdr *iph = ip_hdr(skb);
2434 iph->tot_len = 0;
2435 iph->check = 0;
2436 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2437 iph->daddr, 0,
2438 IPPROTO_TCP,
2439 0);
2440 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
2441 ipv6_hdr(skb)->payload_len = 0;
2442 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2443 &ipv6_hdr(skb)->daddr,
2444 0, IPPROTO_TCP, 0);
2445 }
2446
2447 i = tx_ring->next_to_use;
2448
2449 buffer_info = &tx_ring->buffer_info[i];
2450 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
2451 /* VLAN MACLEN IPLEN */
2452 if (tx_flags & IGB_TX_FLAGS_VLAN)
2453 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
2454 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
2455 *hdr_len += skb_network_offset(skb);
2456 info |= skb_network_header_len(skb);
2457 *hdr_len += skb_network_header_len(skb);
2458 context_desc->vlan_macip_lens = cpu_to_le32(info);
2459
2460 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2461 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
2462
2463 if (skb->protocol == htons(ETH_P_IP))
2464 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
2465 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
2466
2467 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
2468
2469 /* MSS L4LEN IDX */
2470 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
2471 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
2472
2473 /* Context index must be unique per ring. Luckily, so is the interrupt
2474 * mask value. */
2475 mss_l4len_idx |= tx_ring->eims_value >> 4;
2476
2477 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2478 context_desc->seqnum_seed = 0;
2479
2480 buffer_info->time_stamp = jiffies;
2481 buffer_info->dma = 0;
2482 i++;
2483 if (i == tx_ring->count)
2484 i = 0;
2485
2486 tx_ring->next_to_use = i;
2487
2488 return true;
2489}
2490
2491static inline bool igb_tx_csum_adv(struct igb_adapter *adapter,
2492 struct igb_ring *tx_ring,
2493 struct sk_buff *skb, u32 tx_flags)
2494{
2495 struct e1000_adv_tx_context_desc *context_desc;
2496 unsigned int i;
2497 struct igb_buffer *buffer_info;
2498 u32 info = 0, tu_cmd = 0;
2499
2500 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
2501 (tx_flags & IGB_TX_FLAGS_VLAN)) {
2502 i = tx_ring->next_to_use;
2503 buffer_info = &tx_ring->buffer_info[i];
2504 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
2505
2506 if (tx_flags & IGB_TX_FLAGS_VLAN)
2507 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
2508 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
2509 if (skb->ip_summed == CHECKSUM_PARTIAL)
2510 info |= skb_network_header_len(skb);
2511
2512 context_desc->vlan_macip_lens = cpu_to_le32(info);
2513
2514 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
2515
2516 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2517 if (skb->protocol == htons(ETH_P_IP))
2518 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
2519 if (skb->sk && (skb->sk->sk_protocol == IPPROTO_TCP))
2520 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
2521 }
2522
2523 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
2524 context_desc->seqnum_seed = 0;
2525 context_desc->mss_l4len_idx =
2526 cpu_to_le32(tx_ring->eims_value >> 4);
2527
2528 buffer_info->time_stamp = jiffies;
2529 buffer_info->dma = 0;
2530
2531 i++;
2532 if (i == tx_ring->count)
2533 i = 0;
2534 tx_ring->next_to_use = i;
2535
2536 return true;
2537 }
2538
2539
2540 return false;
2541}
2542
2543#define IGB_MAX_TXD_PWR 16
2544#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
2545
2546static inline int igb_tx_map_adv(struct igb_adapter *adapter,
2547 struct igb_ring *tx_ring,
2548 struct sk_buff *skb)
2549{
2550 struct igb_buffer *buffer_info;
2551 unsigned int len = skb_headlen(skb);
2552 unsigned int count = 0, i;
2553 unsigned int f;
2554
2555 i = tx_ring->next_to_use;
2556
2557 buffer_info = &tx_ring->buffer_info[i];
2558 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
2559 buffer_info->length = len;
2560 /* set time_stamp *before* dma to help avoid a possible race */
2561 buffer_info->time_stamp = jiffies;
2562 buffer_info->dma = pci_map_single(adapter->pdev, skb->data, len,
2563 PCI_DMA_TODEVICE);
2564 count++;
2565 i++;
2566 if (i == tx_ring->count)
2567 i = 0;
2568
2569 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2570 struct skb_frag_struct *frag;
2571
2572 frag = &skb_shinfo(skb)->frags[f];
2573 len = frag->size;
2574
2575 buffer_info = &tx_ring->buffer_info[i];
2576 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
2577 buffer_info->length = len;
2578 buffer_info->time_stamp = jiffies;
2579 buffer_info->dma = pci_map_page(adapter->pdev,
2580 frag->page,
2581 frag->page_offset,
2582 len,
2583 PCI_DMA_TODEVICE);
2584
2585 count++;
2586 i++;
2587 if (i == tx_ring->count)
2588 i = 0;
2589 }
2590
2591 i = (i == 0) ? tx_ring->count - 1 : i - 1;
2592 tx_ring->buffer_info[i].skb = skb;
2593
2594 return count;
2595}
2596
2597static inline void igb_tx_queue_adv(struct igb_adapter *adapter,
2598 struct igb_ring *tx_ring,
2599 int tx_flags, int count, u32 paylen,
2600 u8 hdr_len)
2601{
2602 union e1000_adv_tx_desc *tx_desc = NULL;
2603 struct igb_buffer *buffer_info;
2604 u32 olinfo_status = 0, cmd_type_len;
2605 unsigned int i;
2606
2607 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2608 E1000_ADVTXD_DCMD_DEXT);
2609
2610 if (tx_flags & IGB_TX_FLAGS_VLAN)
2611 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2612
2613 if (tx_flags & IGB_TX_FLAGS_TSO) {
2614 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2615
2616 /* insert tcp checksum */
2617 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2618
2619 /* insert ip checksum */
2620 if (tx_flags & IGB_TX_FLAGS_IPV4)
2621 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2622
2623 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
2624 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2625 }
2626
2627 if (tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_TSO |
2628 IGB_TX_FLAGS_VLAN))
2629 olinfo_status |= tx_ring->eims_value >> 4;
2630
2631 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2632
2633 i = tx_ring->next_to_use;
2634 while (count--) {
2635 buffer_info = &tx_ring->buffer_info[i];
2636 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
2637 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2638 tx_desc->read.cmd_type_len =
2639 cpu_to_le32(cmd_type_len | buffer_info->length);
2640 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2641 i++;
2642 if (i == tx_ring->count)
2643 i = 0;
2644 }
2645
2646 tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2647 /* Force memory writes to complete before letting h/w
2648 * know there are new descriptors to fetch. (Only
2649 * applicable for weak-ordered memory model archs,
2650 * such as IA-64). */
2651 wmb();
2652
2653 tx_ring->next_to_use = i;
2654 writel(i, adapter->hw.hw_addr + tx_ring->tail);
2655 /* we need this if more than one processor can write to our tail
2656 * at a time, it syncronizes IO on IA64/Altix systems */
2657 mmiowb();
2658}
2659
2660static int __igb_maybe_stop_tx(struct net_device *netdev,
2661 struct igb_ring *tx_ring, int size)
2662{
2663 struct igb_adapter *adapter = netdev_priv(netdev);
2664
2665 netif_stop_queue(netdev);
2666 /* Herbert's original patch had:
2667 * smp_mb__after_netif_stop_queue();
2668 * but since that doesn't exist yet, just open code it. */
2669 smp_mb();
2670
2671 /* We need to check again in a case another CPU has just
2672 * made room available. */
2673 if (IGB_DESC_UNUSED(tx_ring) < size)
2674 return -EBUSY;
2675
2676 /* A reprieve! */
2677 netif_start_queue(netdev);
2678 ++adapter->restart_queue;
2679 return 0;
2680}
2681
2682static int igb_maybe_stop_tx(struct net_device *netdev,
2683 struct igb_ring *tx_ring, int size)
2684{
2685 if (IGB_DESC_UNUSED(tx_ring) >= size)
2686 return 0;
2687 return __igb_maybe_stop_tx(netdev, tx_ring, size);
2688}
2689
2690#define TXD_USE_COUNT(S) (((S) >> (IGB_MAX_TXD_PWR)) + 1)
2691
2692static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
2693 struct net_device *netdev,
2694 struct igb_ring *tx_ring)
2695{
2696 struct igb_adapter *adapter = netdev_priv(netdev);
2697 unsigned int tx_flags = 0;
2698 unsigned int len;
2699 unsigned long irq_flags;
2700 u8 hdr_len = 0;
2701 int tso = 0;
2702
2703 len = skb_headlen(skb);
2704
2705 if (test_bit(__IGB_DOWN, &adapter->state)) {
2706 dev_kfree_skb_any(skb);
2707 return NETDEV_TX_OK;
2708 }
2709
2710 if (skb->len <= 0) {
2711 dev_kfree_skb_any(skb);
2712 return NETDEV_TX_OK;
2713 }
2714
2715 if (!spin_trylock_irqsave(&tx_ring->tx_lock, irq_flags))
2716 /* Collision - tell upper layer to requeue */
2717 return NETDEV_TX_LOCKED;
2718
2719 /* need: 1 descriptor per page,
2720 * + 2 desc gap to keep tail from touching head,
2721 * + 1 desc for skb->data,
2722 * + 1 desc for context descriptor,
2723 * otherwise try next time */
2724 if (igb_maybe_stop_tx(netdev, tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
2725 /* this is a hard error */
2726 spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
2727 return NETDEV_TX_BUSY;
2728 }
2729
2730 if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
2731 tx_flags |= IGB_TX_FLAGS_VLAN;
2732 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
2733 }
2734
2735 tso = skb_is_gso(skb) ? igb_tso_adv(adapter, tx_ring, skb, tx_flags,
2736 &hdr_len) : 0;
2737
2738 if (tso < 0) {
2739 dev_kfree_skb_any(skb);
2740 spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
2741 return NETDEV_TX_OK;
2742 }
2743
2744 if (tso)
2745 tx_flags |= IGB_TX_FLAGS_TSO;
2746 else if (igb_tx_csum_adv(adapter, tx_ring, skb, tx_flags))
2747 if (skb->ip_summed == CHECKSUM_PARTIAL)
2748 tx_flags |= IGB_TX_FLAGS_CSUM;
2749
2750 if (skb->protocol == htons(ETH_P_IP))
2751 tx_flags |= IGB_TX_FLAGS_IPV4;
2752
2753 igb_tx_queue_adv(adapter, tx_ring, tx_flags,
2754 igb_tx_map_adv(adapter, tx_ring, skb),
2755 skb->len, hdr_len);
2756
2757 netdev->trans_start = jiffies;
2758
2759 /* Make sure there is space in the ring for the next send. */
2760 igb_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 4);
2761
2762 spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
2763 return NETDEV_TX_OK;
2764}
2765
2766static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *netdev)
2767{
2768 struct igb_adapter *adapter = netdev_priv(netdev);
2769 struct igb_ring *tx_ring = &adapter->tx_ring[0];
2770
2771 /* This goes back to the question of how to logically map a tx queue
2772 * to a flow. Right now, performance is impacted slightly negatively
2773 * if using multiple tx queues. If the stack breaks away from a
2774 * single qdisc implementation, we can look at this again. */
2775 return (igb_xmit_frame_ring_adv(skb, netdev, tx_ring));
2776}
2777
2778/**
2779 * igb_tx_timeout - Respond to a Tx Hang
2780 * @netdev: network interface device structure
2781 **/
2782static void igb_tx_timeout(struct net_device *netdev)
2783{
2784 struct igb_adapter *adapter = netdev_priv(netdev);
2785 struct e1000_hw *hw = &adapter->hw;
2786
2787 /* Do the reset outside of interrupt context */
2788 adapter->tx_timeout_count++;
2789 schedule_work(&adapter->reset_task);
2790 wr32(E1000_EICS, adapter->eims_enable_mask &
2791 ~(E1000_EIMS_TCP_TIMER | E1000_EIMS_OTHER));
2792}
2793
2794static void igb_reset_task(struct work_struct *work)
2795{
2796 struct igb_adapter *adapter;
2797 adapter = container_of(work, struct igb_adapter, reset_task);
2798
2799 igb_reinit_locked(adapter);
2800}
2801
2802/**
2803 * igb_get_stats - Get System Network Statistics
2804 * @netdev: network interface device structure
2805 *
2806 * Returns the address of the device statistics structure.
2807 * The statistics are actually updated from the timer callback.
2808 **/
2809static struct net_device_stats *
2810igb_get_stats(struct net_device *netdev)
2811{
2812 struct igb_adapter *adapter = netdev_priv(netdev);
2813
2814 /* only return the current stats */
2815 return &adapter->net_stats;
2816}
2817
2818/**
2819 * igb_change_mtu - Change the Maximum Transfer Unit
2820 * @netdev: network interface device structure
2821 * @new_mtu: new value for maximum frame size
2822 *
2823 * Returns 0 on success, negative on failure
2824 **/
2825static int igb_change_mtu(struct net_device *netdev, int new_mtu)
2826{
2827 struct igb_adapter *adapter = netdev_priv(netdev);
2828 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2829
2830 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
2831 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
2832 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
2833 return -EINVAL;
2834 }
2835
2836#define MAX_STD_JUMBO_FRAME_SIZE 9234
2837 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
2838 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
2839 return -EINVAL;
2840 }
2841
2842 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
2843 msleep(1);
2844 /* igb_down has a dependency on max_frame_size */
2845 adapter->max_frame_size = max_frame;
2846 if (netif_running(netdev))
2847 igb_down(adapter);
2848
2849 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2850 * means we reserve 2 more, this pushes us to allocate from the next
2851 * larger slab size.
2852 * i.e. RXBUFFER_2048 --> size-4096 slab
2853 */
2854
2855 if (max_frame <= IGB_RXBUFFER_256)
2856 adapter->rx_buffer_len = IGB_RXBUFFER_256;
2857 else if (max_frame <= IGB_RXBUFFER_512)
2858 adapter->rx_buffer_len = IGB_RXBUFFER_512;
2859 else if (max_frame <= IGB_RXBUFFER_1024)
2860 adapter->rx_buffer_len = IGB_RXBUFFER_1024;
2861 else if (max_frame <= IGB_RXBUFFER_2048)
2862 adapter->rx_buffer_len = IGB_RXBUFFER_2048;
2863 else
2864 adapter->rx_buffer_len = IGB_RXBUFFER_4096;
2865 /* adjust allocation if LPE protects us, and we aren't using SBP */
2866 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2867 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE))
2868 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
2869
2870 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
2871 netdev->mtu, new_mtu);
2872 netdev->mtu = new_mtu;
2873
2874 if (netif_running(netdev))
2875 igb_up(adapter);
2876 else
2877 igb_reset(adapter);
2878
2879 clear_bit(__IGB_RESETTING, &adapter->state);
2880
2881 return 0;
2882}
2883
2884/**
2885 * igb_update_stats - Update the board statistics counters
2886 * @adapter: board private structure
2887 **/
2888
2889void igb_update_stats(struct igb_adapter *adapter)
2890{
2891 struct e1000_hw *hw = &adapter->hw;
2892 struct pci_dev *pdev = adapter->pdev;
2893 u16 phy_tmp;
2894
2895#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
2896
2897 /*
2898 * Prevent stats update while adapter is being reset, or if the pci
2899 * connection is down.
2900 */
2901 if (adapter->link_speed == 0)
2902 return;
2903 if (pci_channel_offline(pdev))
2904 return;
2905
2906 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
2907 adapter->stats.gprc += rd32(E1000_GPRC);
2908 adapter->stats.gorc += rd32(E1000_GORCL);
2909 rd32(E1000_GORCH); /* clear GORCL */
2910 adapter->stats.bprc += rd32(E1000_BPRC);
2911 adapter->stats.mprc += rd32(E1000_MPRC);
2912 adapter->stats.roc += rd32(E1000_ROC);
2913
2914 adapter->stats.prc64 += rd32(E1000_PRC64);
2915 adapter->stats.prc127 += rd32(E1000_PRC127);
2916 adapter->stats.prc255 += rd32(E1000_PRC255);
2917 adapter->stats.prc511 += rd32(E1000_PRC511);
2918 adapter->stats.prc1023 += rd32(E1000_PRC1023);
2919 adapter->stats.prc1522 += rd32(E1000_PRC1522);
2920 adapter->stats.symerrs += rd32(E1000_SYMERRS);
2921 adapter->stats.sec += rd32(E1000_SEC);
2922
2923 adapter->stats.mpc += rd32(E1000_MPC);
2924 adapter->stats.scc += rd32(E1000_SCC);
2925 adapter->stats.ecol += rd32(E1000_ECOL);
2926 adapter->stats.mcc += rd32(E1000_MCC);
2927 adapter->stats.latecol += rd32(E1000_LATECOL);
2928 adapter->stats.dc += rd32(E1000_DC);
2929 adapter->stats.rlec += rd32(E1000_RLEC);
2930 adapter->stats.xonrxc += rd32(E1000_XONRXC);
2931 adapter->stats.xontxc += rd32(E1000_XONTXC);
2932 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
2933 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
2934 adapter->stats.fcruc += rd32(E1000_FCRUC);
2935 adapter->stats.gptc += rd32(E1000_GPTC);
2936 adapter->stats.gotc += rd32(E1000_GOTCL);
2937 rd32(E1000_GOTCH); /* clear GOTCL */
2938 adapter->stats.rnbc += rd32(E1000_RNBC);
2939 adapter->stats.ruc += rd32(E1000_RUC);
2940 adapter->stats.rfc += rd32(E1000_RFC);
2941 adapter->stats.rjc += rd32(E1000_RJC);
2942 adapter->stats.tor += rd32(E1000_TORH);
2943 adapter->stats.tot += rd32(E1000_TOTH);
2944 adapter->stats.tpr += rd32(E1000_TPR);
2945
2946 adapter->stats.ptc64 += rd32(E1000_PTC64);
2947 adapter->stats.ptc127 += rd32(E1000_PTC127);
2948 adapter->stats.ptc255 += rd32(E1000_PTC255);
2949 adapter->stats.ptc511 += rd32(E1000_PTC511);
2950 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
2951 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
2952
2953 adapter->stats.mptc += rd32(E1000_MPTC);
2954 adapter->stats.bptc += rd32(E1000_BPTC);
2955
2956 /* used for adaptive IFS */
2957
2958 hw->mac.tx_packet_delta = rd32(E1000_TPT);
2959 adapter->stats.tpt += hw->mac.tx_packet_delta;
2960 hw->mac.collision_delta = rd32(E1000_COLC);
2961 adapter->stats.colc += hw->mac.collision_delta;
2962
2963 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
2964 adapter->stats.rxerrc += rd32(E1000_RXERRC);
2965 adapter->stats.tncrs += rd32(E1000_TNCRS);
2966 adapter->stats.tsctc += rd32(E1000_TSCTC);
2967 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
2968
2969 adapter->stats.iac += rd32(E1000_IAC);
2970 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
2971 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
2972 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
2973 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
2974 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
2975 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
2976 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
2977 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
2978
2979 /* Fill out the OS statistics structure */
2980 adapter->net_stats.multicast = adapter->stats.mprc;
2981 adapter->net_stats.collisions = adapter->stats.colc;
2982
2983 /* Rx Errors */
2984
2985 /* RLEC on some newer hardware can be incorrect so build
2986 * our own version based on RUC and ROC */
2987 adapter->net_stats.rx_errors = adapter->stats.rxerrc +
2988 adapter->stats.crcerrs + adapter->stats.algnerrc +
2989 adapter->stats.ruc + adapter->stats.roc +
2990 adapter->stats.cexterr;
2991 adapter->net_stats.rx_length_errors = adapter->stats.ruc +
2992 adapter->stats.roc;
2993 adapter->net_stats.rx_crc_errors = adapter->stats.crcerrs;
2994 adapter->net_stats.rx_frame_errors = adapter->stats.algnerrc;
2995 adapter->net_stats.rx_missed_errors = adapter->stats.mpc;
2996
2997 /* Tx Errors */
2998 adapter->net_stats.tx_errors = adapter->stats.ecol +
2999 adapter->stats.latecol;
3000 adapter->net_stats.tx_aborted_errors = adapter->stats.ecol;
3001 adapter->net_stats.tx_window_errors = adapter->stats.latecol;
3002 adapter->net_stats.tx_carrier_errors = adapter->stats.tncrs;
3003
3004 /* Tx Dropped needs to be maintained elsewhere */
3005
3006 /* Phy Stats */
3007 if (hw->phy.media_type == e1000_media_type_copper) {
3008 if ((adapter->link_speed == SPEED_1000) &&
3009 (!hw->phy.ops.read_phy_reg(hw, PHY_1000T_STATUS,
3010 &phy_tmp))) {
3011 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3012 adapter->phy_stats.idle_errors += phy_tmp;
3013 }
3014 }
3015
3016 /* Management Stats */
3017 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3018 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3019 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3020}
3021
3022
3023static irqreturn_t igb_msix_other(int irq, void *data)
3024{
3025 struct net_device *netdev = data;
3026 struct igb_adapter *adapter = netdev_priv(netdev);
3027 struct e1000_hw *hw = &adapter->hw;
3028 u32 eicr;
3029 /* disable interrupts from the "other" bit, avoid re-entry */
3030 wr32(E1000_EIMC, E1000_EIMS_OTHER);
3031
3032 eicr = rd32(E1000_EICR);
3033
3034 if (eicr & E1000_EIMS_OTHER) {
3035 u32 icr = rd32(E1000_ICR);
3036 /* reading ICR causes bit 31 of EICR to be cleared */
3037 if (!(icr & E1000_ICR_LSC))
3038 goto no_link_interrupt;
3039 hw->mac.get_link_status = 1;
3040 /* guard against interrupt when we're going down */
3041 if (!test_bit(__IGB_DOWN, &adapter->state))
3042 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3043 }
3044
3045no_link_interrupt:
3046 wr32(E1000_IMS, E1000_IMS_LSC);
3047 wr32(E1000_EIMS, E1000_EIMS_OTHER);
3048
3049 return IRQ_HANDLED;
3050}
3051
3052static irqreturn_t igb_msix_tx(int irq, void *data)
3053{
3054 struct igb_ring *tx_ring = data;
3055 struct igb_adapter *adapter = tx_ring->adapter;
3056 struct e1000_hw *hw = &adapter->hw;
3057
3058 if (!tx_ring->itr_val)
3059 wr32(E1000_EIMC, tx_ring->eims_value);
3060
3061 tx_ring->total_bytes = 0;
3062 tx_ring->total_packets = 0;
3063 if (!igb_clean_tx_irq(adapter, tx_ring))
3064 /* Ring was not completely cleaned, so fire another interrupt */
3065 wr32(E1000_EICS, tx_ring->eims_value);
3066
3067 if (!tx_ring->itr_val)
3068 wr32(E1000_EIMS, tx_ring->eims_value);
3069 return IRQ_HANDLED;
3070}
3071
3072static irqreturn_t igb_msix_rx(int irq, void *data)
3073{
3074 struct igb_ring *rx_ring = data;
3075 struct igb_adapter *adapter = rx_ring->adapter;
3076 struct e1000_hw *hw = &adapter->hw;
3077
3078 if (!rx_ring->itr_val)
3079 wr32(E1000_EIMC, rx_ring->eims_value);
3080
3081 if (netif_rx_schedule_prep(adapter->netdev, &rx_ring->napi)) {
3082 rx_ring->total_bytes = 0;
3083 rx_ring->total_packets = 0;
3084 rx_ring->no_itr_adjust = 0;
3085 __netif_rx_schedule(adapter->netdev, &rx_ring->napi);
3086 } else {
3087 if (!rx_ring->no_itr_adjust) {
3088 igb_lower_rx_eitr(adapter, rx_ring);
3089 rx_ring->no_itr_adjust = 1;
3090 }
3091 }
3092
3093 return IRQ_HANDLED;
3094}
3095
3096
3097/**
3098 * igb_intr_msi - Interrupt Handler
3099 * @irq: interrupt number
3100 * @data: pointer to a network interface device structure
3101 **/
3102static irqreturn_t igb_intr_msi(int irq, void *data)
3103{
3104 struct net_device *netdev = data;
3105 struct igb_adapter *adapter = netdev_priv(netdev);
3106 struct napi_struct *napi = &adapter->napi;
3107 struct e1000_hw *hw = &adapter->hw;
3108 /* read ICR disables interrupts using IAM */
3109 u32 icr = rd32(E1000_ICR);
3110
3111 /* Write the ITR value calculated at the end of the
3112 * previous interrupt.
3113 */
3114 if (adapter->set_itr) {
3115 wr32(E1000_ITR,
3116 1000000000 / (adapter->itr * 256));
3117 adapter->set_itr = 0;
3118 }
3119
3120 /* read ICR disables interrupts using IAM */
3121 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
3122 hw->mac.get_link_status = 1;
3123 if (!test_bit(__IGB_DOWN, &adapter->state))
3124 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3125 }
3126
3127 if (netif_rx_schedule_prep(netdev, napi)) {
3128 adapter->tx_ring->total_bytes = 0;
3129 adapter->tx_ring->total_packets = 0;
3130 adapter->rx_ring->total_bytes = 0;
3131 adapter->rx_ring->total_packets = 0;
3132 __netif_rx_schedule(netdev, napi);
3133 }
3134
3135 return IRQ_HANDLED;
3136}
3137
3138/**
3139 * igb_intr - Interrupt Handler
3140 * @irq: interrupt number
3141 * @data: pointer to a network interface device structure
3142 **/
3143static irqreturn_t igb_intr(int irq, void *data)
3144{
3145 struct net_device *netdev = data;
3146 struct igb_adapter *adapter = netdev_priv(netdev);
3147 struct napi_struct *napi = &adapter->napi;
3148 struct e1000_hw *hw = &adapter->hw;
3149 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
3150 * need for the IMC write */
3151 u32 icr = rd32(E1000_ICR);
3152 u32 eicr = 0;
3153 if (!icr)
3154 return IRQ_NONE; /* Not our interrupt */
3155
3156 /* Write the ITR value calculated at the end of the
3157 * previous interrupt.
3158 */
3159 if (adapter->set_itr) {
3160 wr32(E1000_ITR,
3161 1000000000 / (adapter->itr * 256));
3162 adapter->set_itr = 0;
3163 }
3164
3165 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
3166 * not set, then the adapter didn't send an interrupt */
3167 if (!(icr & E1000_ICR_INT_ASSERTED))
3168 return IRQ_NONE;
3169
3170 eicr = rd32(E1000_EICR);
3171
3172 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
3173 hw->mac.get_link_status = 1;
3174 /* guard against interrupt when we're going down */
3175 if (!test_bit(__IGB_DOWN, &adapter->state))
3176 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3177 }
3178
3179 if (netif_rx_schedule_prep(netdev, napi)) {
3180 adapter->tx_ring->total_bytes = 0;
3181 adapter->rx_ring->total_bytes = 0;
3182 adapter->tx_ring->total_packets = 0;
3183 adapter->rx_ring->total_packets = 0;
3184 __netif_rx_schedule(netdev, napi);
3185 }
3186
3187 return IRQ_HANDLED;
3188}
3189
3190/**
3191 * igb_clean - NAPI Rx polling callback
3192 * @adapter: board private structure
3193 **/
3194static int igb_clean(struct napi_struct *napi, int budget)
3195{
3196 struct igb_adapter *adapter = container_of(napi, struct igb_adapter,
3197 napi);
3198 struct net_device *netdev = adapter->netdev;
3199 int tx_clean_complete = 1, work_done = 0;
3200 int i;
3201
3202 /* Must NOT use netdev_priv macro here. */
3203 adapter = netdev->priv;
3204
3205 /* Keep link state information with original netdev */
3206 if (!netif_carrier_ok(netdev))
3207 goto quit_polling;
3208
3209 /* igb_clean is called per-cpu. This lock protects tx_ring[i] from
3210 * being cleaned by multiple cpus simultaneously. A failure obtaining
3211 * the lock means tx_ring[i] is currently being cleaned anyway. */
3212 for (i = 0; i < adapter->num_tx_queues; i++) {
3213 if (spin_trylock(&adapter->tx_ring[i].tx_clean_lock)) {
3214 tx_clean_complete &= igb_clean_tx_irq(adapter,
3215 &adapter->tx_ring[i]);
3216 spin_unlock(&adapter->tx_ring[i].tx_clean_lock);
3217 }
3218 }
3219
3220 for (i = 0; i < adapter->num_rx_queues; i++)
3221 igb_clean_rx_irq_adv(adapter, &adapter->rx_ring[i], &work_done,
3222 adapter->rx_ring[i].napi.weight);
3223
3224 /* If no Tx and not enough Rx work done, exit the polling mode */
3225 if ((tx_clean_complete && (work_done < budget)) ||
3226 !netif_running(netdev)) {
3227quit_polling:
3228 if (adapter->itr_setting & 3)
3229 igb_set_itr(adapter, E1000_ITR, false);
3230 netif_rx_complete(netdev, napi);
3231 if (!test_bit(__IGB_DOWN, &adapter->state))
3232 igb_irq_enable(adapter);
3233 return 0;
3234 }
3235
3236 return 1;
3237}
3238
3239static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
3240{
3241 struct igb_ring *rx_ring = container_of(napi, struct igb_ring, napi);
3242 struct igb_adapter *adapter = rx_ring->adapter;
3243 struct e1000_hw *hw = &adapter->hw;
3244 struct net_device *netdev = adapter->netdev;
3245 int work_done = 0;
3246
3247 /* Keep link state information with original netdev */
3248 if (!netif_carrier_ok(netdev))
3249 goto quit_polling;
3250
3251 igb_clean_rx_irq_adv(adapter, rx_ring, &work_done, budget);
3252
3253
3254 /* If not enough Rx work done, exit the polling mode */
3255 if ((work_done == 0) || !netif_running(netdev)) {
3256quit_polling:
3257 netif_rx_complete(netdev, napi);
3258
3259 wr32(E1000_EIMS, rx_ring->eims_value);
3260 if ((adapter->itr_setting & 3) && !rx_ring->no_itr_adjust &&
3261 (rx_ring->total_packets > IGB_DYN_ITR_PACKET_THRESHOLD)) {
3262 int mean_size = rx_ring->total_bytes /
3263 rx_ring->total_packets;
3264 if (mean_size < IGB_DYN_ITR_LENGTH_LOW)
3265 igb_raise_rx_eitr(adapter, rx_ring);
3266 else if (mean_size > IGB_DYN_ITR_LENGTH_HIGH)
3267 igb_lower_rx_eitr(adapter, rx_ring);
3268 }
3269 return 0;
3270 }
3271
3272 return 1;
3273}
3274/**
3275 * igb_clean_tx_irq - Reclaim resources after transmit completes
3276 * @adapter: board private structure
3277 * returns true if ring is completely cleaned
3278 **/
3279static bool igb_clean_tx_irq(struct igb_adapter *adapter,
3280 struct igb_ring *tx_ring)
3281{
3282 struct net_device *netdev = adapter->netdev;
3283 struct e1000_hw *hw = &adapter->hw;
3284 struct e1000_tx_desc *tx_desc;
3285 struct igb_buffer *buffer_info;
3286 struct sk_buff *skb;
3287 unsigned int i;
3288 u32 head, oldhead;
3289 unsigned int count = 0;
3290 bool cleaned = false;
3291 bool retval = true;
3292 unsigned int total_bytes = 0, total_packets = 0;
3293
3294 rmb();
3295 head = *(volatile u32 *)((struct e1000_tx_desc *)tx_ring->desc
3296 + tx_ring->count);
3297 head = le32_to_cpu(head);
3298 i = tx_ring->next_to_clean;
3299 while (1) {
3300 while (i != head) {
3301 cleaned = true;
3302 tx_desc = E1000_TX_DESC(*tx_ring, i);
3303 buffer_info = &tx_ring->buffer_info[i];
3304 skb = buffer_info->skb;
3305
3306 if (skb) {
3307 unsigned int segs, bytecount;
3308 /* gso_segs is currently only valid for tcp */
3309 segs = skb_shinfo(skb)->gso_segs ?: 1;
3310 /* multiply data chunks by size of headers */
3311 bytecount = ((segs - 1) * skb_headlen(skb)) +
3312 skb->len;
3313 total_packets += segs;
3314 total_bytes += bytecount;
3315 }
3316
3317 igb_unmap_and_free_tx_resource(adapter, buffer_info);
3318 tx_desc->upper.data = 0;
3319
3320 i++;
3321 if (i == tx_ring->count)
3322 i = 0;
3323
3324 count++;
3325 if (count == IGB_MAX_TX_CLEAN) {
3326 retval = false;
3327 goto done_cleaning;
3328 }
3329 }
3330 oldhead = head;
3331 rmb();
3332 head = *(volatile u32 *)((struct e1000_tx_desc *)tx_ring->desc
3333 + tx_ring->count);
3334 head = le32_to_cpu(head);
3335 if (head == oldhead)
3336 goto done_cleaning;
3337 } /* while (1) */
3338
3339done_cleaning:
3340 tx_ring->next_to_clean = i;
3341
3342 if (unlikely(cleaned &&
3343 netif_carrier_ok(netdev) &&
3344 IGB_DESC_UNUSED(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
3345 /* Make sure that anybody stopping the queue after this
3346 * sees the new next_to_clean.
3347 */
3348 smp_mb();
3349 if (netif_queue_stopped(netdev) &&
3350 !(test_bit(__IGB_DOWN, &adapter->state))) {
3351 netif_wake_queue(netdev);
3352 ++adapter->restart_queue;
3353 }
3354 }
3355
3356 if (tx_ring->detect_tx_hung) {
3357 /* Detect a transmit hang in hardware, this serializes the
3358 * check with the clearing of time_stamp and movement of i */
3359 tx_ring->detect_tx_hung = false;
3360 if (tx_ring->buffer_info[i].time_stamp &&
3361 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
3362 (adapter->tx_timeout_factor * HZ))
3363 && !(rd32(E1000_STATUS) &
3364 E1000_STATUS_TXOFF)) {
3365
3366 tx_desc = E1000_TX_DESC(*tx_ring, i);
3367 /* detected Tx unit hang */
3368 dev_err(&adapter->pdev->dev,
3369 "Detected Tx Unit Hang\n"
3370 " Tx Queue <%lu>\n"
3371 " TDH <%x>\n"
3372 " TDT <%x>\n"
3373 " next_to_use <%x>\n"
3374 " next_to_clean <%x>\n"
3375 " head (WB) <%x>\n"
3376 "buffer_info[next_to_clean]\n"
3377 " time_stamp <%lx>\n"
3378 " jiffies <%lx>\n"
3379 " desc.status <%x>\n",
3380 (unsigned long)((tx_ring - adapter->tx_ring) /
3381 sizeof(struct igb_ring)),
3382 readl(adapter->hw.hw_addr + tx_ring->head),
3383 readl(adapter->hw.hw_addr + tx_ring->tail),
3384 tx_ring->next_to_use,
3385 tx_ring->next_to_clean,
3386 head,
3387 tx_ring->buffer_info[i].time_stamp,
3388 jiffies,
3389 tx_desc->upper.fields.status);
3390 netif_stop_queue(netdev);
3391 }
3392 }
3393 tx_ring->total_bytes += total_bytes;
3394 tx_ring->total_packets += total_packets;
3395 adapter->net_stats.tx_bytes += total_bytes;
3396 adapter->net_stats.tx_packets += total_packets;
3397 return retval;
3398}
3399
3400
3401/**
3402 * igb_receive_skb - helper function to handle rx indications
3403 * @adapter: board private structure
3404 * @status: descriptor status field as written by hardware
3405 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3406 * @skb: pointer to sk_buff to be indicated to stack
3407 **/
3408static void igb_receive_skb(struct igb_adapter *adapter, u8 status, u16 vlan,
3409 struct sk_buff *skb)
3410{
3411 if (adapter->vlgrp && (status & E1000_RXD_STAT_VP))
3412 vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
3413 le16_to_cpu(vlan) &
3414 E1000_RXD_SPC_VLAN_MASK);
3415 else
3416 netif_receive_skb(skb);
3417}
3418
3419
3420static inline void igb_rx_checksum_adv(struct igb_adapter *adapter,
3421 u32 status_err, struct sk_buff *skb)
3422{
3423 skb->ip_summed = CHECKSUM_NONE;
3424
3425 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
3426 if ((status_err & E1000_RXD_STAT_IXSM) || !adapter->rx_csum)
3427 return;
3428 /* TCP/UDP checksum error bit is set */
3429 if (status_err &
3430 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
3431 /* let the stack verify checksum errors */
3432 adapter->hw_csum_err++;
3433 return;
3434 }
3435 /* It must be a TCP or UDP packet with a valid checksum */
3436 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
3437 skb->ip_summed = CHECKSUM_UNNECESSARY;
3438
3439 adapter->hw_csum_good++;
3440}
3441
3442static bool igb_clean_rx_irq_adv(struct igb_adapter *adapter,
3443 struct igb_ring *rx_ring,
3444 int *work_done, int budget)
3445{
3446 struct net_device *netdev = adapter->netdev;
3447 struct pci_dev *pdev = adapter->pdev;
3448 union e1000_adv_rx_desc *rx_desc , *next_rxd;
3449 struct igb_buffer *buffer_info , *next_buffer;
3450 struct sk_buff *skb;
3451 unsigned int i, j;
3452 u32 length, hlen, staterr;
3453 bool cleaned = false;
3454 int cleaned_count = 0;
3455 unsigned int total_bytes = 0, total_packets = 0;
3456
3457 i = rx_ring->next_to_clean;
3458 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
3459 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
3460
3461 while (staterr & E1000_RXD_STAT_DD) {
3462 if (*work_done >= budget)
3463 break;
3464 (*work_done)++;
3465 buffer_info = &rx_ring->buffer_info[i];
3466
3467 /* HW will not DMA in data larger than the given buffer, even
3468 * if it parses the (NFS, of course) header to be larger. In
3469 * that case, it fills the header buffer and spills the rest
3470 * into the page.
3471 */
3472 hlen = le16_to_cpu((rx_desc->wb.lower.lo_dword.hdr_info &
3473 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT);
3474 if (hlen > adapter->rx_ps_hdr_size)
3475 hlen = adapter->rx_ps_hdr_size;
3476
3477 length = le16_to_cpu(rx_desc->wb.upper.length);
3478 cleaned = true;
3479 cleaned_count++;
3480
3481 if (rx_ring->pending_skb != NULL) {
3482 skb = rx_ring->pending_skb;
3483 rx_ring->pending_skb = NULL;
3484 j = rx_ring->pending_skb_page;
3485 } else {
3486 skb = buffer_info->skb;
3487 prefetch(skb->data - NET_IP_ALIGN);
3488 buffer_info->skb = NULL;
3489 if (hlen) {
3490 pci_unmap_single(pdev, buffer_info->dma,
3491 adapter->rx_ps_hdr_size +
3492 NET_IP_ALIGN,
3493 PCI_DMA_FROMDEVICE);
3494 skb_put(skb, hlen);
3495 } else {
3496 pci_unmap_single(pdev, buffer_info->dma,
3497 adapter->rx_buffer_len +
3498 NET_IP_ALIGN,
3499 PCI_DMA_FROMDEVICE);
3500 skb_put(skb, length);
3501 goto send_up;
3502 }
3503 j = 0;
3504 }
3505
3506 while (length) {
3507 pci_unmap_page(pdev, buffer_info->page_dma,
3508 PAGE_SIZE, PCI_DMA_FROMDEVICE);
3509 buffer_info->page_dma = 0;
3510 skb_fill_page_desc(skb, j, buffer_info->page,
3511 0, length);
3512 buffer_info->page = NULL;
3513
3514 skb->len += length;
3515 skb->data_len += length;
3516 skb->truesize += length;
3517 rx_desc->wb.upper.status_error = 0;
3518 if (staterr & E1000_RXD_STAT_EOP)
3519 break;
3520
3521 j++;
3522 cleaned_count++;
3523 i++;
3524 if (i == rx_ring->count)
3525 i = 0;
3526
3527 buffer_info = &rx_ring->buffer_info[i];
3528 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
3529 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
3530 length = le16_to_cpu(rx_desc->wb.upper.length);
3531 if (!(staterr & E1000_RXD_STAT_DD)) {
3532 rx_ring->pending_skb = skb;
3533 rx_ring->pending_skb_page = j;
3534 goto out;
3535 }
3536 }
3537send_up:
3538 pskb_trim(skb, skb->len - 4);
3539 i++;
3540 if (i == rx_ring->count)
3541 i = 0;
3542 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
3543 prefetch(next_rxd);
3544 next_buffer = &rx_ring->buffer_info[i];
3545
3546 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
3547 dev_kfree_skb_irq(skb);
3548 goto next_desc;
3549 }
3550 rx_ring->no_itr_adjust |= (staterr & E1000_RXD_STAT_DYNINT);
3551
3552 total_bytes += skb->len;
3553 total_packets++;
3554
3555 igb_rx_checksum_adv(adapter, staterr, skb);
3556
3557 skb->protocol = eth_type_trans(skb, netdev);
3558
3559 igb_receive_skb(adapter, staterr, rx_desc->wb.upper.vlan, skb);
3560
3561 netdev->last_rx = jiffies;
3562
3563next_desc:
3564 rx_desc->wb.upper.status_error = 0;
3565
3566 /* return some buffers to hardware, one at a time is too slow */
3567 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3568 igb_alloc_rx_buffers_adv(adapter, rx_ring,
3569 cleaned_count);
3570 cleaned_count = 0;
3571 }
3572
3573 /* use prefetched values */
3574 rx_desc = next_rxd;
3575 buffer_info = next_buffer;
3576
3577 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
3578 }
3579out:
3580 rx_ring->next_to_clean = i;
3581 cleaned_count = IGB_DESC_UNUSED(rx_ring);
3582
3583 if (cleaned_count)
3584 igb_alloc_rx_buffers_adv(adapter, rx_ring, cleaned_count);
3585
3586 rx_ring->total_packets += total_packets;
3587 rx_ring->total_bytes += total_bytes;
3588 rx_ring->rx_stats.packets += total_packets;
3589 rx_ring->rx_stats.bytes += total_bytes;
3590 adapter->net_stats.rx_bytes += total_bytes;
3591 adapter->net_stats.rx_packets += total_packets;
3592 return cleaned;
3593}
3594
3595
3596/**
3597 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
3598 * @adapter: address of board private structure
3599 **/
3600static void igb_alloc_rx_buffers_adv(struct igb_adapter *adapter,
3601 struct igb_ring *rx_ring,
3602 int cleaned_count)
3603{
3604 struct net_device *netdev = adapter->netdev;
3605 struct pci_dev *pdev = adapter->pdev;
3606 union e1000_adv_rx_desc *rx_desc;
3607 struct igb_buffer *buffer_info;
3608 struct sk_buff *skb;
3609 unsigned int i;
3610
3611 i = rx_ring->next_to_use;
3612 buffer_info = &rx_ring->buffer_info[i];
3613
3614 while (cleaned_count--) {
3615 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
3616
3617 if (adapter->rx_ps_hdr_size && !buffer_info->page) {
3618 buffer_info->page = alloc_page(GFP_ATOMIC);
3619 if (!buffer_info->page) {
3620 adapter->alloc_rx_buff_failed++;
3621 goto no_buffers;
3622 }
3623 buffer_info->page_dma =
3624 pci_map_page(pdev,
3625 buffer_info->page,
3626 0, PAGE_SIZE,
3627 PCI_DMA_FROMDEVICE);
3628 }
3629
3630 if (!buffer_info->skb) {
3631 int bufsz;
3632
3633 if (adapter->rx_ps_hdr_size)
3634 bufsz = adapter->rx_ps_hdr_size;
3635 else
3636 bufsz = adapter->rx_buffer_len;
3637 bufsz += NET_IP_ALIGN;
3638 skb = netdev_alloc_skb(netdev, bufsz);
3639
3640 if (!skb) {
3641 adapter->alloc_rx_buff_failed++;
3642 goto no_buffers;
3643 }
3644
3645 /* Make buffer alignment 2 beyond a 16 byte boundary
3646 * this will result in a 16 byte aligned IP header after
3647 * the 14 byte MAC header is removed
3648 */
3649 skb_reserve(skb, NET_IP_ALIGN);
3650
3651 buffer_info->skb = skb;
3652 buffer_info->dma = pci_map_single(pdev, skb->data,
3653 bufsz,
3654 PCI_DMA_FROMDEVICE);
3655
3656 }
3657 /* Refresh the desc even if buffer_addrs didn't change because
3658 * each write-back erases this info. */
3659 if (adapter->rx_ps_hdr_size) {
3660 rx_desc->read.pkt_addr =
3661 cpu_to_le64(buffer_info->page_dma);
3662 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
3663 } else {
3664 rx_desc->read.pkt_addr =
3665 cpu_to_le64(buffer_info->dma);
3666 rx_desc->read.hdr_addr = 0;
3667 }
3668
3669 i++;
3670 if (i == rx_ring->count)
3671 i = 0;
3672 buffer_info = &rx_ring->buffer_info[i];
3673 }
3674
3675no_buffers:
3676 if (rx_ring->next_to_use != i) {
3677 rx_ring->next_to_use = i;
3678 if (i == 0)
3679 i = (rx_ring->count - 1);
3680 else
3681 i--;
3682
3683 /* Force memory writes to complete before letting h/w
3684 * know there are new descriptors to fetch. (Only
3685 * applicable for weak-ordered memory model archs,
3686 * such as IA-64). */
3687 wmb();
3688 writel(i, adapter->hw.hw_addr + rx_ring->tail);
3689 }
3690}
3691
3692/**
3693 * igb_mii_ioctl -
3694 * @netdev:
3695 * @ifreq:
3696 * @cmd:
3697 **/
3698static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
3699{
3700 struct igb_adapter *adapter = netdev_priv(netdev);
3701 struct mii_ioctl_data *data = if_mii(ifr);
3702
3703 if (adapter->hw.phy.media_type != e1000_media_type_copper)
3704 return -EOPNOTSUPP;
3705
3706 switch (cmd) {
3707 case SIOCGMIIPHY:
3708 data->phy_id = adapter->hw.phy.addr;
3709 break;
3710 case SIOCGMIIREG:
3711 if (!capable(CAP_NET_ADMIN))
3712 return -EPERM;
3713 if (adapter->hw.phy.ops.read_phy_reg(&adapter->hw,
3714 data->reg_num
3715 & 0x1F, &data->val_out))
3716 return -EIO;
3717 break;
3718 case SIOCSMIIREG:
3719 default:
3720 return -EOPNOTSUPP;
3721 }
3722 return 0;
3723}
3724
3725/**
3726 * igb_ioctl -
3727 * @netdev:
3728 * @ifreq:
3729 * @cmd:
3730 **/
3731static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
3732{
3733 switch (cmd) {
3734 case SIOCGMIIPHY:
3735 case SIOCGMIIREG:
3736 case SIOCSMIIREG:
3737 return igb_mii_ioctl(netdev, ifr, cmd);
3738 default:
3739 return -EOPNOTSUPP;
3740 }
3741}
3742
3743static void igb_vlan_rx_register(struct net_device *netdev,
3744 struct vlan_group *grp)
3745{
3746 struct igb_adapter *adapter = netdev_priv(netdev);
3747 struct e1000_hw *hw = &adapter->hw;
3748 u32 ctrl, rctl;
3749
3750 igb_irq_disable(adapter);
3751 adapter->vlgrp = grp;
3752
3753 if (grp) {
3754 /* enable VLAN tag insert/strip */
3755 ctrl = rd32(E1000_CTRL);
3756 ctrl |= E1000_CTRL_VME;
3757 wr32(E1000_CTRL, ctrl);
3758
3759 /* enable VLAN receive filtering */
3760 rctl = rd32(E1000_RCTL);
3761 rctl |= E1000_RCTL_VFE;
3762 rctl &= ~E1000_RCTL_CFIEN;
3763 wr32(E1000_RCTL, rctl);
3764 igb_update_mng_vlan(adapter);
3765 wr32(E1000_RLPML,
3766 adapter->max_frame_size + VLAN_TAG_SIZE);
3767 } else {
3768 /* disable VLAN tag insert/strip */
3769 ctrl = rd32(E1000_CTRL);
3770 ctrl &= ~E1000_CTRL_VME;
3771 wr32(E1000_CTRL, ctrl);
3772
3773 /* disable VLAN filtering */
3774 rctl = rd32(E1000_RCTL);
3775 rctl &= ~E1000_RCTL_VFE;
3776 wr32(E1000_RCTL, rctl);
3777 if (adapter->mng_vlan_id != (u16)IGB_MNG_VLAN_NONE) {
3778 igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
3779 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
3780 }
3781 wr32(E1000_RLPML,
3782 adapter->max_frame_size);
3783 }
3784
3785 if (!test_bit(__IGB_DOWN, &adapter->state))
3786 igb_irq_enable(adapter);
3787}
3788
3789static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
3790{
3791 struct igb_adapter *adapter = netdev_priv(netdev);
3792 struct e1000_hw *hw = &adapter->hw;
3793 u32 vfta, index;
3794
3795 if ((adapter->hw.mng_cookie.status &
3796 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
3797 (vid == adapter->mng_vlan_id))
3798 return;
3799 /* add VID to filter table */
3800 index = (vid >> 5) & 0x7F;
3801 vfta = array_rd32(E1000_VFTA, index);
3802 vfta |= (1 << (vid & 0x1F));
3803 igb_write_vfta(&adapter->hw, index, vfta);
3804}
3805
3806static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
3807{
3808 struct igb_adapter *adapter = netdev_priv(netdev);
3809 struct e1000_hw *hw = &adapter->hw;
3810 u32 vfta, index;
3811
3812 igb_irq_disable(adapter);
3813 vlan_group_set_device(adapter->vlgrp, vid, NULL);
3814
3815 if (!test_bit(__IGB_DOWN, &adapter->state))
3816 igb_irq_enable(adapter);
3817
3818 if ((adapter->hw.mng_cookie.status &
3819 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
3820 (vid == adapter->mng_vlan_id)) {
3821 /* release control to f/w */
3822 igb_release_hw_control(adapter);
3823 return;
3824 }
3825
3826 /* remove VID from filter table */
3827 index = (vid >> 5) & 0x7F;
3828 vfta = array_rd32(E1000_VFTA, index);
3829 vfta &= ~(1 << (vid & 0x1F));
3830 igb_write_vfta(&adapter->hw, index, vfta);
3831}
3832
3833static void igb_restore_vlan(struct igb_adapter *adapter)
3834{
3835 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
3836
3837 if (adapter->vlgrp) {
3838 u16 vid;
3839 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
3840 if (!vlan_group_get_device(adapter->vlgrp, vid))
3841 continue;
3842 igb_vlan_rx_add_vid(adapter->netdev, vid);
3843 }
3844 }
3845}
3846
3847int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
3848{
3849 struct e1000_mac_info *mac = &adapter->hw.mac;
3850
3851 mac->autoneg = 0;
3852
3853 /* Fiber NICs only allow 1000 gbps Full duplex */
3854 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) &&
3855 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
3856 dev_err(&adapter->pdev->dev,
3857 "Unsupported Speed/Duplex configuration\n");
3858 return -EINVAL;
3859 }
3860
3861 switch (spddplx) {
3862 case SPEED_10 + DUPLEX_HALF:
3863 mac->forced_speed_duplex = ADVERTISE_10_HALF;
3864 break;
3865 case SPEED_10 + DUPLEX_FULL:
3866 mac->forced_speed_duplex = ADVERTISE_10_FULL;
3867 break;
3868 case SPEED_100 + DUPLEX_HALF:
3869 mac->forced_speed_duplex = ADVERTISE_100_HALF;
3870 break;
3871 case SPEED_100 + DUPLEX_FULL:
3872 mac->forced_speed_duplex = ADVERTISE_100_FULL;
3873 break;
3874 case SPEED_1000 + DUPLEX_FULL:
3875 mac->autoneg = 1;
3876 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
3877 break;
3878 case SPEED_1000 + DUPLEX_HALF: /* not supported */
3879 default:
3880 dev_err(&adapter->pdev->dev,
3881 "Unsupported Speed/Duplex configuration\n");
3882 return -EINVAL;
3883 }
3884 return 0;
3885}
3886
3887
3888static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
3889{
3890 struct net_device *netdev = pci_get_drvdata(pdev);
3891 struct igb_adapter *adapter = netdev_priv(netdev);
3892 struct e1000_hw *hw = &adapter->hw;
3893 u32 ctrl, ctrl_ext, rctl, status;
3894 u32 wufc = adapter->wol;
3895#ifdef CONFIG_PM
3896 int retval = 0;
3897#endif
3898
3899 netif_device_detach(netdev);
3900
3901 if (netif_running(netdev)) {
3902 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
3903 igb_down(adapter);
3904 igb_free_irq(adapter);
3905 }
3906
3907#ifdef CONFIG_PM
3908 retval = pci_save_state(pdev);
3909 if (retval)
3910 return retval;
3911#endif
3912
3913 status = rd32(E1000_STATUS);
3914 if (status & E1000_STATUS_LU)
3915 wufc &= ~E1000_WUFC_LNKC;
3916
3917 if (wufc) {
3918 igb_setup_rctl(adapter);
3919 igb_set_multi(netdev);
3920
3921 /* turn on all-multi mode if wake on multicast is enabled */
3922 if (wufc & E1000_WUFC_MC) {
3923 rctl = rd32(E1000_RCTL);
3924 rctl |= E1000_RCTL_MPE;
3925 wr32(E1000_RCTL, rctl);
3926 }
3927
3928 ctrl = rd32(E1000_CTRL);
3929 /* advertise wake from D3Cold */
3930 #define E1000_CTRL_ADVD3WUC 0x00100000
3931 /* phy power management enable */
3932 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
3933 ctrl |= E1000_CTRL_ADVD3WUC;
3934 wr32(E1000_CTRL, ctrl);
3935
3936 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
3937 adapter->hw.phy.media_type ==
3938 e1000_media_type_internal_serdes) {
3939 /* keep the laser running in D3 */
3940 ctrl_ext = rd32(E1000_CTRL_EXT);
3941 ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
3942 wr32(E1000_CTRL_EXT, ctrl_ext);
3943 }
3944
3945 /* Allow time for pending master requests to run */
3946 igb_disable_pcie_master(&adapter->hw);
3947
3948 wr32(E1000_WUC, E1000_WUC_PME_EN);
3949 wr32(E1000_WUFC, wufc);
3950 pci_enable_wake(pdev, PCI_D3hot, 1);
3951 pci_enable_wake(pdev, PCI_D3cold, 1);
3952 } else {
3953 wr32(E1000_WUC, 0);
3954 wr32(E1000_WUFC, 0);
3955 pci_enable_wake(pdev, PCI_D3hot, 0);
3956 pci_enable_wake(pdev, PCI_D3cold, 0);
3957 }
3958
3959 igb_release_manageability(adapter);
3960
3961 /* make sure adapter isn't asleep if manageability is enabled */
3962 if (adapter->en_mng_pt) {
3963 pci_enable_wake(pdev, PCI_D3hot, 1);
3964 pci_enable_wake(pdev, PCI_D3cold, 1);
3965 }
3966
3967 /* Release control of h/w to f/w. If f/w is AMT enabled, this
3968 * would have already happened in close and is redundant. */
3969 igb_release_hw_control(adapter);
3970
3971 pci_disable_device(pdev);
3972
3973 pci_set_power_state(pdev, pci_choose_state(pdev, state));
3974
3975 return 0;
3976}
3977
3978#ifdef CONFIG_PM
3979static int igb_resume(struct pci_dev *pdev)
3980{
3981 struct net_device *netdev = pci_get_drvdata(pdev);
3982 struct igb_adapter *adapter = netdev_priv(netdev);
3983 struct e1000_hw *hw = &adapter->hw;
3984 u32 err;
3985
3986 pci_set_power_state(pdev, PCI_D0);
3987 pci_restore_state(pdev);
3988 err = pci_enable_device(pdev);
3989 if (err) {
3990 dev_err(&pdev->dev,
3991 "igb: Cannot enable PCI device from suspend\n");
3992 return err;
3993 }
3994 pci_set_master(pdev);
3995
3996 pci_enable_wake(pdev, PCI_D3hot, 0);
3997 pci_enable_wake(pdev, PCI_D3cold, 0);
3998
3999 if (netif_running(netdev)) {
4000 err = igb_request_irq(adapter);
4001 if (err)
4002 return err;
4003 }
4004
4005 /* e1000_power_up_phy(adapter); */
4006
4007 igb_reset(adapter);
4008 wr32(E1000_WUS, ~0);
4009
4010 igb_init_manageability(adapter);
4011
4012 if (netif_running(netdev))
4013 igb_up(adapter);
4014
4015 netif_device_attach(netdev);
4016
4017 /* let the f/w know that the h/w is now under the control of the
4018 * driver. */
4019 igb_get_hw_control(adapter);
4020
4021 return 0;
4022}
4023#endif
4024
4025static void igb_shutdown(struct pci_dev *pdev)
4026{
4027 igb_suspend(pdev, PMSG_SUSPEND);
4028}
4029
4030#ifdef CONFIG_NET_POLL_CONTROLLER
4031/*
4032 * Polling 'interrupt' - used by things like netconsole to send skbs
4033 * without having to re-enable interrupts. It's not called while
4034 * the interrupt routine is executing.
4035 */
4036static void igb_netpoll(struct net_device *netdev)
4037{
4038 struct igb_adapter *adapter = netdev_priv(netdev);
4039 int i;
4040 int work_done = 0;
4041
4042 igb_irq_disable(adapter);
4043 for (i = 0; i < adapter->num_tx_queues; i++)
4044 igb_clean_tx_irq(adapter, &adapter->tx_ring[i]);
4045
4046 for (i = 0; i < adapter->num_rx_queues; i++)
4047 igb_clean_rx_irq_adv(adapter, &adapter->rx_ring[i],
4048 &work_done,
4049 adapter->rx_ring[i].napi.weight);
4050
4051 igb_irq_enable(adapter);
4052}
4053#endif /* CONFIG_NET_POLL_CONTROLLER */
4054
4055/**
4056 * igb_io_error_detected - called when PCI error is detected
4057 * @pdev: Pointer to PCI device
4058 * @state: The current pci connection state
4059 *
4060 * This function is called after a PCI bus error affecting
4061 * this device has been detected.
4062 */
4063static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
4064 pci_channel_state_t state)
4065{
4066 struct net_device *netdev = pci_get_drvdata(pdev);
4067 struct igb_adapter *adapter = netdev_priv(netdev);
4068
4069 netif_device_detach(netdev);
4070
4071 if (netif_running(netdev))
4072 igb_down(adapter);
4073 pci_disable_device(pdev);
4074
4075 /* Request a slot slot reset. */
4076 return PCI_ERS_RESULT_NEED_RESET;
4077}
4078
4079/**
4080 * igb_io_slot_reset - called after the pci bus has been reset.
4081 * @pdev: Pointer to PCI device
4082 *
4083 * Restart the card from scratch, as if from a cold-boot. Implementation
4084 * resembles the first-half of the igb_resume routine.
4085 */
4086static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
4087{
4088 struct net_device *netdev = pci_get_drvdata(pdev);
4089 struct igb_adapter *adapter = netdev_priv(netdev);
4090 struct e1000_hw *hw = &adapter->hw;
4091
4092 if (pci_enable_device(pdev)) {
4093 dev_err(&pdev->dev,
4094 "Cannot re-enable PCI device after reset.\n");
4095 return PCI_ERS_RESULT_DISCONNECT;
4096 }
4097 pci_set_master(pdev);
4098
4099 pci_enable_wake(pdev, PCI_D3hot, 0);
4100 pci_enable_wake(pdev, PCI_D3cold, 0);
4101
4102 igb_reset(adapter);
4103 wr32(E1000_WUS, ~0);
4104
4105 return PCI_ERS_RESULT_RECOVERED;
4106}
4107
4108/**
4109 * igb_io_resume - called when traffic can start flowing again.
4110 * @pdev: Pointer to PCI device
4111 *
4112 * This callback is called when the error recovery driver tells us that
4113 * its OK to resume normal operation. Implementation resembles the
4114 * second-half of the igb_resume routine.
4115 */
4116static void igb_io_resume(struct pci_dev *pdev)
4117{
4118 struct net_device *netdev = pci_get_drvdata(pdev);
4119 struct igb_adapter *adapter = netdev_priv(netdev);
4120
4121 igb_init_manageability(adapter);
4122
4123 if (netif_running(netdev)) {
4124 if (igb_up(adapter)) {
4125 dev_err(&pdev->dev, "igb_up failed after reset\n");
4126 return;
4127 }
4128 }
4129
4130 netif_device_attach(netdev);
4131
4132 /* let the f/w know that the h/w is now under the control of the
4133 * driver. */
4134 igb_get_hw_control(adapter);
4135
4136}
4137
4138/* igb_main.c */
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-26 11:17:50 -0500