1 files changed, 203 insertions, 102 deletions

diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c
index 3732dd6c4b2a..ead49e54f31b 100644
--- a/drivers/net/ixgbe/ixgbe_main.c
+++ b/drivers/net/ixgbe/ixgbe_main.c
@@ -87,6 +87,25 @@ MODULE_VERSION(DRV_VERSION);
 
 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
 
+static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
+{
+        u32 ctrl_ext;
+
+        /* Let firmware take over control of h/w */
+        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
+        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
+                        ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
+}
+
+static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
+{
+        u32 ctrl_ext;
+
+        /* Let firmware know the driver has taken over */
+        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
+        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
+                        ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
+}
 
 #ifdef DEBUG
 /**
@@ -165,6 +184,15 @@ static inline bool ixgbe_check_tx_hang(struct ixgbe_adapter *adapter,
         return false;
 }
 
+#define IXGBE_MAX_TXD_PWR       14
+#define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
+
+/* Tx Descriptors needed, worst case */
+#define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \
+                         (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
+#define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \
+        MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1)   /* for context */
+
 /**
  * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
  * @adapter: board private structure
@@ -177,18 +205,34 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_adapter *adapter,
         struct ixgbe_tx_buffer *tx_buffer_info;
         unsigned int i, eop;
         bool cleaned = false;
-        int count = 0;
+        unsigned int total_tx_bytes = 0, total_tx_packets = 0;
 
         i = tx_ring->next_to_clean;
         eop = tx_ring->tx_buffer_info[i].next_to_watch;
         eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop);
         while (eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) {
-                for (cleaned = false; !cleaned;) {
+                cleaned = false;
+                while (!cleaned) {
                         tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);
                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
                         cleaned = (i == eop);
 
                         tx_ring->stats.bytes += tx_buffer_info->length;
+                        if (cleaned) {
+                                struct sk_buff *skb = tx_buffer_info->skb;
+#ifdef NETIF_F_TSO
+                                unsigned int segs, bytecount;
+                                segs = skb_shinfo(skb)->gso_segs ?: 1;
+                                /* multiply data chunks by size of headers */
+                                bytecount = ((segs - 1) * skb_headlen(skb)) +
+                                            skb->len;
+                                total_tx_packets += segs;
+                                total_tx_bytes += bytecount;
+#else
+                                total_tx_packets++;
+                                total_tx_bytes += skb->len;
+#endif
+                        }
                         ixgbe_unmap_and_free_tx_resource(adapter,
                                                          tx_buffer_info);
                         tx_desc->wb.status = 0;
@@ -204,29 +248,36 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_adapter *adapter,
                 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop);
 
                 /* weight of a sort for tx, avoid endless transmit cleanup */
-                if (count++ >= tx_ring->work_limit)
+                if (total_tx_packets >= tx_ring->work_limit)
                         break;
         }
 
         tx_ring->next_to_clean = i;
 
-#define TX_WAKE_THRESHOLD 32
-        spin_lock(&tx_ring->tx_lock);
-
-        if (cleaned && netif_carrier_ok(netdev) &&
-            (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD) &&
-            !test_bit(__IXGBE_DOWN, &adapter->state))
-                netif_wake_queue(netdev);
-
-        spin_unlock(&tx_ring->tx_lock);
+#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
+        if (total_tx_packets && netif_carrier_ok(netdev) &&
+            (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
+                /* Make sure that anybody stopping the queue after this
+                 * sees the new next_to_clean.
+                 */
+                smp_mb();
+                if (netif_queue_stopped(netdev) &&
+                    !test_bit(__IXGBE_DOWN, &adapter->state)) {
+                        netif_wake_queue(netdev);
+                        adapter->restart_queue++;
+                }
+        }
 
         if (adapter->detect_tx_hung)
                 if (ixgbe_check_tx_hang(adapter, tx_ring, eop, eop_desc))
                         netif_stop_queue(netdev);
 
-        if (count >= tx_ring->work_limit)
+        if (total_tx_packets >= tx_ring->work_limit)
                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, tx_ring->eims_value);
 
+        adapter->net_stats.tx_bytes += total_tx_bytes;
+        adapter->net_stats.tx_packets += total_tx_packets;
+        cleaned = total_tx_packets ? true : false;
         return cleaned;
 }
 
@@ -255,25 +306,40 @@ static void ixgbe_receive_skb(struct ixgbe_adapter *adapter,
         }
 }
 
+/**
+ * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
+ * @adapter: address of board private structure
+ * @status_err: hardware indication of status of receive
+ * @skb: skb currently being received and modified
+ **/
 static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
                                          u32 status_err,
                                          struct sk_buff *skb)
 {
         skb->ip_summed = CHECKSUM_NONE;
 
-        /* Ignore Checksum bit is set */
+        /* Ignore Checksum bit is set, or rx csum disabled */
         if ((status_err & IXGBE_RXD_STAT_IXSM) ||
-                     !(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
+            !(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
                 return;
-        /* TCP/UDP checksum error bit is set */
-        if (status_err & (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE)) {
-                /* let the stack verify checksum errors */
+
+        /* if IP and error */
+        if ((status_err & IXGBE_RXD_STAT_IPCS) &&
+            (status_err & IXGBE_RXDADV_ERR_IPE)) {
                 adapter->hw_csum_rx_error++;
                 return;
         }
+
+        if (!(status_err & IXGBE_RXD_STAT_L4CS))
+                return;
+
+        if (status_err & IXGBE_RXDADV_ERR_TCPE) {
+                adapter->hw_csum_rx_error++;
+                return;
+        }
+
         /* It must be a TCP or UDP packet with a valid checksum */
-        if (status_err & (IXGBE_RXD_STAT_L4CS | IXGBE_RXD_STAT_UDPCS))
-                skb->ip_summed = CHECKSUM_UNNECESSARY;
+        skb->ip_summed = CHECKSUM_UNNECESSARY;
         adapter->hw_csum_rx_good++;
 }
 
@@ -379,6 +445,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_adapter *adapter,
         u16 hdr_info, vlan_tag;
         bool is_vlan, cleaned = false;
         int cleaned_count = 0;
+        unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 
         i = rx_ring->next_to_clean;
         upper_len = 0;
@@ -458,6 +525,11 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_adapter *adapter,
                 }
 
                 ixgbe_rx_checksum(adapter, staterr, skb);
+
+                /* probably a little skewed due to removing CRC */
+                total_rx_bytes += skb->len;
+                total_rx_packets++;
+
                 skb->protocol = eth_type_trans(skb, netdev);
                 ixgbe_receive_skb(adapter, skb, is_vlan, vlan_tag);
                 netdev->last_rx = jiffies;
@@ -486,6 +558,9 @@ next_desc:
         if (cleaned_count)
                 ixgbe_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
 
+        adapter->net_stats.rx_bytes += total_rx_bytes;
+        adapter->net_stats.rx_packets += total_rx_packets;
+
         return cleaned;
 }
 
@@ -535,7 +610,9 @@ static irqreturn_t ixgbe_msix_lsc(int irq, void *data)
                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
                         mod_timer(&adapter->watchdog_timer, jiffies);
         }
-        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
+
+        if (!test_bit(__IXGBE_DOWN, &adapter->state))
+                IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
 
         return IRQ_HANDLED;
 }
@@ -713,7 +790,6 @@ static irqreturn_t ixgbe_intr(int irq, void *data)
         if (netif_rx_schedule_prep(netdev, &adapter->napi)) {
                 /* Disable interrupts and register for poll. The flush of the
                  * posted write is intentionally left out. */
-                atomic_inc(&adapter->irq_sem);
                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
                 __netif_rx_schedule(netdev, &adapter->napi);
         }
@@ -801,7 +877,6 @@ static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
  **/
 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
 {
-        atomic_inc(&adapter->irq_sem);
         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
         IXGBE_WRITE_FLUSH(&adapter->hw);
         synchronize_irq(adapter->pdev->irq);
@@ -813,15 +888,13 @@ static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
  **/
 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter)
 {
-        if (atomic_dec_and_test(&adapter->irq_sem)) {
-                if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
-                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC,
-                                        (IXGBE_EIMS_ENABLE_MASK &
-                                         ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC)));
-                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS,
-                                IXGBE_EIMS_ENABLE_MASK);
-                IXGBE_WRITE_FLUSH(&adapter->hw);
-        }
+        if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
+                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC,
+                                (IXGBE_EIMS_ENABLE_MASK &
+                                 ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC)));
+        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS,
+                        IXGBE_EIMS_ENABLE_MASK);
+        IXGBE_WRITE_FLUSH(&adapter->hw);
 }
 
 /**
@@ -1040,7 +1113,8 @@ static void ixgbe_vlan_rx_register(struct net_device *netdev,
         struct ixgbe_adapter *adapter = netdev_priv(netdev);
         u32 ctrl;
 
-        ixgbe_irq_disable(adapter);
+        if (!test_bit(__IXGBE_DOWN, &adapter->state))
+                ixgbe_irq_disable(adapter);
         adapter->vlgrp = grp;
 
         if (grp) {
@@ -1051,7 +1125,8 @@ static void ixgbe_vlan_rx_register(struct net_device *netdev,
                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl);
         }
 
-        ixgbe_irq_enable(adapter);
+        if (!test_bit(__IXGBE_DOWN, &adapter->state))
+                ixgbe_irq_enable(adapter);
 }
 
 static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
@@ -1066,9 +1141,13 @@ static void ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
 {
         struct ixgbe_adapter *adapter = netdev_priv(netdev);
 
-        ixgbe_irq_disable(adapter);
+        if (!test_bit(__IXGBE_DOWN, &adapter->state))
+                ixgbe_irq_disable(adapter);
+
         vlan_group_set_device(adapter->vlgrp, vid, NULL);
-        ixgbe_irq_enable(adapter);
+
+        if (!test_bit(__IXGBE_DOWN, &adapter->state))
+                ixgbe_irq_enable(adapter);
 
         /* remove VID from filter table */
         ixgbe_set_vfta(&adapter->hw, vid, 0, false);
@@ -1170,6 +1249,8 @@ static int ixgbe_up_complete(struct ixgbe_adapter *adapter)
         u32 txdctl, rxdctl, mhadd;
         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
 
+        ixgbe_get_hw_control(adapter);
+
         if (adapter->flags & (IXGBE_FLAG_MSIX_ENABLED |
                               IXGBE_FLAG_MSI_ENABLED)) {
                 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
@@ -1224,6 +1305,16 @@ static int ixgbe_up_complete(struct ixgbe_adapter *adapter)
         return 0;
 }
 
+void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
+{
+        WARN_ON(in_interrupt());
+        while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
+                msleep(1);
+        ixgbe_down(adapter);
+        ixgbe_up(adapter);
+        clear_bit(__IXGBE_RESETTING, &adapter->state);
+}
+
 int ixgbe_up(struct ixgbe_adapter *adapter)
 {
         /* hardware has been reset, we need to reload some things */
@@ -1408,7 +1499,6 @@ void ixgbe_down(struct ixgbe_adapter *adapter)
         msleep(10);
 
         napi_disable(&adapter->napi);
-        atomic_set(&adapter->irq_sem, 0);
 
         ixgbe_irq_disable(adapter);
 
@@ -1447,6 +1537,8 @@ static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
         pci_enable_wake(pdev, PCI_D3hot, 0);
         pci_enable_wake(pdev, PCI_D3cold, 0);
 
+        ixgbe_release_hw_control(adapter);
+
         pci_disable_device(pdev);
 
         pci_set_power_state(pdev, pci_choose_state(pdev, state));
@@ -1481,7 +1573,8 @@ static int ixgbe_clean(struct napi_struct *napi, int budget)
         /* If budget not fully consumed, exit the polling mode */
         if (work_done < budget) {
                 netif_rx_complete(netdev, napi);
-                ixgbe_irq_enable(adapter);
+                if (!test_bit(__IXGBE_DOWN, &adapter->state))
+                        ixgbe_irq_enable(adapter);
         }
 
         return work_done;
@@ -1506,8 +1599,7 @@ static void ixgbe_reset_task(struct work_struct *work)
 
         adapter->tx_timeout_count++;
 
-        ixgbe_down(adapter);
-        ixgbe_up(adapter);
+        ixgbe_reinit_locked(adapter);
 }
 
 /**
@@ -1590,7 +1682,6 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
                 return -ENOMEM;
         }
 
-        atomic_set(&adapter->irq_sem, 1);
         set_bit(__IXGBE_DOWN, &adapter->state);
 
         return 0;
@@ -1634,7 +1725,6 @@ int ixgbe_setup_tx_resources(struct ixgbe_adapter *adapter,
         txdr->next_to_use = 0;
         txdr->next_to_clean = 0;
         txdr->work_limit = txdr->count;
-        spin_lock_init(&txdr->tx_lock);
 
         return 0;
 }
@@ -1828,10 +1918,8 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
 
         netdev->mtu = new_mtu;
 
-        if (netif_running(netdev)) {
-                ixgbe_down(adapter);
-                ixgbe_up(adapter);
-        }
+        if (netif_running(netdev))
+                ixgbe_reinit_locked(adapter);
 
         return 0;
 }
@@ -1852,14 +1940,8 @@ static int ixgbe_open(struct net_device *netdev)
 {
         struct ixgbe_adapter *adapter = netdev_priv(netdev);
         int err;
-        u32 ctrl_ext;
         u32 num_rx_queues = adapter->num_rx_queues;
 
-        /* Let firmware know the driver has taken over */
-        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
-        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
-                        ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
-
 try_intr_reinit:
         /* allocate transmit descriptors */
         err = ixgbe_setup_all_tx_resources(adapter);
@@ -1910,6 +1992,7 @@ try_intr_reinit:
         return 0;
 
 err_up:
+        ixgbe_release_hw_control(adapter);
         ixgbe_free_irq(adapter);
 err_req_irq:
         ixgbe_free_all_rx_resources(adapter);
@@ -1935,7 +2018,6 @@ err_setup_tx:
 static int ixgbe_close(struct net_device *netdev)
 {
         struct ixgbe_adapter *adapter = netdev_priv(netdev);
-        u32 ctrl_ext;
 
         ixgbe_down(adapter);
         ixgbe_free_irq(adapter);
@@ -1943,9 +2025,7 @@ static int ixgbe_close(struct net_device *netdev)
         ixgbe_free_all_tx_resources(adapter);
         ixgbe_free_all_rx_resources(adapter);
 
-        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
-        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
-                        ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
+        ixgbe_release_hw_control(adapter);
 
         return 0;
 }
@@ -1957,22 +2037,26 @@ static int ixgbe_close(struct net_device *netdev)
 void ixgbe_update_stats(struct ixgbe_adapter *adapter)
 {
         struct ixgbe_hw *hw = &adapter->hw;
-        u64 good_rx, missed_rx, bprc;
+        u64 total_mpc = 0;
+        u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
 
         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
-        good_rx = IXGBE_READ_REG(hw, IXGBE_GPRC);
-        missed_rx = IXGBE_READ_REG(hw, IXGBE_MPC(0));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(1));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(2));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(3));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(4));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(5));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(6));
-        missed_rx += IXGBE_READ_REG(hw, IXGBE_MPC(7));
-        adapter->stats.gprc += (good_rx - missed_rx);
-
-        adapter->stats.mpc[0] += missed_rx;
+        for (i = 0; i < 8; i++) {
+                /* for packet buffers not used, the register should read 0 */
+                mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
+                missed_rx += mpc;
+                adapter->stats.mpc[i] += mpc;
+                total_mpc += adapter->stats.mpc[i];
+                adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
+        }
+        adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
+        /* work around hardware counting issue */
+        adapter->stats.gprc -= missed_rx;
+
+        /* 82598 hardware only has a 32 bit counter in the high register */
         adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
+        adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
+        adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
         adapter->stats.bprc += bprc;
         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
@@ -1984,35 +2068,37 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter)
         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
-
         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
         adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
-        adapter->stats.lxontxc += IXGBE_READ_REG(hw, IXGBE_LXONTXC);
         adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
-        adapter->stats.lxofftxc += IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
+        lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
+        adapter->stats.lxontxc += lxon;
+        lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
+        adapter->stats.lxofftxc += lxoff;
         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
-        adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
-        adapter->stats.rnbc[0] += IXGBE_READ_REG(hw, IXGBE_RNBC(0));
+        adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
+        /*
+         * 82598 errata - tx of flow control packets is included in tx counters
+         */
+        xon_off_tot = lxon + lxoff;
+        adapter->stats.gptc -= xon_off_tot;
+        adapter->stats.mptc -= xon_off_tot;
+        adapter->stats.gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
-        adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
+        adapter->stats.ptc64 -= xon_off_tot;
         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
-        adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
 
         /* Fill out the OS statistics structure */
-        adapter->net_stats.rx_packets = adapter->stats.gprc;
-        adapter->net_stats.tx_packets = adapter->stats.gptc;
-        adapter->net_stats.rx_bytes = adapter->stats.gorc;
-        adapter->net_stats.tx_bytes = adapter->stats.gotc;
         adapter->net_stats.multicast = adapter->stats.mprc;
 
         /* Rx Errors */
@@ -2021,8 +2107,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter)
         adapter->net_stats.rx_dropped = 0;
         adapter->net_stats.rx_length_errors = adapter->stats.rlec;
         adapter->net_stats.rx_crc_errors = adapter->stats.crcerrs;
-        adapter->net_stats.rx_missed_errors = adapter->stats.mpc[0];
-
+        adapter->net_stats.rx_missed_errors = total_mpc;
 }
 
 /**
@@ -2076,15 +2161,6 @@ static void ixgbe_watchdog(unsigned long data)
                           round_jiffies(jiffies + 2 * HZ));
 }
 
-#define IXGBE_MAX_TXD_PWR       14
-#define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
-
-/* Tx Descriptors needed, worst case */
-#define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \
-                         (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
-#define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \
-        MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1)   /* for context */
-
 static int ixgbe_tso(struct ixgbe_adapter *adapter,
                          struct ixgbe_ring *tx_ring, struct sk_buff *skb,
                          u32 tx_flags, u8 *hdr_len)
@@ -2356,6 +2432,37 @@ static void ixgbe_tx_queue(struct ixgbe_adapter *adapter,
         writel(i, adapter->hw.hw_addr + tx_ring->tail);
 }
 
+static int __ixgbe_maybe_stop_tx(struct net_device *netdev,
+                                 struct ixgbe_ring *tx_ring, int size)
+{
+        struct ixgbe_adapter *adapter = netdev_priv(netdev);
+
+        netif_stop_queue(netdev);
+        /* Herbert's original patch had:
+         *  smp_mb__after_netif_stop_queue();
+         * but since that doesn't exist yet, just open code it. */
+        smp_mb();
+
+        /* We need to check again in a case another CPU has just
+         * made room available. */
+        if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
+                return -EBUSY;
+
+        /* A reprieve! - use start_queue because it doesn't call schedule */
+        netif_wake_queue(netdev);
+        ++adapter->restart_queue;
+        return 0;
+}
+
+static int ixgbe_maybe_stop_tx(struct net_device *netdev,
+                               struct ixgbe_ring *tx_ring, int size)
+{
+        if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
+                return 0;
+        return __ixgbe_maybe_stop_tx(netdev, tx_ring, size);
+}
+
+
 static int ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 {
         struct ixgbe_adapter *adapter = netdev_priv(netdev);
@@ -2363,7 +2470,6 @@ static int ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
         unsigned int len = skb->len;
         unsigned int first;
         unsigned int tx_flags = 0;
-        unsigned long flags = 0;
         u8 hdr_len;
         int tso;
         unsigned int mss = 0;
@@ -2389,14 +2495,10 @@ static int ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
         for (f = 0; f < nr_frags; f++)
                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
 
-        spin_lock_irqsave(&tx_ring->tx_lock, flags);
-        if (IXGBE_DESC_UNUSED(tx_ring) < (count + 2)) {
+        if (ixgbe_maybe_stop_tx(netdev, tx_ring, count)) {
                 adapter->tx_busy++;
-                netif_stop_queue(netdev);
-                spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
                 return NETDEV_TX_BUSY;
         }
-        spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
         if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
                 tx_flags |= (vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT);
@@ -2423,11 +2525,7 @@ static int ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 
         netdev->trans_start = jiffies;
 
-        spin_lock_irqsave(&tx_ring->tx_lock, flags);
-        /* Make sure there is space in the ring for the next send. */
-        if (IXGBE_DESC_UNUSED(tx_ring) < DESC_NEEDED)
-                netif_stop_queue(netdev);
-        spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
+        ixgbe_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED);
 
         return NETDEV_TX_OK;
 }
@@ -2697,6 +2795,7 @@ static int __devinit ixgbe_probe(struct pci_dev *pdev,
         return 0;
 
 err_register:
+        ixgbe_release_hw_control(adapter);
 err_hw_init:
 err_sw_init:
 err_eeprom:
@@ -2732,6 +2831,8 @@ static void __devexit ixgbe_remove(struct pci_dev *pdev)
 
         unregister_netdev(netdev);
 
+        ixgbe_release_hw_control(adapter);
+
         kfree(adapter->tx_ring);
         kfree(adapter->rx_ring);