IB/ipath: Prevent link-recovery code from negating admin disable

The link can be put in LINKDOWN_DISABLE state either locally or via a
MAD.  However, the link-recovery code will take it out of that state as
a side-effect of attempts to clear SerDes/XGXS issues.

We add a flag to indicate "link is down on purpose, leave it alone."

Signed-off-by: Michael Albaugh <michael.albaugh@qlogic.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

6 files changed, 55 insertions, 29 deletions

diff --git a/drivers/infiniband/hw/ipath/ipath_debug.h b/drivers/infiniband/hw/ipath/ipath_debug.h
index 7170bd20cfbb..65926cd35759 100644
--- a/drivers/infiniband/hw/ipath/ipath_debug.h
+++ b/drivers/infiniband/hw/ipath/ipath_debug.h
@@ -90,6 +90,7 @@
 #define __IPATH_IPATHERR  0x0   /* Ethernet (IPATH) errors on   */
 #define __IPATH_IPATHPD   0x0   /* Ethernet (IPATH) packet dump on   */
 #define __IPATH_IPATHTABLE 0x0  /* Ethernet (IPATH) packet dump on   */
+#define __IPATH_LINKVERBDBG 0x0 /* very verbose linkchange debug */
 
 #endif                          /* _IPATH_DEBUGGING */
 
diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c
index 130f4db7b8c1..1299171c1e50 100644
--- a/drivers/infiniband/hw/ipath/ipath_driver.c
+++ b/drivers/infiniband/hw/ipath/ipath_driver.c
@@ -1664,30 +1664,48 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 }
 
-
-static void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
+static void ipath_set_ib_lstate(struct ipath_devdata *dd, int linkcmd,
+                                int linitcmd)
 {
+        u64 mod_wd;
         static const char *what[4] = {
                 [0] = "NOP",
                 [INFINIPATH_IBCC_LINKCMD_DOWN] = "DOWN",
                 [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
                 [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
         };
-        int linkcmd = (which >> INFINIPATH_IBCC_LINKCMD_SHIFT) &
-                        INFINIPATH_IBCC_LINKCMD_MASK;
 
-        ipath_cdbg(VERBOSE, "Trying to move unit %u to %s, current ltstate "
-                   "is %s\n", dd->ipath_unit,
-                   what[linkcmd],
-                   ipath_ibcstatus_str[ipath_ib_linktrstate(dd,
+        if (linitcmd == INFINIPATH_IBCC_LINKINITCMD_DISABLE) {
+                /*
+                 * If we are told to disable, note that so link-recovery
+                 * code does not attempt to bring us back up.
+                 */
+                preempt_disable();
+                dd->ipath_flags |= IPATH_IB_LINK_DISABLED;
+                preempt_enable();
+        } else if (linitcmd) {
+                /*
+                 * Any other linkinitcmd will lead to LINKDOWN and then
+                 * to INIT (if all is well), so clear flag to let
+                 * link-recovery code attempt to bring us back up.
+                 */
+                preempt_disable();
+                dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED;
+                preempt_enable();
+        }
+
+        mod_wd = (linkcmd << dd->ibcc_lc_shift) |
+                (linitcmd << INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+        ipath_cdbg(VERBOSE,
+                "Moving unit %u to %s (initcmd=0x%x), current ltstate is %s\n",
+                dd->ipath_unit, what[linkcmd], linitcmd,
+                ipath_ibcstatus_str[ipath_ib_linktrstate(dd,
                         ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus))]);
-        /* flush all queued sends when going to DOWN to be sure that
-         * they don't block MAD packets */
-        if (linkcmd == INFINIPATH_IBCC_LINKCMD_DOWN)
-                ipath_cancel_sends(dd, 1);
 
         ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
-                         dd->ipath_ibcctrl | which);
+                         dd->ipath_ibcctrl | mod_wd);
+        /* read from chip so write is flushed */
+        (void) ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
 }
 
 int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
@@ -1697,30 +1715,28 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
 
         switch (newstate) {
         case IPATH_IB_LINKDOWN_ONLY:
-                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN <<
-                                    INFINIPATH_IBCC_LINKCMD_SHIFT);
+                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, 0);
                 /* don't wait */
                 ret = 0;
                 goto bail;
 
         case IPATH_IB_LINKDOWN:
-                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_POLL <<
-                                    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
+                                        INFINIPATH_IBCC_LINKINITCMD_POLL);
                 /* don't wait */
                 ret = 0;
                 goto bail;
 
         case IPATH_IB_LINKDOWN_SLEEP:
-                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_SLEEP <<
-                                    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
+                                        INFINIPATH_IBCC_LINKINITCMD_SLEEP);
                 /* don't wait */
                 ret = 0;
                 goto bail;
 
         case IPATH_IB_LINKDOWN_DISABLE:
-                ipath_set_ib_lstate(dd,
-                                    INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
-                                    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
+                                        INFINIPATH_IBCC_LINKINITCMD_DISABLE);
                 /* don't wait */
                 ret = 0;
                 goto bail;
@@ -1735,8 +1751,8 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
                         ret = -EINVAL;
                         goto bail;
                 }
-                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED <<
-                                    INFINIPATH_IBCC_LINKCMD_SHIFT);
+                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED, 0);
+
                 /*
                  * Since the port can transition to ACTIVE by receiving
                  * a non VL 15 packet, wait for either state.
@@ -1753,8 +1769,7 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
                         ret = -EINVAL;
                         goto bail;
                 }
-                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE <<
-                                    INFINIPATH_IBCC_LINKCMD_SHIFT);
+                ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE, 0);
                 lstate = IPATH_LINKACTIVE;
                 break;
 
@@ -2026,8 +2041,7 @@ void ipath_shutdown_device(struct ipath_devdata *dd)
          */
         udelay(5);
 
-        ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
-                            INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+        ipath_set_ib_lstate(dd, 0, INFINIPATH_IBCC_LINKINITCMD_DISABLE);
         ipath_cancel_sends(dd, 0);
 
         signal_ib_event(dd, IB_EVENT_PORT_ERR);
diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c
index 524fdf7d8c88..46c70656d538 100644
--- a/drivers/infiniband/hw/ipath/ipath_init_chip.c
+++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c
@@ -180,10 +180,13 @@ static int bringup_link(struct ipath_devdata *dd)
         /*
          * Want to start out with both LINKCMD and LINKINITCMD in NOP
          * (0 and 0).  Don't put linkinitcmd in ipath_ibcctrl, want that
-         * to stay a NOP
+         * to stay a NOP. Flag that we are disabled, for the (unlikely)
+         * case that some recovery path is trying to bring the link up
+         * before we are ready.
          */
         ibc |= INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
                 INFINIPATH_IBCC_LINKINITCMD_SHIFT;
+        dd->ipath_flags |= IPATH_IB_LINK_DISABLED;
         ipath_cdbg(VERBOSE, "Writing 0x%llx to ibcctrl\n",
                    (unsigned long long) ibc);
         ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, ibc);
diff --git a/drivers/infiniband/hw/ipath/ipath_intr.c b/drivers/infiniband/hw/ipath/ipath_intr.c
index ed82ecbb02da..5608e3268a62 100644
--- a/drivers/infiniband/hw/ipath/ipath_intr.c
+++ b/drivers/infiniband/hw/ipath/ipath_intr.c
@@ -309,6 +309,8 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
                 lastlstate == INFINIPATH_IBCS_L_STATE_DOWN) {
                 /* transitioned to UP */
                 if (dd->ipath_f_ib_updown(dd, 1, ibcs)) {
+                        /* link came up, so we must no longer be disabled */
+                        dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED;
                         ipath_cdbg(LINKVERB, "LinkUp handled, skipped\n");
                         goto skip_ibchange; /* chip-code handled */
                 }
diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index b8b81cb745b9..2510359b1c83 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -848,6 +848,8 @@ void ipath_hol_event(unsigned long);
                 /* Suppress heartbeat, even if turning off loopback */
 #define IPATH_NO_HRTBT      0x1000000
 #define IPATH_HAS_MULT_IB_SPEED 0x8000000
+                /* Linkdown-disable intentionally, Do not attempt to bring up */
+#define IPATH_IB_LINK_DISABLED 0x40000000
 #define IPATH_IB_FORCE_NOTIFY 0x80000000 /* force notify on next ib change */
 
 /* Bits in GPIO for the added interrupts */
diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c
index 7516a2608988..30b2f44406b1 100644
--- a/drivers/infiniband/hw/ipath/ipath_mad.c
+++ b/drivers/infiniband/hw/ipath/ipath_mad.c
@@ -590,6 +590,10 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
                 else
                         goto err;
                 ipath_set_linkstate(dd, lstate);
+                if (lstate == IPATH_IB_LINKDOWN_DISABLE) {
+                        ret = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
+                        goto done;
+                }
                 ipath_wait_linkstate(dd, IPATH_LINKINIT | IPATH_LINKARMED |
                                 IPATH_LINKACTIVE, 1000);
                 break;