5 files changed, 28 insertions, 9 deletions

diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 6b811e3e8bd1..2d638877c4af 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -873,7 +873,14 @@ struct qib_devdata {
          * pio_writing.
          */
         spinlock_t pioavail_lock;
-
+        /*
+         * index of last buffer to optimize search for next
+         */
+        u32 last_pio;
+        /*
+         * min kernel pio buffer to optimize search
+         */
+        u32 min_kernel_pio;
         /*
          * Shadow copies of registers; size indicates read access size.
          * Most of them are readonly, but some are write-only register,
diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c
index d0c64d514813..4d352b90750a 100644
--- a/drivers/infiniband/hw/qib/qib_iba6120.c
+++ b/drivers/infiniband/hw/qib/qib_iba6120.c
@@ -3132,6 +3132,7 @@ static void get_6120_chip_params(struct qib_devdata *dd)
         val = qib_read_kreg64(dd, kr_sendpiobufcnt);
         dd->piobcnt2k = val & ~0U;
         dd->piobcnt4k = val >> 32;
+        dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
         /* these may be adjusted in init_chip_wc_pat() */
         dd->pio2kbase = (u32 __iomem *)
                 (((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c
index 3c722f79d6f6..86a0ba7ca0c2 100644
--- a/drivers/infiniband/hw/qib/qib_iba7220.c
+++ b/drivers/infiniband/hw/qib/qib_iba7220.c
@@ -4157,6 +4157,7 @@ static int qib_init_7220_variables(struct qib_devdata *dd)
                 dd->cspec->sdmabufcnt;
         dd->lastctxt_piobuf = dd->cspec->lastbuf_for_pio - sbufs;
         dd->cspec->lastbuf_for_pio--; /* range is <= , not < */
+        dd->last_pio = dd->cspec->lastbuf_for_pio;
         dd->pbufsctxt = dd->lastctxt_piobuf /
                 (dd->cfgctxts - dd->first_user_ctxt);
 
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index 060b96064469..e7b9ad34fe2e 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -6379,6 +6379,7 @@ static int qib_init_7322_variables(struct qib_devdata *dd)
                 dd->cspec->sdmabufcnt;
         dd->lastctxt_piobuf = dd->cspec->lastbuf_for_pio - sbufs;
         dd->cspec->lastbuf_for_pio--; /* range is <= , not < */
+        dd->last_pio = dd->cspec->lastbuf_for_pio;
         dd->pbufsctxt = (dd->cfgctxts > dd->first_user_ctxt) ?
                 dd->lastctxt_piobuf / (dd->cfgctxts - dd->first_user_ctxt) : 0;
 
diff --git a/drivers/infiniband/hw/qib/qib_tx.c b/drivers/infiniband/hw/qib/qib_tx.c
index 1bf626c40172..31d3561400a4 100644
--- a/drivers/infiniband/hw/qib/qib_tx.c
+++ b/drivers/infiniband/hw/qib/qib_tx.c
@@ -295,6 +295,7 @@ u32 __iomem *qib_getsendbuf_range(struct qib_devdata *dd, u32 *pbufnum,
 
         nbufs = last - first + 1; /* number in range to check */
         if (dd->upd_pio_shadow) {
+update_shadow:
                 /*
                  * Minor optimization.  If we had no buffers on last call,
                  * start out by doing the update; continue and do scan even
@@ -304,37 +305,39 @@ u32 __iomem *qib_getsendbuf_range(struct qib_devdata *dd, u32 *pbufnum,
                 updated++;
         }
         i = first;
-rescan:
         /*
          * While test_and_set_bit() is atomic, we do that and then the
          * change_bit(), and the pair is not.  See if this is the cause
          * of the remaining armlaunch errors.
          */
         spin_lock_irqsave(&dd->pioavail_lock, flags);
+        if (dd->last_pio >= first && dd->last_pio <= last)
+                i = dd->last_pio + 1;
+        if (!first)
+                /* adjust to min possible  */
+                nbufs = last - dd->min_kernel_pio + 1;
         for (j = 0; j < nbufs; j++, i++) {
                 if (i > last)
-                        i = first;
+                        i = !first ? dd->min_kernel_pio : first;
                 if (__test_and_set_bit((2 * i) + 1, shadow))
                         continue;
                 /* flip generation bit */
                 __change_bit(2 * i, shadow);
                 /* remember that the buffer can be written to now */
                 __set_bit(i, dd->pio_writing);
+                if (!first && first != last) /* first == last on VL15, avoid */
+                        dd->last_pio = i;
                 break;
         }
         spin_unlock_irqrestore(&dd->pioavail_lock, flags);
 
         if (j == nbufs) {
-                if (!updated) {
+                if (!updated)
                         /*
                          * First time through; shadow exhausted, but may be
                          * buffers available, try an update and then rescan.
                          */
-                        update_send_bufs(dd);
-                        updated++;
-                        i = first;
-                        goto rescan;
-                }
+                        goto update_shadow;
                 no_send_bufs(dd);
                 buf = NULL;
         } else {
@@ -422,14 +425,20 @@ void qib_chg_pioavailkernel(struct qib_devdata *dd, unsigned start,
                                 __clear_bit(QLOGIC_IB_SENDPIOAVAIL_CHECK_SHIFT
                                             + start, dd->pioavailshadow);
                         __set_bit(start, dd->pioavailkernel);
+                        if ((start >> 1) < dd->min_kernel_pio)
+                                dd->min_kernel_pio = start >> 1;
                 } else {
                         __set_bit(start + QLOGIC_IB_SENDPIOAVAIL_BUSY_SHIFT,
                                   dd->pioavailshadow);
                         __clear_bit(start, dd->pioavailkernel);
+                        if ((start >> 1) > dd->min_kernel_pio)
+                                dd->min_kernel_pio = start >> 1;
                 }
                 start += 2;
         }
 
+        if (dd->min_kernel_pio > 0 && dd->last_pio < dd->min_kernel_pio - 1)
+                dd->last_pio = dd->min_kernel_pio - 1;
         spin_unlock_irqrestore(&dd->pioavail_lock, flags);
 
         dd->f_txchk_change(dd, ostart, len, avail, rcd);