15 files changed, 199 insertions, 132 deletions

diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 6b811e3e8bd1..7e62f4137148 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -530,8 +530,6 @@ struct qib_pportdata {
         /* qib_lflags driver is waiting for */
         u32 state_wanted;
         spinlock_t lflags_lock;
-        /* number of (port-specific) interrupts for this port -- saturates... */
-        u32 int_counter;
 
         /* ref count for each pkey */
         atomic_t pkeyrefs[4];
@@ -543,24 +541,26 @@ struct qib_pportdata {
         u64 *statusp;
 
         /* SendDMA related entries */
-        spinlock_t            sdma_lock;
-        struct qib_sdma_state sdma_state;
-        unsigned long         sdma_buf_jiffies;
+
+        /* read mostly */
         struct qib_sdma_desc *sdma_descq;
+        struct qib_sdma_state sdma_state;
+        dma_addr_t       sdma_descq_phys;
+        volatile __le64 *sdma_head_dma; /* DMA'ed by chip */
+        dma_addr_t       sdma_head_phys;
+        u16                   sdma_descq_cnt;
+
+        /* read/write using lock */
+        spinlock_t            sdma_lock ____cacheline_aligned_in_smp;
+        struct list_head      sdma_activelist;
         u64                   sdma_descq_added;
         u64                   sdma_descq_removed;
-        u16                   sdma_descq_cnt;
         u16                   sdma_descq_tail;
         u16                   sdma_descq_head;
-        u16                   sdma_next_intr;
-        u16                   sdma_reset_wait;
         u8                    sdma_generation;
-        struct tasklet_struct sdma_sw_clean_up_task;
-        struct list_head      sdma_activelist;
 
-        dma_addr_t       sdma_descq_phys;
-        volatile __le64 *sdma_head_dma; /* DMA'ed by chip */
-        dma_addr_t       sdma_head_phys;
+        struct tasklet_struct sdma_sw_clean_up_task
+                ____cacheline_aligned_in_smp;
 
         wait_queue_head_t state_wait; /* for state_wanted */
 
@@ -873,7 +873,14 @@ struct qib_devdata {
          * pio_writing.
          */
         spinlock_t pioavail_lock;
-
+        /*
+         * index of last buffer to optimize search for next
+         */
+        u32 last_pio;
+        /*
+         * min kernel pio buffer to optimize search
+         */
+        u32 min_kernel_pio;
         /*
          * Shadow copies of registers; size indicates read access size.
          * Most of them are readonly, but some are write-only register,
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 6fc9365ba8a6..8895cfec5019 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -38,6 +38,7 @@
 #include <linux/netdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/module.h>
+#include <linux/prefetch.h>
 
 #include "qib.h"
 
@@ -481,8 +482,10 @@ u32 qib_kreceive(struct qib_ctxtdata *rcd, u32 *llic, u32 *npkts)
                         etail = qib_hdrget_index(rhf_addr);
                         updegr = 1;
                         if (tlen > sizeof(*hdr) ||
-                            etype >= RCVHQ_RCV_TYPE_NON_KD)
+                            etype >= RCVHQ_RCV_TYPE_NON_KD) {
                                 ebuf = qib_get_egrbuf(rcd, etail);
+                                prefetch_range(ebuf, tlen - sizeof(*hdr));
+                        }
                 }
                 if (!eflags) {
                         u16 lrh_len = be16_to_cpu(hdr->lrh[2]) << 2;
diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c
index d0c64d514813..4d352b90750a 100644
--- a/drivers/infiniband/hw/qib/qib_iba6120.c
+++ b/drivers/infiniband/hw/qib/qib_iba6120.c
@@ -3132,6 +3132,7 @@ static void get_6120_chip_params(struct qib_devdata *dd)
         val = qib_read_kreg64(dd, kr_sendpiobufcnt);
         dd->piobcnt2k = val & ~0U;
         dd->piobcnt4k = val >> 32;
+        dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
         /* these may be adjusted in init_chip_wc_pat() */
         dd->pio2kbase = (u32 __iomem *)
                 (((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c
index 3c722f79d6f6..86a0ba7ca0c2 100644
--- a/drivers/infiniband/hw/qib/qib_iba7220.c
+++ b/drivers/infiniband/hw/qib/qib_iba7220.c
@@ -4157,6 +4157,7 @@ static int qib_init_7220_variables(struct qib_devdata *dd)
                 dd->cspec->sdmabufcnt;
         dd->lastctxt_piobuf = dd->cspec->lastbuf_for_pio - sbufs;
         dd->cspec->lastbuf_for_pio--; /* range is <= , not < */
+        dd->last_pio = dd->cspec->lastbuf_for_pio;
         dd->pbufsctxt = dd->lastctxt_piobuf /
                 (dd->cfgctxts - dd->first_user_ctxt);
 
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index 060b96064469..c881e744c091 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -6379,6 +6379,7 @@ static int qib_init_7322_variables(struct qib_devdata *dd)
                 dd->cspec->sdmabufcnt;
         dd->lastctxt_piobuf = dd->cspec->lastbuf_for_pio - sbufs;
         dd->cspec->lastbuf_for_pio--; /* range is <= , not < */
+        dd->last_pio = dd->cspec->lastbuf_for_pio;
         dd->pbufsctxt = (dd->cfgctxts > dd->first_user_ctxt) ?
                 dd->lastctxt_piobuf / (dd->cfgctxts - dd->first_user_ctxt) : 0;
 
@@ -7708,7 +7709,7 @@ static int serdes_7322_init_new(struct qib_pportdata *ppd)
         ibsd_wr_allchans(ppd, 5, 0, BMASK(0, 0));
         msleep(20);
         /*       Set Frequency Loop Bandwidth */
-        ibsd_wr_allchans(ppd, 2, (7 << 5), BMASK(8, 5));
+        ibsd_wr_allchans(ppd, 2, (15 << 5), BMASK(8, 5));
         /*       Enable Frequency Loop */
         ibsd_wr_allchans(ppd, 2, (1 << 4), BMASK(4, 4));
         /*       Set Timing Loop Bandwidth */
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index cf0cd30adc8d..dc14e100a7f1 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -102,6 +102,8 @@ void qib_set_ctxtcnt(struct qib_devdata *dd)
                 dd->cfgctxts = qib_cfgctxts;
         else
                 dd->cfgctxts = dd->ctxtcnt;
+        dd->freectxts = (dd->first_user_ctxt > dd->cfgctxts) ? 0 :
+                dd->cfgctxts - dd->first_user_ctxt;
 }
 
 /*
@@ -402,7 +404,6 @@ static void enable_chip(struct qib_devdata *dd)
                 if (rcd)
                         dd->f_rcvctrl(rcd->ppd, rcvmask, i);
         }
-        dd->freectxts = dd->cfgctxts - dd->first_user_ctxt;
 }
 
 static void verify_interrupt(unsigned long opaque)
diff --git a/drivers/infiniband/hw/qib/qib_mad.c b/drivers/infiniband/hw/qib/qib_mad.c
index c4ff788823b5..43390217a026 100644
--- a/drivers/infiniband/hw/qib/qib_mad.c
+++ b/drivers/infiniband/hw/qib/qib_mad.c
@@ -396,6 +396,7 @@ static int get_linkdowndefaultstate(struct qib_pportdata *ppd)
 
 static int check_mkey(struct qib_ibport *ibp, struct ib_smp *smp, int mad_flags)
 {
+        int valid_mkey = 0;
         int ret = 0;
 
         /* Is the mkey in the process of expiring? */
@@ -406,23 +407,36 @@ static int check_mkey(struct qib_ibport *ibp, struct ib_smp *smp, int mad_flags)
                 ibp->mkeyprot = 0;
         }
 
-        /* M_Key checking depends on Portinfo:M_Key_protect_bits */
-        if ((mad_flags & IB_MAD_IGNORE_MKEY) == 0 && ibp->mkey != 0 &&
-            ibp->mkey != smp->mkey &&
-            (smp->method == IB_MGMT_METHOD_SET ||
-             smp->method == IB_MGMT_METHOD_TRAP_REPRESS ||
-             (smp->method == IB_MGMT_METHOD_GET && ibp->mkeyprot >= 2))) {
-                if (ibp->mkey_violations != 0xFFFF)
-                        ++ibp->mkey_violations;
-                if (!ibp->mkey_lease_timeout && ibp->mkey_lease_period)
-                        ibp->mkey_lease_timeout = jiffies +
-                                ibp->mkey_lease_period * HZ;
-                /* Generate a trap notice. */
-                qib_bad_mkey(ibp, smp);
-                ret = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
-        } else if (ibp->mkey_lease_timeout)
+        if ((mad_flags & IB_MAD_IGNORE_MKEY) ||  ibp->mkey == 0 ||
+            ibp->mkey == smp->mkey)
+                valid_mkey = 1;
+
+        /* Unset lease timeout on any valid Get/Set/TrapRepress */
+        if (valid_mkey && ibp->mkey_lease_timeout &&
+            (smp->method == IB_MGMT_METHOD_GET ||
+             smp->method == IB_MGMT_METHOD_SET ||
+             smp->method == IB_MGMT_METHOD_TRAP_REPRESS))
                 ibp->mkey_lease_timeout = 0;
 
+        if (!valid_mkey) {
+                switch (smp->method) {
+                case IB_MGMT_METHOD_GET:
+                        /* Bad mkey not a violation below level 2 */
+                        if (ibp->mkeyprot < 2)
+                                break;
+                case IB_MGMT_METHOD_SET:
+                case IB_MGMT_METHOD_TRAP_REPRESS:
+                        if (ibp->mkey_violations != 0xFFFF)
+                                ++ibp->mkey_violations;
+                        if (!ibp->mkey_lease_timeout && ibp->mkey_lease_period)
+                                ibp->mkey_lease_timeout = jiffies +
+                                        ibp->mkey_lease_period * HZ;
+                        /* Generate a trap notice. */
+                        qib_bad_mkey(ibp, smp);
+                        ret = 1;
+                }
+        }
+
         return ret;
 }
 
@@ -450,6 +464,7 @@ static int subn_get_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
                         ibp = to_iport(ibdev, port_num);
                         ret = check_mkey(ibp, smp, 0);
                         if (ret)
+                                ret = IB_MAD_RESULT_FAILURE;
                                 goto bail;
                 }
         }
@@ -631,7 +646,7 @@ static int subn_set_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
         struct qib_devdata *dd;
         struct qib_pportdata *ppd;
         struct qib_ibport *ibp;
-        char clientrereg = 0;
+        u8 clientrereg = (pip->clientrereg_resv_subnetto & 0x80);
         unsigned long flags;
         u16 lid, smlid;
         u8 lwe;
@@ -781,12 +796,6 @@ static int subn_set_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
 
         ibp->subnet_timeout = pip->clientrereg_resv_subnetto & 0x1F;
 
-        if (pip->clientrereg_resv_subnetto & 0x80) {
-                clientrereg = 1;
-                event.event = IB_EVENT_CLIENT_REREGISTER;
-                ib_dispatch_event(&event);
-        }
-
         /*
          * Do the port state change now that the other link parameters
          * have been set.
@@ -844,10 +853,15 @@ static int subn_set_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
                 smp->status |= IB_SMP_INVALID_FIELD;
         }
 
+        if (clientrereg) {
+                event.event = IB_EVENT_CLIENT_REREGISTER;
+                ib_dispatch_event(&event);
+        }
+
         ret = subn_get_portinfo(smp, ibdev, port);
 
-        if (clientrereg)
-                pip->clientrereg_resv_subnetto |= 0x80;
+        /* restore re-reg bit per o14-12.2.1 */
+        pip->clientrereg_resv_subnetto |= clientrereg;
 
         goto get_only;
 
@@ -1835,6 +1849,7 @@ static int process_subn(struct ib_device *ibdev, int mad_flags,
                     port_num && port_num <= ibdev->phys_port_cnt &&
                     port != port_num)
                         (void) check_mkey(to_iport(ibdev, port_num), smp, 0);
+                ret = IB_MAD_RESULT_FAILURE;
                 goto bail;
         }
 
diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c
index 7e7e16fbee99..1ce56b51ab1a 100644
--- a/drivers/infiniband/hw/qib/qib_qp.c
+++ b/drivers/infiniband/hw/qib/qib_qp.c
@@ -1038,6 +1038,11 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
                         goto bail_swq;
                 }
                 RCU_INIT_POINTER(qp->next, NULL);
+                qp->s_hdr = kzalloc(sizeof(*qp->s_hdr), GFP_KERNEL);
+                if (!qp->s_hdr) {
+                        ret = ERR_PTR(-ENOMEM);
+                        goto bail_qp;
+                }
                 qp->timeout_jiffies =
                         usecs_to_jiffies((4096UL * (1UL << qp->timeout)) /
                                 1000UL);
@@ -1159,6 +1164,7 @@ bail_ip:
                 vfree(qp->r_rq.wq);
         free_qpn(&dev->qpn_table, qp->ibqp.qp_num);
 bail_qp:
+        kfree(qp->s_hdr);
         kfree(qp);
 bail_swq:
         vfree(swq);
@@ -1214,6 +1220,7 @@ int qib_destroy_qp(struct ib_qp *ibqp)
         else
                 vfree(qp->r_rq.wq);
         vfree(qp->s_wq);
+        kfree(qp->s_hdr);
         kfree(qp);
         return 0;
 }
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 765b4cbaa020..b641416148eb 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -244,9 +244,9 @@ int qib_make_rc_req(struct qib_qp *qp)
         int ret = 0;
         int delta;
 
-        ohdr = &qp->s_hdr.u.oth;
+        ohdr = &qp->s_hdr->u.oth;
         if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
-                ohdr = &qp->s_hdr.u.l.oth;
+                ohdr = &qp->s_hdr->u.l.oth;
 
         /*
          * The lock is needed to synchronize between the sending tasklet,
diff --git a/drivers/infiniband/hw/qib/qib_ruc.c b/drivers/infiniband/hw/qib/qib_ruc.c
index b4b37e47321a..c0ee7e095d81 100644
--- a/drivers/infiniband/hw/qib/qib_ruc.c
+++ b/drivers/infiniband/hw/qib/qib_ruc.c
@@ -688,17 +688,17 @@ void qib_make_ruc_header(struct qib_qp *qp, struct qib_other_headers *ohdr,
         nwords = (qp->s_cur_size + extra_bytes) >> 2;
         lrh0 = QIB_LRH_BTH;
         if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
-                qp->s_hdrwords += qib_make_grh(ibp, &qp->s_hdr.u.l.grh,
+                qp->s_hdrwords += qib_make_grh(ibp, &qp->s_hdr->u.l.grh,
                                                &qp->remote_ah_attr.grh,
                                                qp->s_hdrwords, nwords);
                 lrh0 = QIB_LRH_GRH;
         }
         lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 |
                 qp->remote_ah_attr.sl << 4;
-        qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
-        qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
-        qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
-        qp->s_hdr.lrh[3] = cpu_to_be16(ppd_from_ibp(ibp)->lid |
+        qp->s_hdr->lrh[0] = cpu_to_be16(lrh0);
+        qp->s_hdr->lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+        qp->s_hdr->lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
+        qp->s_hdr->lrh[3] = cpu_to_be16(ppd_from_ibp(ibp)->lid |
                                        qp->remote_ah_attr.src_path_bits);
         bth0 |= qib_get_pkey(ibp, qp->s_pkey_index);
         bth0 |= extra_bytes << 20;
@@ -758,7 +758,7 @@ void qib_do_send(struct work_struct *work)
                          * If the packet cannot be sent now, return and
                          * the send tasklet will be woken up later.
                          */
-                        if (qib_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords,
+                        if (qib_verbs_send(qp, qp->s_hdr, qp->s_hdrwords,
                                            qp->s_cur_sge, qp->s_cur_size))
                                 break;
                         /* Record that s_hdr is empty. */
diff --git a/drivers/infiniband/hw/qib/qib_sysfs.c b/drivers/infiniband/hw/qib/qib_sysfs.c
index dae51604cfcd..dd9cd49d0979 100644
--- a/drivers/infiniband/hw/qib/qib_sysfs.c
+++ b/drivers/infiniband/hw/qib/qib_sysfs.c
@@ -503,8 +503,11 @@ static ssize_t show_nctxts(struct device *device,
         struct qib_devdata *dd = dd_from_dev(dev);
 
         /* Return the number of user ports (contexts) available. */
-        return scnprintf(buf, PAGE_SIZE, "%u\n", dd->cfgctxts -
-                dd->first_user_ctxt);
+        /* The calculation below deals with a special case where
+         * cfgctxts is set to 1 on a single-port board. */
+        return scnprintf(buf, PAGE_SIZE, "%u\n",
+                        (dd->first_user_ctxt > dd->cfgctxts) ? 0 :
+                        (dd->cfgctxts - dd->first_user_ctxt));
 }
 
 static ssize_t show_nfreectxts(struct device *device,
diff --git a/drivers/infiniband/hw/qib/qib_tx.c b/drivers/infiniband/hw/qib/qib_tx.c
index 1bf626c40172..31d3561400a4 100644
--- a/drivers/infiniband/hw/qib/qib_tx.c
+++ b/drivers/infiniband/hw/qib/qib_tx.c
@@ -295,6 +295,7 @@ u32 __iomem *qib_getsendbuf_range(struct qib_devdata *dd, u32 *pbufnum,
 
         nbufs = last - first + 1; /* number in range to check */
         if (dd->upd_pio_shadow) {
+update_shadow:
                 /*
                  * Minor optimization.  If we had no buffers on last call,
                  * start out by doing the update; continue and do scan even
@@ -304,37 +305,39 @@ u32 __iomem *qib_getsendbuf_range(struct qib_devdata *dd, u32 *pbufnum,
                 updated++;
         }
         i = first;
-rescan:
         /*
          * While test_and_set_bit() is atomic, we do that and then the
          * change_bit(), and the pair is not.  See if this is the cause
          * of the remaining armlaunch errors.
          */
         spin_lock_irqsave(&dd->pioavail_lock, flags);
+        if (dd->last_pio >= first && dd->last_pio <= last)
+                i = dd->last_pio + 1;
+        if (!first)
+                /* adjust to min possible  */
+                nbufs = last - dd->min_kernel_pio + 1;
         for (j = 0; j < nbufs; j++, i++) {
                 if (i > last)
-                        i = first;
+                        i = !first ? dd->min_kernel_pio : first;
                 if (__test_and_set_bit((2 * i) + 1, shadow))
                         continue;
                 /* flip generation bit */
                 __change_bit(2 * i, shadow);
                 /* remember that the buffer can be written to now */
                 __set_bit(i, dd->pio_writing);
+                if (!first && first != last) /* first == last on VL15, avoid */
+                        dd->last_pio = i;
                 break;
         }
         spin_unlock_irqrestore(&dd->pioavail_lock, flags);
 
         if (j == nbufs) {
-                if (!updated) {
+                if (!updated)
                         /*
                          * First time through; shadow exhausted, but may be
                          * buffers available, try an update and then rescan.
                          */
-                        update_send_bufs(dd);
-                        updated++;
-                        i = first;
-                        goto rescan;
-                }
+                        goto update_shadow;
                 no_send_bufs(dd);
                 buf = NULL;
         } else {
@@ -422,14 +425,20 @@ void qib_chg_pioavailkernel(struct qib_devdata *dd, unsigned start,
                                 __clear_bit(QLOGIC_IB_SENDPIOAVAIL_CHECK_SHIFT
                                             + start, dd->pioavailshadow);
                         __set_bit(start, dd->pioavailkernel);
+                        if ((start >> 1) < dd->min_kernel_pio)
+                                dd->min_kernel_pio = start >> 1;
                 } else {
                         __set_bit(start + QLOGIC_IB_SENDPIOAVAIL_BUSY_SHIFT,
                                   dd->pioavailshadow);
                         __clear_bit(start, dd->pioavailkernel);
+                        if ((start >> 1) > dd->min_kernel_pio)
+                                dd->min_kernel_pio = start >> 1;
                 }
                 start += 2;
         }
 
+        if (dd->min_kernel_pio > 0 && dd->last_pio < dd->min_kernel_pio - 1)
+                dd->last_pio = dd->min_kernel_pio - 1;
         spin_unlock_irqrestore(&dd->pioavail_lock, flags);
 
         dd->f_txchk_change(dd, ostart, len, avail, rcd);
diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c
index 7ce2ac2ed219..ce7387ff5d91 100644
--- a/drivers/infiniband/hw/qib/qib_uc.c
+++ b/drivers/infiniband/hw/qib/qib_uc.c
@@ -72,9 +72,9 @@ int qib_make_uc_req(struct qib_qp *qp)
                 goto done;
         }
 
-        ohdr = &qp->s_hdr.u.oth;
+        ohdr = &qp->s_hdr->u.oth;
         if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
-                ohdr = &qp->s_hdr.u.l.oth;
+                ohdr = &qp->s_hdr->u.l.oth;
 
         /* header size in 32-bit words LRH+BTH = (8+12)/4. */
         hwords = 5;
diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c
index 828609fa4d28..a468bf2d4465 100644
--- a/drivers/infiniband/hw/qib/qib_ud.c
+++ b/drivers/infiniband/hw/qib/qib_ud.c
@@ -321,11 +321,11 @@ int qib_make_ud_req(struct qib_qp *qp)
 
         if (ah_attr->ah_flags & IB_AH_GRH) {
                 /* Header size in 32-bit words. */
-                qp->s_hdrwords += qib_make_grh(ibp, &qp->s_hdr.u.l.grh,
+                qp->s_hdrwords += qib_make_grh(ibp, &qp->s_hdr->u.l.grh,
                                                &ah_attr->grh,
                                                qp->s_hdrwords, nwords);
                 lrh0 = QIB_LRH_GRH;
-                ohdr = &qp->s_hdr.u.l.oth;
+                ohdr = &qp->s_hdr->u.l.oth;
                 /*
                  * Don't worry about sending to locally attached multicast
                  * QPs.  It is unspecified by the spec. what happens.
@@ -333,7 +333,7 @@ int qib_make_ud_req(struct qib_qp *qp)
         } else {
                 /* Header size in 32-bit words. */
                 lrh0 = QIB_LRH_BTH;
-                ohdr = &qp->s_hdr.u.oth;
+                ohdr = &qp->s_hdr->u.oth;
         }
         if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
                 qp->s_hdrwords++;
@@ -346,15 +346,15 @@ int qib_make_ud_req(struct qib_qp *qp)
                 lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */
         else
                 lrh0 |= ibp->sl_to_vl[ah_attr->sl] << 12;
-        qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
-        qp->s_hdr.lrh[1] = cpu_to_be16(ah_attr->dlid);  /* DEST LID */
-        qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
+        qp->s_hdr->lrh[0] = cpu_to_be16(lrh0);
+        qp->s_hdr->lrh[1] = cpu_to_be16(ah_attr->dlid);  /* DEST LID */
+        qp->s_hdr->lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
         lid = ppd->lid;
         if (lid) {
                 lid |= ah_attr->src_path_bits & ((1 << ppd->lmc) - 1);
-                qp->s_hdr.lrh[3] = cpu_to_be16(lid);
+                qp->s_hdr->lrh[3] = cpu_to_be16(lid);
         } else
-                qp->s_hdr.lrh[3] = IB_LID_PERMISSIVE;
+                qp->s_hdr->lrh[3] = IB_LID_PERMISSIVE;
         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
                 bth0 |= IB_BTH_SOLICITED;
         bth0 |= extra_bytes << 20;
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index 0c19ef0c4123..487606024659 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -367,9 +367,10 @@ struct qib_rwq {
 
 struct qib_rq {
         struct qib_rwq *wq;
-        spinlock_t lock; /* protect changes in this struct */
         u32 size;               /* size of RWQE array */
         u8 max_sge;
+        spinlock_t lock /* protect changes in this struct */
+                ____cacheline_aligned_in_smp;
 };
 
 struct qib_srq {
@@ -412,31 +413,75 @@ struct qib_ack_entry {
  */
 struct qib_qp {
         struct ib_qp ibqp;
-        struct qib_qp *next;            /* link list for QPN hash table */
-        struct qib_qp *timer_next;      /* link list for qib_ib_timer() */
-        struct list_head iowait;        /* link for wait PIO buf */
-        struct list_head rspwait;       /* link for waititing to respond */
+        /* read mostly fields above and below */
         struct ib_ah_attr remote_ah_attr;
         struct ib_ah_attr alt_ah_attr;
-        struct qib_ib_header s_hdr;     /* next packet header to send */
-        atomic_t refcount;
-        wait_queue_head_t wait;
-        wait_queue_head_t wait_dma;
-        struct timer_list s_timer;
-        struct work_struct s_work;
+        struct qib_qp *next;            /* link list for QPN hash table */
+        struct qib_swqe *s_wq;  /* send work queue */
         struct qib_mmap_info *ip;
+        struct qib_ib_header *s_hdr;     /* next packet header to send */
+        unsigned long timeout_jiffies;  /* computed from timeout */
+
+        enum ib_mtu path_mtu;
+        u32 remote_qpn;
+        u32 pmtu;               /* decoded from path_mtu */
+        u32 qkey;               /* QKEY for this QP (for UD or RD) */
+        u32 s_size;             /* send work queue size */
+        u32 s_rnr_timeout;      /* number of milliseconds for RNR timeout */
+
+        u8 state;               /* QP state */
+        u8 qp_access_flags;
+        u8 alt_timeout;         /* Alternate path timeout for this QP */
+        u8 timeout;             /* Timeout for this QP */
+        u8 s_srate;
+        u8 s_mig_state;
+        u8 port_num;
+        u8 s_pkey_index;        /* PKEY index to use */
+        u8 s_alt_pkey_index;    /* Alternate path PKEY index to use */
+        u8 r_max_rd_atomic;     /* max number of RDMA read/atomic to receive */
+        u8 s_max_rd_atomic;     /* max number of RDMA read/atomic to send */
+        u8 s_retry_cnt;         /* number of times to retry */
+        u8 s_rnr_retry_cnt;
+        u8 r_min_rnr_timer;     /* retry timeout value for RNR NAKs */
+        u8 s_max_sge;           /* size of s_wq->sg_list */
+        u8 s_draining;
+
+        /* start of read/write fields */
+
+        atomic_t refcount ____cacheline_aligned_in_smp;
+        wait_queue_head_t wait;
+
+
+        struct qib_ack_entry s_ack_queue[QIB_MAX_RDMA_ATOMIC + 1]
+                ____cacheline_aligned_in_smp;
+        struct qib_sge_state s_rdma_read_sge;
+
+        spinlock_t r_lock ____cacheline_aligned_in_smp;      /* used for APM */
+        unsigned long r_aflags;
+        u64 r_wr_id;            /* ID for current receive WQE */
+        u32 r_ack_psn;          /* PSN for next ACK or atomic ACK */
+        u32 r_len;              /* total length of r_sge */
+        u32 r_rcv_len;          /* receive data len processed */
+        u32 r_psn;              /* expected rcv packet sequence number */
+        u32 r_msn;              /* message sequence number */
+
+        u8 r_state;             /* opcode of last packet received */
+        u8 r_flags;
+        u8 r_head_ack_queue;    /* index into s_ack_queue[] */
+
+        struct list_head rspwait;       /* link for waititing to respond */
+
+        struct qib_sge_state r_sge;     /* current receive data */
+        struct qib_rq r_rq;             /* receive work queue */
+
+        spinlock_t s_lock ____cacheline_aligned_in_smp;
         struct qib_sge_state *s_cur_sge;
+        u32 s_flags;
         struct qib_verbs_txreq *s_tx;
-        struct qib_mregion *s_rdma_mr;
+        struct qib_swqe *s_wqe;
         struct qib_sge_state s_sge;     /* current send request data */
-        struct qib_ack_entry s_ack_queue[QIB_MAX_RDMA_ATOMIC + 1];
-        struct qib_sge_state s_ack_rdma_sge;
-        struct qib_sge_state s_rdma_read_sge;
-        struct qib_sge_state r_sge;     /* current receive data */
-        spinlock_t r_lock;      /* used for APM */
-        spinlock_t s_lock;
+        struct qib_mregion *s_rdma_mr;
         atomic_t s_dma_busy;
-        u32 s_flags;
         u32 s_cur_size;         /* size of send packet in bytes */
         u32 s_len;              /* total length of s_sge */
         u32 s_rdma_read_len;    /* total length of s_rdma_read_sge */
@@ -447,60 +492,34 @@ struct qib_qp {
         u32 s_psn;              /* current packet sequence number */
         u32 s_ack_rdma_psn;     /* PSN for sending RDMA read responses */
         u32 s_ack_psn;          /* PSN for acking sends and RDMA writes */
-        u32 s_rnr_timeout;      /* number of milliseconds for RNR timeout */
-        u32 r_ack_psn;          /* PSN for next ACK or atomic ACK */
-        u64 r_wr_id;            /* ID for current receive WQE */
-        unsigned long r_aflags;
-        u32 r_len;              /* total length of r_sge */
-        u32 r_rcv_len;          /* receive data len processed */
-        u32 r_psn;              /* expected rcv packet sequence number */
-        u32 r_msn;              /* message sequence number */
+        u32 s_head;             /* new entries added here */
+        u32 s_tail;             /* next entry to process */
+        u32 s_cur;              /* current work queue entry */
+        u32 s_acked;            /* last un-ACK'ed entry */
+        u32 s_last;             /* last completed entry */
+        u32 s_ssn;              /* SSN of tail entry */
+        u32 s_lsn;              /* limit sequence number (credit) */
         u16 s_hdrwords;         /* size of s_hdr in 32 bit words */
         u16 s_rdma_ack_cnt;
-        u8 state;               /* QP state */
         u8 s_state;             /* opcode of last packet sent */
         u8 s_ack_state;         /* opcode of packet to ACK */
         u8 s_nak_state;         /* non-zero if NAK is pending */
-        u8 r_state;             /* opcode of last packet received */
         u8 r_nak_state;         /* non-zero if NAK is pending */
-        u8 r_min_rnr_timer;     /* retry timeout value for RNR NAKs */
-        u8 r_flags;
-        u8 r_max_rd_atomic;     /* max number of RDMA read/atomic to receive */
-        u8 r_head_ack_queue;    /* index into s_ack_queue[] */
-        u8 qp_access_flags;
-        u8 s_max_sge;           /* size of s_wq->sg_list */
-        u8 s_retry_cnt;         /* number of times to retry */
-        u8 s_rnr_retry_cnt;
         u8 s_retry;             /* requester retry counter */
         u8 s_rnr_retry;         /* requester RNR retry counter */
-        u8 s_pkey_index;        /* PKEY index to use */
-        u8 s_alt_pkey_index;    /* Alternate path PKEY index to use */
-        u8 s_max_rd_atomic;     /* max number of RDMA read/atomic to send */
         u8 s_num_rd_atomic;     /* number of RDMA read/atomic pending */
         u8 s_tail_ack_queue;    /* index into s_ack_queue[] */
-        u8 s_srate;
-        u8 s_draining;
-        u8 s_mig_state;
-        u8 timeout;             /* Timeout for this QP */
-        u8 alt_timeout;         /* Alternate path timeout for this QP */
-        u8 port_num;
-        enum ib_mtu path_mtu;
-        u32 pmtu;               /* decoded from path_mtu */
-        u32 remote_qpn;
-        u32 qkey;               /* QKEY for this QP (for UD or RD) */
-        u32 s_size;             /* send work queue size */
-        u32 s_head;             /* new entries added here */
-        u32 s_tail;             /* next entry to process */
-        u32 s_cur;              /* current work queue entry */
-        u32 s_acked;            /* last un-ACK'ed entry */
-        u32 s_last;             /* last completed entry */
-        u32 s_ssn;              /* SSN of tail entry */
-        u32 s_lsn;              /* limit sequence number (credit) */
-        unsigned long timeout_jiffies;  /* computed from timeout */
-        struct qib_swqe *s_wq;  /* send work queue */
-        struct qib_swqe *s_wqe;
-        struct qib_rq r_rq;             /* receive work queue */
-        struct qib_sge r_sg_list[0];    /* verified SGEs */
+
+        struct qib_sge_state s_ack_rdma_sge;
+        struct timer_list s_timer;
+        struct list_head iowait;        /* link for wait PIO buf */
+
+        struct work_struct s_work;
+
+        wait_queue_head_t wait_dma;
+
+        struct qib_sge r_sg_list[0] /* verified SGEs */
+                ____cacheline_aligned_in_smp;
 };
 
 /*