Merge branch 'wr-cleanup' into k.o/for-4.4

5 files changed, 120 insertions, 105 deletions

diff --git a/net/rds/ib.h b/net/rds/ib.h
index aae60fda77f6..301c48385166 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -69,7 +69,11 @@ struct rds_ib_connect_private {
 
 struct rds_ib_send_work {
         void                    *s_op;
-        struct ib_send_wr       s_wr;
+        union {
+                struct ib_send_wr       s_wr;
+                struct ib_rdma_wr       s_rdma_wr;
+                struct ib_atomic_wr     s_atomic_wr;
+        };
         struct ib_sge           s_sge[RDS_IB_MAX_SGE];
         unsigned long           s_queued;
 };
diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index 4e88047086b6..987386e9931b 100644
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -785,23 +785,23 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)
         send->s_queued = jiffies;
 
         if (op->op_type == RDS_ATOMIC_TYPE_CSWP) {
-                send->s_wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;
-                send->s_wr.wr.atomic.compare_add = op->op_m_cswp.compare;
-                send->s_wr.wr.atomic.swap = op->op_m_cswp.swap;
-                send->s_wr.wr.atomic.compare_add_mask = op->op_m_cswp.compare_mask;
-                send->s_wr.wr.atomic.swap_mask = op->op_m_cswp.swap_mask;
+                send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;
+                send->s_atomic_wr.compare_add = op->op_m_cswp.compare;
+                send->s_atomic_wr.swap = op->op_m_cswp.swap;
+                send->s_atomic_wr.compare_add_mask = op->op_m_cswp.compare_mask;
+                send->s_atomic_wr.swap_mask = op->op_m_cswp.swap_mask;
         } else { /* FADD */
-                send->s_wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;
-                send->s_wr.wr.atomic.compare_add = op->op_m_fadd.add;
-                send->s_wr.wr.atomic.swap = 0;
-                send->s_wr.wr.atomic.compare_add_mask = op->op_m_fadd.nocarry_mask;
-                send->s_wr.wr.atomic.swap_mask = 0;
+                send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;
+                send->s_atomic_wr.compare_add = op->op_m_fadd.add;
+                send->s_atomic_wr.swap = 0;
+                send->s_atomic_wr.compare_add_mask = op->op_m_fadd.nocarry_mask;
+                send->s_atomic_wr.swap_mask = 0;
         }
         nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);
-        send->s_wr.num_sge = 1;
-        send->s_wr.next = NULL;
-        send->s_wr.wr.atomic.remote_addr = op->op_remote_addr;
-        send->s_wr.wr.atomic.rkey = op->op_rkey;
+        send->s_atomic_wr.wr.num_sge = 1;
+        send->s_atomic_wr.wr.next = NULL;
+        send->s_atomic_wr.remote_addr = op->op_remote_addr;
+        send->s_atomic_wr.rkey = op->op_rkey;
         send->s_op = op;
         rds_message_addref(container_of(send->s_op, struct rds_message, atomic));
 
@@ -826,11 +826,11 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)
         if (nr_sig)
                 atomic_add(nr_sig, &ic->i_signaled_sends);
 
-        failed_wr = &send->s_wr;
-        ret = ib_post_send(ic->i_cm_id->qp, &send->s_wr, &failed_wr);
+        failed_wr = &send->s_atomic_wr.wr;
+        ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);
         rdsdebug("ic %p send %p (wr %p) ret %d wr %p\n", ic,
-                 send, &send->s_wr, ret, failed_wr);
-        BUG_ON(failed_wr != &send->s_wr);
+                 send, &send->s_atomic_wr, ret, failed_wr);
+        BUG_ON(failed_wr != &send->s_atomic_wr.wr);
         if (ret) {
                 printk(KERN_WARNING "RDS/IB: atomic ib_post_send to %pI4 "
                        "returned %d\n", &conn->c_faddr, ret);
@@ -839,9 +839,9 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)
                 goto out;
         }
 
-        if (unlikely(failed_wr != &send->s_wr)) {
+        if (unlikely(failed_wr != &send->s_atomic_wr.wr)) {
                 printk(KERN_WARNING "RDS/IB: atomic ib_post_send() rc=%d, but failed_wqe updated!\n", ret);
-                BUG_ON(failed_wr != &send->s_wr);
+                BUG_ON(failed_wr != &send->s_atomic_wr.wr);
         }
 
 out:
@@ -912,22 +912,23 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                 nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);
 
                 send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
-                send->s_wr.wr.rdma.remote_addr = remote_addr;
-                send->s_wr.wr.rdma.rkey = op->op_rkey;
+                send->s_rdma_wr.remote_addr = remote_addr;
+                send->s_rdma_wr.rkey = op->op_rkey;
 
                 if (num_sge > max_sge) {
-                        send->s_wr.num_sge = max_sge;
+                        send->s_rdma_wr.wr.num_sge = max_sge;
                         num_sge -= max_sge;
                 } else {
-                        send->s_wr.num_sge = num_sge;
+                        send->s_rdma_wr.wr.num_sge = num_sge;
                 }
 
-                send->s_wr.next = NULL;
+                send->s_rdma_wr.wr.next = NULL;
 
                 if (prev)
-                        prev->s_wr.next = &send->s_wr;
+                        prev->s_rdma_wr.wr.next = &send->s_rdma_wr.wr;
 
-                for (j = 0; j < send->s_wr.num_sge && scat != &op->op_sg[op->op_count]; j++) {
+                for (j = 0; j < send->s_rdma_wr.wr.num_sge &&
+                     scat != &op->op_sg[op->op_count]; j++) {
                         len = ib_sg_dma_len(ic->i_cm_id->device, scat);
                         send->s_sge[j].addr =
                                  ib_sg_dma_address(ic->i_cm_id->device, scat);
@@ -942,7 +943,9 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                 }
 
                 rdsdebug("send %p wr %p num_sge %u next %p\n", send,
-                        &send->s_wr, send->s_wr.num_sge, send->s_wr.next);
+                        &send->s_rdma_wr.wr,
+                        send->s_rdma_wr.wr.num_sge,
+                        send->s_rdma_wr.wr.next);
 
                 prev = send;
                 if (++send == &ic->i_sends[ic->i_send_ring.w_nr])
@@ -963,11 +966,11 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
         if (nr_sig)
                 atomic_add(nr_sig, &ic->i_signaled_sends);
 
-        failed_wr = &first->s_wr;
-        ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
+        failed_wr = &first->s_rdma_wr.wr;
+        ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);
         rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
-                 first, &first->s_wr, ret, failed_wr);
-        BUG_ON(failed_wr != &first->s_wr);
+                 first, &first->s_rdma_wr.wr, ret, failed_wr);
+        BUG_ON(failed_wr != &first->s_rdma_wr.wr);
         if (ret) {
                 printk(KERN_WARNING "RDS/IB: rdma ib_post_send to %pI4 "
                        "returned %d\n", &conn->c_faddr, ret);
@@ -976,9 +979,9 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                 goto out;
         }
 
-        if (unlikely(failed_wr != &first->s_wr)) {
+        if (unlikely(failed_wr != &first->s_rdma_wr.wr)) {
                 printk(KERN_WARNING "RDS/IB: ib_post_send() rc=%d, but failed_wqe updated!\n", ret);
-                BUG_ON(failed_wr != &first->s_wr);
+                BUG_ON(failed_wr != &first->s_rdma_wr.wr);
         }
 
 
diff --git a/net/rds/iw.h b/net/rds/iw.h
index cbe6674e31ee..fe858e5dd8d1 100644
--- a/net/rds/iw.h
+++ b/net/rds/iw.h
@@ -77,7 +77,11 @@ struct rds_iw_send_work {
         struct ib_fast_reg_page_list *s_page_list;
         unsigned char           s_remap_count;
 
-        struct ib_send_wr       s_wr;
+        union {
+                struct ib_send_wr       s_send_wr;
+                struct ib_rdma_wr       s_rdma_wr;
+                struct ib_fast_reg_wr   s_fast_reg_wr;
+        };
         struct ib_sge           s_sge[RDS_IW_MAX_SGE];
         unsigned long           s_queued;
 };
diff --git a/net/rds/iw_rdma.c b/net/rds/iw_rdma.c
index 6a8fbd6e69e7..f8a612cc69e6 100644
--- a/net/rds/iw_rdma.c
+++ b/net/rds/iw_rdma.c
@@ -696,7 +696,8 @@ static int rds_iw_init_fastreg(struct rds_iw_mr_pool *pool,
 static int rds_iw_rdma_build_fastreg(struct rds_iw_mapping *mapping)
 {
         struct rds_iw_mr *ibmr = mapping->m_mr;
-        struct ib_send_wr f_wr, *failed_wr;
+        struct ib_fast_reg_wr f_wr;
+        struct ib_send_wr *failed_wr;
         int ret;
 
         /*
@@ -709,22 +710,22 @@ static int rds_iw_rdma_build_fastreg(struct rds_iw_mapping *mapping)
         mapping->m_rkey = ibmr->mr->rkey;
 
         memset(&f_wr, 0, sizeof(f_wr));
-        f_wr.wr_id = RDS_IW_FAST_REG_WR_ID;
-        f_wr.opcode = IB_WR_FAST_REG_MR;
-        f_wr.wr.fast_reg.length = mapping->m_sg.bytes;
-        f_wr.wr.fast_reg.rkey = mapping->m_rkey;
-        f_wr.wr.fast_reg.page_list = ibmr->page_list;
-        f_wr.wr.fast_reg.page_list_len = mapping->m_sg.dma_len;
-        f_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
-        f_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE |
+        f_wr.wr.wr_id = RDS_IW_FAST_REG_WR_ID;
+        f_wr.wr.opcode = IB_WR_FAST_REG_MR;
+        f_wr.length = mapping->m_sg.bytes;
+        f_wr.rkey = mapping->m_rkey;
+        f_wr.page_list = ibmr->page_list;
+        f_wr.page_list_len = mapping->m_sg.dma_len;
+        f_wr.page_shift = PAGE_SHIFT;
+        f_wr.access_flags = IB_ACCESS_LOCAL_WRITE |
                                 IB_ACCESS_REMOTE_READ |
                                 IB_ACCESS_REMOTE_WRITE;
-        f_wr.wr.fast_reg.iova_start = 0;
-        f_wr.send_flags = IB_SEND_SIGNALED;
+        f_wr.iova_start = 0;
+        f_wr.wr.send_flags = IB_SEND_SIGNALED;
 
-        failed_wr = &f_wr;
-        ret = ib_post_send(ibmr->cm_id->qp, &f_wr, &failed_wr);
-        BUG_ON(failed_wr != &f_wr);
+        failed_wr = &f_wr.wr;
+        ret = ib_post_send(ibmr->cm_id->qp, &f_wr.wr, &failed_wr);
+        BUG_ON(failed_wr != &f_wr.wr);
         if (ret)
                 printk_ratelimited(KERN_WARNING "RDS/IW: %s:%d ib_post_send returned %d\n",
                         __func__, __LINE__, ret);
diff --git a/net/rds/iw_send.c b/net/rds/iw_send.c
index 86152ec3b887..f6e23c515b44 100644
--- a/net/rds/iw_send.c
+++ b/net/rds/iw_send.c
@@ -137,13 +137,13 @@ void rds_iw_send_init_ring(struct rds_iw_connection *ic)
                 send->s_op = NULL;
                 send->s_mapping = NULL;
 
-                send->s_wr.next = NULL;
-                send->s_wr.wr_id = i;
-                send->s_wr.sg_list = send->s_sge;
-                send->s_wr.num_sge = 1;
-                send->s_wr.opcode = IB_WR_SEND;
-                send->s_wr.send_flags = 0;
-                send->s_wr.ex.imm_data = 0;
+                send->s_send_wr.next = NULL;
+                send->s_send_wr.wr_id = i;
+                send->s_send_wr.sg_list = send->s_sge;
+                send->s_send_wr.num_sge = 1;
+                send->s_send_wr.opcode = IB_WR_SEND;
+                send->s_send_wr.send_flags = 0;
+                send->s_send_wr.ex.imm_data = 0;
 
                 sge = rds_iw_data_sge(ic, send->s_sge);
                 sge->lkey = 0;
@@ -179,7 +179,7 @@ void rds_iw_send_clear_ring(struct rds_iw_connection *ic)
                 ib_dereg_mr(send->s_mr);
                 BUG_ON(!send->s_page_list);
                 ib_free_fast_reg_page_list(send->s_page_list);
-                if (send->s_wr.opcode == 0xdead)
+                if (send->s_send_wr.opcode == 0xdead)
                         continue;
                 if (send->s_rm)
                         rds_iw_send_unmap_rm(ic, send, IB_WC_WR_FLUSH_ERR);
@@ -247,7 +247,7 @@ void rds_iw_send_cq_comp_handler(struct ib_cq *cq, void *context)
                         send = &ic->i_sends[oldest];
 
                         /* In the error case, wc.opcode sometimes contains garbage */
-                        switch (send->s_wr.opcode) {
+                        switch (send->s_send_wr.opcode) {
                         case IB_WR_SEND:
                                 if (send->s_rm)
                                         rds_iw_send_unmap_rm(ic, send, wc.status);
@@ -262,12 +262,12 @@ void rds_iw_send_cq_comp_handler(struct ib_cq *cq, void *context)
                         default:
                                 printk_ratelimited(KERN_NOTICE
                                                 "RDS/IW: %s: unexpected opcode 0x%x in WR!\n",
-                                                __func__, send->s_wr.opcode);
+                                                __func__, send->s_send_wr.opcode);
                                 break;
                         }
 
-                        send->s_wr.opcode = 0xdead;
-                        send->s_wr.num_sge = 1;
+                        send->s_send_wr.opcode = 0xdead;
+                        send->s_send_wr.num_sge = 1;
                         if (time_after(jiffies, send->s_queued + HZ/2))
                                 rds_iw_stats_inc(s_iw_tx_stalled);
 
@@ -455,10 +455,10 @@ rds_iw_xmit_populate_wr(struct rds_iw_connection *ic,
 
         WARN_ON(pos != send - ic->i_sends);
 
-        send->s_wr.send_flags = send_flags;
-        send->s_wr.opcode = IB_WR_SEND;
-        send->s_wr.num_sge = 2;
-        send->s_wr.next = NULL;
+        send->s_send_wr.send_flags = send_flags;
+        send->s_send_wr.opcode = IB_WR_SEND;
+        send->s_send_wr.num_sge = 2;
+        send->s_send_wr.next = NULL;
         send->s_queued = jiffies;
         send->s_op = NULL;
 
@@ -472,7 +472,7 @@ rds_iw_xmit_populate_wr(struct rds_iw_connection *ic,
         } else {
                 /* We're sending a packet with no payload. There is only
                  * one SGE */
-                send->s_wr.num_sge = 1;
+                send->s_send_wr.num_sge = 1;
                 sge = &send->s_sge[0];
         }
 
@@ -672,23 +672,23 @@ int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm,
                  */
                 if (ic->i_unsignaled_wrs-- == 0) {
                         ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
-                        send->s_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
+                        send->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
                 }
 
                 ic->i_unsignaled_bytes -= len;
                 if (ic->i_unsignaled_bytes <= 0) {
                         ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;
-                        send->s_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
+                        send->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
                 }
 
                 /*
                  * Always signal the last one if we're stopping due to flow control.
                  */
                 if (flow_controlled && i == (work_alloc-1))
-                        send->s_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
+                        send->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
 
                 rdsdebug("send %p wr %p num_sge %u next %p\n", send,
-                         &send->s_wr, send->s_wr.num_sge, send->s_wr.next);
+                         &send->s_send_wr, send->s_send_wr.num_sge, send->s_send_wr.next);
 
                 sent += len;
                 rm->data.op_dmaoff += len;
@@ -722,7 +722,7 @@ add_header:
                 }
 
                 if (prev)
-                        prev->s_wr.next = &send->s_wr;
+                        prev->s_send_wr.next = &send->s_send_wr;
                 prev = send;
 
                 pos = (pos + 1) % ic->i_send_ring.w_nr;
@@ -736,7 +736,7 @@ add_header:
         /* if we finished the message then send completion owns it */
         if (scat == &rm->data.op_sg[rm->data.op_count]) {
                 prev->s_rm = ic->i_rm;
-                prev->s_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
+                prev->s_send_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;
                 ic->i_rm = NULL;
         }
 
@@ -748,11 +748,11 @@ add_header:
                 rds_iw_send_add_credits(conn, credit_alloc - i);
 
         /* XXX need to worry about failed_wr and partial sends. */
-        failed_wr = &first->s_wr;
-        ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
+        failed_wr = &first->s_send_wr;
+        ret = ib_post_send(ic->i_cm_id->qp, &first->s_send_wr, &failed_wr);
         rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
-                 first, &first->s_wr, ret, failed_wr);
-        BUG_ON(failed_wr != &first->s_wr);
+                 first, &first->s_send_wr, ret, failed_wr);
+        BUG_ON(failed_wr != &first->s_send_wr);
         if (ret) {
                 printk(KERN_WARNING "RDS/IW: ib_post_send to %pI4 "
                        "returned %d\n", &conn->c_faddr, ret);
@@ -778,14 +778,14 @@ static void rds_iw_build_send_fastreg(struct rds_iw_device *rds_iwdev, struct rd
          * in the sg list is added to the fast reg page list and placed
          * inside the fast_reg_mr WR.
          */
-        send->s_wr.opcode = IB_WR_FAST_REG_MR;
-        send->s_wr.wr.fast_reg.length = len;
-        send->s_wr.wr.fast_reg.rkey = send->s_mr->rkey;
-        send->s_wr.wr.fast_reg.page_list = send->s_page_list;
-        send->s_wr.wr.fast_reg.page_list_len = nent;
-        send->s_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
-        send->s_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE;
-        send->s_wr.wr.fast_reg.iova_start = sg_addr;
+        send->s_fast_reg_wr.wr.opcode = IB_WR_FAST_REG_MR;
+        send->s_fast_reg_wr.length = len;
+        send->s_fast_reg_wr.rkey = send->s_mr->rkey;
+        send->s_fast_reg_wr.page_list = send->s_page_list;
+        send->s_fast_reg_wr.page_list_len = nent;
+        send->s_fast_reg_wr.page_shift = PAGE_SHIFT;
+        send->s_fast_reg_wr.access_flags = IB_ACCESS_REMOTE_WRITE;
+        send->s_fast_reg_wr.iova_start = sg_addr;
 
         ib_update_fast_reg_key(send->s_mr, send->s_remap_count++);
 }
@@ -863,7 +863,7 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
         num_sge = op->op_count;
 
         for (i = 0; i < work_alloc && scat != &op->op_sg[op->op_count]; i++) {
-                send->s_wr.send_flags = 0;
+                send->s_rdma_wr.wr.send_flags = 0;
                 send->s_queued = jiffies;
 
                 /*
@@ -872,7 +872,7 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                  */
                 if (ic->i_unsignaled_wrs-- == 0) {
                         ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
-                        send->s_wr.send_flags = IB_SEND_SIGNALED;
+                        send->s_rdma_wr.wr.send_flags = IB_SEND_SIGNALED;
                 }
 
                 /* To avoid the need to have the plumbing to invalidate the fastreg_mr used
@@ -880,29 +880,30 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                  * IB_WR_RDMA_READ_WITH_INV will invalidate it after the read has completed.
                  */
                 if (op->op_write)
-                        send->s_wr.opcode = IB_WR_RDMA_WRITE;
+                        send->s_rdma_wr.wr.opcode = IB_WR_RDMA_WRITE;
                 else
-                        send->s_wr.opcode = IB_WR_RDMA_READ_WITH_INV;
+                        send->s_rdma_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV;
 
-                send->s_wr.wr.rdma.remote_addr = remote_addr;
-                send->s_wr.wr.rdma.rkey = op->op_rkey;
+                send->s_rdma_wr.remote_addr = remote_addr;
+                send->s_rdma_wr.rkey = op->op_rkey;
                 send->s_op = op;
 
                 if (num_sge > rds_iwdev->max_sge) {
-                        send->s_wr.num_sge = rds_iwdev->max_sge;
+                        send->s_rdma_wr.wr.num_sge = rds_iwdev->max_sge;
                         num_sge -= rds_iwdev->max_sge;
                 } else
-                        send->s_wr.num_sge = num_sge;
+                        send->s_rdma_wr.wr.num_sge = num_sge;
 
-                send->s_wr.next = NULL;
+                send->s_rdma_wr.wr.next = NULL;
 
                 if (prev)
-                        prev->s_wr.next = &send->s_wr;
+                        prev->s_send_wr.next = &send->s_rdma_wr.wr;
 
-                for (j = 0; j < send->s_wr.num_sge && scat != &op->op_sg[op->op_count]; j++) {
+                for (j = 0; j < send->s_rdma_wr.wr.num_sge &&
+                     scat != &op->op_sg[op->op_count]; j++) {
                         len = ib_sg_dma_len(ic->i_cm_id->device, scat);
 
-                        if (send->s_wr.opcode == IB_WR_RDMA_READ_WITH_INV)
+                        if (send->s_rdma_wr.wr.opcode == IB_WR_RDMA_READ_WITH_INV)
                                 send->s_page_list->page_list[j] = ib_sg_dma_address(ic->i_cm_id->device, scat);
                         else {
                                 send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat);
@@ -917,15 +918,17 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                         scat++;
                 }
 
-                if (send->s_wr.opcode == IB_WR_RDMA_READ_WITH_INV) {
-                        send->s_wr.num_sge = 1;
+                if (send->s_rdma_wr.wr.opcode == IB_WR_RDMA_READ_WITH_INV) {
+                        send->s_rdma_wr.wr.num_sge = 1;
                         send->s_sge[0].addr = conn->c_xmit_rm->m_rs->rs_user_addr;
                         send->s_sge[0].length = conn->c_xmit_rm->m_rs->rs_user_bytes;
                         send->s_sge[0].lkey = ic->i_sends[fr_pos].s_mr->lkey;
                 }
 
                 rdsdebug("send %p wr %p num_sge %u next %p\n", send,
-                        &send->s_wr, send->s_wr.num_sge, send->s_wr.next);
+                        &send->s_rdma_wr,
+                        send->s_rdma_wr.wr.num_sge,
+                        send->s_rdma_wr.wr.next);
 
                 prev = send;
                 if (++send == &ic->i_sends[ic->i_send_ring.w_nr])
@@ -934,7 +937,7 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
 
         /* if we finished the message then send completion owns it */
         if (scat == &op->op_sg[op->op_count])
-                first->s_wr.send_flags = IB_SEND_SIGNALED;
+                first->s_rdma_wr.wr.send_flags = IB_SEND_SIGNALED;
 
         if (i < work_alloc) {
                 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);
@@ -953,11 +956,11 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
                 work_alloc++;
         }
 
-        failed_wr = &first->s_wr;
-        ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
+        failed_wr = &first->s_rdma_wr.wr;
+        ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);
         rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
-                 first, &first->s_wr, ret, failed_wr);
-        BUG_ON(failed_wr != &first->s_wr);
+                 first, &first->s_rdma_wr, ret, failed_wr);
+        BUG_ON(failed_wr != &first->s_rdma_wr.wr);
         if (ret) {
                 printk(KERN_WARNING "RDS/IW: rdma ib_post_send to %pI4 "
                        "returned %d\n", &conn->c_faddr, ret);