15 files changed, 46 insertions, 31 deletions

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 9e98cec6230f..2c3bff5fe867 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -1463,7 +1463,6 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
                 next->num_sge    = user_wr->num_sge;
                 next->opcode     = user_wr->opcode;
                 next->send_flags = user_wr->send_flags;
-                next->imm_data   = (__be32 __force) user_wr->imm_data;
 
                 if (is_ud) {
                         next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah,
@@ -1476,14 +1475,24 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
                         next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey;
                 } else {
                         switch (next->opcode) {
-                        case IB_WR_RDMA_WRITE:
                         case IB_WR_RDMA_WRITE_WITH_IMM:
+                                next->ex.imm_data =
+                                        (__be32 __force) user_wr->ex.imm_data;
+                        case IB_WR_RDMA_WRITE:
                         case IB_WR_RDMA_READ:
                                 next->wr.rdma.remote_addr =
                                         user_wr->wr.rdma.remote_addr;
                                 next->wr.rdma.rkey        =
                                         user_wr->wr.rdma.rkey;
                                 break;
+                        case IB_WR_SEND_WITH_IMM:
+                                next->ex.imm_data =
+                                        (__be32 __force) user_wr->ex.imm_data;
+                                break;
+                        case IB_WR_SEND_WITH_INV:
+                                next->ex.invalidate_rkey =
+                                        user_wr->ex.invalidate_rkey;
+                                break;
                         case IB_WR_ATOMIC_CMP_AND_SWP:
                         case IB_WR_ATOMIC_FETCH_AND_ADD:
                                 next->wr.atomic.remote_addr =
diff --git a/drivers/infiniband/hw/amso1100/c2_rnic.c b/drivers/infiniband/hw/amso1100/c2_rnic.c
index 7a625524e0c5..b1441aeb60c2 100644
--- a/drivers/infiniband/hw/amso1100/c2_rnic.c
+++ b/drivers/infiniband/hw/amso1100/c2_rnic.c
@@ -455,7 +455,7 @@ int __devinit c2_rnic_init(struct c2_dev *c2dev)
              IB_DEVICE_CURR_QP_STATE_MOD |
              IB_DEVICE_SYS_IMAGE_GUID |
              IB_DEVICE_ZERO_STAG |
-             IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
+             IB_DEVICE_MEM_WINDOW);
 
         /* Allocate the qptr_array */
         c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *));
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 50e1f2a16e0c..ca7265443c05 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -1109,8 +1109,7 @@ int iwch_register_device(struct iwch_dev *dev)
         memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
         dev->ibdev.owner = THIS_MODULE;
         dev->device_cap_flags =
-            (IB_DEVICE_ZERO_STAG |
-             IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
+            (IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW);
 
         dev->ibdev.uverbs_cmd_mask =
             (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c
index bc5d9b0813e5..8891c3b0a3d5 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_qp.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_qp.c
@@ -72,7 +72,7 @@ static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr,
         wqe->send.reserved[2] = 0;
         if (wr->opcode == IB_WR_SEND_WITH_IMM) {
                 plen = 4;
-                wqe->send.sgl[0].stag = wr->imm_data;
+                wqe->send.sgl[0].stag = wr->ex.imm_data;
                 wqe->send.sgl[0].len = __constant_cpu_to_be32(0);
                 wqe->send.num_sgle = __constant_cpu_to_be32(0);
                 *flit_cnt = 5;
@@ -112,7 +112,7 @@ static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr,
 
         if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
                 plen = 4;
-                wqe->write.sgl[0].stag = wr->imm_data;
+                wqe->write.sgl[0].stag = wr->ex.imm_data;
                 wqe->write.sgl[0].len = __constant_cpu_to_be32(0);
                 wqe->write.num_sgle = __constant_cpu_to_be32(0);
                 *flit_cnt = 6;
diff --git a/drivers/infiniband/hw/ehca/ehca_reqs.c b/drivers/infiniband/hw/ehca/ehca_reqs.c
index 2ce8cffb8664..a20bbf466188 100644
--- a/drivers/infiniband/hw/ehca/ehca_reqs.c
+++ b/drivers/infiniband/hw/ehca/ehca_reqs.c
@@ -188,7 +188,7 @@ static inline int ehca_write_swqe(struct ehca_qp *qp,
         if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
             send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
                 /* this might not work as long as HW does not support it */
-                wqe_p->immediate_data = be32_to_cpu(send_wr->imm_data);
+                wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
                 wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
         }
 
diff --git a/drivers/infiniband/hw/ipath/ipath_rc.c b/drivers/infiniband/hw/ipath/ipath_rc.c
index 467981905bbe..c405dfba5531 100644
--- a/drivers/infiniband/hw/ipath/ipath_rc.c
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c
@@ -308,7 +308,7 @@ int ipath_make_rc_req(struct ipath_qp *qp)
                         else {
                                 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
                                 /* Immediate data comes after the BTH */
-                                ohdr->u.imm_data = wqe->wr.imm_data;
+                                ohdr->u.imm_data = wqe->wr.ex.imm_data;
                                 hwords += 1;
                         }
                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
@@ -346,7 +346,7 @@ int ipath_make_rc_req(struct ipath_qp *qp)
                                 qp->s_state =
                                         OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
                                 /* Immediate data comes after RETH */
-                                ohdr->u.rc.imm_data = wqe->wr.imm_data;
+                                ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
                                 hwords += 1;
                                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
                                         bth0 |= 1 << 23;
@@ -490,7 +490,7 @@ int ipath_make_rc_req(struct ipath_qp *qp)
                 else {
                         qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
                         /* Immediate data comes after the BTH */
-                        ohdr->u.imm_data = wqe->wr.imm_data;
+                        ohdr->u.imm_data = wqe->wr.ex.imm_data;
                         hwords += 1;
                 }
                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
@@ -526,7 +526,7 @@ int ipath_make_rc_req(struct ipath_qp *qp)
                 else {
                         qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
                         /* Immediate data comes after the BTH */
-                        ohdr->u.imm_data = wqe->wr.imm_data;
+                        ohdr->u.imm_data = wqe->wr.ex.imm_data;
                         hwords += 1;
                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
                                 bth0 |= 1 << 23;
diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
index bcaa2914e341..8ac5c1d82ccd 100644
--- a/drivers/infiniband/hw/ipath/ipath_ruc.c
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -310,7 +310,7 @@ again:
         switch (wqe->wr.opcode) {
         case IB_WR_SEND_WITH_IMM:
                 wc.wc_flags = IB_WC_WITH_IMM;
-                wc.imm_data = wqe->wr.imm_data;
+                wc.imm_data = wqe->wr.ex.imm_data;
                 /* FALLTHROUGH */
         case IB_WR_SEND:
                 if (!ipath_get_rwqe(qp, 0)) {
@@ -339,7 +339,7 @@ again:
                         goto err;
                 }
                 wc.wc_flags = IB_WC_WITH_IMM;
-                wc.imm_data = wqe->wr.imm_data;
+                wc.imm_data = wqe->wr.ex.imm_data;
                 if (!ipath_get_rwqe(qp, 1))
                         goto rnr_nak;
                 /* FALLTHROUGH */
diff --git a/drivers/infiniband/hw/ipath/ipath_uc.c b/drivers/infiniband/hw/ipath/ipath_uc.c
index 2dd8de20d221..bfe8926b5514 100644
--- a/drivers/infiniband/hw/ipath/ipath_uc.c
+++ b/drivers/infiniband/hw/ipath/ipath_uc.c
@@ -94,7 +94,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
                                 qp->s_state =
                                         OP(SEND_ONLY_WITH_IMMEDIATE);
                                 /* Immediate data comes after the BTH */
-                                ohdr->u.imm_data = wqe->wr.imm_data;
+                                ohdr->u.imm_data = wqe->wr.ex.imm_data;
                                 hwords += 1;
                         }
                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
@@ -123,7 +123,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
                                 qp->s_state =
                                         OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
                                 /* Immediate data comes after the RETH */
-                                ohdr->u.rc.imm_data = wqe->wr.imm_data;
+                                ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
                                 hwords += 1;
                                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
                                         bth0 |= 1 << 23;
@@ -152,7 +152,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
                 else {
                         qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
                         /* Immediate data comes after the BTH */
-                        ohdr->u.imm_data = wqe->wr.imm_data;
+                        ohdr->u.imm_data = wqe->wr.ex.imm_data;
                         hwords += 1;
                 }
                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
@@ -177,7 +177,7 @@ int ipath_make_uc_req(struct ipath_qp *qp)
                         qp->s_state =
                                 OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
                         /* Immediate data comes after the BTH */
-                        ohdr->u.imm_data = wqe->wr.imm_data;
+                        ohdr->u.imm_data = wqe->wr.ex.imm_data;
                         hwords += 1;
                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
                                 bth0 |= 1 << 23;
diff --git a/drivers/infiniband/hw/ipath/ipath_ud.c b/drivers/infiniband/hw/ipath/ipath_ud.c
index 918f52070653..8b6a261c89e3 100644
--- a/drivers/infiniband/hw/ipath/ipath_ud.c
+++ b/drivers/infiniband/hw/ipath/ipath_ud.c
@@ -95,7 +95,7 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe)
 
         if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
                 wc.wc_flags = IB_WC_WITH_IMM;
-                wc.imm_data = swqe->wr.imm_data;
+                wc.imm_data = swqe->wr.ex.imm_data;
         } else {
                 wc.wc_flags = 0;
                 wc.imm_data = 0;
@@ -327,7 +327,7 @@ int ipath_make_ud_req(struct ipath_qp *qp)
         }
         if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
                 qp->s_hdrwords++;
-                ohdr->u.ud.imm_data = wqe->wr.imm_data;
+                ohdr->u.ud.imm_data = wqe->wr.ex.imm_data;
                 bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
         } else
                 bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index f5210c17e312..38e651a67589 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -1249,7 +1249,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
         case IB_WR_SEND_WITH_IMM:
                 sqp->ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
                 sqp->ud_header.immediate_present = 1;
-                sqp->ud_header.immediate_data    = wr->imm_data;
+                sqp->ud_header.immediate_data    = wr->ex.imm_data;
                 break;
         default:
                 return -EINVAL;
@@ -1492,7 +1492,7 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 
                 if (wr->opcode == IB_WR_SEND_WITH_IMM ||
                     wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
-                        ctrl->imm = wr->imm_data;
+                        ctrl->imm = wr->ex.imm_data;
                 else
                         ctrl->imm = 0;
 
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 8433897624bc..b3fd6b05d79d 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -1532,7 +1532,7 @@ static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp,
         case IB_WR_SEND_WITH_IMM:
                 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
                 sqp->ud_header.immediate_present = 1;
-                sqp->ud_header.immediate_data = wr->imm_data;
+                sqp->ud_header.immediate_data = wr->ex.imm_data;
                 break;
         default:
                 return -EINVAL;
@@ -1679,7 +1679,7 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                         cpu_to_be32(1);
                 if (wr->opcode == IB_WR_SEND_WITH_IMM ||
                     wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
-                        ((struct mthca_next_seg *) wqe)->imm = wr->imm_data;
+                        ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data;
 
                 wqe += sizeof (struct mthca_next_seg);
                 size = sizeof (struct mthca_next_seg) / 16;
@@ -2020,7 +2020,7 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                         cpu_to_be32(1);
                 if (wr->opcode == IB_WR_SEND_WITH_IMM ||
                     wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
-                        ((struct mthca_next_seg *) wqe)->imm = wr->imm_data;
+                        ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data;
 
                 wqe += sizeof (struct mthca_next_seg);
                 size = sizeof (struct mthca_next_seg) / 16;
diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c
index 134189d77ed3..aa53aab91bf8 100644
--- a/drivers/infiniband/hw/nes/nes_hw.c
+++ b/drivers/infiniband/hw/nes/nes_hw.c
@@ -393,7 +393,7 @@ struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
         nesadapter->base_pd = 1;
 
         nesadapter->device_cap_flags =
-                        IB_DEVICE_ZERO_STAG | IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW;
+                IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 
         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h
index 64a721fcbc1c..8d65bf0a625b 100644
--- a/include/rdma/ib_user_verbs.h
+++ b/include/rdma/ib_user_verbs.h
@@ -533,7 +533,10 @@ struct ib_uverbs_send_wr {
         __u32 num_sge;
         __u32 opcode;
         __u32 send_flags;
-        __u32 imm_data;
+        union {
+                __u32 imm_data;
+                __u32 invalidate_rkey;
+        } ex;
         union {
                 struct {
                         __u64 remote_addr;
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 66928e9cab19..c48f6af5ef9a 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -94,7 +94,7 @@ enum ib_device_cap_flags {
         IB_DEVICE_SRQ_RESIZE            = (1<<13),
         IB_DEVICE_N_NOTIFY_CQ           = (1<<14),
         IB_DEVICE_ZERO_STAG             = (1<<15),
-        IB_DEVICE_SEND_W_INV            = (1<<16),
+        IB_DEVICE_RESERVED              = (1<<16), /* old SEND_W_INV */
         IB_DEVICE_MEM_WINDOW            = (1<<17),
         /*
          * Devices should set IB_DEVICE_UD_IP_SUM if they support
@@ -105,6 +105,7 @@ enum ib_device_cap_flags {
          */
         IB_DEVICE_UD_IP_CSUM            = (1<<18),
         IB_DEVICE_UD_TSO                = (1<<19),
+        IB_DEVICE_SEND_W_INV            = (1<<21),
 };
 
 enum ib_atomic_cap {
@@ -625,7 +626,8 @@ enum ib_wr_opcode {
         IB_WR_RDMA_READ,
         IB_WR_ATOMIC_CMP_AND_SWP,
         IB_WR_ATOMIC_FETCH_AND_ADD,
-        IB_WR_LSO
+        IB_WR_LSO,
+        IB_WR_SEND_WITH_INV,
 };
 
 enum ib_send_flags {
@@ -649,7 +651,10 @@ struct ib_send_wr {
         int                     num_sge;
         enum ib_wr_opcode       opcode;
         int                     send_flags;
-        __be32                  imm_data;
+        union {
+                __be32          imm_data;
+                u32             invalidate_rkey;
+        } ex;
         union {
                 struct {
                         u64     remote_addr;
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index ffbf22a1d2ca..8ea283ecc522 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -1573,7 +1573,6 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
         send_wr.sg_list = req->rl_send_iov;
         send_wr.num_sge = req->rl_niovs;
         send_wr.opcode = IB_WR_SEND;
-        send_wr.imm_data = 0;
         if (send_wr.num_sge == 4)       /* no need to sync any pad (constant) */
                 ib_dma_sync_single_for_device(ia->ri_id->device,
                         req->rl_send_iov[3].addr, req->rl_send_iov[3].length,