10 files changed, 69 insertions, 23 deletions

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 112b37cd6895..56feab6c251e 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -917,7 +917,7 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
                 resp->wc[i].opcode         = wc[i].opcode;
                 resp->wc[i].vendor_err     = wc[i].vendor_err;
                 resp->wc[i].byte_len       = wc[i].byte_len;
-                resp->wc[i].imm_data       = (__u32 __force) wc[i].imm_data;
+                resp->wc[i].ex.imm_data    = (__u32 __force) wc[i].ex.imm_data;
                 resp->wc[i].qp_num         = wc[i].qp->qp_num;
                 resp->wc[i].src_qp         = wc[i].src_qp;
                 resp->wc[i].wc_flags       = wc[i].wc_flags;
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 9f399d3a42b4..e0fbe5975866 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -753,6 +753,52 @@ int ib_dereg_mr(struct ib_mr *mr)
 }
 EXPORT_SYMBOL(ib_dereg_mr);
 
+struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
+{
+        struct ib_mr *mr;
+
+        if (!pd->device->alloc_fast_reg_mr)
+                return ERR_PTR(-ENOSYS);
+
+        mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
+
+        if (!IS_ERR(mr)) {
+                mr->device  = pd->device;
+                mr->pd      = pd;
+                mr->uobject = NULL;
+                atomic_inc(&pd->usecnt);
+                atomic_set(&mr->usecnt, 0);
+        }
+
+        return mr;
+}
+EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
+
+struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
+                                                          int max_page_list_len)
+{
+        struct ib_fast_reg_page_list *page_list;
+
+        if (!device->alloc_fast_reg_page_list)
+                return ERR_PTR(-ENOSYS);
+
+        page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
+
+        if (!IS_ERR(page_list)) {
+                page_list->device = device;
+                page_list->max_page_list_len = max_page_list_len;
+        }
+
+        return page_list;
+}
+EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
+
+void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
+{
+        page_list->device->free_fast_reg_page_list(page_list);
+}
+EXPORT_SYMBOL(ib_free_fast_reg_page_list);
+
 /* Memory windows */
 
 struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
diff --git a/drivers/infiniband/hw/ehca/ehca_reqs.c b/drivers/infiniband/hw/ehca/ehca_reqs.c
index f093b0033daf..b799b2710210 100644
--- a/drivers/infiniband/hw/ehca/ehca_reqs.c
+++ b/drivers/infiniband/hw/ehca/ehca_reqs.c
@@ -681,7 +681,7 @@ poll_cq_one_read_cqe:
         wc->dlid_path_bits = cqe->dlid;
         wc->src_qp = cqe->remote_qp_number;
         wc->wc_flags = cqe->w_completion_flags;
-        wc->imm_data = cpu_to_be32(cqe->immediate_data);
+        wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
         wc->sl = cqe->service_level;
 
 poll_cq_one_exit0:
diff --git a/drivers/infiniband/hw/ipath/ipath_cq.c b/drivers/infiniband/hw/ipath/ipath_cq.c
index a03bd28d9b48..d385e4168c97 100644
--- a/drivers/infiniband/hw/ipath/ipath_cq.c
+++ b/drivers/infiniband/hw/ipath/ipath_cq.c
@@ -82,7 +82,7 @@ void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)
                 wc->uqueue[head].opcode = entry->opcode;
                 wc->uqueue[head].vendor_err = entry->vendor_err;
                 wc->uqueue[head].byte_len = entry->byte_len;
-                wc->uqueue[head].imm_data = (__u32 __force)entry->imm_data;
+                wc->uqueue[head].ex.imm_data = (__u32 __force) entry->ex.imm_data;
                 wc->uqueue[head].qp_num = entry->qp->qp_num;
                 wc->uqueue[head].src_qp = entry->src_qp;
                 wc->uqueue[head].wc_flags = entry->wc_flags;
diff --git a/drivers/infiniband/hw/ipath/ipath_rc.c b/drivers/infiniband/hw/ipath/ipath_rc.c
index 108df667d2ee..97710522624d 100644
--- a/drivers/infiniband/hw/ipath/ipath_rc.c
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c
@@ -1703,11 +1703,11 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
         case OP(SEND_LAST_WITH_IMMEDIATE):
         send_last_imm:
                 if (header_in_data) {
-                        wc.imm_data = *(__be32 *) data;
+                        wc.ex.imm_data = *(__be32 *) data;
                         data += sizeof(__be32);
                 } else {
                         /* Immediate data comes after BTH */
-                        wc.imm_data = ohdr->u.imm_data;
+                        wc.ex.imm_data = ohdr->u.imm_data;
                 }
                 hdrsize += 4;
                 wc.wc_flags = IB_WC_WITH_IMM;
diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
index a4b5521567fe..af051f757663 100644
--- a/drivers/infiniband/hw/ipath/ipath_ruc.c
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -331,7 +331,7 @@ again:
         switch (wqe->wr.opcode) {
         case IB_WR_SEND_WITH_IMM:
                 wc.wc_flags = IB_WC_WITH_IMM;
-                wc.imm_data = wqe->wr.ex.imm_data;
+                wc.ex.imm_data = wqe->wr.ex.imm_data;
                 /* FALLTHROUGH */
         case IB_WR_SEND:
                 if (!ipath_get_rwqe(qp, 0))
@@ -342,7 +342,7 @@ again:
                 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
                         goto inv_err;
                 wc.wc_flags = IB_WC_WITH_IMM;
-                wc.imm_data = wqe->wr.ex.imm_data;
+                wc.ex.imm_data = wqe->wr.ex.imm_data;
                 if (!ipath_get_rwqe(qp, 1))
                         goto rnr_nak;
                 /* FALLTHROUGH */
diff --git a/drivers/infiniband/hw/ipath/ipath_uc.c b/drivers/infiniband/hw/ipath/ipath_uc.c
index 0596ec16fcbd..82cc588b8bf2 100644
--- a/drivers/infiniband/hw/ipath/ipath_uc.c
+++ b/drivers/infiniband/hw/ipath/ipath_uc.c
@@ -379,11 +379,11 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
         case OP(SEND_LAST_WITH_IMMEDIATE):
         send_last_imm:
                 if (header_in_data) {
-                        wc.imm_data = *(__be32 *) data;
+                        wc.ex.imm_data = *(__be32 *) data;
                         data += sizeof(__be32);
                 } else {
                         /* Immediate data comes after BTH */
-                        wc.imm_data = ohdr->u.imm_data;
+                        wc.ex.imm_data = ohdr->u.imm_data;
                 }
                 hdrsize += 4;
                 wc.wc_flags = IB_WC_WITH_IMM;
@@ -483,11 +483,11 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
         case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
         rdma_last_imm:
                 if (header_in_data) {
-                        wc.imm_data = *(__be32 *) data;
+                        wc.ex.imm_data = *(__be32 *) data;
                         data += sizeof(__be32);
                 } else {
                         /* Immediate data comes after BTH */
-                        wc.imm_data = ohdr->u.imm_data;
+                        wc.ex.imm_data = ohdr->u.imm_data;
                 }
                 hdrsize += 4;
                 wc.wc_flags = IB_WC_WITH_IMM;
diff --git a/drivers/infiniband/hw/ipath/ipath_ud.c b/drivers/infiniband/hw/ipath/ipath_ud.c
index 77ca8ca74e78..36aa242c487c 100644
--- a/drivers/infiniband/hw/ipath/ipath_ud.c
+++ b/drivers/infiniband/hw/ipath/ipath_ud.c
@@ -96,7 +96,7 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe)
 
         if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
                 wc.wc_flags = IB_WC_WITH_IMM;
-                wc.imm_data = swqe->wr.ex.imm_data;
+                wc.ex.imm_data = swqe->wr.ex.imm_data;
         }
 
         /*
@@ -492,14 +492,14 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
         if (qp->ibqp.qp_num > 1 &&
             opcode == IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE) {
                 if (header_in_data) {
-                        wc.imm_data = *(__be32 *) data;
+                        wc.ex.imm_data = *(__be32 *) data;
                         data += sizeof(__be32);
                 } else
-                        wc.imm_data = ohdr->u.ud.imm_data;
+                        wc.ex.imm_data = ohdr->u.ud.imm_data;
                 wc.wc_flags = IB_WC_WITH_IMM;
                 hdrsize += sizeof(u32);
         } else if (opcode == IB_OPCODE_UD_SEND_ONLY) {
-                wc.imm_data = 0;
+                wc.ex.imm_data = 0;
                 wc.wc_flags = 0;
         } else {
                 dev->n_pkt_drops++;
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 4521319b1406..299f20832ab6 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -663,18 +663,18 @@ repoll:
 
                 switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
                 case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
-                        wc->opcode   = IB_WC_RECV_RDMA_WITH_IMM;
-                        wc->wc_flags = IB_WC_WITH_IMM;
-                        wc->imm_data = cqe->immed_rss_invalid;
+                        wc->opcode      = IB_WC_RECV_RDMA_WITH_IMM;
+                        wc->wc_flags    = IB_WC_WITH_IMM;
+                        wc->ex.imm_data = cqe->immed_rss_invalid;
                         break;
                 case MLX4_RECV_OPCODE_SEND:
                         wc->opcode   = IB_WC_RECV;
                         wc->wc_flags = 0;
                         break;
                 case MLX4_RECV_OPCODE_SEND_IMM:
-                        wc->opcode   = IB_WC_RECV;
-                        wc->wc_flags = IB_WC_WITH_IMM;
-                        wc->imm_data = cqe->immed_rss_invalid;
+                        wc->opcode      = IB_WC_RECV;
+                        wc->wc_flags    = IB_WC_WITH_IMM;
+                        wc->ex.imm_data = cqe->immed_rss_invalid;
                         break;
                 }
 
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index f788fce71ac7..d9f4735c2b37 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -620,13 +620,13 @@ static inline int mthca_poll_one(struct mthca_dev *dev,
                 case IB_OPCODE_SEND_LAST_WITH_IMMEDIATE:
                 case IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE:
                         entry->wc_flags = IB_WC_WITH_IMM;
-                        entry->imm_data = cqe->imm_etype_pkey_eec;
+                        entry->ex.imm_data = cqe->imm_etype_pkey_eec;
                         entry->opcode = IB_WC_RECV;
                         break;
                 case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
                 case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
                         entry->wc_flags = IB_WC_WITH_IMM;
-                        entry->imm_data = cqe->imm_etype_pkey_eec;
+                        entry->ex.imm_data = cqe->imm_etype_pkey_eec;
                         entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
                         break;
                 default: