IB/ehca: Support small QP queues

eHCA2 supports QP queues that can be as small as 512 bytes. This
greatly reduces memory overhead for consumers that use lots of QPs
with small queues (e.g. RDMA-only QPs). Apart from dealing with
firmware, this code needs to manage bite-sized chunks of kernel pages,
making sure that no kernel page is shared between different protection
domains.

Signed-off-by: Hoang-Nam Nguyen <hnguyen@de.ibm.com>

1 files changed, 19 insertions, 11 deletions

diff --git a/drivers/infiniband/hw/ehca/hcp_if.c b/drivers/infiniband/hw/ehca/hcp_if.c
index 358796ccf008..fdbfebea7d11 100644
--- a/drivers/infiniband/hw/ehca/hcp_if.c
+++ b/drivers/infiniband/hw/ehca/hcp_if.c
@@ -52,10 +52,13 @@
 #define H_ALL_RES_QP_ENHANCED_OPS       EHCA_BMASK_IBM(9, 11)
 #define H_ALL_RES_QP_PTE_PIN            EHCA_BMASK_IBM(12, 12)
 #define H_ALL_RES_QP_SERVICE_TYPE       EHCA_BMASK_IBM(13, 15)
+#define H_ALL_RES_QP_STORAGE            EHCA_BMASK_IBM(16, 17)
 #define H_ALL_RES_QP_LL_RQ_CQE_POSTING  EHCA_BMASK_IBM(18, 18)
 #define H_ALL_RES_QP_LL_SQ_CQE_POSTING  EHCA_BMASK_IBM(19, 21)
 #define H_ALL_RES_QP_SIGNALING_TYPE     EHCA_BMASK_IBM(22, 23)
 #define H_ALL_RES_QP_UD_AV_LKEY_CTRL    EHCA_BMASK_IBM(31, 31)
+#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
+#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
 #define H_ALL_RES_QP_RESOURCE_TYPE      EHCA_BMASK_IBM(56, 63)
 
 #define H_ALL_RES_QP_MAX_OUTST_SEND_WR  EHCA_BMASK_IBM(0, 15)
@@ -299,6 +302,11 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
                 | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
                 | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
                 | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
+                | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
+                | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
+                                 parms->squeue.page_size)
+                | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
+                                 parms->rqueue.page_size)
                 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
                                  !!(parms->ll_comp_flags & LLQP_RECV_COMP))
                 | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
@@ -309,13 +317,13 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
 
         max_r10_reg =
                 EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
-                               parms->max_send_wr + 1)
+                               parms->squeue.max_wr + 1)
                 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
-                                 parms->max_recv_wr + 1)
+                                 parms->rqueue.max_wr + 1)
                 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
-                                 parms->max_send_sge)
+                                 parms->squeue.max_sge)
                 | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
-                                 parms->max_recv_sge);
+                                 parms->rqueue.max_sge);
 
         r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
 
@@ -335,17 +343,17 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
 
         parms->qp_handle.handle = outs[0];
         parms->real_qp_num = (u32)outs[1];
-        parms->act_nr_send_wqes =
+        parms->squeue.act_nr_wqes =
                 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
-        parms->act_nr_recv_wqes =
+        parms->rqueue.act_nr_wqes =
                 (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
-        parms->act_nr_send_sges =
+        parms->squeue.act_nr_sges =
                 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
-        parms->act_nr_recv_sges =
+        parms->rqueue.act_nr_sges =
                 (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
-        parms->nr_sq_pages =
+        parms->squeue.queue_size =
                 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
-        parms->nr_rq_pages =
+        parms->rqueue.queue_size =
                 (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
 
         if (ret == H_SUCCESS)
@@ -497,7 +505,7 @@ u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
                              const u64 count,
                              const struct h_galpa galpa)
 {
-        if (count != 1) {
+        if (count > 1) {
                 ehca_gen_err("Page counter=%lx", count);
                 return H_PARAMETER;
         }