IB/core: Add "type 2" memory windows support

This patch enhances the IB core support for Memory Windows (MWs).

MWs allow an application to have better/flexible control over remote
access to memory.

Two types of MWs are supported, with the second type having two flavors:

    Type 1  - associated with PD only
    Type 2A - associated with QPN only
    Type 2B - associated with PD and QPN

Applications can allocate a MW once, and then repeatedly bind the MW
to different ranges in MRs that are associated to the same PD. Type 1
windows are bound through a verb, while type 2 windows are bound by
posting a work request.

The 32-bit memory key is composed of a 24-bit index and an 8-bit
key. The key is changed with each bind, thus allowing more control
over the peer's use of the memory key.

The changes introduced are the following:

* add memory window type enum and a corresponding parameter to ib_alloc_mw.
* type 2 memory window bind work request support.
* create a struct that contains the common part of the bind verb struct
  ibv_mw_bind and the bind work request into a single struct.
* add the ib_inc_rkey helper function to advance the tag part of an rkey.

Consumer interface details:

* new device capability flags IB_DEVICE_MEM_WINDOW_TYPE_2A and
  IB_DEVICE_MEM_WINDOW_TYPE_2B are added to indicate device support
  for these features.

  Devices can set either IB_DEVICE_MEM_WINDOW_TYPE_2A or
  IB_DEVICE_MEM_WINDOW_TYPE_2B if it supports type 2A or type 2B
  memory windows. It can set neither to indicate it doesn't support
  type 2 windows at all.

* modify existing provides and consumers code to the new param of
  ib_alloc_mw and the ib_mw_bind_info structure

Signed-off-by: Haggai Eran <haggaie@mellanox.com>
Signed-off-by: Shani Michaeli <shanim@mellanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>

1 files changed, 11 insertions, 8 deletions

diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 07e4fbad987a..8f67fe2e91e6 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -55,7 +55,8 @@ static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
 /**
  * nes_alloc_mw
  */
-static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd) {
+static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type)
+{
         struct nes_pd *nespd = to_nespd(ibpd);
         struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
         struct nes_device *nesdev = nesvnic->nesdev;
@@ -71,6 +72,9 @@ static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd) {
         u32 driver_key = 0;
         u8 stag_key = 0;
 
+        if (type != IB_MW_TYPE_1)
+                return ERR_PTR(-EINVAL);
+
         get_random_bytes(&next_stag_index, sizeof(next_stag_index));
         stag_key = (u8)next_stag_index;
 
@@ -244,20 +248,19 @@ static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw,
         if (ibmw_bind->send_flags & IB_SEND_SIGNALED)
                 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL;
 
-        if (ibmw_bind->mw_access_flags & IB_ACCESS_REMOTE_WRITE) {
+        if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_WRITE)
                 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE;
-        }
-        if (ibmw_bind->mw_access_flags & IB_ACCESS_REMOTE_READ) {
+        if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_READ)
                 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_READ;
-        }
 
         set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_MISC_IDX, wqe_misc);
-        set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MR_IDX, ibmw_bind->mr->lkey);
+        set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MR_IDX,
+                            ibmw_bind->bind_info.mr->lkey);
         set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MW_IDX, ibmw->rkey);
         set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX,
-                        ibmw_bind->length);
+                        ibmw_bind->bind_info.length);
         wqe->wqe_words[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX] = 0;
-        u64temp = (u64)ibmw_bind->addr;
+        u64temp = (u64)ibmw_bind->bind_info.addr;
         set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX, u64temp);
 
         head++;