RDMA/cxgb3: MEM_MGT_EXTENSIONS support

- set IB_DEVICE_MEM_MGT_EXTENSIONS capability bit if fw supports it.
- set max_fast_reg_page_list_len device attribute.
- add iwch_alloc_fast_reg_mr function.
- add iwch_alloc_fastreg_pbl
- add iwch_free_fastreg_pbl
- adjust the WQ depth for kernel mode work queues to account for
  fastreg possibly taking 2 WR slots.
- add fastreg_mr work request support.
- add local_inv work request support.
- add send_with_inv and send_with_se_inv work request support.
- removed useless duplicate enums/defines for TPT/MW/MR stuff.

Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

1 files changed, 70 insertions, 20 deletions

diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h
index f1a25a821a45..de760e9f1cc6 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_wr.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_wr.h
@@ -72,7 +72,8 @@ enum t3_wr_opcode {
         T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
         T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
         T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
-        T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
+        T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP,
+        T3_WR_FASTREG = FW_WROPCODE_RI_FASTREGISTER_MR
 } __attribute__ ((packed));
 
 enum t3_rdma_opcode {
@@ -89,7 +90,8 @@ enum t3_rdma_opcode {
         T3_FAST_REGISTER,
         T3_LOCAL_INV,
         T3_QP_MOD,
-        T3_BYPASS
+        T3_BYPASS,
+        T3_RDMA_READ_REQ_WITH_INV,
 } __attribute__ ((packed));
 
 static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
@@ -103,6 +105,7 @@ static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
                 case T3_WR_BIND: return T3_BIND_MW;
                 case T3_WR_INIT: return T3_RDMA_INIT;
                 case T3_WR_QP_MOD: return T3_QP_MOD;
+                case T3_WR_FASTREG: return T3_FAST_REGISTER;
                 default: break;
         }
         return -1;
@@ -170,11 +173,54 @@ struct t3_send_wr {
         struct t3_sge sgl[T3_MAX_SGE];  /* 4+ */
 };
 
+#define T3_MAX_FASTREG_DEPTH 24
+#define T3_MAX_FASTREG_FRAG 10
+
+struct t3_fastreg_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        __be32 stag;            /* 2 */
+        __be32 len;
+        __be32 va_base_hi;      /* 3 */
+        __be32 va_base_lo_fbo;
+        __be32 page_type_perms; /* 4 */
+        __be32 reserved1;
+        __be64 pbl_addrs[0];    /* 5+ */
+};
+
+/*
+ * If a fastreg wr spans multiple wqes, then the 2nd fragment look like this.
+ */
+struct t3_pbl_frag {
+        struct fw_riwrh wrh;    /* 0 */
+        __be64 pbl_addrs[14];   /* 1..14 */
+};
+
+#define S_FR_PAGE_COUNT         24
+#define M_FR_PAGE_COUNT         0xff
+#define V_FR_PAGE_COUNT(x)      ((x) << S_FR_PAGE_COUNT)
+#define G_FR_PAGE_COUNT(x)      ((((x) >> S_FR_PAGE_COUNT)) & M_FR_PAGE_COUNT)
+
+#define S_FR_PAGE_SIZE          16
+#define M_FR_PAGE_SIZE          0x1f
+#define V_FR_PAGE_SIZE(x)       ((x) << S_FR_PAGE_SIZE)
+#define G_FR_PAGE_SIZE(x)       ((((x) >> S_FR_PAGE_SIZE)) & M_FR_PAGE_SIZE)
+
+#define S_FR_TYPE               8
+#define M_FR_TYPE               0x1
+#define V_FR_TYPE(x)            ((x) << S_FR_TYPE)
+#define G_FR_TYPE(x)            ((((x) >> S_FR_TYPE)) & M_FR_TYPE)
+
+#define S_FR_PERMS              0
+#define M_FR_PERMS              0xff
+#define V_FR_PERMS(x)           ((x) << S_FR_PERMS)
+#define G_FR_PERMS(x)           ((((x) >> S_FR_PERMS)) & M_FR_PERMS)
+
 struct t3_local_inv_wr {
         struct fw_riwrh wrh;    /* 0 */
         union t3_wrid wrid;     /* 1 */
         __be32 stag;            /* 2 */
-        __be32 reserved3;
+        __be32 reserved;
 };
 
 struct t3_rdma_write_wr {
@@ -193,7 +239,8 @@ struct t3_rdma_read_wr {
         struct fw_riwrh wrh;    /* 0 */
         union t3_wrid wrid;     /* 1 */
         u8 rdmaop;              /* 2 */
-        u8 reserved[3];
+        u8 local_inv;
+        u8 reserved[2];
         __be32 rem_stag;
         __be64 rem_to;          /* 3 */
         __be32 local_stag;      /* 4 */
@@ -201,18 +248,6 @@ struct t3_rdma_read_wr {
         __be64 local_to;        /* 5 */
 };
 
-enum t3_addr_type {
-        T3_VA_BASED_TO = 0x0,
-        T3_ZERO_BASED_TO = 0x1
-} __attribute__ ((packed));
-
-enum t3_mem_perms {
-        T3_MEM_ACCESS_LOCAL_READ = 0x1,
-        T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
-        T3_MEM_ACCESS_REM_READ = 0x4,
-        T3_MEM_ACCESS_REM_WRITE = 0x8
-} __attribute__ ((packed));
-
 struct t3_bind_mw_wr {
         struct fw_riwrh wrh;    /* 0 */
         union t3_wrid wrid;     /* 1 */
@@ -336,6 +371,11 @@ struct t3_genbit {
         __be64 genbit;
 };
 
+struct t3_wq_in_err {
+        u64 flit[13];
+        u64 err;
+};
+
 enum rdma_init_wr_flags {
         MPA_INITIATOR = (1<<0),
         PRIV_QP = (1<<1),
@@ -346,13 +386,16 @@ union t3_wr {
         struct t3_rdma_write_wr write;
         struct t3_rdma_read_wr read;
         struct t3_receive_wr recv;
+        struct t3_fastreg_wr fastreg;
+        struct t3_pbl_frag pbl_frag;
         struct t3_local_inv_wr local_inv;
         struct t3_bind_mw_wr bind;
         struct t3_bypass_wr bypass;
         struct t3_rdma_init_wr init;
         struct t3_modify_qp_wr qp_mod;
         struct t3_genbit genbit;
-        u64 flit[16];
+        struct t3_wq_in_err wq_in_err;
+        __be64 flit[16];
 };
 
 #define T3_SQ_CQE_FLIT    13
@@ -366,12 +409,18 @@ static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
         return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
 }
 
+enum t3_wr_hdr_bits {
+        T3_EOP = 1,
+        T3_SOP = 2,
+        T3_SOPEOP = T3_EOP|T3_SOP,
+};
+
 static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
                                   enum t3_wr_flags flags, u8 genbit, u32 tid,
-                                  u8 len)
+                                  u8 len, u8 sopeop)
 {
         wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
-                                         V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
+                                         V_FW_RIWR_SOPEOP(sopeop) |
                                          V_FW_RIWR_FLAGS(flags));
         wmb();
         wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
@@ -404,6 +453,7 @@ enum tpt_addr_type {
 };
 
 enum tpt_mem_perm {
+        TPT_MW_BIND = 0x10,
         TPT_LOCAL_READ = 0x8,
         TPT_LOCAL_WRITE = 0x4,
         TPT_REMOTE_READ = 0x2,
@@ -659,7 +709,7 @@ struct t3_cq {
 
 static inline void cxio_set_wq_in_error(struct t3_wq *wq)
 {
-        wq->queue->flit[13] = 1;
+        wq->queue->wq_in_err.err = 1;
 }
 
 static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)