1 files changed, 685 insertions, 0 deletions

diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h
new file mode 100644
index 000000000000..103fc42d6976
--- /dev/null
+++ b/drivers/infiniband/hw/cxgb3/cxio_wr.h
@@ -0,0 +1,685 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef __CXIO_WR_H__
+#define __CXIO_WR_H__
+
+#include <asm/io.h>
+#include <linux/pci.h>
+#include <linux/timer.h>
+#include "firmware_exports.h"
+
+#define T3_MAX_SGE      4
+
+#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
+#define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))>>(size_log2)) && \
+                                       ((rptr)!=(wptr)) )
+#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
+#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
+#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
+#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
+
+static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)
+{
+        writel(((1<<31) | qpid), doorbell);
+}
+
+#define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))
+
+enum t3_wr_flags {
+        T3_COMPLETION_FLAG = 0x01,
+        T3_NOTIFY_FLAG = 0x02,
+        T3_SOLICITED_EVENT_FLAG = 0x04,
+        T3_READ_FENCE_FLAG = 0x08,
+        T3_LOCAL_FENCE_FLAG = 0x10
+} __attribute__ ((packed));
+
+enum t3_wr_opcode {
+        T3_WR_BP = FW_WROPCODE_RI_BYPASS,
+        T3_WR_SEND = FW_WROPCODE_RI_SEND,
+        T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
+        T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
+        T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
+        T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
+        T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
+        T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
+        T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
+} __attribute__ ((packed));
+
+enum t3_rdma_opcode {
+        T3_RDMA_WRITE,          /* IETF RDMAP v1.0 ... */
+        T3_READ_REQ,
+        T3_READ_RESP,
+        T3_SEND,
+        T3_SEND_WITH_INV,
+        T3_SEND_WITH_SE,
+        T3_SEND_WITH_SE_INV,
+        T3_TERMINATE,
+        T3_RDMA_INIT,           /* CHELSIO RI specific ... */
+        T3_BIND_MW,
+        T3_FAST_REGISTER,
+        T3_LOCAL_INV,
+        T3_QP_MOD,
+        T3_BYPASS
+} __attribute__ ((packed));
+
+static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
+{
+        switch (wrop) {
+                case T3_WR_BP: return T3_BYPASS;
+                case T3_WR_SEND: return T3_SEND;
+                case T3_WR_WRITE: return T3_RDMA_WRITE;
+                case T3_WR_READ: return T3_READ_REQ;
+                case T3_WR_INV_STAG: return T3_LOCAL_INV;
+                case T3_WR_BIND: return T3_BIND_MW;
+                case T3_WR_INIT: return T3_RDMA_INIT;
+                case T3_WR_QP_MOD: return T3_QP_MOD;
+                default: break;
+        }
+        return -1;
+}
+
+
+/* Work request id */
+union t3_wrid {
+        struct {
+                u32 hi;
+                u32 low;
+        } id0;
+        u64 id1;
+};
+
+#define WRID(wrid)              (wrid.id1)
+#define WRID_GEN(wrid)          (wrid.id0.wr_gen)
+#define WRID_IDX(wrid)          (wrid.id0.wr_idx)
+#define WRID_LO(wrid)           (wrid.id0.wr_lo)
+
+struct fw_riwrh {
+        __be32 op_seop_flags;
+        __be32 gen_tid_len;
+};
+
+#define S_FW_RIWR_OP            24
+#define M_FW_RIWR_OP            0xff
+#define V_FW_RIWR_OP(x)         ((x) << S_FW_RIWR_OP)
+#define G_FW_RIWR_OP(x) ((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)
+
+#define S_FW_RIWR_SOPEOP        22
+#define M_FW_RIWR_SOPEOP        0x3
+#define V_FW_RIWR_SOPEOP(x)     ((x) << S_FW_RIWR_SOPEOP)
+
+#define S_FW_RIWR_FLAGS         8
+#define M_FW_RIWR_FLAGS         0x3fffff
+#define V_FW_RIWR_FLAGS(x)      ((x) << S_FW_RIWR_FLAGS)
+#define G_FW_RIWR_FLAGS(x)      ((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)
+
+#define S_FW_RIWR_TID           8
+#define V_FW_RIWR_TID(x)        ((x) << S_FW_RIWR_TID)
+
+#define S_FW_RIWR_LEN           0
+#define V_FW_RIWR_LEN(x)        ((x) << S_FW_RIWR_LEN)
+
+#define S_FW_RIWR_GEN           31
+#define V_FW_RIWR_GEN(x)        ((x)  << S_FW_RIWR_GEN)
+
+struct t3_sge {
+        __be32 stag;
+        __be32 len;
+        __be64 to;
+};
+
+/* If num_sgle is zero, flit 5+ contains immediate data.*/
+struct t3_send_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+
+        u8 rdmaop;              /* 2 */
+        u8 reserved[3];
+        __be32 rem_stag;
+        __be32 plen;            /* 3 */
+        __be32 num_sgle;
+        struct t3_sge sgl[T3_MAX_SGE];  /* 4+ */
+};
+
+struct t3_local_inv_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        __be32 stag;            /* 2 */
+        __be32 reserved3;
+};
+
+struct t3_rdma_write_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        u8 rdmaop;              /* 2 */
+        u8 reserved[3];
+        __be32 stag_sink;
+        __be64 to_sink;         /* 3 */
+        __be32 plen;            /* 4 */
+        __be32 num_sgle;
+        struct t3_sge sgl[T3_MAX_SGE];  /* 5+ */
+};
+
+struct t3_rdma_read_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        u8 rdmaop;              /* 2 */
+        u8 reserved[3];
+        __be32 rem_stag;
+        __be64 rem_to;          /* 3 */
+        __be32 local_stag;      /* 4 */
+        __be32 local_len;
+        __be64 local_to;        /* 5 */
+};
+
+enum t3_addr_type {
+        T3_VA_BASED_TO = 0x0,
+        T3_ZERO_BASED_TO = 0x1
+} __attribute__ ((packed));
+
+enum t3_mem_perms {
+        T3_MEM_ACCESS_LOCAL_READ = 0x1,
+        T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
+        T3_MEM_ACCESS_REM_READ = 0x4,
+        T3_MEM_ACCESS_REM_WRITE = 0x8
+} __attribute__ ((packed));
+
+struct t3_bind_mw_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        u16 reserved;           /* 2 */
+        u8 type;
+        u8 perms;
+        __be32 mr_stag;
+        __be32 mw_stag;         /* 3 */
+        __be32 mw_len;
+        __be64 mw_va;           /* 4 */
+        __be32 mr_pbl_addr;     /* 5 */
+        u8 reserved2[3];
+        u8 mr_pagesz;
+};
+
+struct t3_receive_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        u8 pagesz[T3_MAX_SGE];
+        __be32 num_sgle;                /* 2 */
+        struct t3_sge sgl[T3_MAX_SGE];  /* 3+ */
+        __be32 pbl_addr[T3_MAX_SGE];
+};
+
+struct t3_bypass_wr {
+        struct fw_riwrh wrh;
+        union t3_wrid wrid;     /* 1 */
+};
+
+struct t3_modify_qp_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        __be32 flags;           /* 2 */
+        __be32 quiesce;         /* 2 */
+        __be32 max_ird;         /* 3 */
+        __be32 max_ord;         /* 3 */
+        __be64 sge_cmd;         /* 4 */
+        __be64 ctx1;            /* 5 */
+        __be64 ctx0;            /* 6 */
+};
+
+enum t3_modify_qp_flags {
+        MODQP_QUIESCE  = 0x01,
+        MODQP_MAX_IRD  = 0x02,
+        MODQP_MAX_ORD  = 0x04,
+        MODQP_WRITE_EC = 0x08,
+        MODQP_READ_EC  = 0x10,
+};
+
+
+enum t3_mpa_attrs {
+        uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
+        uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
+        uP_RI_MPA_CRC_ENABLE = 0x4,
+        uP_RI_MPA_IETF_ENABLE = 0x8
+} __attribute__ ((packed));
+
+enum t3_qp_caps {
+        uP_RI_QP_RDMA_READ_ENABLE = 0x01,
+        uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
+        uP_RI_QP_BIND_ENABLE = 0x04,
+        uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
+        uP_RI_QP_STAG0_ENABLE = 0x10
+} __attribute__ ((packed));
+
+struct t3_rdma_init_attr {
+        u32 tid;
+        u32 qpid;
+        u32 pdid;
+        u32 scqid;
+        u32 rcqid;
+        u32 rq_addr;
+        u32 rq_size;
+        enum t3_mpa_attrs mpaattrs;
+        enum t3_qp_caps qpcaps;
+        u16 tcp_emss;
+        u32 ord;
+        u32 ird;
+        u64 qp_dma_addr;
+        u32 qp_dma_size;
+        u32 flags;
+};
+
+struct t3_rdma_init_wr {
+        struct fw_riwrh wrh;    /* 0 */
+        union t3_wrid wrid;     /* 1 */
+        __be32 qpid;            /* 2 */
+        __be32 pdid;
+        __be32 scqid;           /* 3 */
+        __be32 rcqid;
+        __be32 rq_addr;         /* 4 */
+        __be32 rq_size;
+        u8 mpaattrs;            /* 5 */
+        u8 qpcaps;
+        __be16 ulpdu_size;
+        __be32 flags;           /* bits 31-1 - reservered */
+                                /* bit     0 - set if RECV posted */
+        __be32 ord;             /* 6 */
+        __be32 ird;
+        __be64 qp_dma_addr;     /* 7 */
+        __be32 qp_dma_size;     /* 8 */
+        u32 rsvd;
+};
+
+struct t3_genbit {
+        u64 flit[15];
+        __be64 genbit;
+};
+
+enum rdma_init_wr_flags {
+        RECVS_POSTED = 1,
+};
+
+union t3_wr {
+        struct t3_send_wr send;
+        struct t3_rdma_write_wr write;
+        struct t3_rdma_read_wr read;
+        struct t3_receive_wr recv;
+        struct t3_local_inv_wr local_inv;
+        struct t3_bind_mw_wr bind;
+        struct t3_bypass_wr bypass;
+        struct t3_rdma_init_wr init;
+        struct t3_modify_qp_wr qp_mod;
+        struct t3_genbit genbit;
+        u64 flit[16];
+};
+
+#define T3_SQ_CQE_FLIT    13
+#define T3_SQ_COOKIE_FLIT 14
+
+#define T3_RQ_COOKIE_FLIT 13
+#define T3_RQ_CQE_FLIT    14
+
+static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
+{
+        return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
+}
+
+static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
+                                  enum t3_wr_flags flags, u8 genbit, u32 tid,
+                                  u8 len)
+{
+        wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
+                                         V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
+                                         V_FW_RIWR_FLAGS(flags));
+        wmb();
+        wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
+                                       V_FW_RIWR_TID(tid) |
+                                       V_FW_RIWR_LEN(len));
+        /* 2nd gen bit... */
+        ((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit);
+}
+
+/*
+ * T3 ULP2_TX commands
+ */
+enum t3_utx_mem_op {
+        T3_UTX_MEM_READ = 2,
+        T3_UTX_MEM_WRITE = 3
+};
+
+/* T3 MC7 RDMA TPT entry format */
+
+enum tpt_mem_type {
+        TPT_NON_SHARED_MR = 0x0,
+        TPT_SHARED_MR = 0x1,
+        TPT_MW = 0x2,
+        TPT_MW_RELAXED_PROTECTION = 0x3
+};
+
+enum tpt_addr_type {
+        TPT_ZBTO = 0,
+        TPT_VATO = 1
+};
+
+enum tpt_mem_perm {
+        TPT_LOCAL_READ = 0x8,
+        TPT_LOCAL_WRITE = 0x4,
+        TPT_REMOTE_READ = 0x2,
+        TPT_REMOTE_WRITE = 0x1
+};
+
+struct tpt_entry {
+        __be32 valid_stag_pdid;
+        __be32 flags_pagesize_qpid;
+
+        __be32 rsvd_pbl_addr;
+        __be32 len;
+        __be32 va_hi;
+        __be32 va_low_or_fbo;
+
+        __be32 rsvd_bind_cnt_or_pstag;
+        __be32 rsvd_pbl_size;
+};
+
+#define S_TPT_VALID             31
+#define V_TPT_VALID(x)          ((x) << S_TPT_VALID)
+#define F_TPT_VALID             V_TPT_VALID(1U)
+
+#define S_TPT_STAG_KEY          23
+#define M_TPT_STAG_KEY          0xFF
+#define V_TPT_STAG_KEY(x)       ((x) << S_TPT_STAG_KEY)
+#define G_TPT_STAG_KEY(x)       (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
+
+#define S_TPT_STAG_STATE        22
+#define V_TPT_STAG_STATE(x)     ((x) << S_TPT_STAG_STATE)
+#define F_TPT_STAG_STATE        V_TPT_STAG_STATE(1U)
+
+#define S_TPT_STAG_TYPE         20
+#define M_TPT_STAG_TYPE         0x3
+#define V_TPT_STAG_TYPE(x)      ((x) << S_TPT_STAG_TYPE)
+#define G_TPT_STAG_TYPE(x)      (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
+
+#define S_TPT_PDID              0
+#define M_TPT_PDID              0xFFFFF
+#define V_TPT_PDID(x)           ((x) << S_TPT_PDID)
+#define G_TPT_PDID(x)           (((x) >> S_TPT_PDID) & M_TPT_PDID)
+
+#define S_TPT_PERM              28
+#define M_TPT_PERM              0xF
+#define V_TPT_PERM(x)           ((x) << S_TPT_PERM)
+#define G_TPT_PERM(x)           (((x) >> S_TPT_PERM) & M_TPT_PERM)
+
+#define S_TPT_REM_INV_DIS       27
+#define V_TPT_REM_INV_DIS(x)    ((x) << S_TPT_REM_INV_DIS)
+#define F_TPT_REM_INV_DIS       V_TPT_REM_INV_DIS(1U)
+
+#define S_TPT_ADDR_TYPE         26
+#define V_TPT_ADDR_TYPE(x)      ((x) << S_TPT_ADDR_TYPE)
+#define F_TPT_ADDR_TYPE         V_TPT_ADDR_TYPE(1U)
+
+#define S_TPT_MW_BIND_ENABLE    25
+#define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE)
+#define F_TPT_MW_BIND_ENABLE    V_TPT_MW_BIND_ENABLE(1U)
+
+#define S_TPT_PAGE_SIZE         20
+#define M_TPT_PAGE_SIZE         0x1F
+#define V_TPT_PAGE_SIZE(x)      ((x) << S_TPT_PAGE_SIZE)
+#define G_TPT_PAGE_SIZE(x)      (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
+
+#define S_TPT_PBL_ADDR          0
+#define M_TPT_PBL_ADDR          0x1FFFFFFF
+#define V_TPT_PBL_ADDR(x)       ((x) << S_TPT_PBL_ADDR)
+#define G_TPT_PBL_ADDR(x)       (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
+
+#define S_TPT_QPID              0
+#define M_TPT_QPID              0xFFFFF
+#define V_TPT_QPID(x)           ((x) << S_TPT_QPID)
+#define G_TPT_QPID(x)           (((x) >> S_TPT_QPID) & M_TPT_QPID)
+
+#define S_TPT_PSTAG             0
+#define M_TPT_PSTAG             0xFFFFFF
+#define V_TPT_PSTAG(x)          ((x) << S_TPT_PSTAG)
+#define G_TPT_PSTAG(x)          (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
+
+#define S_TPT_PBL_SIZE          0
+#define M_TPT_PBL_SIZE          0xFFFFF
+#define V_TPT_PBL_SIZE(x)       ((x) << S_TPT_PBL_SIZE)
+#define G_TPT_PBL_SIZE(x)       (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
+
+/*
+ * CQE defs
+ */
+struct t3_cqe {
+        __be32 header;
+        __be32 len;
+        union {
+                struct {
+                        __be32 stag;
+                        __be32 msn;
+                } rcqe;
+                struct {
+                        u32 wrid_hi;
+                        u32 wrid_low;
+                } scqe;
+        } u;
+};
+
+#define S_CQE_OOO         31
+#define M_CQE_OOO         0x1
+#define G_CQE_OOO(x)      ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
+#define V_CEQ_OOO(x)      ((x)<<S_CQE_OOO)
+
+#define S_CQE_QPID        12
+#define M_CQE_QPID        0x7FFFF
+#define G_CQE_QPID(x)     ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
+#define V_CQE_QPID(x)     ((x)<<S_CQE_QPID)
+
+#define S_CQE_SWCQE       11
+#define M_CQE_SWCQE       0x1
+#define G_CQE_SWCQE(x)    ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
+#define V_CQE_SWCQE(x)    ((x)<<S_CQE_SWCQE)
+
+#define S_CQE_GENBIT      10
+#define M_CQE_GENBIT      0x1
+#define G_CQE_GENBIT(x)   (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
+#define V_CQE_GENBIT(x)   ((x)<<S_CQE_GENBIT)
+
+#define S_CQE_STATUS      5
+#define M_CQE_STATUS      0x1F
+#define G_CQE_STATUS(x)   ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
+#define V_CQE_STATUS(x)   ((x)<<S_CQE_STATUS)
+
+#define S_CQE_TYPE        4
+#define M_CQE_TYPE        0x1
+#define G_CQE_TYPE(x)     ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
+#define V_CQE_TYPE(x)     ((x)<<S_CQE_TYPE)
+
+#define S_CQE_OPCODE      0
+#define M_CQE_OPCODE      0xF
+#define G_CQE_OPCODE(x)   ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
+#define V_CQE_OPCODE(x)   ((x)<<S_CQE_OPCODE)
+
+#define SW_CQE(x)         (G_CQE_SWCQE(be32_to_cpu((x).header)))
+#define CQE_OOO(x)        (G_CQE_OOO(be32_to_cpu((x).header)))
+#define CQE_QPID(x)       (G_CQE_QPID(be32_to_cpu((x).header)))
+#define CQE_GENBIT(x)     (G_CQE_GENBIT(be32_to_cpu((x).header)))
+#define CQE_TYPE(x)       (G_CQE_TYPE(be32_to_cpu((x).header)))
+#define SQ_TYPE(x)        (CQE_TYPE((x)))
+#define RQ_TYPE(x)        (!CQE_TYPE((x)))
+#define CQE_STATUS(x)     (G_CQE_STATUS(be32_to_cpu((x).header)))
+#define CQE_OPCODE(x)     (G_CQE_OPCODE(be32_to_cpu((x).header)))
+
+#define CQE_LEN(x)        (be32_to_cpu((x).len))
+
+/* used for RQ completion processing */
+#define CQE_WRID_STAG(x)  (be32_to_cpu((x).u.rcqe.stag))
+#define CQE_WRID_MSN(x)   (be32_to_cpu((x).u.rcqe.msn))
+
+/* used for SQ completion processing */
+#define CQE_WRID_SQ_WPTR(x)     ((x).u.scqe.wrid_hi)
+#define CQE_WRID_WPTR(x)        ((x).u.scqe.wrid_low)
+
+/* generic accessor macros */
+#define CQE_WRID_HI(x)          ((x).u.scqe.wrid_hi)
+#define CQE_WRID_LOW(x)         ((x).u.scqe.wrid_low)
+
+#define TPT_ERR_SUCCESS                     0x0
+#define TPT_ERR_STAG                        0x1  /* STAG invalid: either the */
+                                                 /* STAG is offlimt, being 0, */
+                                                 /* or STAG_key mismatch */
+#define TPT_ERR_PDID                        0x2  /* PDID mismatch */
+#define TPT_ERR_QPID                        0x3  /* QPID mismatch */
+#define TPT_ERR_ACCESS                      0x4  /* Invalid access right */
+#define TPT_ERR_WRAP                        0x5  /* Wrap error */
+#define TPT_ERR_BOUND                       0x6  /* base and bounds voilation */
+#define TPT_ERR_INVALIDATE_SHARED_MR        0x7  /* attempt to invalidate a  */
+                                                 /* shared memory region */
+#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8  /* attempt to invalidate a  */
+                                                 /* shared memory region */
+#define TPT_ERR_ECC                         0x9  /* ECC error detected */
+#define TPT_ERR_ECC_PSTAG                   0xA  /* ECC error detected when  */
+                                                 /* reading PSTAG for a MW  */
+                                                 /* Invalidate */
+#define TPT_ERR_PBL_ADDR_BOUND              0xB  /* pbl addr out of bounds:  */
+                                                 /* software error */
+#define TPT_ERR_SWFLUSH                     0xC  /* SW FLUSHED */
+#define TPT_ERR_CRC                         0x10 /* CRC error */
+#define TPT_ERR_MARKER                      0x11 /* Marker error */
+#define TPT_ERR_PDU_LEN_ERR                 0x12 /* invalid PDU length */
+#define TPT_ERR_OUT_OF_RQE                  0x13 /* out of RQE */
+#define TPT_ERR_DDP_VERSION                 0x14 /* wrong DDP version */
+#define TPT_ERR_RDMA_VERSION                0x15 /* wrong RDMA version */
+#define TPT_ERR_OPCODE                      0x16 /* invalid rdma opcode */
+#define TPT_ERR_DDP_QUEUE_NUM               0x17 /* invalid ddp queue number */
+#define TPT_ERR_MSN                         0x18 /* MSN error */
+#define TPT_ERR_TBIT                        0x19 /* tag bit not set correctly */
+#define TPT_ERR_MO                          0x1A /* MO not 0 for TERMINATE  */
+                                                 /* or READ_REQ */
+#define TPT_ERR_MSN_GAP                     0x1B
+#define TPT_ERR_MSN_RANGE                   0x1C
+#define TPT_ERR_IRD_OVERFLOW                0x1D
+#define TPT_ERR_RQE_ADDR_BOUND              0x1E /* RQE addr out of bounds:  */
+                                                 /* software error */
+#define TPT_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */
+                                                 /* mismatch) */
+
+struct t3_swsq {
+        __u64                   wr_id;
+        struct t3_cqe           cqe;
+        __u32                   sq_wptr;
+        __be32                  read_len;
+        int                     opcode;
+        int                     complete;
+        int                     signaled;
+};
+
+/*
+ * A T3 WQ implements both the SQ and RQ.
+ */
+struct t3_wq {
+        union t3_wr *queue;             /* DMA accessable memory */
+        dma_addr_t dma_addr;            /* DMA address for HW */
+        DECLARE_PCI_UNMAP_ADDR(mapping) /* unmap kruft */
+        u32 error;                      /* 1 once we go to ERROR */
+        u32 qpid;
+        u32 wptr;                       /* idx to next available WR slot */
+        u32 size_log2;                  /* total wq size */
+        struct t3_swsq *sq;             /* SW SQ */
+        struct t3_swsq *oldest_read;    /* tracks oldest pending read */
+        u32 sq_wptr;                    /* sq_wptr - sq_rptr == count of */
+        u32 sq_rptr;                    /* pending wrs */
+        u32 sq_size_log2;               /* sq size */
+        u64 *rq;                        /* SW RQ (holds consumer wr_ids */
+        u32 rq_wptr;                    /* rq_wptr - rq_rptr == count of */
+        u32 rq_rptr;                    /* pending wrs */
+        u64 *rq_oldest_wr;              /* oldest wr on the SW RQ */
+        u32 rq_size_log2;               /* rq size */
+        u32 rq_addr;                    /* rq adapter address */
+        void __iomem *doorbell;         /* kernel db */
+        u64 udb;                        /* user db if any */
+};
+
+struct t3_cq {
+        u32 cqid;
+        u32 rptr;
+        u32 wptr;
+        u32 size_log2;
+        dma_addr_t dma_addr;
+        DECLARE_PCI_UNMAP_ADDR(mapping)
+        struct t3_cqe *queue;
+        struct t3_cqe *sw_queue;
+        u32 sw_rptr;
+        u32 sw_wptr;
+};
+
+#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
+                                         CQE_GENBIT(*cqe))
+
+static inline void cxio_set_wq_in_error(struct t3_wq *wq)
+{
+        wq->queue->flit[13] = 1;
+}
+
+static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
+{
+        struct t3_cqe *cqe;
+
+        cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
+        if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
+                return cqe;
+        return NULL;
+}
+
+static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
+{
+        struct t3_cqe *cqe;
+
+        if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
+                cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
+                return cqe;
+        }
+        return NULL;
+}
+
+static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
+{
+        struct t3_cqe *cqe;
+
+        if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
+                cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
+                return cqe;
+        }
+        cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
+        if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
+                return cqe;
+        return NULL;
+}
+
+#endif