1 files changed, 67 insertions, 5 deletions

diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index eeca8b1e6376..2178f3198410 100644
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -84,7 +84,14 @@ struct t4_status_page {
                         sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))
 #define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \
                         sizeof(struct fw_ri_immd)) & ~31UL)
-#define T4_MAX_FR_DEPTH (1024 / sizeof(u64))
+#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))
+#define T4_MAX_FR_DSGL 1024
+#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64))
+
+static inline int t4_max_fr_depth(int use_dsgl)
+{
+        return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH;
+}
 
 #define T4_RQ_NUM_SLOTS 2
 #define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS)
@@ -292,7 +299,7 @@ struct t4_sq {
         unsigned long phys_addr;
         struct t4_swsqe *sw_sq;
         struct t4_swsqe *oldest_read;
-        u64 udb;
+        u64 __iomem *udb;
         size_t memsize;
         u32 qid;
         u16 in_use;
@@ -314,7 +321,7 @@ struct t4_rq {
         dma_addr_t dma_addr;
         DEFINE_DMA_UNMAP_ADDR(mapping);
         struct t4_swrqe *sw_rq;
-        u64 udb;
+        u64 __iomem *udb;
         size_t memsize;
         u32 qid;
         u32 msn;
@@ -435,15 +442,67 @@ static inline u16 t4_sq_wq_size(struct t4_wq *wq)
                 return wq->sq.size * T4_SQ_NUM_SLOTS;
 }
 
-static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc)
+/* This function copies 64 byte coalesced work request to memory
+ * mapped BAR2 space. For coalesced WRs, the SGE fetches data
+ * from the FIFO instead of from Host.
+ */
+static inline void pio_copy(u64 __iomem *dst, u64 *src)
+{
+        int count = 8;
+
+        while (count) {
+                writeq(*src, dst);
+                src++;
+                dst++;
+                count--;
+        }
+}
+
+static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, u8 t5,
+                                 union t4_wr *wqe)
 {
+
+        /* Flush host queue memory writes. */
         wmb();
+        if (t5) {
+                if (inc == 1 && wqe) {
+                        PDBG("%s: WC wq->sq.pidx = %d\n",
+                             __func__, wq->sq.pidx);
+                        pio_copy(wq->sq.udb + 7, (void *)wqe);
+                } else {
+                        PDBG("%s: DB wq->sq.pidx = %d\n",
+                             __func__, wq->sq.pidx);
+                        writel(PIDX_T5(inc), wq->sq.udb);
+                }
+
+                /* Flush user doorbell area writes. */
+                wmb();
+                return;
+        }
         writel(QID(wq->sq.qid) | PIDX(inc), wq->db);
 }
 
-static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc)
+static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t5,
+                                 union t4_recv_wr *wqe)
 {
+
+        /* Flush host queue memory writes. */
         wmb();
+        if (t5) {
+                if (inc == 1 && wqe) {
+                        PDBG("%s: WC wq->rq.pidx = %d\n",
+                             __func__, wq->rq.pidx);
+                        pio_copy(wq->rq.udb + 7, (void *)wqe);
+                } else {
+                        PDBG("%s: DB wq->rq.pidx = %d\n",
+                             __func__, wq->rq.pidx);
+                        writel(PIDX_T5(inc), wq->rq.udb);
+                }
+
+                /* Flush user doorbell area writes. */
+                wmb();
+                return;
+        }
         writel(QID(wq->rq.qid) | PIDX(inc), wq->db);
 }
 
@@ -568,6 +627,9 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
                 printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);
                 BUG_ON(1);
         } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {
+
+                /* Ensure CQE is flushed to memory */
+                rmb();
                 *cqe = &cq->queue[cq->cidx];
                 ret = 0;
         } else