NTB: Use DMA Engine to Transmit and Receive

Allocate and use a DMA engine channel to transmit and receive data over
NTB.  If none is allocated, fall back to using the CPU to transfer data.

Signed-off-by: Jon Mason <jon.mason@intel.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Reviewed-by: Dave Jiang <dave.jiang@intel.com>

3 files changed, 295 insertions, 47 deletions

diff --git a/drivers/ntb/ntb_hw.c b/drivers/ntb/ntb_hw.c
index ab34795cf125..0345817a8355 100644
--- a/drivers/ntb/ntb_hw.c
+++ b/drivers/ntb/ntb_hw.c
@@ -350,6 +350,23 @@ int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 }
 
 /**
+ * ntb_get_mw_base() - get addr for the NTB memory window
+ * @ndev: pointer to ntb_device instance
+ * @mw: memory window number
+ *
+ * This function provides the base address of the memory window specified.
+ *
+ * RETURNS: address, or NULL on error.
+ */
+resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
+{
+        if (mw >= ntb_max_mw(ndev))
+                return 0;
+
+        return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
+}
+
+/**
  * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
  * @ndev: pointer to ntb_device instance
  * @mw: memory window number
diff --git a/drivers/ntb/ntb_hw.h b/drivers/ntb/ntb_hw.h
index d838bc13b956..4f42ed18103a 100644
--- a/drivers/ntb/ntb_hw.h
+++ b/drivers/ntb/ntb_hw.h
@@ -240,6 +240,7 @@ int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val);
 int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val);
 int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val);
 int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val);
+resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw);
 void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw);
 u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw);
 void ntb_ring_sdb(struct ntb_device *ndev, unsigned int idx);
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index f7380e959656..ae8657259ca0 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -47,6 +47,7 @@
  */
 #include <linux/debugfs.h>
 #include <linux/delay.h>
+#include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
 #include <linux/errno.h>
 #include <linux/export.h>
@@ -68,6 +69,10 @@ static unsigned char max_num_clients;
 module_param(max_num_clients, byte, 0644);
 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");
 
+static unsigned int copy_bytes = 1024;
+module_param(copy_bytes, uint, 0644);
+MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");
+
 struct ntb_queue_entry {
         /* ntb_queue list reference */
         struct list_head entry;
@@ -76,6 +81,13 @@ struct ntb_queue_entry {
         void *buf;
         unsigned int len;
         unsigned int flags;
+
+        struct ntb_transport_qp *qp;
+        union {
+                struct ntb_payload_header __iomem *tx_hdr;
+                struct ntb_payload_header *rx_hdr;
+        };
+        unsigned int index;
 };
 
 struct ntb_rx_info {
@@ -86,6 +98,7 @@ struct ntb_transport_qp {
         struct ntb_transport *transport;
         struct ntb_device *ndev;
         void *cb_data;
+        struct dma_chan *dma_chan;
 
         bool client_ready;
         bool qp_link;
@@ -99,6 +112,7 @@ struct ntb_transport_qp {
         struct list_head tx_free_q;
         spinlock_t ntb_tx_free_q_lock;
         void __iomem *tx_mw;
+        dma_addr_t tx_mw_phys;
         unsigned int tx_index;
         unsigned int tx_max_entry;
         unsigned int tx_max_frame;
@@ -114,6 +128,7 @@ struct ntb_transport_qp {
         unsigned int rx_index;
         unsigned int rx_max_entry;
         unsigned int rx_max_frame;
+        dma_cookie_t last_cookie;
 
         void (*event_handler) (void *data, int status);
         struct delayed_work link_work;
@@ -129,9 +144,14 @@ struct ntb_transport_qp {
         u64 rx_err_no_buf;
         u64 rx_err_oflow;
         u64 rx_err_ver;
+        u64 rx_memcpy;
+        u64 rx_async;
         u64 tx_bytes;
         u64 tx_pkts;
         u64 tx_ring_full;
+        u64 tx_err_no_buf;
+        u64 tx_memcpy;
+        u64 tx_async;
 };
 
 struct ntb_transport_mw {
@@ -381,7 +401,7 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
         char *buf;
         ssize_t ret, out_offset, out_count;
 
-        out_count = 600;
+        out_count = 1000;
 
         buf = kmalloc(out_count, GFP_KERNEL);
         if (!buf)
@@ -396,6 +416,10 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "rx_pkts - \t%llu\n", qp->rx_pkts);
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
+                               "rx_memcpy - \t%llu\n", qp->rx_memcpy);
+        out_offset += snprintf(buf + out_offset, out_count - out_offset,
+                               "rx_async - \t%llu\n", qp->rx_async);
+        out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "rx_ring_empty - %llu\n", qp->rx_ring_empty);
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
@@ -415,8 +439,14 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "tx_pkts - \t%llu\n", qp->tx_pkts);
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
+                               "tx_memcpy - \t%llu\n", qp->tx_memcpy);
+        out_offset += snprintf(buf + out_offset, out_count - out_offset,
+                               "tx_async - \t%llu\n", qp->tx_async);
+        out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "tx_ring_full - \t%llu\n", qp->tx_ring_full);
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
+                               "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
+        out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "tx_mw - \t%p\n", qp->tx_mw);
         out_offset += snprintf(buf + out_offset, out_count - out_offset,
                                "tx_index - \t%u\n", qp->tx_index);
@@ -488,11 +518,11 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt,
                 num_qps_mw = nt->max_qps / mw_max;
 
         rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;
-        qp->remote_rx_info = nt->mw[mw_num].virt_addr +
+        qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size;