rapidio/tsi721_dma: rework scatter-gather list handling

Rework Tsi721 RapidIO DMA engine support to allow handling data
scatter/gather lists longer than number of hardware buffer descriptors in
the DMA channel's descriptor list.

The current implementation of Tsi721 DMA transfers requires that number of
entries in a scatter/gather list provided by a caller of
dmaengine_prep_rio_sg() should not exceed number of allocated hardware
buffer descriptors.

This patch removes the limitation by processing long scatter/gather lists
by sections that can be transferred using hardware descriptor ring of
configured size.  It also introduces a module parameter
"dma_desc_per_channel" to allow run-time configuration of Tsi721 hardware
buffer descriptor rings.

Signed-off-by: Alexandre Bounine <alexandre.bounine@idt.com>
Cc: Matt Porter <mporter@kernel.crashing.org>
Cc: Andre van Herk <andre.van.herk@prodrive-technologies.com>
Cc: Stef van Os <stef.van.os@prodrive-technologies.com>
Cc: Vinod Koul <vinod.koul@intel.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

3 files changed, 394 insertions, 355 deletions

diff --git a/Documentation/rapidio/tsi721.txt b/Documentation/rapidio/tsi721.txt
index 335f3c6087dc..626052f403bb 100644
--- a/Documentation/rapidio/tsi721.txt
+++ b/Documentation/rapidio/tsi721.txt
@@ -20,13 +20,26 @@ II. Known problems
 
   None.
 
-III. To do
+III. DMA Engine Support
 
- Add DMA data transfers (non-messaging).
- Add inbound region (SRIO-to-PCIe) mapping.
+Tsi721 mport driver supports DMA data transfers between local system memory and
+remote RapidIO devices. This functionality is implemented according to SLAVE
+mode API defined by common Linux kernel DMA Engine framework.
+
+Depending on system requirements RapidIO DMA operations can be included/excluded
+by setting CONFIG_RAPIDIO_DMA_ENGINE option. Tsi721 miniport driver uses seven
+out of eight available BDMA channels to support DMA data transfers.
+One BDMA channel is reserved for generation of maintenance read/write requests.
+
+If Tsi721 mport driver have been built with RAPIDIO_DMA_ENGINE support included,
+this driver will accept DMA-specific module parameter:
+  "dma_desc_per_channel" - defines number of hardware buffer descriptors used by
+                           each BDMA channel of Tsi721 (by default - 128).
 
 IV. Version History
 
+  1.1.0 - DMA operations re-worked to support data scatter/gather lists larger
+          than hardware buffer descriptors ring.
   1.0.0 - Initial driver release.
 
 V.  License
diff --git a/drivers/rapidio/devices/tsi721.h b/drivers/rapidio/devices/tsi721.h
index 0305675270ee..a7b42680a06a 100644
--- a/drivers/rapidio/devices/tsi721.h
+++ b/drivers/rapidio/devices/tsi721.h
@@ -644,27 +644,26 @@ enum tsi721_smsg_int_flag {
 
 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 
-#define TSI721_BDMA_BD_RING_SZ  128
 #define TSI721_BDMA_MAX_BCOUNT  (TSI721_DMAD_BCOUNT1 + 1)
 
 struct tsi721_tx_desc {
         struct dma_async_tx_descriptor  txd;
-        struct tsi721_dma_desc          *hw_desc;
         u16                             destid;
         /* low 64-bits of 66-bit RIO address */
         u64                             rio_addr;
         /* upper 2-bits of 66-bit RIO address */
         u8                              rio_addr_u;
-        u32                             bcount;
-        bool                            interrupt;
+        enum dma_rtype                  rtype;
         struct list_head                desc_node;
-        struct list_head                tx_list;
+        struct scatterlist              *sg;
+        unsigned int                    sg_len;
+        enum dma_status                 status;
 };
 
 struct tsi721_bdma_chan {
         int             id;
         void __iomem    *regs;
-        int             bd_num;         /* number of buffer descriptors */
+        int             bd_num;         /* number of HW buffer descriptors */
         void            *bd_base;       /* start of DMA descriptors */
         dma_addr_t      bd_phys;
         void            *sts_base;      /* start of DMA BD status FIFO */
@@ -680,7 +679,6 @@ struct tsi721_bdma_chan {
         struct list_head        active_list;
         struct list_head        queue;
         struct list_head        free_list;
-        dma_cookie_t            completed_cookie;
         struct tasklet_struct   tasklet;
         bool                    active;
 };
diff --git a/drivers/rapidio/devices/tsi721_dma.c b/drivers/rapidio/devices/tsi721_dma.c
index 44341dc5b148..f64c5decb747 100644
--- a/drivers/rapidio/devices/tsi721_dma.c
+++ b/drivers/rapidio/devices/tsi721_dma.c
@@ -1,7 +1,7 @@
 /*
  * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge
  *
- * Copyright 2011 Integrated Device Technology, Inc.
+ * Copyright (c) 2011-2014 Integrated Device Technology, Inc.
  * Alexandre Bounine <alexandre.bounine@idt.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -14,9 +14,8 @@
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59
- * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
  */
 
 #include <linux/io.h>
@@ -32,9 +31,22 @@
 #include <linux/interrupt.h>
 #include <linux/kfifo.h>
 #include <linux/delay.h>
+#include "../../dma/dmaengine.h"
 
 #include "tsi721.h"
 
+#define TSI721_DMA_TX_QUEUE_SZ  16      /* number of transaction descriptors */
+
+#ifdef CONFIG_PCI_MSI
+static irqreturn_t tsi721_bdma_msix(int irq, void *ptr);
+#endif
+static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
+
+static unsigned int dma_desc_per_channel = 128;
+module_param(dma_desc_per_channel, uint, S_IWUSR | S_IRUGO);
+MODULE_PARM_DESC(dma_desc_per_channel,
+                 "Number of DMA descriptors per channel (default: 128)");
+
 static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan)
 {
         return container_of(chan, struct tsi721_bdma_chan, dchan);
@@ -59,7 +71,7 @@ struct tsi721_tx_desc *tsi721_dma_first_active(
                                 struct tsi721_tx_desc, desc_node);
 }
 
-static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
+static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num)
 {
         struct tsi721_dma_desc *bd_ptr;
         struct device *dev = bdma_chan->dchan.device->dev;
@@ -67,17 +79,23 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
         dma_addr_t      bd_phys;
         dma_addr_t      sts_phys;
         int             sts_size;
-        int             bd_num = bdma_chan->bd_num;
+#ifdef CONFIG_PCI_MSI
+        struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
+#endif
 
         dev_dbg(dev, "Init Block DMA Engine, CH%d\n", bdma_chan->id);
 
-        /* Allocate space for DMA descriptors */
+        /*
+         * Allocate space for DMA descriptors
+         * (add an extra element for link descriptor)
+         */
         bd_ptr = dma_zalloc_coherent(dev,
-                                bd_num * sizeof(struct tsi721_dma_desc),
+                                (bd_num + 1) * sizeof(struct tsi721_dma_desc),
                                 &bd_phys, GFP_KERNEL);
         if (!bd_ptr)
                 return -ENOMEM;
 
+        bdma_chan->bd_num = bd_num;
         bdma_chan->bd_phys = bd_phys;
         bdma_chan->bd_base = bd_ptr;
 
@@ -85,8 +103,8 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
                 bd_ptr, (unsigned long long)bd_phys);
 
         /* Allocate space for descriptor status FIFO */
-        sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
-                                        bd_num : TSI721_DMA_MINSTSSZ;
+        sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ?
+                                        (bd_num + 1) : TSI721_DMA_MINSTSSZ;
         sts_size = roundup_pow_of_two(sts_size);
         sts_ptr = dma_zalloc_coherent(dev,
                                      sts_size * sizeof(struct tsi721_dma_sts),
@@ -94,7 +112,7 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
         if (!sts_ptr) {
                 /* Free space allocated for DMA descriptors */
                 dma_free_coherent(dev,
-                                  bd_num * sizeof(struct tsi721_dma_desc),
+                                  (bd_num + 1) * sizeof(struct tsi721_dma_desc),
                                   bd_ptr, bd_phys);
                 bdma_chan->bd_base = NULL;
                 return -ENOMEM;
@@ -108,11 +126,11 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
                 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
                 sts_ptr, (unsigned long long)sts_phys, sts_size);
 
-        /* Initialize DMA descriptors ring */
-        bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
-        bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
+        /* Initialize DMA descriptors ring using added link descriptor */
+        bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29);
+        bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys &
                                                  TSI721_DMAC_DPTRL_MASK);
-        bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
+        bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32);
 
         /* Setup DMA descriptor pointers */
         iowrite32(((u64)bd_phys >> 32),
@@ -134,6 +152,55 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
 
         ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 
+#ifdef CONFIG_PCI_MSI
+        /* Request interrupt service if we are in MSI-X mode */
+        if (priv->flags & TSI721_USING_MSIX) {
+                int rc, idx;
+
+                idx = TSI721_VECT_DMA0_DONE + bdma_chan->id;
+
+                rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
+                                 priv->msix[idx].irq_name, (void *)bdma_chan);
+
+                if (rc) {
+                        dev_dbg(dev, "Unable to get MSI-X for BDMA%d-DONE\n",
+                                bdma_chan->id);
+                        goto err_out;
+                }
+
+                idx = TSI721_VECT_DMA0_INT + bdma_chan->id;
+
+                rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
+                                priv->msix[idx].irq_name, (void *)bdma_chan);
+
+                if (rc) {
+                        dev_dbg(dev, "Unable to get MSI-X for BDMA%d-INT\n",
+                                bdma_chan->id);
+                        free_irq(
+                                priv->msix[TSI721_VECT_DMA0_DONE +
+                                            bdma_chan->id].vector,
+                                (void *)bdma_chan);
+                }
+
+err_out:
+                if (rc) {
+                        /* Free space allocated for DMA descriptors */
+                        dma_free_coherent(dev,
+                                (bd_num + 1) * sizeof(struct tsi721_dma_desc),
+                                bd_ptr, bd_phys);
+                        bdma_chan->bd_base = NULL;
+
+                        /* Free space allocated for status descriptors */
+                        dma_free_coherent(dev,
+                                sts_size * sizeof(struct tsi721_dma_sts),
+                                sts_ptr, sts_phys);
+                        bdma_chan->sts_base = NULL;
+
+                        return -EIO;
+                }
+        }
+#endif /* CONFIG_PCI_MSI */
+
         /* Toggle DMA channel initialization */
         iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
         ioread32(bdma_chan->regs + TSI721_DMAC_CTL);
@@ -147,6 +214,9 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
 static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan)
 {
         u32 ch_stat;
+#ifdef CONFIG_PCI_MSI
+        struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
+#endif
 
         if (bdma_chan->bd_base == NULL)
                 return 0;
@@ -159,9 +229,18 @@ static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan)
         /* Put DMA channel into init state */
         iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
 
+#ifdef CONFIG_PCI_MSI
+        if (priv->flags & TSI721_USING_MSIX) {
+                free_irq(priv->msix[TSI721_VECT_DMA0_DONE +
+                                    bdma_chan->id].vector, (void *)bdma_chan);
+                free_irq(priv->msix[TSI721_VECT_DMA0_INT +
+                                    bdma_chan->id].vector, (void *)bdma_chan);
+        }
+#endif /* CONFIG_PCI_MSI */
+
         /* Free space allocated for DMA descriptors */
         dma_free_coherent(bdma_chan->dchan.device->dev,
-                bdma_chan->bd_num * sizeof(struct tsi721_dma_desc),
+                (bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc),
                 bdma_chan->bd_base, bdma_chan->bd_phys);
         bdma_chan->bd_base = NULL;
 
@@ -243,8 +322,8 @@ static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan)
         }
 
         dev_dbg(bdma_chan->dchan.device->dev,
-                "tx_chan: %p, chan: %d, regs: %p\n",
-                bdma_chan, bdma_chan->dchan.chan_id, bdma_chan->regs);
+                "%s: chan_%d (wrc=%d)\n", __func__, bdma_chan->id,
+                bdma_chan->wr_count_next);
 
         iowrite32(bdma_chan->wr_count_next,
                 bdma_chan->regs + TSI721_DMAC_DWRCNT);
@@ -253,72 +332,19 @@ static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan)
         bdma_chan->wr_count = bdma_chan->wr_count_next;
 }
 
-static void tsi721_desc_put(struct tsi721_bdma_chan *bdma_chan,
-                            struct tsi721_tx_desc *desc)
-{
-        dev_dbg(bdma_chan->dchan.device->dev,
-                "Put desc: %p into free list\n", desc);
-
-        if (desc) {
-                spin_lock_bh(&bdma_chan->lock);
-                list_splice_init(&desc->tx_list, &bdma_chan->free_list);
-                list_add(&desc->desc_node, &bdma_chan->free_list);
-                bdma_chan->wr_count_next = bdma_chan->wr_count;
-                spin_unlock_bh(&bdma_chan->lock);
-        }
-}
-
-static
-struct tsi721_tx_desc *tsi721_desc_get(struct tsi721_bdma_chan *bdma_chan)
-{
-        struct tsi721_tx_desc *tx_desc, *_tx_desc;
-        struct tsi721_tx_desc *ret = NULL;
-        int i;
-
-        spin_lock_bh(&bdma_chan->lock);
-        list_for_each_entry_safe(tx_desc, _tx_desc,
-                                 &bdma_chan->free_list, desc_node) {
-                if (async_tx_test_ack(&tx_desc->txd)) {
-                        list_del(&tx_desc->desc_node);
-                        ret = tx_desc;
-                        break;
-                }
-                dev_dbg(bdma_chan->dchan.device->dev,
-                        "desc %p not ACKed\n", tx_desc);
-        }
-
-        if (ret == NULL) {
-                dev_dbg(bdma_chan->dchan.device->dev,
-                        "%s: unable to obtain tx descriptor\n", __func__);
-                goto err_out;
-        }
-
-        i = bdma_chan->wr_count_next % bdma_chan->bd_num;
-        if (i == bdma_chan->bd_num - 1) {
-                i = 0;
-                bdma_chan->wr_count_next++; /* skip link descriptor */
-        }
-
-        bdma_chan->wr_count_next++;
-        tx_desc->txd.phys = bdma_chan->bd_phys +
-                                i * sizeof(struct tsi721_dma_desc);
-        tx_desc->hw_desc = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[i];
-err_out:
-        spin_unlock_bh(&bdma_chan->lock);
-
-        return ret;
-}
-
 static int
-tsi721_desc_fill_init(struct tsi721_tx_desc *desc, struct scatterlist *sg,
-        enum dma_rtype rtype, u32 sys_size)
+tsi721_desc_fill_init(struct tsi721_tx_desc *desc,
+                      struct tsi721_dma_desc *bd_ptr,
+                      struct scatterlist *sg, u32 sys_size)
 {
-        struct tsi721_dma_desc *bd_ptr = desc->hw_desc;
         u64 rio_addr;
 
+        if (bd_ptr == NULL)
+                return -EINVAL;
+
         /* Initialize DMA descriptor */
         bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) |
-                                        (rtype << 19) | desc->destid);
+                                      (desc->rtype << 19) | desc->destid);
         bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |
                                      (sys_size << 26));
         rio_addr = (desc->rio_addr >> 2) |
@@ -335,51 +361,32 @@ tsi721_desc_fill_init(struct tsi721_tx_desc *desc, struct scatterlist *sg,
 }
 
 static int
-tsi721_desc_fill_end(struct tsi721_tx_desc *desc)
+tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt)
 {
-        struct tsi721_dma_desc *bd_ptr = desc->hw_desc;
+        if (bd_ptr == NULL)
+                return -EINVAL;
 
         /* Update DMA descriptor */
-        if (desc->interrupt)
+        if (interrupt)
                 bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF);
-        bd_ptr->bcount |= cpu_to_le32(desc->bcount & TSI721_DMAD_BCOUNT1);
+        bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1);
 
         return 0;
 }
 
-
-static void tsi721_dma_chain_complete(struct tsi721_bdma_chan *bdma_chan,
-                                      struct tsi721_tx_desc *desc)
+static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan,
+                              struct tsi721_tx_desc *desc)
 {
         struct dma_async_tx_descriptor *txd = &desc->txd;
         dma_async_tx_callback callback = txd->callback;
         void *param = txd->callback_param;
 
-        list_splice_init(&desc->tx_list, &bdma_chan->free_list);
         list_move(&desc->desc_node, &bdma_chan->free_list);
-        bdma_chan->completed_cookie = txd->cookie;
 
         if (callback)
                 callback(param);
 }
 
-static void tsi721_dma_complete_all(struct tsi721_bdma_chan *bdma_chan)
-{
-        struct tsi721_tx_desc *desc, *_d;
-        LIST_HEAD(list);
-
-        BUG_ON(!tsi721_dma_is_idle(bdma_chan));
-
-        if (!list_empty(&bdma_chan->queue))
-                tsi721_start_dma(bdma_chan);
-
-        list_splice_init(&bdma_chan->active_list, &list);
-        list_splice_init(&bdma_chan->queue, &bdma_chan->active_list);
-
-        list_for_each_entry_safe(desc, _d, &list, desc_node)
-                tsi721_dma_chain_complete(bdma_chan, desc);
-}
-
 static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan)
 {
         u32 srd_ptr;
@@ -403,20 +410,159 @@ static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan)
         bdma_chan->sts_rdptr = srd_ptr;
 }
 
+/* Must be called with the channel spinlock held */
+static int tsi721_submit_sg(struct tsi721_tx_desc *desc)
+{
+        struct dma_chan *dchan = desc->txd.chan;
+        struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
+        u32 sys_size;
+        u64 rio_addr;
+        dma_addr_t next_addr;
+        u32 bcount;
+        struct scatterlist *sg;
+        unsigned int i;
+        int err = 0;
+        struct tsi721_dma_desc *bd_ptr = NULL;
+        u32 idx, rd_idx;
+        u32 add_count = 0;
+
+        if (!tsi721_dma_is_idle(bdma_chan)) {
+                dev_err(bdma_chan->dchan.device->dev,
+                        "BUG: Attempt to use non-idle channel\n");
+                return -EIO;
+        }
+
+        /*
+         * Fill DMA channel's hardware buffer descriptors.
+         * (NOTE: RapidIO destination address is limited to 64 bits for now)
+         */
+        rio_addr = desc->rio_addr;
+        next_addr = -1;
+        bcount = 0;
+        sys_size = dma_to_mport(bdma_chan->dchan.device)->sys_size;
+
+        rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT);
+        rd_idx %= (bdma_chan->bd_num + 1);
+
+        idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1);
+        if (idx == bdma_chan->bd_num) {
+                /* wrap around link descriptor */
+                idx = 0;
+                add_count++;
+        }
+
+        dev_dbg(dchan->device->dev, "%s: BD ring status: rdi=%d wri=%d\n",
+                __func__, rd_idx, idx);
+
+        for_each_sg(desc->sg, sg, desc->sg_len, i) {
+
+                dev_dbg(dchan->device->dev, "sg%d/%d addr: 0x%llx len: %d\n",
+                        i, desc->sg_len,
+                        (unsigned long long)sg_dma_address(sg), sg_dma_len(sg));
+
+                if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) {
+                        dev_err(dchan->device->dev,
+                                "%s: SG entry %d is too large\n", __func__, i);
+                        err = -EINVAL;
+                        break;
+                }
+
+                /*
+                 * If this sg entry forms contiguous block with previous one,
+                 * try to merge it into existing DMA descriptor
+                 */
+                if (next_addr == sg_dma_address(sg) &&
+                    bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) {
+                        /* Adjust byte count of the descriptor */
+                        bcount += sg_dma_len(sg);
+                        goto entry_done;
+                } else if (next_addr != -1) {
+                        /* Finalize descriptor using total byte count value */
+                        tsi721_desc_fill_end(bd_ptr, bcount, 0);
+                        dev_dbg(dchan->device->dev,
+                                "%s: prev desc final len: %d\n",
+                                __func__, bcount);
+                }
+
+                desc->rio_addr = rio_addr;
+
+                if (i && idx == rd_idx) {
+                        dev_dbg(dchan->device->dev,
+                                "%s: HW descriptor ring is full @ %d\n",
+                                __func__, i);
+                        desc->sg = sg;
+                        desc->sg_len -= i;
+                        break;
+                }
+
+                bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx];
+                err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);
+                if (err) {
+                        dev_err(dchan->device->dev,
+                                "Failed to build desc: err=%d\n", err);
+                        break;
+                }
+
+                dev_dbg(dchan->device->dev, "bd_ptr = %p did=%d raddr=0x%llx\n",
+                        bd_ptr, desc->destid, desc->rio_addr);
+
+                next_addr = sg_dma_address(sg);
+                bcount = sg_dma_len(sg);
+
+                add_count++;
+                if (++idx == bdma_chan->bd_num) {
+                        /* wrap around link descriptor */
+                        idx = 0;
+                        add_count++;
+                }
+
+entry_done:
+                if (sg_is_last(sg)) {
+                        tsi721_desc_fill_end(bd_ptr, bcount, 0);
+                        dev_dbg(dchan->device->dev, "%s: last desc final len: %d\n",
+                                __func__, bcount);
+                        desc->sg_len = 0;
+                } else {
+                        rio_addr += sg_dma_len(sg);
+                        next_addr += sg_dma_len(sg);
+                }
+        }
+
+        if (!err)
+                bdma_chan->wr_count_next += add_count;
+
+        return err;
+}
+
 static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan)
 {
-        if (list_empty(&bdma_chan->active_list) ||
-                list_is_singular(&bdma_chan->active_list)) {
-                dev_dbg(bdma_chan->dchan.device->dev,
-                        "%s: Active_list empty\n", __func__);
-                tsi721_dma_complete_all(bdma_chan);
-        } else {
-                dev_dbg(bdma_chan->dchan.device->dev,
-                        "%s: Active_list NOT empty\n", __func__);
-                tsi721_dma_chain_complete(bdma_chan,
-                                        tsi721_dma_first_active(bdma_chan));
-                tsi721_start_dma(bdma_chan);
+        struct tsi721_tx_desc *desc;
+        int err;
+
+        dev_dbg(bdma_chan->dchan.device->dev, "%s: Enter\n", __func__);
+
+        /*
+         * If there are any new transactions in the queue add them
+         * into the processing list
+         */
+        if (!list_empty(&bdma_chan->queue))
+                list_splice_init(&bdma_chan->queue, &bdma_chan->active_list);
+
+        /* Start new transaction (if available) */
+        if (!list_empty(&bdma_chan->active_list)) {
+                desc = tsi721_dma_first_active(bdma_chan);
+                err = tsi721_submit_sg(desc);
+                if (!err)
+                        tsi721_start_dma(bdma_chan);
+                else {
+                        tsi721_dma_tx_err(bdma_chan, desc);
+                        dev_dbg(bdma_chan->dchan.device->dev,
+                                "ERR: tsi721_submit_sg failed with err=%d\n",
+                                err);
+                }
         }
+
+        dev_dbg(bdma_chan->dchan.device->dev, "%s: Exit\n", __func__);
 }
 
 static void tsi721_dma_tasklet(unsigned long data)
@@ -444,8 +590,29 @@ static void tsi721_dma_tasklet(unsigned long data)
         }
 
         if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) {
+                struct tsi721_tx_desc *desc;
+
                 tsi721_clr_stat(bdma_chan);
                 spin_lock(&bdma_chan->lock);
+                desc = tsi721_dma_first_active(bdma_chan);
+
+                if (desc->sg_len == 0) {
+                        dma_async_tx_callback callback = NULL;
+                        void *param = NULL;
+
+                        desc->status = DMA_COMPLETE;
+                        dma_cookie_complete(&desc->txd);
+                        if (desc->txd.flags & DMA_PREP_INTERRUPT) {
+                                callback = desc->txd.callback;
+                                param = desc->txd.callback_param;
+                        }
+                        list_move(&desc->desc_node, &bdma_chan->free_list);
+                        spin_unlock(&bdma_chan->lock);
+                        if (callback)
+                                callback(param);
+                        spin_lock(&bdma_chan->lock);
+                }
+
                 tsi721_advance_work(bdma_chan);
                 spin_unlock(&bdma_chan->lock);
         }
@@ -460,21 +627,24 @@ static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd)
         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan);
         dma_cookie_t cookie;
 
-        spin_lock_bh(&bdma_chan->lock);
+        /* Check if the descriptor is detached from any lists */
+        if (!list_empty(&desc->desc_node)) {
+                dev_err(bdma_chan->dchan.device->dev,
+                        "%s: wrong state of descriptor %p\n", __func__, txd);
+                return -EIO;
+        }
 
-        cookie = txd->chan->cookie;
-        if (++cookie < 0)
-                cookie = 1;
-        txd->chan->cookie = cookie;
-        txd->cookie = cookie;
+        spin_lock_bh(&bdma_chan->lock);
 
-        if (list_empty(&bdma_chan->active_list)) {
-                list_add_tail(&desc->desc_node, &bdma_chan->active_list);
-                tsi721_start_dma(bdma_chan);
-        } else {
-                list_add_tail(&desc->desc_node, &bdma_chan->queue);
+        if (!bdma_chan->active) {
+                spin_unlock_bh(&bdma_chan->lock);
+                return -ENODEV;
         }
 
+        cookie = dma_cookie_assign(txd);
+        desc->status = DMA_IN_PROGRESS;
+        list_add_tail(&desc->desc_node, &bdma_chan->queue);
+
         spin_unlock_bh(&bdma_chan->lock);
         return cookie;
 }
@@ -482,115 +652,52 @@ static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd)
 static int tsi721_alloc_chan_resources(struct dma_chan *dchan)
 {
         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
-#ifdef CONFIG_PCI_MSI
-        struct tsi721_device *priv = to_tsi721(dchan->device);
-#endif
         struct tsi721_tx_desc *desc = NULL;
-        LIST_HEAD(tmp_list);
         int i;
-        int rc;
+
+        dev_dbg(dchan->device->dev, "%s: for channel %d\n",
+                __func__, bdma_chan->id);
 
         if (bdma_chan->bd_base)
-                return bdma_chan->bd_num - 1;
+                return TSI721_DMA_TX_QUEUE_SZ;
 
         /* Initialize BDMA channel */
-        if (tsi721_bdma_ch_init(bdma_chan)) {
+        if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) {
                 dev_err(dchan->device->dev, "Unable to initialize data DMA"
                         " channel %d, aborting\n", bdma_chan->id);
-                return -ENOMEM;
+                return -ENODEV;
         }
 
-        /* Alocate matching number of logical descriptors */
-        desc = kcalloc((bdma_chan->bd_num - 1), sizeof(struct tsi721_tx_desc),
+        /* Allocate queue of transaction descriptors */
+        desc = kcalloc(TSI721_DMA_TX_QUEUE_SZ, sizeof(struct tsi721_tx_desc),
                         GFP_KERNEL);
         if (!desc) {
                 dev_err(dchan->device->dev,
                         "Failed to allocate logical descriptors\n");
-                rc = -ENOMEM;
-                goto err_out;
+                tsi721_bdma_ch_free(bdma_chan);
+                return -ENOMEM;
         }
 
         bdma_chan->tx_desc = desc;
 
-        for (i = 0; i < bdma_chan->bd_num - 1; i++) {
+        for (i = 0; i < TSI721_DMA_TX_QUEUE_SZ; i++) {
                 dma_async_tx_descriptor_init(&desc[i].txd, dchan);
                 desc[i].txd.tx_submit = tsi721_tx_submit;
                 desc[i].txd.flags = DMA_CTRL_ACK;
-                INIT_LIST_HEAD(&desc[i].tx_list);
-                list_add_tail(&desc[i].desc_node, &tmp_list);
+                list_add(&desc[i].desc_node, &bdma_chan->free_list);
         }
 
-        spin_lock_bh(&bdma_chan->lock);
-        list_splice(&tmp_list, &bdma_chan->free_list);
-        bdma_chan->completed_cookie = dchan->cookie = 1;
-        spin_unlock_bh(&bdma_chan->lock);
-
-#ifdef CONFIG_PCI_MSI
-        if (priv->flags & TSI721_USING_MSIX) {
-                /* Request interrupt service if we are in MSI-X mode */
-                rc = request_irq(
-                        priv->msix[TSI721_VECT_DMA0_DONE +
-                                   bdma_chan->id].vector,
-                        tsi721_bdma_msix, 0,
-                        priv->msix[TSI721_VECT_DMA0_DONE +
-                                   bdma_chan->id].irq_name,
-                        (void *)bdma_chan);
-
-                if (rc) {
-                        dev_dbg(dchan->device->dev,
-                                "Unable to allocate MSI-X interrupt for "
-                                "BDMA%d-DONE\n", bdma_chan->id);
-                        goto err_out;
-                }
-
-                rc = request_irq(priv->msix[TSI721_VECT_DMA0_INT +
-                                            bdma_chan->id].vector,
-                                tsi721_bdma_msix, 0,
-                                priv->msix[TSI721_VECT_DMA0_INT +
-                                           bdma_chan->id].irq_name,
-                                (void *)bdma_chan);
-
-                if (rc) {
-                        dev_dbg(dchan->device->dev,
-                                "Unable to allocate MSI-X interrupt for "
-                                "BDMA%d-INT\n", bdma_chan->id);
-                        free_irq(
-                                priv->msix[TSI721_VECT_DMA0_DONE +
-                                           bdma_chan->id].vector,
-                                (void *)bdma_chan);
-                        rc = -EIO;
-                        goto err_out;
-                }
-        }
-#endif /* CONFIG_PCI_MSI */
+        dma_cookie_init(dchan);
 
         bdma_chan->active = true;
         tsi721_bdma_interrupt_enable(bdma_chan, 1);
 
-        return bdma_chan->bd_num - 1;
-
-err_out:
-        kfree(desc);
-        tsi721_bdma_ch_free(bdma_chan);
-        return rc;
+        return TSI721_DMA_TX_QUEUE_SZ;
 }
 
-static void tsi721_free_chan_resources(struct dma_chan *dchan)
+static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan)
 {
-        struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
-        struct tsi721_device *priv = to_tsi721(dchan->device);
-        LIST_HEAD(list);
-
-        dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
-
-        if (bdma_chan->bd_base == NULL)
-                return;
-
-        BUG_ON(!list_empty(&bdma_chan->active_list));
-        BUG_ON(!list_empty(&bdma_chan->queue));
-
-        tsi721_bdma_interrupt_enable(bdma_chan, 0);
-        bdma_chan->active = false;
+        struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
 
 #ifdef CONFIG_PCI_MSI
         if (priv->flags & TSI721_USING_MSIX) {
@@ -601,64 +708,48 @@ static void tsi721_free_chan_resources(struct dma_chan *dchan)
         } else
 #endif
         synchronize_irq(priv->pdev->irq);
+}
 
-        tasklet_kill(&bdma_chan->tasklet);
+static void tsi721_free_chan_resources(struct dma_chan *dchan)
+{
+        struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 
-        spin_lock_bh(&bdma_chan->lock);
-        list_splice_init(&bdma_chan->free_list, &list);
-        spin_unlock_bh(&bdma_chan->lock);
+        dev_dbg(dchan->device->dev, "%s: for channel %d\n",
+                __func__, bdma_chan->id);
 
-#ifdef CONFIG_PCI_MSI
-        if (priv->flags & TSI721_USING_MSIX) {
-                free_irq(priv->msix[TSI721_VECT_DMA0_DONE +
-                                    bdma_chan->id].vector, (void *)bdma_chan);
-                free_irq(priv->msix[TSI721_VECT_DMA0_INT +
-                                    bdma_chan->id].vector, (void *)bdma_chan);
-        }
-#endif /* CONFIG_PCI_MSI */
+        if (bdma_chan->bd_base == NULL)
+                return;
 
-        tsi721_bdma_ch_free(bdma_chan);
+        BUG_ON(!list_empty(&bdma_chan->active_list));
+        BUG_ON(!list_empty(&bdma_chan->queue));
+
+        tsi721_bdma_interrupt_enable(bdma_chan, 0);
+        bdma_chan->active = false;
+        tsi721_sync_dma_irq(bdma_chan);
+        tasklet_kill(&bdma_chan->tasklet);
+        INIT_LIST_HEAD(&bdma_chan->free_list);
         kfree(bdma_chan->tx_desc);
+        tsi721_bdma_ch_free(bdma_chan);
 }
 
 static
 enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
                                  struct dma_tx_state *txstate)
 {
-        struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
-        dma_cookie_t            last_used;
-        dma_cookie_t            last_completed;
-        int                     ret;
-
-        spin_lock_bh(&bdma_chan->lock);
-        last_completed = bdma_chan->completed_cookie;
-        last_used = dchan->cookie;
-        spin_unlock_bh(&bdma_chan->lock);
-
-        ret = dma_async_is_complete(cookie, last_completed, last_used);
-
-        dma_set_tx_state(txstate, last_completed, last_used, 0);
-
-        dev_dbg(dchan->device->dev,
-                "%s: exit, ret: %d, last_completed: %d, last_used: %d\n",
-                __func__, ret, last_completed, last_used);
-
-        return ret;
+        return dma_cookie_status(dchan, cookie, txstate);
 }
 
 static void tsi721_issue_pending(struct dma_chan *dchan)
 {
         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 
-        dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
+        dev_dbg(dchan->device->dev, "%s: Enter\n", __func__);
 
-        if (tsi721_dma_is_idle(bdma_chan)) {
+        if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) {
                 spin_lock_bh(&bdma_chan->lock);
                 tsi721_advance_work(bdma_chan);
                 spin_unlock_bh(&bdma_chan->lock);
-        } else
-                dev_dbg(dchan->device->dev,
-                        "%s: DMA channel still busy\n", __func__);
+        }
 }
 
 static
@@ -668,21 +759,19 @@ struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan,
                         void *tinfo)
 {
         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
-        struct tsi721_tx_desc *desc = NULL;
-        struct tsi721_tx_desc *first = NULL;
-        struct scatterlist *sg;
+        struct tsi721_tx_desc *desc, *_d;
         struct rio_dma_ext *rext = tinfo;
-        u64 rio_addr = rext->rio_addr; /* limited to 64-bit rio_addr for now */
-        unsigned int i;
-        u32 sys_size = dma_to_mport(dchan->device)->sys_size;
         enum dma_rtype rtype;
-        dma_addr_t next_addr = -1;
+        struct dma_async_tx_descriptor *txd = NULL;
 
         if (!sgl || !sg_len) {
                 dev_err(dchan->device->dev, "%s: No SG list\n", __func__);
                 return NULL;
         }
 
+        dev_dbg(dchan->device->dev, "%s: %s\n", __func__,
+                (dir == DMA_DEV_TO_MEM)?"READ":"WRITE");
+
         if (dir == DMA_DEV_TO_MEM)
                 rtype = NREAD;
         else if (dir == DMA_MEM_TO_DEV) {
@@ -704,97 +793,26 @@ struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan,
                 return NULL;
         }
 
-        for_each_sg(sgl, sg, sg_len, i) {
-                int err;
-
-                if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) {
-                        dev_err(dchan->device->dev,
-                                "%s: SG entry %d is too large\n", __func__, i);
-                        goto err_desc_put;
-                }
-
-                /*
-                 * If this sg entry forms contiguous block with previous one,
-                 * try to merge it into existing DMA descriptor
-                 */
-                if (desc) {
-                        if (next_addr == sg_dma_address(sg) &&
-                            desc->bcount + sg_dma_len(sg) <=
-                                                TSI721_BDMA_MAX_BCOUNT) {
-                                /* Adjust byte count of the descriptor */
-                                desc->bcount += sg_dma_len(sg);
-                                goto entry_done;
-                        }
-
-                        /*
-                         * Finalize this descriptor using total
-                         * byte count value.
-                         */
-                        tsi721_desc_fill_end(desc);
-                        dev_dbg(dchan->device->dev, "%s: desc final len: %d\n",
-                                __func__, desc->bcount);
-                }
-
-                /*
-                 * Obtain and initialize a new descriptor
-                 */
-                desc = tsi721_desc_get(bdma_chan);
-                if (!desc) {
-                        dev_err(dchan->device->dev,
-                                "%s: Failed to get new descriptor for SG %d\n",
-                                __func__, i);
-                        goto err_desc_put;
-                }
-
-                desc->destid = rext->destid;
-                desc->rio_addr = rio_addr;
-                desc->rio_addr_u = 0;
-                desc->bcount = sg_dma_len(sg);
-
-                dev_dbg(dchan->device->dev,
-                        "sg%d desc: 0x%llx, addr: 0x%llx len: %d\n",
-                        i, (u64)desc->txd.phys,
-                        (unsigned long long)sg_dma_address(sg),
-                        sg_dma_len(sg));
-
-                dev_dbg(dchan->device->dev,
-                        "bd_ptr = %p did=%d raddr=0x%llx\n",
-                        desc->hw_desc, desc->destid, desc->rio_addr);
-
-                err = tsi721_desc_fill_init(desc, sg, rtype, sys_size);
-                if (err) {
-                        dev_err(dchan->device->dev,
-                                "Failed to build desc: %d\n", err);
-                        goto err_desc_put;
-                }
-
-                next_addr = sg_dma_address(sg);
-
-                if (!first)
-                        first = desc;
-                else
-                        list_add_tail(&desc->desc_node, &first->tx_list);
+        spin_lock_bh(&bdma_chan->lock);
 
-entry_done:
-                if (sg_is_last(sg)) {
-                        desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0;
-                        tsi721_desc_fill_end(desc);
-                        dev_dbg(dchan->device->dev, "%s: desc final len: %d\n",
-                                __func__, desc->bcount);
-                } else {
-                        rio_addr += sg_dma_len(sg);
-                        next_addr += sg_dma_len(sg);
+        list_for_each_entry_safe(desc, _d, &bdma_chan->free_list, desc_node) {
+                if (async_tx_test_ack(&desc->txd)) {
+                        list_del_init(&desc->desc_node);
+                        desc->destid = rext->destid;
+                        desc->rio_addr = rext->rio_addr;
+                        desc->rio_addr_u = 0;
+                        desc->rtype = rtype;
+                        desc->sg_len    = sg_len;
+                        desc->sg        = sgl;
+                        txd             = &desc->txd;
+                        txd->flags      = flags;
+                        break;
                 }
         }
 
-        first->txd.cookie = -EBUSY;
-        desc->txd.flags = flags;
-
-        return &first->txd;
+        spin_unlock_bh(&bdma_chan->lock);
 
-err_desc_put:
-        tsi721_desc_put(bdma_chan, first);
-        return NULL;
+        return txd;
 }
 
 static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd,
@@ -802,23 +820,34 @@ static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd,
 {
         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
         struct tsi721_tx_desc *desc, *_d;
+        u32 dmac_int;
         LIST_HEAD(list);
 
         dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
 
         if (cmd != DMA_TERMINATE_ALL)
-                return -ENXIO;
+                return -ENOSYS;
 
         spin_lock_bh(&bdma_chan->lock);
 
-        /* make sure to stop the transfer */
-        iowrite32(TSI721_DMAC_CTL_SUSP, bdma_chan->regs + TSI721_DMAC_CTL);
+        bdma_chan->active = false;
+
+        if (!tsi721_dma_is_idle(bdma_chan)) {
+                /* make sure to stop the transfer */
+                iowrite32(TSI721_DMAC_CTL_SUSP,
+                          bdma_chan->regs + TSI721_DMAC_CTL);
+
+                /* Wait until DMA channel stops */
+                do {
+                        dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
+                } while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0);
+        }
 
         list_splice_init(&bdma_chan->active_list, &list);
         list_splice_init(&bdma_chan->queue, &list);
 
         list_for_each_entry_safe(desc, _d, &list, desc_node)
-                tsi721_dma_chain_complete(bdma_chan, desc);
+                tsi721_dma_tx_err(bdma_chan, desc);
 
         spin_unlock_bh(&bdma_chan->lock);
 
@@ -828,22 +857,18 @@ static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd,
 int tsi721_register_dma(struct tsi721_device *priv)
 {
         int i;
-        int nr_channels = TSI721_DMA_MAXCH;
+        int nr_channels = 0;
         int err;
         struct rio_mport *mport = priv->mport;
 
-        mport->dma.dev = &priv->pdev->dev;
-        mport->dma.chancnt = nr_channels;
-
         INIT_LIST_HEAD(&mport->dma.channels);
 
-        for (i = 0; i < nr_channels; i++) {
+        for (i = 0; i < TSI721_DMA_MAXCH; i++) {
                 struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i];
 
                 if (i == TSI721_DMACH_MAINT)
                         continue;
 
-                bdma_chan->bd_num = TSI721_BDMA_BD_RING_SZ;
                 bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i);
 
                 bdma_chan->dchan.device = &mport->dma;
@@ -862,12 +887,15 @@ int tsi721_register_dma(struct tsi721_device *priv)
                              (unsigned long)bdma_chan);
                 list_add_tail(&bdma_chan->dchan.device_node,
                               &mport->dma.channels);
+                nr_channels++;
         }
 
+        mport->dma.chancnt = nr_channels;
         dma_cap_zero(mport->dma.cap_mask);
         dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask);
         dma_cap_set(DMA_SLAVE, mport->dma.cap_mask);
 
+        mport->dma.dev = &priv->pdev->dev;
         mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources;
         mport->dma.device_free_chan_resources = tsi721_free_chan_resources;
         mport->dma.device_tx_status = tsi721_tx_status;