Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx

* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: (63 commits)
  ARM: PL08x: cleanup comments
  Update CONFIG_MD_RAID6_PQ to CONFIG_RAID6_PQ in drivers/dma/iop-adma.c
  ARM: PL08x: fix a warning
  Fix dmaengine_submit() return type
  dmaengine: at_hdmac: fix race while monitoring channel status
  dmaengine: at_hdmac: flags located in first descriptor
  dmaengine: at_hdmac: use subsys_initcall instead of module_init
  dmaengine: at_hdmac: no need set ACK in new descriptor
  dmaengine: at_hdmac: trivial add precision to unmapping comment
  dmaengine: at_hdmac: use dma_address to program DMA hardware
  pch_dma: support new device ML7213 IOH
  ARM: PL08x: prevent dma_set_runtime_config() reconfiguring memcpy channels
  ARM: PL08x: allow dma_set_runtime_config() to return errors
  ARM: PL08x: fix locking between prepare function and submit function
  ARM: PL08x: introduce 'phychan_hold' to hold on to physical channels
  ARM: PL08x: put txd's on the pending list in pl08x_tx_submit()
  ARM: PL08x: rename 'desc_list' as 'pend_list'
  ARM: PL08x: implement unmapping of memcpy buffers
  ARM: PL08x: store prep_* flags in async_tx structure
  ARM: PL08x: shrink srcbus/dstbus in txd structure
  ...

10 files changed, 935 insertions, 794 deletions

diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index ef138731c0e..1c28816152f 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -200,11 +200,16 @@ config PL330_DMA
           platform_data for a dma-pl330 device.
 
 config PCH_DMA
-        tristate "Topcliff (Intel EG20T) PCH DMA support"
+        tristate "Intel EG20T PCH / OKI SEMICONDUCTOR ML7213 IOH DMA support"
         depends on PCI && X86
         select DMA_ENGINE
         help
-          Enable support for the Topcliff (Intel EG20T) PCH DMA engine.
+          Enable support for Intel EG20T PCH DMA engine.
+
+          This driver also can be used for OKI SEMICONDUCTOR ML7213 IOH(Input/
+          Output Hub) which is for IVI(In-Vehicle Infotainment) use.
+          ML7213 is companion chip for Intel Atom E6xx series.
+          ML7213 is completely compatible for Intel EG20T PCH.
 
 config IMX_SDMA
         tristate "i.MX SDMA support"
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index b605cc9ac3a..297f48b0cba 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -19,14 +19,14 @@
  * this program; if not, write to the Free Software Foundation, Inc., 59
  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  *
- * The full GNU General Public License is iin this distribution in the
- * file called COPYING.
+ * The full GNU General Public License is in this distribution in the file
+ * called COPYING.
  *
  * Documentation: ARM DDI 0196G == PL080
- * Documentation: ARM DDI 0218E == PL081
+ * Documentation: ARM DDI 0218E == PL081
  *
- * PL080 & PL081 both have 16 sets of DMA signals that can be routed to
- * any channel.
+ * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
+ * channel.
  *
  * The PL080 has 8 channels available for simultaneous use, and the PL081
  * has only two channels. So on these DMA controllers the number of channels
@@ -53,7 +53,23 @@
  *
  * ASSUMES default (little) endianness for DMA transfers
  *
- * Only DMAC flow control is implemented
+ * The PL08x has two flow control settings:
+ *  - DMAC flow control: the transfer size defines the number of transfers
+ *    which occur for the current LLI entry, and the DMAC raises TC at the
+ *    end of every LLI entry.  Observed behaviour shows the DMAC listening
+ *    to both the BREQ and SREQ signals (contrary to documented),
+ *    transferring data if either is active.  The LBREQ and LSREQ signals
+ *    are ignored.
+ *
+ *  - Peripheral flow control: the transfer size is ignored (and should be
+ *    zero).  The data is transferred from the current LLI entry, until
+ *    after the final transfer signalled by LBREQ or LSREQ.  The DMAC
+ *    will then move to the next LLI entry.
+ *
+ * Only the former works sanely with scatter lists, so we only implement
+ * the DMAC flow control method.  However, peripherals which use the LBREQ
+ * and LSREQ signals (eg, MMCI) are unable to use this mode, which through
+ * these hardware restrictions prevents them from using scatter DMA.
  *
  * Global TODO:
  * - Break out common code from arch/arm/mach-s3c64xx and share
@@ -61,50 +77,39 @@
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/module.h>
-#include <linux/pci.h>
 #include <linux/interrupt.h>
 #include <linux/slab.h>
 #include <linux/dmapool.h>
-#include <linux/amba/bus.h>
 #include <linux/dmaengine.h>
+#include <linux/amba/bus.h>
 #include <linux/amba/pl08x.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 
 #include <asm/hardware/pl080.h>
-#include <asm/dma.h>
-#include <asm/mach/dma.h>
-#include <asm/atomic.h>
-#include <asm/processor.h>
-#include <asm/cacheflush.h>
 
 #define DRIVER_NAME     "pl08xdmac"
 
 /**
- * struct vendor_data - vendor-specific config parameters
- * for PL08x derivates
- * @name: the name of this specific variant
+ * struct vendor_data - vendor-specific config parameters for PL08x derivatives
  * @channels: the number of channels available in this variant
- * @dualmaster: whether this version supports dual AHB masters
- * or not.
+ * @dualmaster: whether this version supports dual AHB masters or not.
  */
 struct vendor_data {
-        char *name;
         u8 channels;
         bool dualmaster;
 };
 
 /*
  * PL08X private data structures
- * An LLI struct - see pl08x TRM
- * Note that next uses bit[0] as a bus bit,
- * start & end do not - their bus bit info
- * is in cctl
+ * An LLI struct - see PL08x TRM.  Note that next uses bit[0] as a bus bit,
+ * start & end do not - their bus bit info is in cctl.  Also note that these
+ * are fixed 32-bit quantities.
  */
-struct lli {
-        dma_addr_t src;
-        dma_addr_t dst;
-        dma_addr_t next;
+struct pl08x_lli {
+        u32 src;
+        u32 dst;
+        u32 lli;
         u32 cctl;
 };
 
@@ -119,6 +124,8 @@ struct lli {
  * @phy_chans: array of data for the physical channels
  * @pool: a pool for the LLI descriptors
  * @pool_ctr: counter of LLIs in the pool
+ * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI fetches
+ * @mem_buses: set to indicate memory transfers on AHB2.
  * @lock: a spinlock for this struct
  */
 struct pl08x_driver_data {
@@ -126,11 +133,13 @@ struct pl08x_driver_data {
         struct dma_device memcpy;
         void __iomem *base;
         struct amba_device *adev;
-        struct vendor_data *vd;
+        const struct vendor_data *vd;
         struct pl08x_platform_data *pd;
         struct pl08x_phy_chan *phy_chans;
         struct dma_pool *pool;
         int pool_ctr;
+        u8 lli_buses;
+        u8 mem_buses;
         spinlock_t lock;
 };
 
@@ -152,9 +161,9 @@ struct pl08x_driver_data {
 /* Size (bytes) of each LLI buffer allocated for one transfer */
 # define PL08X_LLI_TSFR_SIZE    0x2000
 
-/* Maximimum times we call dma_pool_alloc on this pool without freeing */
+/* Maximum times we call dma_pool_alloc on this pool without freeing */
 #define PL08X_MAX_ALLOCS        0x40
-#define MAX_NUM_TSFR_LLIS       (PL08X_LLI_TSFR_SIZE/sizeof(struct lli))
+#define MAX_NUM_TSFR_LLIS       (PL08X_LLI_TSFR_SIZE/sizeof(struct pl08x_lli))
 #define PL08X_ALIGN             8
 
 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
@@ -162,6 +171,11 @@ static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
         return container_of(chan, struct pl08x_dma_chan, chan);
 }
 
+static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
+{
+        return container_of(tx, struct pl08x_txd, tx);
+}
+
 /*
  * Physical channel handling
  */
@@ -177,88 +191,47 @@ static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
 
 /*
  * Set the initial DMA register values i.e. those for the first LLI
- * The next lli pointer and the configuration interrupt bit have
- * been set when the LLIs were constructed
+ * The next LLI pointer and the configuration interrupt bit have
+ * been set when the LLIs were constructed.  Poke them into the hardware
+ * and start the transfer.
  */
-static void pl08x_set_cregs(struct pl08x_driver_data *pl08x,
-                            struct pl08x_phy_chan *ch)
-{
-        /* Wait for channel inactive */
-        while (pl08x_phy_channel_busy(ch))
-                ;
-
-        dev_vdbg(&pl08x->adev->dev,
-                "WRITE channel %d: csrc=%08x, cdst=%08x, "
-                 "cctl=%08x, clli=%08x, ccfg=%08x\n",
-                ch->id,
-                ch->csrc,
-                ch->cdst,
-                ch->cctl,
-                ch->clli,
-                ch->ccfg);
-
-        writel(ch->csrc, ch->base + PL080_CH_SRC_ADDR);
-        writel(ch->cdst, ch->base + PL080_CH_DST_ADDR);
-        writel(ch->clli, ch->base + PL080_CH_LLI);
-        writel(ch->cctl, ch->base + PL080_CH_CONTROL);
-        writel(ch->ccfg, ch->base + PL080_CH_CONFIG);
-}
-
-static inline void pl08x_config_phychan_for_txd(struct pl08x_dma_chan *plchan)
+static void pl08x_start_txd(struct pl08x_dma_chan *plchan,
+        struct pl08x_txd *txd)
 {
-        struct pl08x_channel_data *cd = plchan->cd;
+        struct pl08x_driver_data *pl08x = plchan->host;
         struct pl08x_phy_chan *phychan = plchan->phychan;
-        struct pl08x_txd *txd = plchan->at;
-
-        /* Copy the basic control register calculated at transfer config */
-        phychan->csrc = txd->csrc;
-        phychan->cdst = txd->cdst;
-        phychan->clli = txd->clli;
-        phychan->cctl = txd->cctl;
-
-        /* Assign the signal to the proper control registers */
-        phychan->ccfg = cd->ccfg;
-        phychan->ccfg &= ~PL080_CONFIG_SRC_SEL_MASK;
-        phychan->ccfg &= ~PL080_CONFIG_DST_SEL_MASK;
-        /* If it wasn't set from AMBA, ignore it */
-        if (txd->direction == DMA_TO_DEVICE)
-                /* Select signal as destination */
-                phychan->ccfg |=
-                        (phychan->signal << PL080_CONFIG_DST_SEL_SHIFT);
-        else if (txd->direction == DMA_FROM_DEVICE)
-                /* Select signal as source */
-                phychan->ccfg |=
-                        (phychan->signal << PL080_CONFIG_SRC_SEL_SHIFT);
-        /* Always enable error interrupts */
-        phychan->ccfg |= PL080_CONFIG_ERR_IRQ_MASK;
-        /* Always enable terminal interrupts */
-        phychan->ccfg |= PL080_CONFIG_TC_IRQ_MASK;
-}
-
-/*
- * Enable the DMA channel
- * Assumes all other configuration bits have been set
- * as desired before this code is called
- */
-static void pl08x_enable_phy_chan(struct pl08x_driver_data *pl08x,
-                                  struct pl08x_phy_chan *ch)
-{
+        struct pl08x_lli *lli = &txd->llis_va[0];
         u32 val;
 
-        /*
-         * Do not access config register until channel shows as disabled
-         */
-        while (readl(pl08x->base + PL080_EN_CHAN) & (1 << ch->id))
-                ;
+        plchan->at = txd;
 
-        /*
-         * Do not access config register until channel shows as inactive
-         */
-        val = readl(ch->base + PL080_CH_CONFIG);
+        /* Wait for channel inactive */
+        while (pl08x_phy_channel_busy(phychan))
+                cpu_relax();
+
+        dev_vdbg(&pl08x->adev->dev,
+                "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
+                "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
+                phychan->id, lli->src, lli->dst, lli->lli, lli->cctl,
+                txd->ccfg);
+
+        writel(lli->src, phychan->base + PL080_CH_SRC_ADDR);
+        writel(lli->dst, phychan->base + PL080_CH_DST_ADDR);
+        writel(lli->lli, phychan->base + PL080_CH_LLI);
+        writel(lli->cctl, phychan->base + PL080_CH_CONTROL);
+        writel(txd->ccfg, phychan->base + PL080_CH_CONFIG);
+
+        /* Enable the DMA channel */
+        /* Do not access config register until channel shows as disabled */
+        while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
+                cpu_relax();
+
+        /* Do not access config register until channel shows as inactive */
+        val = readl(phychan->base + PL080_CH_CONFIG);
         while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
-                val = readl(ch->base + PL080_CH_CONFIG);
+                val = readl(phychan->base + PL080_CH_CONFIG);
 
-        writel(val | PL080_CONFIG_ENABLE, ch->base + PL080_CH_CONFIG);
+        writel(val | PL080_CONFIG_ENABLE, phychan->base + PL080_CH_CONFIG);
 }
 
 /*
@@ -266,10 +239,8 @@ static void pl08x_enable_phy_chan(struct pl08x_driver_data *pl08x,
  *
  * Disabling individual channels could lose data.
  *
- * Disable the peripheral DMA after disabling the DMAC
- * in order to allow the DMAC FIFO to drain, and
- * hence allow the channel to show inactive
- *
+ * Disable the peripheral DMA after disabling the DMAC in order to allow
+ * the DMAC FIFO to drain, and hence allow the channel to show inactive
  */
 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
 {
@@ -282,7 +253,7 @@ static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
 
         /* Wait for channel inactive */
         while (pl08x_phy_channel_busy(ch))
-                ;
+                cpu_relax();
 }
 
 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
@@ -333,54 +304,56 @@ static inline u32 get_bytes_in_cctl(u32 cctl)
 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
 {
         struct pl08x_phy_chan *ch;
-        struct pl08x_txd *txdi = NULL;
         struct pl08x_txd *txd;
         unsigned long flags;
-        u32 bytes = 0;
+        size_t bytes = 0;
 
         spin_lock_irqsave(&plchan->lock, flags);
-
         ch = plchan->phychan;
         txd = plchan->at;
 
         /*
-         * Next follow the LLIs to get the number of pending bytes in the
-         * currently active transaction.
+         * Follow the LLIs to get the number of remaining
+         * bytes in the currently active transaction.
          */
         if (ch && txd) {
-                struct lli *llis_va = txd->llis_va;
-                struct lli *llis_bus = (struct lli *) txd->llis_bus;
-                u32 clli = readl(ch->base + PL080_CH_LLI);
+                u32 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
 
-                /* First get the bytes in the current active LLI */
+                /* First get the remaining bytes in the active transfer */
                 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));
 
                 if (clli) {
-                        int i = 0;
+                        struct pl08x_lli *llis_va = txd->llis_va;
+                        dma_addr_t llis_bus = txd->llis_bus;
+                        int index;
+
+                        BUG_ON(clli < llis_bus || clli >= llis_bus +
+                                sizeof(struct pl08x_lli) * MAX_NUM_TSFR_LLIS);
+
+                        /*
+                         * Locate the next LLI - as this is an array,
+                         * it's simple maths to find.
+                         */
+                        index = (clli - llis_bus) / sizeof(struct pl08x_lli);
 
-                        /* Forward to the LLI pointed to by clli */
-                        while ((clli != (u32) &(llis_bus[i])) &&
-                               (i < MAX_NUM_TSFR_LLIS))
-                                i++;
+                        for (; index < MAX_NUM_TSFR_LLIS; index++) {
+                                bytes += get_bytes_in_cctl(llis_va[index].cctl);
 
-                        while (clli) {
-                                bytes += get_bytes_in_cctl(llis_va[i].cctl);
                                 /*
-                                 * A clli of 0x00000000 will terminate the
-                                 * LLI list
+                                 * A LLI pointer of 0 terminates the LLI list
                                  */
-                                clli = llis_va[i].next;
-                                i++;
+                                if (!llis_va[index].lli)
+                                        break;
                         }
                 }
         }
 
         /* Sum up all queued transactions */
-        if (!list_empty(&plchan->desc_list)) {
-                list_for_each_entry(txdi, &plchan->desc_list, node) {
+        if (!list_empty(&plchan->pend_list)) {
+                struct pl08x_txd *txdi;
+                list_for_each_entry(txdi, &plchan->pend_list, node) {
                         bytes += txdi->len;
                 }
-
         }
 
         spin_unlock_irqrestore(&plchan->lock, flags);
@@ -390,6 +363,10 @@ static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
 
 /*
  * Allocate a physical channel for a virtual channel
+ *
+ * Try to locate a physical channel to be used for this transfer. If all
+ * are taken return NULL and the requester will have to cope by using
+ * some fallback PIO mode or retrying later.
  */
 static struct pl08x_phy_chan *
 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
@@ -399,12 +376,6 @@ pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
         unsigned long flags;
         int i;
 
-        /*
-         * Try to locate a physical channel to be used for
-         * this transfer. If all are taken return NULL and
-         * the requester will have to cope by using some fallback
-         * PIO mode or retrying later.
-         */
         for (i = 0; i < pl08x->vd->channels; i++) {
                 ch = &pl08x->phy_chans[i];
 
@@ -465,11 +436,11 @@ static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
 }
 
 static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
-                                  u32 tsize)
+                                  size_t tsize)
 {
         u32 retbits = cctl;
 
-        /* Remove all src, dst and transfersize bits */
+        /* Remove all src, dst and transfer size bits */
         retbits &= ~PL080_CONTROL_DWIDTH_MASK;
         retbits &= ~PL080_CONTROL_SWIDTH_MASK;
         retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
@@ -509,95 +480,87 @@ static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
         return retbits;
 }
 
+struct pl08x_lli_build_data {
+        struct pl08x_txd *txd;
+        struct pl08x_driver_data *pl08x;
+        struct pl08x_bus_data srcbus;
+        struct pl08x_bus_data dstbus;
+        size_t remainder;
+};
+
 /*
- * Autoselect a master bus to use for the transfer
- * this prefers the destination bus if both available
- * if fixed address on one bus the other will be chosen
+ * Autoselect a master bus to use for the transfer this prefers the
+ * destination bus if both available if fixed address on one bus the
+ * other will be chosen
  */
-void pl08x_choose_master_bus(struct pl08x_bus_data *src_bus,
-        struct pl08x_bus_data *dst_bus, struct pl08x_bus_data **mbus,
-        struct pl08x_bus_data **sbus, u32 cctl)
+static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
+        struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
 {
         if (!(cctl & PL080_CONTROL_DST_INCR)) {
-                *mbus = src_bus;
-                *sbus = dst_bus;
+                *mbus = &bd->srcbus;
+                *sbus = &bd->dstbus;
         } else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
-                *mbus = dst_bus;
-                *sbus = src_bus;
+                *mbus = &bd->dstbus;
+                *sbus = &bd->srcbus;
         } else {
-                if (dst_bus->buswidth == 4) {
-                        *mbus = dst_bus;
-                        *sbus = src_bus;
-                } else if (src_bus->buswidth == 4) {
-                        *mbus = src_bus;
-                        *sbus = dst_bus;
-                } else if (dst_bus->buswidth == 2) {
-                        *mbus = dst_bus;
-                        *sbus = src_bus;
-                } else if (src_bus->buswidth == 2) {
-                        *mbus = src_bus;
-                        *sbus = dst_bus;
+                if (bd->dstbus.buswidth == 4) {
+                        *mbus = &bd->dstbus;
+                        *sbus = &bd->srcbus;
+                } else if (bd->srcbus.buswidth == 4) {
+                        *mbus = &bd->srcbus;
+                        *sbus = &bd->dstbus;
+                } else if (bd->dstbus.buswidth == 2) {
+                        *mbus = &bd->dstbus;
+                        *sbus = &bd->srcbus;
+                } else if (bd->srcbus.buswidth == 2) {
+                        *mbus = &bd->srcbus;
+                        *sbus = &bd->dstbus;
                 } else {
-                        /* src_bus->buswidth == 1 */
-                        *mbus = dst_bus;
-                        *sbus = src_bus;
+                        /* bd->srcbus.buswidth == 1 */
+                        *mbus = &bd->dstbus;
+                        *sbus = &bd->srcbus;
                 }
         }
 }
 
 /*
- * Fills in one LLI for a certain transfer descriptor
- * and advance the counter
+ * Fills in one LLI for a certain transfer descriptor and advance the counter
  */
-int pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
-                            struct pl08x_txd *txd, int num_llis, int len,
-                            u32 cctl, u32 *remainder)
+static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd,
+        int num_llis, int len, u32 cctl)
 {
-        struct lli *llis_va = txd->llis_va;
-        struct lli *llis_bus = (struct lli *) txd->llis_bus;
+        struct pl08x_lli *llis_va = bd->txd->llis_va;
+        dma_addr_t llis_bus = bd->txd->llis_bus;
 
         BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
 
-        llis_va[num_llis].cctl          = cctl;
-        llis_va[num_llis].src           = txd->srcbus.addr;
-        llis_va[num_llis].dst           = txd->dstbus.addr;
-
-        /*
-         * On versions with dual masters, you can optionally AND on
-         * PL080_LLI_LM_AHB2 to the LLI to tell the hardware to read
-         * in new LLIs with that controller, but we always try to
-         * choose AHB1 to point into memory. The idea is to have AHB2
-         * fixed on the peripheral and AHB1 messing around in the
-         * memory. So we don't manipulate this bit currently.
-         */
-
-        llis_va[num_llis].next =
-                (dma_addr_t)((u32) &(llis_bus[num_llis + 1]));
+        llis_va[num_llis].cctl = cctl;
+        llis_va[num_llis].src = bd->srcbus.addr;
+        llis_va[num_llis].dst = bd->dstbus.addr;
+        llis_va[num_llis].lli = llis_bus + (num_llis + 1) * sizeof(struct pl08x_lli);
+        if (bd->pl08x->lli_buses & PL08X_AHB2)
+                llis_va[num_llis].lli |= PL080_LLI_LM_AHB2;
 
         if (cctl & PL080_CONTROL_SRC_INCR)
-                txd->srcbus.addr += len;
+                bd->srcbus.addr += len;
         if (cctl & PL080_CONTROL_DST_INCR)
-                txd->dstbus.addr += len;
+                bd->dstbus.addr += len;
 
-        *remainder -= len;
+        BUG_ON(bd->remainder < len);
 
-        return num_llis + 1;
+        bd->remainder -= len;
 }
 
 /*
- * Return number of bytes to fill to boundary, or len
+ * Return number of bytes to fill to boundary, or len.
+ * This calculation works for any value of addr.
  */
-static inline u32 pl08x_pre_boundary(u32 addr, u32 len)
+static inline size_t pl08x_pre_boundary(u32 addr, size_t len)
 {
-        u32 boundary;
-
-        boundary = ((addr >> PL08X_BOUNDARY_SHIFT) + 1)
-                << PL08X_BOUNDARY_SHIFT;
+        size_t boundary_len = PL08X_BOUNDARY_SIZE -
+                        (addr & (PL08X_BOUNDARY_SIZE - 1));
 
-        if (boundary < addr + len)
-                return boundary - addr;
-        else
-                return len;
+        return min(boundary_len, len);
 }
 
 /*
@@ -608,20 +571,13 @@ static inline u32 pl08x_pre_boundary(u32 addr, u32 len)
 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
                               struct pl08x_txd *txd)
 {
-        struct pl08x_channel_data *cd = txd->cd;
         struct pl08x_bus_data *mbus, *sbus;
-        u32 remainder;
+        struct pl08x_lli_build_data bd;
         int num_llis = 0;
         u32 cctl;
-        int max_bytes_per_lli;
-        int total_bytes = 0;
-        struct lli *llis_va;
-        struct lli *llis_bus;
-
-        if (!txd) {
-                dev_err(&pl08x->adev->dev, "%s no descriptor\n", __func__);
-                return 0;
-        }
+        size_t max_bytes_per_lli;
+        size_t total_bytes = 0;
+        struct pl08x_lli *llis_va;
 
         txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT,
                                       &txd->llis_bus);
@@ -632,121 +588,79 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
 
         pl08x->pool_ctr++;
 
-        /*
-         * Initialize bus values for this transfer
-         * from the passed optimal values
-         */
-        if (!cd) {
-                dev_err(&pl08x->adev->dev, "%s no channel data\n", __func__);
-                return 0;
-        }
+        /* Get the default CCTL */
+        cctl = txd->cctl;
 
-        /* Get the default CCTL from the platform data */
-        cctl = cd->cctl;
-
-        /*
-         * On the PL080 we have two bus masters and we
-         * should select one for source and one for
-         * destination. We try to use AHB2 for the
-         * bus which does not increment (typically the
-         * peripheral) else we just choose something.
-         */
-        cctl &= ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
-        if (pl08x->vd->dualmaster) {
-                if (cctl & PL080_CONTROL_SRC_INCR)
-                        /* Source increments, use AHB2 for destination */
-                        cctl |= PL080_CONTROL_DST_AHB2;
-                else if (cctl & PL080_CONTROL_DST_INCR)
-                        /* Destination increments, use AHB2 for source */
-                        cctl |= PL080_CONTROL_SRC_AHB2;
-                else
-                        /* Just pick something, source AHB1 dest AHB2 */
-                        cctl |= PL080_CONTROL_DST_AHB2;
-        }
+        bd.txd = txd;
+        bd.pl08x = pl08x;
+        bd.srcbus.addr = txd->src_addr;
+        bd.dstbus.addr = txd->dst_addr;
 
         /* Find maximum width of the source bus */
-        txd->srcbus.maxwidth =
+        bd.srcbus.maxwidth =
                 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
                                        PL080_CONTROL_SWIDTH_SHIFT);
 
         /* Find maximum width of the destination bus */
-        txd->dstbus.maxwidth =
+        bd.dstbus.maxwidth =
                 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
                                        PL080_CONTROL_DWIDTH_SHIFT);
 
         /* Set up the bus widths to the maximum */
-        txd->srcbus.buswidth = txd->srcbus.maxwidth;
-        txd->dstbus.buswidth = txd->dstbus.maxwidth;
+        bd.srcbus.buswidth = bd.srcbus.maxwidth;
+        bd.dstbus.buswidth = bd.dstbus.maxwidth;
         dev_vdbg(&pl08x->adev->dev,
                  "%s source bus is %d bytes wide, dest bus is %d bytes wide\n",
-                 __func__, txd->srcbus.buswidth, txd->dstbus.buswidth);
+                 __func__, bd.srcbus.buswidth, bd.dstbus.buswidth);
 
 
         /*
          * Bytes transferred == tsize * MIN(buswidths), not max(buswidths)
          */
-        max_bytes_per_lli = min(txd->srcbus.buswidth, txd->dstbus.buswidth) *
+        max_bytes_per_lli = min(bd.srcbus.buswidth, bd.dstbus.buswidth) *
                 PL080_CONTROL_TRANSFER_SIZE_MASK;
         dev_vdbg(&pl08x->adev->dev,
-                 "%s max bytes per lli = %d\n",
+                 "%s max bytes per lli = %zu\n",
                  __func__, max_bytes_per_lli);
 
         /* We need to count this down to zero */
-        remainder = txd->len;
+        bd.remainder = txd->len;
         dev_vdbg(&pl08x->adev->dev,
-                 "%s remainder = %d\n",
-                 __func__, remainder);
+                 "%s remainder = %zu\n",
+                 __func__, bd.remainder);
 
         /*
          * Choose bus to align to
          * - prefers destination bus if both available
          * - if fixed address on one bus chooses other
-         * - modifies cctl to choose an apropriate master
-         */
-        pl08x_choose_master_bus(&txd->srcbus, &txd->dstbus,
-                                &mbus, &sbus, cctl);
-
-
-        /*
-         * The lowest bit of the LLI register
-         * is also used to indicate which master to
-         * use for reading the LLIs.
          */
+        pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
 
         if (txd->len < mbus->buswidth) {
-                /*
-                 * Less than a bus width available
-                 * - send as single bytes
-                 */
-                while (remainder) {
+                /* Less than a bus width available - send as single bytes */
+                while (bd.remainder) {
                         dev_vdbg(&pl08x->adev->dev,
                                  "%s single byte LLIs for a transfer of "
-                                 "less than a bus width (remain %08x)\n",
-                                 __func__, remainder);
+                                 "less than a bus width (remain 0x%08x)\n",
+                                 __func__, bd.remainder);
                         cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
-                        num_llis =
-                                pl08x_fill_lli_for_desc(pl08x, txd, num_llis, 1,
-                                        cctl, &remainder);
+                        pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl);
                         total_bytes++;
                 }
         } else {
-                /*
-                 *  Make one byte LLIs until master bus is aligned
-                 *  - slave will then be aligned also
-                 */
+                /* Make one byte LLIs until master bus is aligned */
                 while ((mbus->addr) % (mbus->buswidth)) {
                         dev_vdbg(&pl08x->adev->dev,
                                 "%s adjustment lli for less than bus width "
-                                 "(remain %08x)\n",
-                                 __func__, remainder);
+                                 "(remain 0x%08x)\n",
+                                 __func__, bd.remainder);
                         cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
-                        num_llis = pl08x_fill_lli_for_desc
-                                (pl08x, txd, num_llis, 1, cctl, &remainder);
+                        pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl);
                         total_bytes++;
                 }
 
                 /*
-                 *  Master now aligned
+                 * Master now aligned
                  * - if slave is not then we must set its width down
                  */
                 if (sbus->addr % sbus->buswidth) {
@@ -761,63 +675,51 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
                  * Make largest possible LLIs until less than one bus
                  * width left
                  */
-                while (remainder > (mbus->buswidth - 1)) {
-                        int lli_len, target_len;
-                        int tsize;
-                        int odd_bytes;
+                while (bd.remainder > (mbus->buswidth - 1)) {
+                        size_t lli_len, target_len, tsize, odd_bytes;
 
                         /*
                          * If enough left try to send max possible,
                          * otherwise try to send the remainder
                          */
-                        target_len = remainder;
-                        if (remainder > max_bytes_per_lli)
-                                target_len = max_bytes_per_lli;
+                        target_len = min(bd.remainder, max_bytes_per_lli);
 
                         /*
-                         * Set bus lengths for incrementing busses
-                         * to number of bytes which fill to next memory
-                         * boundary
+                         * Set bus lengths for incrementing buses to the
+                         * number of bytes which fill to next memory boundary,
+                         * limiting on the target length calculated above.
                          */
                         if (cctl & PL080_CONTROL_SRC_INCR)
-                                txd->srcbus.fill_bytes =
-                                        pl08x_pre_boundary(
-                                                txd->srcbus.addr,
-                                                remainder);
+                                bd.srcbus.fill_bytes =
+                                        pl08x_pre_boundary(bd.srcbus.addr,
+                                                target_len);
                         else
-                                txd->srcbus.fill_bytes =
-                                        max_bytes_per_lli;
+                                bd.srcbus.fill_bytes = target_len;
 
                         if (cctl & PL080_CONTROL_DST_INCR)
-                                txd->dstbus.fill_bytes =
-                                        pl08x_pre_boundary(
-                                                txd->dstbus.addr,
-                                                remainder);
+                                bd.dstbus.fill_bytes =
+                                        pl08x_pre_boundary(bd.dstbus.addr,
+                                                target_len);
                         else
-                                txd->dstbus.fill_bytes =
-                                                max_bytes_per_lli;
+                                bd.dstbus.fill_bytes = target_len;
 
-                        /*
-                         *  Find the nearest
-                         */
-                        lli_len = min(txd->srcbus.fill_bytes,
-                                txd->dstbus.fill_bytes);
+                        /* Find the nearest */
+                        lli_len = min(bd.srcbus.fill_bytes,
+                                      bd.dstbus.fill_bytes);
 
-                        BUG_ON(lli_len > remainder);
+                        BUG_ON(lli_len > bd.remainder);
 
                         if (lli_len <= 0) {
                                 dev_err(&pl08x->adev->dev,
-                                        "%s lli_len is %d, <= 0\n",
+                                        "%s lli_len is %zu, <= 0\n",
                                                 __func__, lli_len);
                                 return 0;
                         }
 
                         if (lli_len == target_len) {
                                 /*
-                                 * Can send what we wanted
-                                 */
-                                /*
-                                 *  Maintain alignment
+                                 * Can send what we wanted.
+                                 * Maintain alignment
                                  */
                                 lli_len = (lli_len/mbus->buswidth) *
                                                         mbus->buswidth;
@@ -825,17 +727,14 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
                         } else {
                                 /*
                                  * So now we know how many bytes to transfer
-                                 * to get to the nearest boundary
-                                 * The next lli will past the boundary
-                                 * - however we may be working to a boundary
-                                 *   on the slave bus
-                                 *   We need to ensure the master stays aligned
+                                 * to get to the nearest boundary.  The next
+                                 * LLI will past the boundary.  However, we
+                                 * may be working to a boundary on the slave
+                                 * bus.  We need to ensure the master stays
+                                 * aligned, and that we are working in
+                                 * multiples of the bus widths.
                                  */
                                 odd_bytes = lli_len % mbus->buswidth;
-                                /*
-                                 * - and that we are working in multiples
-                                 *   of the bus widths
-                                 */
                                 lli_len -= odd_bytes;
 
                         }
@@ -855,41 +754,38 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
 
                                 if (target_len != lli_len) {
                                         dev_vdbg(&pl08x->adev->dev,
-                                        "%s can't send what we want. Desired %08x, lli of %08x bytes in txd of %08x\n",
+                                        "%s can't send what we want. Desired 0x%08zx, lli of 0x%08zx bytes in txd of 0x%08zx\n",
                                         __func__, target_len, lli_len, txd->len);
                                 }
 
                                 cctl = pl08x_cctl_bits(cctl,
-                                                       txd->srcbus.buswidth,
-                                                       txd->dstbus.buswidth,
+                                                       bd.srcbus.buswidth,
+                                                       bd.dstbus.buswidth,
                                                        tsize);
 
                                 dev_vdbg(&pl08x->adev->dev,
-                                        "%s fill lli with single lli chunk of size %08x (remainder %08x)\n",
-                                        __func__, lli_len, remainder);
-                                num_llis = pl08x_fill_lli_for_desc(pl08x, txd,
-                                                num_llis, lli_len, cctl,
-                                                &remainder);
+                                        "%s fill lli with single lli chunk of size 0x%08zx (remainder 0x%08zx)\n",
+                                        __func__, lli_len, bd.remainder);
+                                pl08x_fill_lli_for_desc(&bd, num_llis++,
+                                        lli_len, cctl);
                                 total_bytes += lli_len;
                         }
 
 
                         if (odd_bytes) {
                                 /*
-                                 * Creep past the boundary,
-                                 * maintaining master alignment
+                                 * Creep past the boundary, maintaining
+                                 * master alignment
                                  */
                                 int j;
                                 for (j = 0; (j < mbus->buswidth)
-                                                && (remainder); j++) {
+                                                && (bd.remainder); j++) {
                                         cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
                                         dev_vdbg(&pl08x->adev->dev,
-                                                "%s align with boundardy, single byte (remain %08x)\n",
-                                                __func__, remainder);
-                                        num_llis =
-                                                pl08x_fill_lli_for_desc(pl08x,
-                                                        txd, num_llis, 1,
-                                                        cctl, &remainder);
+                                                "%s align with boundary, single byte (remain 0x%08zx)\n",
+                                                __func__, bd.remainder);
+                                        pl08x_fill_lli_for_desc(&bd,
+                                                num_llis++, 1, cctl);
                                         total_bytes++;
                                 }
                         }
@@ -898,25 +794,18 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
                 /*
                  * Send any odd bytes
                  */
-                if (remainder < 0) {
-                        dev_err(&pl08x->adev->dev, "%s remainder not fitted 0x%08x bytes\n",
-                                        __func__, remainder);
-                        return 0;
-                }
-
-                while (remainder) {
+                while (bd.remainder) {
                         cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
                         dev_vdbg(&pl08x->adev->dev,
-                                "%s align with boundardy, single odd byte (remain %d)\n",
-                                __func__, remainder);
-                        num_llis = pl08x_fill_lli_for_desc(pl08x, txd, num_llis,
-                                        1, cctl, &remainder);
+                                "%s align with boundary, single odd byte (remain %zu)\n",
+                                __func__, bd.remainder);
+                        pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl);
                         total_bytes++;
                 }
         }
         if (total_bytes != txd->len) {
                 dev_err(&pl08x->adev->dev,
-                        "%s size of encoded lli:s don't match total txd, transferred 0x%08x from size 0x%08x\n",
+                        "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
                         __func__, total_bytes, txd->len);
                 return 0;
         }
@@ -927,41 +816,12 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
                         __func__, (u32) MAX_NUM_TSFR_LLIS);
                 return 0;
         }
-        /*
-         * Decide whether this is a loop or a terminated transfer
-         */
-        llis_va = txd->llis_va;
-        llis_bus = (struct lli *) txd->llis_bus;
 
-        if (cd->circular_buffer) {
-                /*
-                 * Loop the circular buffer so that the next element
-                 * points back to the beginning of the LLI.
-                 */
-                llis_va[num_llis - 1].next =
-                        (dma_addr_t)((unsigned int)&(llis_bus[0]));
-        } else {
-                /*
-                 * On non-circular buffers, the final LLI terminates
-                 * the LLI.
-                 */
-                llis_va[num_llis - 1].next = 0;
-                /*
-                 * The final LLI element shall also fire an interrupt
-                 */
-                llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN;
-        }
-
-        /* Now store the channel register values */
-        txd->csrc = llis_va[0].src;
-        txd->cdst = llis_va[0].dst;
-        if (num_llis > 1)
-                txd->clli = llis_va[0].next;
-        else
-                txd->clli = 0;
-
-        txd->cctl = llis_va[0].cctl;
-        /* ccfg will be set at physical channel allocation time */
+        llis_va = txd->llis_va;
+        /* The final LLI terminates the LLI. */
+        llis_va[num_llis - 1].lli = 0;
+        /* The final LLI element shall also fire an interrupt. */
+        llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN;
 
 #ifdef VERBOSE_DEBUG
         {
@@ -969,13 +829,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
 
                 for (i = 0; i < num_llis; i++) {
                         dev_vdbg(&pl08x->adev->dev,
-                                 "lli %d @%p: csrc=%08x, cdst=%08x, cctl=%08x, clli=%08x\n",
+                                 "lli %d @%p: csrc=0x%08x, cdst=0x%08x, cctl=0x%08x, clli=0x%08x\n",
                                  i,
                                  &llis_va[i],
                                  llis_va[i].src,
                                  llis_va[i].dst,
                                  llis_va[i].cctl,
-                                 llis_va[i].next
+                                 llis_va[i].lli
                                 );
                 }
         }
@@ -988,14 +848,8 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
 static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
                            struct pl08x_txd *txd)
 {
-        if (!txd)
-                dev_err(&pl08x->adev->dev,
-                        "%s no descriptor to free\n",
-                        __func__);
-
         /* Free the LLI */
-        dma_pool_free(pl08x->pool, txd->llis_va,
-                      txd->llis_bus);
+        dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
 
         pl08x->pool_ctr--;
 
@@ -1008,13 +862,12 @@ static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
         struct pl08x_txd *txdi = NULL;
         struct pl08x_txd *next;
 
-        if (!list_empty(&plchan->desc_list)) {
+        if (!list_empty(&plchan->pend_list)) {
                 list_for_each_entry_safe(txdi,
-                                         next, &plchan->desc_list, node) {
+                                         next, &plchan->pend_list, node) {
                         list_del(&txdi->node);
                         pl08x_free_txd(pl08x, txdi);
                 }
-
         }
 }
 
@@ -1069,6 +922,12 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan,
                         return -EBUSY;
                 }
                 ch->signal = ret;
+
+                /* Assign the flow control signal to this channel */
+                if (txd->direction == DMA_TO_DEVICE)
+                        txd->ccfg |= ch->signal << PL080_CONFIG_DST_SEL_SHIFT;
+                else if (txd->direction == DMA_FROM_DEVICE)
+                        txd->ccfg |= ch->signal << PL080_CONFIG_SRC_SEL_SHIFT;
         }
 
         dev_dbg(&pl08x->adev->dev, "allocated physical channel %d and signal %d for xfer on %s\n",
@@ -1076,19 +935,54 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan,
                  ch->signal,
                  plchan->name);
 
+        plchan->phychan_hold++;
         plchan->phychan = ch;
 
         return 0;
 }
 
+static void release_phy_channel(struct pl08x_dma_chan *plchan)
+{
+        struct pl08x_driver_data *pl08x = plchan->host;
+
+        if ((plchan->phychan->signal >= 0) && pl08x->pd->put_signal) {
+                pl08x->pd->put_signal(plchan);
+                plchan->phychan->signal = -1;
+        }
+        pl08x_put_phy_channel(pl08x, plchan->phychan);
+        plchan->phychan = NULL;
+}
+
 static dma_cookie_t pl08x_tx_submit(struct dma_async_tx_descriptor *tx)
 {
         struct pl08x_dma_chan *plchan = to_pl08x_chan(tx->chan);
+        struct pl08x_txd *txd = to_pl08x_txd(tx);
+        unsigned long flags;
 
-        atomic_inc(&plchan->last_issued);
-        tx->cookie = atomic_read(&plchan->last_issued);
-        /* This unlock follows the lock in the prep() function */
-        spin_unlock_irqrestore(&plchan->lock, plchan->lockflags);
+        spin_lock_irqsave(&plchan->lock, flags);
+
+        plchan->chan.cookie += 1;
+        if (plchan->chan.cookie < 0)
+                plchan->chan.cookie = 1;
+        tx->cookie = plchan->chan.cookie;
+
+        /* Put this onto the pending list */
+        list_add_tail(&txd->node, &plchan->pend_list);
+
+        /*
+         * If there was no physical channel available for this memcpy,
+         * stack the request up and indicate that the channel is waiting
+         * for a free physical channel.
+         */
+        if (!plchan->slave && !plchan->phychan) {
+                /* Do this memcpy whenever there is a channel ready */
+                plchan->state = PL08X_CHAN_WAITING;
+                plchan->waiting = txd;
+        } else {
+                plchan->phychan_hold--;
+        }
+
+        spin_unlock_irqrestore(&plchan->lock, flags);
 
         return tx->cookie;
 }
@@ -1102,10 +996,9 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
 }
 
 /*
- * Code accessing dma_async_is_complete() in a tight loop
- * may give problems - could schedule where indicated.
- * If slaves are relying on interrupts to signal completion this
- * function must not be called with interrupts disabled
+ * Code accessing dma_async_is_complete() in a tight loop may give problems.
+ * If slaves are relying on interrupts to signal completion this function
+ * must not be called with interrupts disabled.
  */
 static enum dma_status
 pl08x_dma_tx_status(struct dma_chan *chan,
@@ -1118,7 +1011,7 @@ pl08x_dma_tx_status(struct dma_chan *chan,
         enum dma_status ret;
         u32 bytesleft = 0;
 
-        last_used = atomic_read(&plchan->last_issued);
+        last_used = plchan->chan.cookie;
         last_complete = plchan->lc;
 
         ret = dma_async_is_complete(cookie, last_complete, last_used);
@@ -1128,13 +1021,9 @@ pl08x_dma_tx_status(struct dma_chan *chan,
         }
 
         /*
-         * schedule(); could be inserted here
-         */
-
-        /*
          * This cookie not complete yet
          */
-        last_used = atomic_read(&plchan->last_issued);
+        last_used = plchan->chan.cookie;
         last_complete = plchan->lc;
 
         /* Get number of bytes left in the active transactions and queue */
@@ -1199,37 +1088,35 @@ static const struct burst_table burst_sizes[] = {
         },
 };
 
-static void dma_set_runtime_config(struct dma_chan *chan,
-                               struct dma_slave_config *config)
+static int dma_set_runtime_config(struct dma_chan *chan,
+                                  struct dma_slave_config *config)
 {
         struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
         struct pl08x_driver_data *pl08x = plchan->host;
         struct pl08x_channel_data *cd = plchan->cd;
         enum dma_slave_buswidth addr_width;
+        dma_addr_t addr;
         u32 maxburst;
         u32 cctl = 0;
-        /* Mask out all except src and dst channel */
-        u32 ccfg = cd->ccfg & 0x000003DEU;
-        int i = 0;
+        int i;
+
+        if (!plchan->slave)
+                return -EINVAL;
 
         /* Transfer direction */
         plchan->runtime_direction = config->direction;
         if (config->direction == DMA_TO_DEVICE) {
-                plchan->runtime_addr = config->dst_addr;
-                cctl |= PL080_CONTROL_SRC_INCR;
-                ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+                addr = config->dst_addr;
                 addr_width = config->dst_addr_width;
                 maxburst = config->dst_maxburst;
         } else if (config->direction == DMA_FROM_DEVICE) {
-                plchan->runtime_addr = config->src_addr;
-                cctl |= PL080_CONTROL_DST_INCR;
-                ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+                addr = config->src_addr;
                 addr_width = config->src_addr_width;
                 maxburst = config->src_maxburst;
         } else {
                 dev_err(&pl08x->adev->dev,
                         "bad runtime_config: alien transfer direction\n");
-                return;
+                return -EINVAL;
         }
 
         switch (addr_width) {
@@ -1248,42 +1135,40 @@ static void dma_set_runtime_config(struct dma_chan *chan,
         default:
                 dev_err(&pl08x->adev->dev,
                         "bad runtime_config: alien address width\n");
-                return;
+                return -EINVAL;
         }
 
         /*
          * Now decide on a maxburst:
-         * If this channel will only request single transfers, set
-         * this down to ONE element.
+         * If this channel will only request single transfers, set this
+         * down to ONE element.  Also select one element if no maxburst
+         * is specified.
          */
-        if (plchan->cd->single) {
+        if (plchan->cd->single || maxburst == 0) {
                 cctl |= (PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT) |
                         (PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT);
         } else {
-                while (i < ARRAY_SIZE(burst_sizes)) {
+                for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
                         if (burst_sizes[i].burstwords <= maxburst)
                                 break;
-                        i++;
-                }
                 cctl |= burst_sizes[i].reg;
         }
 
-        /* Access the cell in privileged mode, non-bufferable, non-cacheable */
-        cctl &= ~PL080_CONTROL_PROT_MASK;
-        cctl |= PL080_CONTROL_PROT_SYS;
+        plchan->runtime_addr = addr;
 
         /* Modify the default channel data to fit PrimeCell request */
         cd->cctl = cctl;
-        cd->ccfg = ccfg;
 
         dev_dbg(&pl08x->adev->dev,
                 "configured channel %s (%s) for %s, data width %d, "
-                "maxburst %d words, LE, CCTL=%08x, CCFG=%08x\n",
+                "maxburst %d words, LE, CCTL=0x%08x\n",
                 dma_chan_name(chan), plchan->name,
                 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
                 addr_width,
                 maxburst,
-                cctl, ccfg);
+                cctl);
+
+        return 0;
 }
 
 /*
@@ -1293,35 +1178,26 @@ static void dma_set_runtime_config(struct dma_chan *chan,
 static void pl08x_issue_pending(struct dma_chan *chan)
 {
         struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
-        struct pl08x_driver_data *pl08x = plchan->host;
         unsigned long flags;
 
         spin_lock_irqsave(&plchan->lock, flags);
-        /* Something is already active */
-        if (plchan->at) {
-                        spin_unlock_irqrestore(&plchan->lock, flags);
-                        return;
-        }
-
-        /* Didn't get a physical channel so waiting for it ... */
-        if (plchan->state == PL08X_CHAN_WAITING)
+        /* Something is already active, or we're waiting for a channel... */
+        if (plchan->at || plchan->state == PL08X_CHAN_WAITING) {
+                spin_unlock_irqrestore(&plchan->lock, flags);
                 return;
+        }
 
         /* Take the first element in the queue and execute it */
-        if (!list_empty(&plchan->desc_list)) {
+        if (!list_empty(&plchan->pend_list)) {
                 struct pl08x_txd *next;
 
-                next = list_first_entry(&plchan->desc_list,
+                next = list_first_entry(&plchan->pend_list,
                                         struct pl08x_txd,
                                         node);
                 list_del(&next->node);
-                plchan->at = next;
                 plchan->state = PL08X_CHAN_RUNNING;
 
-                /* Configure the physical channel for the active txd */
-                pl08x_config_phychan_for_txd(plchan);
-                pl08x_set_cregs(pl08x, plchan->phychan);
-                pl08x_enable_phy_chan(pl08x, plchan->phychan);
+                pl08x_start_txd(plchan, next);
         }
 
         spin_unlock_irqrestore(&plchan->lock, flags);
@@ -1330,30 +1206,17 @@ static void pl08x_issue_pending(struct dma_chan *chan)
 static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan,
                                         struct pl08x_txd *txd)
 {
-        int num_llis;
         struct pl08x_driver_data *pl08x = plchan->host;
-        int ret;
+        unsigned long flags;
+        int num_llis, ret;
 
         num_llis = pl08x_fill_llis_for_desc(pl08x, txd);
-
-        if (!num_llis)
+        if (!num_llis) {
+                kfree(txd);
                 return -EINVAL;
+        }
 
-        spin_lock_irqsave(&plchan->lock, plchan->lockflags);
-
-        /*
-         * If this device is not using a circular buffer then
-         * queue this new descriptor for transfer.
-         * The descriptor for a circular buffer continues
-         * to be used until the channel is freed.
-         */
-        if (txd->cd->circular_buffer)
-                dev_err(&pl08x->adev->dev,
-                        "%s attempting to queue a circular buffer\n",
-                        __func__);
-        else
-                list_add_tail(&txd->node,
-                              &plchan->desc_list);
+        spin_lock_irqsave(&plchan->lock, flags);
 
         /*
          * See if we already have a physical channel allocated,
@@ -1362,45 +1225,74 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan,
         ret = prep_phy_channel(plchan, txd);
         if (ret) {
                 /*
-                 * No physical channel available, we will
-                 * stack up the memcpy channels until there is a channel
-                 * available to handle it whereas slave transfers may
-                 * have been denied due to platform channel muxing restrictions
-                 * and since there is no guarantee that this will ever be
-                 * resolved, and since the signal must be aquired AFTER
-                 * aquiring the physical channel, we will let them be NACK:ed
-                 * with -EBUSY here. The drivers can alway retry the prep()
-                 * call if they are eager on doing this using DMA.
+                 * No physical channel was available.
+                 *
+                 * memcpy transfers can be sorted out at submission time.
+                 *
+                 * Slave transfers may have been denied due to platform
+                 * channel muxing restrictions.  Since there is no guarantee
+                 * that this will ever be resolved, and the signal must be
+                 * acquired AFTER acquiring the physical channel, we will let
+                 * them be NACK:ed with -EBUSY here. The drivers can retry
+                 * the prep() call if they are eager on doing this using DMA.
                  */
                 if (plchan->slave) {
                         pl08x_free_txd_list(pl08x, plchan);
-                        spin_unlock_irqrestore(&plchan->lock, plchan->lockflags);
+                        pl08x_free_txd(pl08x, txd);
+                        spin_unlock_irqrestore(&plchan->lock, flags);
                         return -EBUSY;
                 }
-                /* Do this memcpy whenever there is a channel ready */
-                plchan->state = PL08X_CHAN_WAITING;
-                plchan->waiting = txd;
         } else
                 /*
-                 * Else we're all set, paused and ready to roll,
-                 * status will switch to PL08X_CHAN_RUNNING when
-                 * we call issue_pending(). If there is something
-                 * running on the channel already we don't change
-                 * its state.
+                 * Else we're all set, paused and ready to roll, status
+                 * will switch to PL08X_CHAN_RUNNING when we call
+                 * issue_pending(). If there is something running on the
+                 * channel already we don't change its state.
                  */
                 if (plchan->state == PL08X_CHAN_IDLE)
                         plchan->state = PL08X_CHAN_PAUSED;
 
-        /*
-         * Notice that we leave plchan->lock locked on purpose:
-         * it will be unlocked in the subsequent tx_submit()
-         * call. This is a consequence of the current API.
-         */
+        spin_unlock_irqrestore(&plchan->lock, flags);
 
         return 0;
 }
 
 /*
+ * Given the source and destination available bus masks, select which
+ * will be routed to each port.  We try to have source and destination
+ * on separate ports, but always respect the allowable settings.
+ */
+static u32 pl08x_select_bus(struct pl08x_driver_data *pl08x, u8 src, u8 dst)
+{
+        u32 cctl = 0;
+
+        if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
+                cctl |= PL080_CONTROL_DST_AHB2;
+        if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
+                cctl |= PL080_CONTROL_SRC_AHB2;
+
+        return cctl;
+}
+
+static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan,
+        unsigned long flags)
+{
+        struct pl08x_txd *txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT);
+
+        if (txd) {
+                dma_async_tx_descriptor_init(&txd->tx, &plchan->chan);
+                txd->tx.flags = flags;
+                txd->tx.tx_submit = pl08x_tx_submit;
+                INIT_LIST_HEAD(&txd->node);
+
+                /* Always enable error and terminal interrupts */
+                txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
+                            PL080_CONFIG_TC_IRQ_MASK;
+        }
+        return txd;
+}
+
+/*
  * Initialize a descriptor to be used by memcpy submit
  */
 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
@@ -1412,40 +1304,38 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
         struct pl08x_txd *txd;
         int ret;
 
-        txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT);
+        txd = pl08x_get_txd(plchan, flags);
         if (!txd) {
                 dev_err(&pl08x->adev->dev,
                         "%s no memory for descriptor\n", __func__);
                 return NULL;
         }
 
-        dma_async_tx_descriptor_init(&txd->tx, chan);
         txd->direction = DMA_NONE;
-        txd->srcbus.addr = src;
-        txd->dstbus.addr = dest;
+        txd->src_addr = src;
+        txd->dst_addr = dest;
+        txd->len = len;
 
         /* Set platform data for m2m */
-        txd->cd = &pl08x->pd->memcpy_channel;
+        txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+        txd->cctl = pl08x->pd->memcpy_channel.cctl &
+                        ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
+
         /* Both to be incremented or the code will break */
-        txd->cd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
-        txd->tx.tx_submit = pl08x_tx_submit;
-        txd->tx.callback = NULL;
-        txd->tx.callback_param = NULL;
-        txd->len = len;
+        txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
+
+        if (pl08x->vd->dualmaster)
+                txd->cctl |= pl08x_select_bus(pl08x,
+                                        pl08x->mem_buses, pl08x->mem_buses);
 
-        INIT_LIST_HEAD(&txd->node);
         ret = pl08x_prep_channel_resources(plchan, txd);
         if (ret)
                 return NULL;
-        /*
-         * NB: the channel lock is held at this point so tx_submit()
-         * must be called in direct succession.
-         */
 
         return &txd->tx;
 }
 
-struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
+static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
                 struct dma_chan *chan, struct scatterlist *sgl,
                 unsigned int sg_len, enum dma_data_direction direction,
                 unsigned long flags)
@@ -1453,6 +1343,7 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
         struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
         struct pl08x_driver_data *pl08x = plchan->host;
         struct pl08x_txd *txd;
+        u8 src_buses, dst_buses;
         int ret;
 
         /*
@@ -1467,14 +1358,12 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
         dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
                 __func__, sgl->length, plchan->name);
 
-        txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT);
+        txd = pl08x_get_txd(plchan, flags);
         if (!txd) {
                 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
                 return NULL;
         }
 
-        dma_async_tx_descriptor_init(&txd->tx, chan);
-
         if (direction != plchan->runtime_direction)
                 dev_err(&pl08x->adev->dev, "%s DMA setup does not match "
                         "the direction configured for the PrimeCell\n",
@@ -1486,37 +1375,47 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
          * channel target address dynamically at runtime.
          */
         txd->direction = direction;
+        txd->len = sgl->length;
+
+        txd->cctl = plchan->cd->cctl &
+                        ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
+                          PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
+                          PL080_CONTROL_PROT_MASK);
+
+        /* Access the cell in privileged mode, non-bufferable, non-cacheable */
+        txd->cctl |= PL080_CONTROL_PROT_SYS;
+
         if (direction == DMA_TO_DEVICE) {
-                txd->srcbus.addr = sgl->dma_address;
+                txd->ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+                txd->cctl |= PL080_CONTROL_SRC_INCR;
+                txd->src_addr = sgl->dma_address;
                 if (plchan->runtime_addr)
-                        txd->dstbus.addr = plchan->runtime_addr;
+                        txd->dst_addr = plchan->runtime_addr;
                 else
-                        txd->dstbus.addr = plchan->cd->addr;
+                        txd->dst_addr = plchan->cd->addr;
+                src_buses = pl08x->mem_buses;
+                dst_buses = plchan->cd->periph_buses;
         } else if (direction == DMA_FROM_DEVICE) {
+                txd->ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+                txd->cctl |= PL080_CONTROL_DST_INCR;
                 if (plchan->runtime_addr)
-                        txd->srcbus.addr = plchan->runtime_addr;
+                        txd->src_addr = plchan->runtime_addr;
                 else
-                        txd->srcbus.addr = plchan->cd->addr;
-                txd->dstbus.addr = sgl->dma_address;
+                        txd->src_addr = plchan->cd->addr;
+                txd->dst_addr = sgl->dma_address;
+                src_buses = plchan->cd->periph_buses;
+                dst_buses = pl08x->mem_buses;
         } else {
                 dev_err(&pl08x->adev->dev,
                         "%s direction unsupported\n", __func__);
                 return NULL;
         }
-        txd->cd = plchan->cd;
-        txd->tx.tx_submit = pl08x_tx_submit;
-        txd->tx.callback = NULL;
-        txd->tx.callback_param = NULL;
-        txd->len = sgl->length;
-        INIT_LIST_HEAD(&txd->node);
+
+        txd->cctl |= pl08x_select_bus(pl08x, src_buses, dst_buses);
 
         ret = pl08x_prep_channel_resources(plchan, txd);
         if (ret)
                 return NULL;
-        /*
-         * NB: the channel lock is held at this point so tx_submit()
-         * must be called in direct succession.
-         */
 
         return &txd->tx;
 }
@@ -1531,10 +1430,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 
         /* Controls applicable to inactive channels */
         if (cmd == DMA_SLAVE_CONFIG) {
-                dma_set_runtime_config(chan,
-                                       (struct dma_slave_config *)
-                                       arg);
-                return 0;
+                return dma_set_runtime_config(chan,
+                                              (struct dma_slave_config *)arg);
         }
 
         /*
@@ -1558,16 +1455,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
                          * Mark physical channel as free and free any slave
                          * signal
                          */
-                        if ((plchan->phychan->signal >= 0) &&
-                            pl08x->pd->put_signal) {
-                                pl08x->pd->put_signal(plchan);
-                                plchan->phychan->signal = -1;
-                        }
-                        pl08x_put_phy_channel(pl08x, plchan->phychan);
-                        plchan->phychan = NULL;
+                        release_phy_channel(plchan);
                 }
-                /* Stop any pending tasklet */
-                tasklet_disable(&plchan->tasklet);
                 /* Dequeue jobs and free LLIs */
                 if (plchan->at) {
                         pl08x_free_txd(pl08x, plchan->at);
@@ -1609,10 +1498,9 @@ bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
 
 /*
  * Just check that the device is there and active
- * TODO: turn this bit on/off depending on the number of
- * physical channels actually used, if it is zero... well
- * shut it off. That will save some power. Cut the clock
- * at the same time.
+ * TODO: turn this bit on/off depending on the number of physical channels
+ * actually used, if it is zero... well shut it off. That will save some
+ * power. Cut the clock at the same time.
  */
 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
 {
@@ -1620,78 +1508,66 @@ static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
 
         val = readl(pl08x->base + PL080_CONFIG);
         val &= ~(PL080_CONFIG_M2_BE | PL080_CONFIG_M1_BE | PL080_CONFIG_ENABLE);
-        /* We implictly clear bit 1 and that means little-endian mode */
+        /* We implicitly clear bit 1 and that means little-endian mode */
         val |= PL080_CONFIG_ENABLE;
         writel(val, pl08x->base + PL080_CONFIG);
 }
 
+static void pl08x_unmap_buffers(struct pl08x_txd *txd)
+{
+        struct device *dev = txd->tx.chan->device->dev;
+
+        if (!(txd->tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                if (txd->tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                        dma_unmap_single(dev, txd->src_addr, txd->len,
+                                DMA_TO_DEVICE);
+                else
+                        dma_unmap_page(dev, txd->src_addr, txd->len,
+                                DMA_TO_DEVICE);
+        }
+        if (!(txd->tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                if (txd->tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                        dma_unmap_single(dev, txd->dst_addr, txd->len,
+                                DMA_FROM_DEVICE);
+                else
+                        dma_unmap_page(dev, txd->dst_addr, txd->len,
+                                DMA_FROM_DEVICE);
+        }
+}
+
 static void pl08x_tasklet(unsigned long data)
 {
         struct pl08x_dma_chan *plchan = (struct pl08x_dma_chan *) data;
-        struct pl08x_phy_chan *phychan = plchan->phychan;
         struct pl08x_driver_data *pl08x = plchan->host;
+        struct pl08x_txd *txd;
+        unsigned long flags;
 
-        if (!plchan)
-                BUG();
-
-        spin_lock(&plchan->lock);
-
-        if (plchan->at) {
-                dma_async_tx_callback callback =
-                        plchan->at->tx.callback;
-                void *callback_param =
-                        plchan->at->tx.callback_param;
-
-                /*
-                 * Update last completed
-                 */
-                plchan->lc =
-                        (plchan->at->tx.cookie);
-
-                /*
-                 * Callback to signal completion
-                 */
-                if (callback)
-                        callback(callback_param);
+        spin_lock_irqsave(&plchan->lock, flags);
 
-                /*
-                 * Device callbacks should NOT clear
-                 * the current transaction on the channel
-                 * Linus: sometimes they should?
-                 */
-                if (!plchan->at)
-                        BUG();
+        txd = plchan->at;
+        plchan->at = NULL;
 
-                /*
-                 * Free the descriptor if it's not for a device
-                 * using a circular buffer
-                 */
-                if (!plchan->at->cd->circular_buffer) {
-                        pl08x_free_txd(pl08x, plchan->at);
-                        plchan->at = NULL;
-                }
-                /*
-                 * else descriptor for circular
-                 * buffers only freed when
-                 * client has disabled dma
-                 */
+        if (txd) {
+                /* Update last completed */
+                plchan->lc = txd->tx.cookie;
         }
-        /*
-         * If a new descriptor is queued, set it up
-         * plchan->at is NULL here
-         */
-        if (!list_empty(&plchan->desc_list)) {
+
+        /* If a new descriptor is queued, set it up plchan->at is NULL here */
+        if (!list_empty(&plchan->pend_list)) {
                 struct pl08x_txd *next;
 
-                next = list_first_entry(&plchan->desc_list,
+                next = list_first_entry(&plchan->pend_list,
                                         struct pl08x_txd,
                                         node);
                 list_del(&next->node);
-                plchan->at = next;
-                /* Configure the physical channel for the next txd */
-                pl08x_config_phychan_for_txd(plchan);
-                pl08x_set_cregs(pl08x, plchan->phychan);
-                pl08x_enable_phy_chan(pl08x, plchan->phychan);
+
+                pl08x_start_txd(plchan, next);
+        } else if (plchan->phychan_hold) {
+                /*
+                 * This channel is still in use - we have a new txd being
+                 * prepared and will soon be queued.  Don't give up the
+                 * physical channel.
+                 */
         } else {
                 struct pl08x_dma_chan *waiting = NULL;
 
@@ -1699,20 +1575,14 @@ static void pl08x_tasklet(unsigned long data)
                  * No more jobs, so free up the physical channel
                  * Free any allocated signal on slave transfers too
                  */
-                if ((phychan->signal >= 0) && pl08x->pd->put_signal) {
-                        pl08x->pd->put_signal(plchan);
-                        phychan->signal = -1;
-                }
-                pl08x_put_phy_channel(pl08x, phychan);
-                plchan->phychan = NULL;
+                release_phy_channel(plchan);
                 plchan->state = PL08X_CHAN_IDLE;
 
                 /*
-                 * And NOW before anyone else can grab that free:d
-                 * up physical channel, see if there is some memcpy
-                 * pending that seriously needs to start because of
-                 * being stacked up while we were choking the
-                 * physical channels with data.
+                 * And NOW before anyone else can grab that free:d up
+                 * physical channel, see if there is some memcpy pending
+                 * that seriously needs to start because of being stacked
+                 * up while we were choking the physical channels with data.
                  */
                 list_for_each_entry(waiting, &pl08x->memcpy.channels,
                                     chan.device_node) {
@@ -1724,6 +1594,7 @@ static void pl08x_tasklet(unsigned long data)
                                 ret = prep_phy_channel(waiting,
                                                        waiting->waiting);
                                 BUG_ON(ret);
+                                waiting->phychan_hold--;
                                 waiting->state = PL08X_CHAN_RUNNING;
                                 waiting->waiting = NULL;
                                 pl08x_issue_pending(&waiting->chan);
@@ -1732,7 +1603,25 @@ static void pl08x_tasklet(unsigned long data)
                 }
         }
 
-        spin_unlock(&plchan->lock);
+        spin_unlock_irqrestore(&plchan->lock, flags);
+
+        if (txd) {
+                dma_async_tx_callback callback = txd->tx.callback;
+                void *callback_param = txd->tx.callback_param;
+
+                /* Don't try to unmap buffers on slave channels */
+                if (!plchan->slave)
+                        pl08x_unmap_buffers(txd);
+
+                /* Free the descriptor */
+                spin_lock_irqsave(&plchan->lock, flags);
+                pl08x_free_txd(pl08x, txd);
+                spin_unlock_irqrestore(&plchan->lock, flags);
+
+                /* Callback to signal completion */
+                if (callback)
+                        callback(callback_param);
+        }
 }
 
 static irqreturn_t pl08x_irq(int irq, void *dev)
@@ -1744,9 +1633,7 @@ static irqreturn_t pl08x_irq(int irq, void *dev)
 
         val = readl(pl08x->base + PL080_ERR_STATUS);
         if (val) {
-                /*
-                 * An error interrupt (on one or more channels)
-                 */
+                /* An error interrupt (on one or more channels) */
                 dev_err(&pl08x->adev->dev,
                         "%s error interrupt, register value 0x%08x\n",
                                 __func__, val);
@@ -1770,9 +1657,7 @@ static irqreturn_t pl08x_irq(int irq, void *dev)
                         mask |= (1 << i);
                 }
         }
-        /*
-         * Clear only the terminal interrupts on channels we processed
-         */
+        /* Clear only the terminal interrupts on channels we processed */
         writel(mask, pl08x->base + PL080_TC_CLEAR);
 
         return mask ? IRQ_HANDLED : IRQ_NONE;
@@ -1791,6 +1676,7 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
         int i;
 
         INIT_LIST_HEAD(&dmadev->channels);
+
         /*
          * Register as many many memcpy as we have physical channels,
          * we won't always be able to use all but the code will have
@@ -1819,16 +1705,23 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
                                 return -ENOMEM;
                         }
                 }
+                if (chan->cd->circular_buffer) {
+                        dev_err(&pl08x->adev->dev,
+                                "channel %s: circular buffers not supported\n",
+                                chan->name);
+                        kfree(chan);
+                        continue;
+                }
                 dev_info(&pl08x->adev->dev,
                          "initialize virtual channel \"%s\"\n",
                          chan->name);
 
                 chan->chan.device = dmadev;
-                atomic_set(&chan->last_issued, 0);
-                chan->lc = atomic_read(&chan->last_issued);
+                chan->chan.cookie = 0;
+                chan->lc = 0;
 
                 spin_lock_init(&chan->lock);
-                INIT_LIST_HEAD(&chan->desc_list);
+                INIT_LIST_HEAD(&chan->pend_list);
                 tasklet_init(&chan->tasklet, pl08x_tasklet,
                              (unsigned long) chan);
 
@@ -1898,7 +1791,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data)
         seq_printf(s, "CHANNEL:\tSTATE:\n");
         seq_printf(s, "--------\t------\n");
         list_for_each_entry(chan, &pl08x->memcpy.channels, chan.device_node) {
-                seq_printf(s, "%s\t\t\%s\n", chan->name,
+                seq_printf(s, "%s\t\t%s\n", chan->name,
                            pl08x_state_str(chan->state));
         }
 
@@ -1906,7 +1799,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data)
         seq_printf(s, "CHANNEL:\tSTATE:\n");
         seq_printf(s, "--------\t------\n");
         list_for_each_entry(chan, &pl08x->slave.channels, chan.device_node) {
-                seq_printf(s, "%s\t\t\%s\n", chan->name,
+                seq_printf(s, "%s\t\t%s\n", chan->name,
                            pl08x_state_str(chan->state));
         }
 
@@ -1942,7 +1835,7 @@ static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
 static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
 {
         struct pl08x_driver_data *pl08x;
-        struct vendor_data *vd = id->data;
+        const struct vendor_data *vd = id->data;
         int ret = 0;
         int i;
 
@@ -1990,6 +1883,14 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
         pl08x->adev = adev;
         pl08x->vd = vd;
 
+        /* By default, AHB1 only.  If dualmaster, from platform */
+        pl08x->lli_buses = PL08X_AHB1;
+        pl08x->mem_buses = PL08X_AHB1;
+        if (pl08x->vd->dualmaster) {
+                pl08x->lli_buses = pl08x->pd->lli_buses;
+                pl08x->mem_buses = pl08x->pd->mem_buses;
+        }
+
         /* A DMA memory pool for LLIs, align on 1-byte boundary */
         pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
                         PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0);
@@ -2009,14 +1910,12 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
         /* Turn on the PL08x */
         pl08x_ensure_on(pl08x);
 
-        /*
-         * Attach the interrupt handler
-         */
+        /* Attach the interrupt handler */
         writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
         writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
 
         ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED,
-                          vd->name, pl08x);
+                          DRIVER_NAME, pl08x);
         if (ret) {
                 dev_err(&adev->dev, "%s failed to request interrupt %d\n",
                         __func__, adev->irq[0]);
@@ -2087,8 +1986,9 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
 
         amba_set_drvdata(adev, pl08x);
         init_pl08x_debugfs(pl08x);
-        dev_info(&pl08x->adev->dev, "ARM(R) %s DMA block initialized @%08x\n",
-                vd->name, adev->res.start);
+        dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n",
+                 amba_part(adev), amba_rev(adev),
+                 (unsigned long long)adev->res.start, adev->irq[0]);
         return 0;
 
 out_no_slave_reg:
@@ -2115,13 +2015,11 @@ out_no_pl08x:
 
 /* PL080 has 8 channels and the PL080 have just 2 */
 static struct vendor_data vendor_pl080 = {
-        .name = "PL080",
         .channels = 8,
         .dualmaster = true,
 };
 
 static struct vendor_data vendor_pl081 = {
-        .name = "PL081",
         .channels = 2,
         .dualmaster = false,
 };
@@ -2160,7 +2058,7 @@ static int __init pl08x_init(void)
         retval = amba_driver_register(&pl08x_amba_driver);
         if (retval)
                 printk(KERN_WARNING DRIVER_NAME
-                       "failed to register as an amba device (%d)\n",
+                       "failed to register as an AMBA device (%d)\n",
                        retval);
         return retval;
 }
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index ea0ee81cff5..3d7d705f026 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -253,7 +253,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
         /* move myself to free_list */
         list_move(&desc->desc_node, &atchan->free_list);
 
-        /* unmap dma addresses */
+        /* unmap dma addresses (not on slave channels) */
         if (!atchan->chan_common.private) {
                 struct device *parent = chan2parent(&atchan->chan_common);
                 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
@@ -583,7 +583,6 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                 desc->lli.ctrlb = ctrlb;
 
                 desc->txd.cookie = 0;
-                async_tx_ack(&desc->txd);
 
                 if (!first) {
                         first = desc;
@@ -604,7 +603,7 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
         /* set end-of-link to the last link descriptor of list*/
         set_desc_eol(desc);
 
-        desc->txd.flags = flags; /* client is in control of this ack */
+        first->txd.flags = flags; /* client is in control of this ack */
 
         return &first->txd;
 
@@ -670,7 +669,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                         if (!desc)
                                 goto err_desc_get;
 
-                        mem = sg_phys(sg);
+                        mem = sg_dma_address(sg);
                         len = sg_dma_len(sg);
                         mem_width = 2;
                         if (unlikely(mem & 3 || len & 3))
@@ -712,7 +711,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                         if (!desc)
                                 goto err_desc_get;
 
-                        mem = sg_phys(sg);
+                        mem = sg_dma_address(sg);
                         len = sg_dma_len(sg);
                         mem_width = 2;
                         if (unlikely(mem & 3 || len & 3))
@@ -749,8 +748,8 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
         first->txd.cookie = -EBUSY;
         first->len = total_len;
 
-        /* last link descriptor of list is responsible of flags */
-        prev->txd.flags = flags; /* client is in control of this ack */
+        /* first link descriptor of list is responsible of flags */
+        first->txd.flags = flags; /* client is in control of this ack */
 
         return &first->txd;
 
@@ -854,11 +853,11 @@ static void atc_issue_pending(struct dma_chan *chan)
 
         dev_vdbg(chan2dev(chan), "issue_pending\n");
 
+        spin_lock_bh(&atchan->lock);
         if (!atc_chan_is_enabled(atchan)) {
-                spin_lock_bh(&atchan->lock);
                 atc_advance_work(atchan);
-                spin_unlock_bh(&atchan->lock);
         }
+        spin_unlock_bh(&atchan->lock);
 }
 
 /**
@@ -1210,7 +1209,7 @@ static int __init at_dma_init(void)
 {
         return platform_driver_probe(&at_dma_driver, at_dma_probe);
 }
-module_init(at_dma_init);
+subsys_initcall(at_dma_init);
 
 static void __exit at_dma_exit(void)
 {
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index e5e172d2169..4de947a450f 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -1,7 +1,7 @@
 /*
  * Freescale MPC85xx, MPC83xx DMA Engine support
  *
- * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
+ * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
  *
  * Author:
  *   Zhang Wei <wei.zhang@freescale.com>, Jul 2007
@@ -1324,6 +1324,8 @@ static int __devinit fsldma_of_probe(struct platform_device *op,
         fdev->common.device_control = fsl_dma_device_control;
         fdev->common.dev = &op->dev;
 
+        dma_set_mask(&(op->dev), DMA_BIT_MASK(36));
+
         dev_set_drvdata(&op->dev, fdev);
 
         /*
diff --git a/drivers/dma/intel_mid_dma.c b/drivers/dma/intel_mid_dma.c
index 78266382797..798f46a4590 100644
--- a/drivers/dma/intel_mid_dma.c
+++ b/drivers/dma/intel_mid_dma.c
@@ -664,11 +664,20 @@ static struct dma_async_tx_descriptor *intel_mid_dma_prep_memcpy(
         /*calculate CTL_LO*/
         ctl_lo.ctl_lo = 0;
         ctl_lo.ctlx.int_en = 1;
-        ctl_lo.ctlx.dst_tr_width = mids->dma_slave.dst_addr_width;
-        ctl_lo.ctlx.src_tr_width = mids->dma_slave.src_addr_width;
         ctl_lo.ctlx.dst_msize = mids->dma_slave.src_maxburst;
         ctl_lo.ctlx.src_msize = mids->dma_slave.dst_maxburst;
 
+        /*
+         * Here we need some translation from "enum dma_slave_buswidth"
+         * to the format for our dma controller
+         *              standard        intel_mid_dmac's format
+         *               1 Byte                 0b000
+         *               2 Bytes                0b001
+         *               4 Bytes                0b010
+         */
+        ctl_lo.ctlx.dst_tr_width = mids->dma_slave.dst_addr_width / 2;
+        ctl_lo.ctlx.src_tr_width = mids->dma_slave.src_addr_width / 2;
+
         if (mids->cfg_mode == LNW_DMA_MEM_TO_MEM) {
                 ctl_lo.ctlx.tt_fc = 0;
                 ctl_lo.ctlx.sinc = 0;
@@ -746,8 +755,18 @@ static struct dma_async_tx_descriptor *intel_mid_dma_prep_slave_sg(
         BUG_ON(!mids);
 
         if (!midc->dma->pimr_mask) {
-                pr_debug("MDMA: SG list is not supported by this controller\n");
-                return  NULL;
+                /* We can still handle sg list with only one item */
+                if (sg_len == 1) {
+                        txd = intel_mid_dma_prep_memcpy(chan,
+                                                mids->dma_slave.dst_addr,
+                                                mids->dma_slave.src_addr,
+                                                sgl->length,
+                                                flags);
+                        return txd;
+                } else {
+                        pr_warn("MDMA: SG list is not supported by this controller\n");
+                        return  NULL;
+                }
         }
 
         pr_debug("MDMA: SG Length = %d, direction = %d, Flags = %#lx\n",
@@ -758,6 +777,7 @@ static struct dma_async_tx_descriptor *intel_mid_dma_prep_slave_sg(
                 pr_err("MDMA: Prep memcpy failed\n");
                 return NULL;
         }
+
         desc = to_intel_mid_dma_desc(txd);
         desc->dirn = direction;
         ctl_lo.ctl_lo = desc->ctl_lo;
@@ -1021,11 +1041,6 @@ static irqreturn_t intel_mid_dma_interrupt(int irq, void *data)
 
         /*DMA Interrupt*/
         pr_debug("MDMA:Got an interrupt on irq %d\n", irq);
-        if (!mid) {
-                pr_err("ERR_MDMA:null pointer mid\n");
-                return -EINVAL;
-        }
-
         pr_debug("MDMA: Status %x, Mask %x\n", tfr_status, mid->intr_mask);
         tfr_status &= mid->intr_mask;
         if (tfr_status) {
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 161c452923b..c6b01f535b2 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -1261,7 +1261,7 @@ out:
         return err;
 }
 
-#ifdef CONFIG_MD_RAID6_PQ
+#ifdef CONFIG_RAID6_PQ
 static int __devinit
 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
 {
@@ -1584,7 +1584,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
 
         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
-                #ifdef CONFIG_MD_RAID6_PQ
+                #ifdef CONFIG_RAID6_PQ
                 ret = iop_adma_pq_zero_sum_self_test(adev);
                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
                 #else
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index c064c89420d..1c38418ae61 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -1,6 +1,7 @@
 /*
  * Topcliff PCH DMA controller driver
  * Copyright (c) 2010 Intel Corporation
+ * Copyright (C) 2011 OKI SEMICONDUCTOR CO., LTD.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -921,12 +922,19 @@ static void __devexit pch_dma_remove(struct pci_dev *pdev)
 }
 
 /* PCI Device ID of DMA device */
-#define PCI_DEVICE_ID_PCH_DMA_8CH        0x8810
-#define PCI_DEVICE_ID_PCH_DMA_4CH        0x8815
+#define PCI_VENDOR_ID_ROHM             0x10DB
+#define PCI_DEVICE_ID_EG20T_PCH_DMA_8CH        0x8810
+#define PCI_DEVICE_ID_EG20T_PCH_DMA_4CH        0x8815
+#define PCI_DEVICE_ID_ML7213_DMA1_8CH   0x8026
+#define PCI_DEVICE_ID_ML7213_DMA2_8CH   0x802B
+#define PCI_DEVICE_ID_ML7213_DMA3_4CH   0x8034
 
 static const struct pci_device_id pch_dma_id_table[] = {
-        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH_DMA_8CH), 8 },
-        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH_DMA_4CH), 4 },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_8CH), 8 },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_4CH), 4 },
+        { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA1_8CH), 8}, /* UART Video */
+        { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA2_8CH), 8}, /* PCMIF SPI */
+        { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA3_4CH), 4}, /* FPGA */
         { 0, },
 };
 
@@ -954,6 +962,7 @@ static void __exit pch_dma_exit(void)
 module_init(pch_dma_init);
 module_exit(pch_dma_exit);
 
-MODULE_DESCRIPTION("Topcliff PCH DMA controller driver");
+MODULE_DESCRIPTION("Intel EG20T PCH / OKI SEMICONDUCTOR ML7213 IOH "
+                   "DMA controller driver");
 MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index fab68a55320..6e1d46a65d0 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -1,5 +1,6 @@
 /*
- * Copyright (C) ST-Ericsson SA 2007-2010
+ * Copyright (C) Ericsson AB 2007-2008
+ * Copyright (C) ST-Ericsson SA 2008-2010
  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
  * License terms: GNU General Public License (GPL) version 2
@@ -554,8 +555,66 @@ static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
         return d;
 }
 
-/* Support functions for logical channels */
+static int d40_psize_2_burst_size(bool is_log, int psize)
+{
+        if (is_log) {
+                if (psize == STEDMA40_PSIZE_LOG_1)
+                        return 1;
+        } else {
+                if (psize == STEDMA40_PSIZE_PHY_1)
+                        return 1;
+        }
+
+        return 2 << psize;
+}
+
+/*
+ * The dma only supports transmitting packages up to
+ * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
+ * dma elements required to send the entire sg list
+ */
+static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
+{
+        int dmalen;
+        u32 max_w = max(data_width1, data_width2);
+        u32 min_w = min(data_width1, data_width2);
+        u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
+
+        if (seg_max > STEDMA40_MAX_SEG_SIZE)
+                seg_max -= (1 << max_w);
+
+        if (!IS_ALIGNED(size, 1 << max_w))
+                return -EINVAL;
+
+        if (size <= seg_max)
+                dmalen = 1;
+        else {
+                dmalen = size / seg_max;
+                if (dmalen * seg_max < size)
+                        dmalen++;
+        }
+        return dmalen;
+}
+
+static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
+                           u32 data_width1, u32 data_width2)
+{
+        struct scatterlist *sg;
+        int i;
+        int len = 0;
+        int ret;
+
+        for_each_sg(sgl, sg, sg_len, i) {
+                ret = d40_size_2_dmalen(sg_dma_len(sg),
+                                        data_width1, data_width2);
+                if (ret < 0)
+                        return ret;
+                len += ret;
+        }
+        return len;
+}
 
+/* Support functions for logical channels */
 
 static int d40_channel_execute_command(struct d40_chan *d40c,
                                        enum d40_command command)
@@ -1241,6 +1300,21 @@ static int d40_validate_conf(struct d40_chan *d40c,
                 res = -EINVAL;
         }
 
+        if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
+            (1 << conf->src_info.data_width) !=
+            d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
+            (1 << conf->dst_info.data_width)) {
+                /*
+                 * The DMAC hardware only supports
+                 * src (burst x width) == dst (burst x width)
+                 */
+
+                dev_err(&d40c->chan.dev->device,
+                        "[%s] src (burst x width) != dst (burst x width)\n",
+                        __func__);
+                res = -EINVAL;
+        }
+
         return res;
 }
 
@@ -1638,13 +1712,21 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
         if (d40d == NULL)
                 goto err;
 
-        d40d->lli_len = sgl_len;
+        d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len,
+                                        d40c->dma_cfg.src_info.data_width,
+                                        d40c->dma_cfg.dst_info.data_width);
+        if (d40d->lli_len < 0) {
+                dev_err(&d40c->chan.dev->device,
+                        "[%s] Unaligned size\n", __func__);
+                goto err;
+        }
+
         d40d->lli_current = 0;
         d40d->txd.flags = dma_flags;
 
         if (d40c->log_num != D40_PHY_CHAN) {
 
-                if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) {
+                if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
                         dev_err(&d40c->chan.dev->device,
                                 "[%s] Out of memory\n", __func__);
                         goto err;
@@ -1654,15 +1736,17 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
                                          sgl_len,
                                          d40d->lli_log.src,
                                          d40c->log_def.lcsp1,
-                                         d40c->dma_cfg.src_info.data_width);
+                                         d40c->dma_cfg.src_info.data_width,
+                                         d40c->dma_cfg.dst_info.data_width);
 
                 (void) d40_log_sg_to_lli(sgl_dst,
                                          sgl_len,
                                          d40d->lli_log.dst,
                                          d40c->log_def.lcsp3,
-                                         d40c->dma_cfg.dst_info.data_width);
+                                         d40c->dma_cfg.dst_info.data_width,
+                                         d40c->dma_cfg.src_info.data_width);
         } else {
-                if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
+                if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
                         dev_err(&d40c->chan.dev->device,
                                 "[%s] Out of memory\n", __func__);
                         goto err;
@@ -1675,6 +1759,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
                                         virt_to_phys(d40d->lli_phy.src),
                                         d40c->src_def_cfg,
                                         d40c->dma_cfg.src_info.data_width,
+                                        d40c->dma_cfg.dst_info.data_width,
                                         d40c->dma_cfg.src_info.psize);
 
                 if (res < 0)
@@ -1687,6 +1772,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
                                         virt_to_phys(d40d->lli_phy.dst),
                                         d40c->dst_def_cfg,
                                         d40c->dma_cfg.dst_info.data_width,
+                                        d40c->dma_cfg.src_info.data_width,
                                         d40c->dma_cfg.dst_info.psize);
 
                 if (res < 0)
@@ -1826,7 +1912,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
         struct d40_chan *d40c = container_of(chan, struct d40_chan,
                                              chan);
         unsigned long flags;
-        int err = 0;
 
         if (d40c->phy_chan == NULL) {
                 dev_err(&d40c->chan.dev->device,
@@ -1844,6 +1929,15 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
         }
 
         d40d->txd.flags = dma_flags;
+        d40d->lli_len = d40_size_2_dmalen(size,
+                                          d40c->dma_cfg.src_info.data_width,
+                                          d40c->dma_cfg.dst_info.data_width);
+        if (d40d->lli_len < 0) {
+                dev_err(&d40c->chan.dev->device,
+                        "[%s] Unaligned size\n", __func__);
+                goto err;
+        }
+
 
         dma_async_tx_descriptor_init(&d40d->txd, chan);
 
@@ -1851,37 +1945,40 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
 
         if (d40c->log_num != D40_PHY_CHAN) {
 
-                if (d40_pool_lli_alloc(d40d, 1, true) < 0) {
+                if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
                         dev_err(&d40c->chan.dev->device,
                                 "[%s] Out of memory\n", __func__);
                         goto err;
                 }
-                d40d->lli_len = 1;
                 d40d->lli_current = 0;
 
-                d40_log_fill_lli(d40d->lli_log.src,
-                                 src,
-                                 size,
-                                 d40c->log_def.lcsp1,
-                                 d40c->dma_cfg.src_info.data_width,
-                                 true);
+                if (d40_log_buf_to_lli(d40d->lli_log.src,
+                                       src,
+                                       size,
+                                       d40c->log_def.lcsp1,
+                                       d40c->dma_cfg.src_info.data_width,
+                                       d40c->dma_cfg.dst_info.data_width,
+                                       true) == NULL)
+                        goto err;
 
-                d40_log_fill_lli(d40d->lli_log.dst,
-                                 dst,
-                                 size,
-                                 d40c->log_def.lcsp3,
-                                 d40c->dma_cfg.dst_info.data_width,
-                                 true);
+                if (d40_log_buf_to_lli(d40d->lli_log.dst,
+                                       dst,
+                                       size,
+                                       d40c->log_def.lcsp3,
+                                       d40c->dma_cfg.dst_info.data_width,
+                                       d40c->dma_cfg.src_info.data_width,
+                                       true) == NULL)
+                        goto err;
 
         } else {
 
-                if (d40_pool_lli_alloc(d40d, 1, false) < 0) {
+                if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
                         dev_err(&d40c->chan.dev->device,
                                 "[%s] Out of memory\n", __func__);
                         goto err;
                 }
 
-                err = d40_phy_fill_lli(d40d->lli_phy.src,
+                if (d40_phy_buf_to_lli(d40d->lli_phy.src,
                                        src,
                                        size,
                                        d40c->dma_cfg.src_info.psize,
@@ -1889,11 +1986,11 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
                                        d40c->src_def_cfg,
                                        true,
                                        d40c->dma_cfg.src_info.data_width,
-                                       false);
-                if (err)
-                        goto err_fill_lli;
+                                       d40c->dma_cfg.dst_info.data_width,
+                                       false) == NULL)
+                        goto err;
 
-                err = d40_phy_fill_lli(d40d->lli_phy.dst,
+                if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
                                        dst,
                                        size,
                                        d40c->dma_cfg.dst_info.psize,
@@ -1901,10 +1998,9 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
                                        d40c->dst_def_cfg,
                                        true,
                                        d40c->dma_cfg.dst_info.data_width,
-                                       false);
-
-                if (err)
-                        goto err_fill_lli;
+                                       d40c->dma_cfg.src_info.data_width,
+                                       false) == NULL)
+                        goto err;
 
                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
                                       d40d->lli_pool.size, DMA_TO_DEVICE);
@@ -1913,9 +2009,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
         spin_unlock_irqrestore(&d40c->lock, flags);
         return &d40d->txd;
 
-err_fill_lli:
-        dev_err(&d40c->chan.dev->device,
-                "[%s] Failed filling in PHY LLI\n", __func__);
 err:
         if (d40d)
                 d40_desc_free(d40c, d40d);
@@ -1945,13 +2038,21 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d,
         dma_addr_t dev_addr = 0;
         int total_size;
 
-        if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) {
+        d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
+                                        d40c->dma_cfg.src_info.data_width,
+                                        d40c->dma_cfg.dst_info.data_width);
+        if (d40d->lli_len < 0) {
+                dev_err(&d40c->chan.dev->device,
+                        "[%s] Unaligned size\n", __func__);
+                return -EINVAL;
+        }
+
+        if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
                 dev_err(&d40c->chan.dev->device,
                         "[%s] Out of memory\n", __func__);
                 return -ENOMEM;
         }
 
-        d40d->lli_len = sg_len;
         d40d->lli_current = 0;
 
         if (direction == DMA_FROM_DEVICE)
@@ -1993,13 +2094,21 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
         dma_addr_t dst_dev_addr;
         int res;
 
-        if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
+        d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
+                                        d40c->dma_cfg.src_info.data_width,
+                                        d40c->dma_cfg.dst_info.data_width);
+        if (d40d->lli_len < 0) {
+                dev_err(&d40c->chan.dev->device,
+                        "[%s] Unaligned size\n", __func__);
+                return -EINVAL;
+        }
+
+        if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
                 dev_err(&d40c->chan.dev->device,
                         "[%s] Out of memory\n", __func__);
                 return -ENOMEM;
         }
 
-        d40d->lli_len = sgl_len;
         d40d->lli_current = 0;
 
         if (direction == DMA_FROM_DEVICE) {
@@ -2024,6 +2133,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
                                 virt_to_phys(d40d->lli_phy.src),
                                 d40c->src_def_cfg,
                                 d40c->dma_cfg.src_info.data_width,
+                                d40c->dma_cfg.dst_info.data_width,
                                 d40c->dma_cfg.src_info.psize);
         if (res < 0)
                 return res;
@@ -2035,6 +2145,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
                                 virt_to_phys(d40d->lli_phy.dst),
                                 d40c->dst_def_cfg,
                                 d40c->dma_cfg.dst_info.data_width,
+                                d40c->dma_cfg.src_info.data_width,
                                 d40c->dma_cfg.dst_info.psize);
         if (res < 0)
                 return res;
@@ -2244,6 +2355,8 @@ static void d40_set_runtime_config(struct dma_chan *chan,
                         psize = STEDMA40_PSIZE_PHY_8;
                 else if (config_maxburst >= 4)
                         psize = STEDMA40_PSIZE_PHY_4;
+                else if (config_maxburst >= 2)
+                        psize = STEDMA40_PSIZE_PHY_2;
                 else
                         psize = STEDMA40_PSIZE_PHY_1;
         }
diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c
index 8557cb88b25..0b096a38322 100644
--- a/drivers/dma/ste_dma40_ll.c
+++ b/drivers/dma/ste_dma40_ll.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) ST-Ericsson SA 2007-2010
- * Author: Per Friden <per.friden@stericsson.com> for ST-Ericsson
+ * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
  * License terms: GNU General Public License (GPL) version 2
  */
@@ -122,15 +122,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
         *dst_cfg = dst;
 }
 
-int d40_phy_fill_lli(struct d40_phy_lli *lli,
-                     dma_addr_t data,
-                     u32 data_size,
-                     int psize,
-                     dma_addr_t next_lli,
-                     u32 reg_cfg,
-                     bool term_int,
-                     u32 data_width,
-                     bool is_device)
+static int d40_phy_fill_lli(struct d40_phy_lli *lli,
+                            dma_addr_t data,
+                            u32 data_size,
+                            int psize,
+                            dma_addr_t next_lli,
+                            u32 reg_cfg,
+                            bool term_int,
+                            u32 data_width,
+                            bool is_device)
 {
         int num_elems;
 
@@ -139,13 +139,6 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli,
         else
                 num_elems = 2 << psize;
 
-        /*
-         * Size is 16bit. data_width is 8, 16, 32 or 64 bit
-         * Block large than 64 KiB must be split.
-         */
-        if (data_size > (0xffff << data_width))
-                return -EINVAL;
-
         /* Must be aligned */
         if (!IS_ALIGNED(data, 0x1 << data_width))
                 return -EINVAL;
@@ -187,55 +180,118 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli,
         return 0;
 }
 
+static int d40_seg_size(int size, int data_width1, int data_width2)
+{
+        u32 max_w = max(data_width1, data_width2);
+        u32 min_w = min(data_width1, data_width2);
+        u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
+
+        if (seg_max > STEDMA40_MAX_SEG_SIZE)
+                seg_max -= (1 << max_w);
+
+        if (size <= seg_max)
+                return size;
+
+        if (size <= 2 * seg_max)
+                return ALIGN(size / 2, 1 << max_w);
+
+        return seg_max;
+}
+
+struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
+                                       dma_addr_t addr,
+                                       u32 size,
+                                       int psize,
+                                       dma_addr_t lli_phys,
+                                       u32 reg_cfg,
+                                       bool term_int,
+                                       u32 data_width1,
+                                       u32 data_width2,
+                                       bool is_device)
+{
+        int err;
+        dma_addr_t next = lli_phys;
+        int size_rest = size;
+        int size_seg = 0;
+
+        do {
+                size_seg = d40_seg_size(size_rest, data_width1, data_width2);
+                size_rest -= size_seg;
+
+                if (term_int && size_rest == 0)
+                        next = 0;
+                else
+                        next = ALIGN(next + sizeof(struct d40_phy_lli),
+                                     D40_LLI_ALIGN);
+
+                err = d40_phy_fill_lli(lli,
+                                       addr,
+                                       size_seg,
+                                       psize,
+                                       next,
+                                       reg_cfg,
+                                       !next,
+                                       data_width1,
+                                       is_device);
+
+                if (err)
+                        goto err;
+
+                lli++;
+                if (!is_device)
+                        addr += size_seg;
+        } while (size_rest);
+
+        return lli;
+
+ err:
+        return NULL;
+}
+
 int d40_phy_sg_to_lli(struct scatterlist *sg,
                       int sg_len,
                       dma_addr_t target,
-                      struct d40_phy_lli *lli,
+                      struct d40_phy_lli *lli_sg,
                       dma_addr_t lli_phys,
                       u32 reg_cfg,
-                      u32 data_width,
+                      u32 data_width1,
+                      u32 data_width2,
                       int psize)
 {
         int total_size = 0;
         int i;
         struct scatterlist *current_sg = sg;
-        dma_addr_t next_lli_phys;
         dma_addr_t dst;
-        int err = 0;
+        struct d40_phy_lli *lli = lli_sg;
+        dma_addr_t l_phys = lli_phys;
 
         for_each_sg(sg, current_sg, sg_len, i) {
 
                 total_size += sg_dma_len(current_sg);
 
-                /* If this scatter list entry is the last one, no next link */
-                if (sg_len - 1 == i)
-                        next_lli_phys = 0;
-                else
-                        next_lli_phys = ALIGN(lli_phys + (i + 1) *
-                                              sizeof(struct d40_phy_lli),
-                                              D40_LLI_ALIGN);
-
                 if (target)
                         dst = target;
                 else
                         dst = sg_phys(current_sg);
 
-                err = d40_phy_fill_lli(&lli[i],
-                                       dst,
-                                       sg_dma_len(current_sg),
-                                       psize,
-                                       next_lli_phys,
-                                       reg_cfg,
-                                       !next_lli_phys,
-                                       data_width,
-                                       target == dst);
-                if (err)
-                        goto err;
+                l_phys = ALIGN(lli_phys + (lli - lli_sg) *
+                               sizeof(struct d40_phy_lli), D40_LLI_ALIGN);
+
+                lli = d40_phy_buf_to_lli(lli,
+                                         dst,
+                                         sg_dma_len(current_sg),
+                                         psize,
+                                         l_phys,
+                                         reg_cfg,
+                                         sg_len - 1 == i,
+                                         data_width1,
+                                         data_width2,
+                                         target == dst);
+                if (lli == NULL)
+                        return -EINVAL;
         }
 
         return total_size;
-err:
-        return err;
 }
 
 
@@ -315,17 +371,20 @@ void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
         writel(lli_dst->lcsp13, &lcla[1].lcsp13);
 }
 
-void d40_log_fill_lli(struct d40_log_lli *lli,
-                      dma_addr_t data, u32 data_size,
-                      u32 reg_cfg,
-                      u32 data_width,
-                      bool addr_inc)
+static void d40_log_fill_lli(struct d40_log_lli *lli,
+                             dma_addr_t data, u32 data_size,
+                             u32 reg_cfg,
+                             u32 data_width,
+                             bool addr_inc)
 {
         lli->lcsp13 = reg_cfg;
 
         /* The number of elements to transfer */
         lli->lcsp02 = ((data_size >> data_width) <<
                        D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK;
+
+        BUG_ON((data_size >> data_width) > STEDMA40_MAX_SEG_SIZE);
+
         /* 16 LSBs address of the current element */
         lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK;
         /* 16 MSBs address of the current element */
@@ -348,55 +407,94 @@ int d40_log_sg_to_dev(struct scatterlist *sg,
         int total_size = 0;
         struct scatterlist *current_sg = sg;
         int i;
+        struct d40_log_lli *lli_src = lli->src;
+        struct d40_log_lli *lli_dst = lli->dst;
 
         for_each_sg(sg, current_sg, sg_len, i) {
                 total_size += sg_dma_len(current_sg);
 
                 if (direction == DMA_TO_DEVICE) {
-                        d40_log_fill_lli(&lli->src[i],
-                                         sg_phys(current_sg),
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp1, src_data_width,
-                                         true);
-                        d40_log_fill_lli(&lli->dst[i],
-                                         dev_addr,
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp3, dst_data_width,
-                                         false);
+                        lli_src =
+                                d40_log_buf_to_lli(lli_src,
+                                                   sg_phys(current_sg),
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp1, src_data_width,
+                                                   dst_data_width,
+                                                   true);
+                        lli_dst =
+                                d40_log_buf_to_lli(lli_dst,
+                                                   dev_addr,
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp3, dst_data_width,
+                                                   src_data_width,
+                                                   false);
                 } else {
-                        d40_log_fill_lli(&lli->dst[i],
-                                         sg_phys(current_sg),
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp3, dst_data_width,
-                                         true);
-                        d40_log_fill_lli(&lli->src[i],
-                                         dev_addr,
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp1, src_data_width,
-                                         false);
+                        lli_dst =
+                                d40_log_buf_to_lli(lli_dst,
+                                                   sg_phys(current_sg),
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp3, dst_data_width,
+                                                   src_data_width,
+                                                   true);
+                        lli_src =
+                                d40_log_buf_to_lli(lli_src,
+                                                   dev_addr,
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp1, src_data_width,
+                                                   dst_data_width,
+                                                   false);
                 }
         }
         return total_size;
 }
 
+struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
+                                       dma_addr_t addr,
+                                       int size,
+                                       u32 lcsp13, /* src or dst*/
+                                       u32 data_width1,
+                                       u32 data_width2,
+                                       bool addr_inc)
+{
+        struct d40_log_lli *lli = lli_sg;
+        int size_rest = size;
+        int size_seg = 0;
+
+        do {
+                size_seg = d40_seg_size(size_rest, data_width1, data_width2);
+                size_rest -= size_seg;
+
+                d40_log_fill_lli(lli,
+                                 addr,
+                                 size_seg,
+                                 lcsp13, data_width1,
+                                 addr_inc);
+                if (addr_inc)
+                        addr += size_seg;
+                lli++;
+        } while (size_rest);
+
+        return lli;
+}
+
 int d40_log_sg_to_lli(struct scatterlist *sg,
                       int sg_len,
                       struct d40_log_lli *lli_sg,
                       u32 lcsp13, /* src or dst*/
-                      u32 data_width)
+                      u32 data_width1, u32 data_width2)
 {
         int total_size = 0;
         struct scatterlist *current_sg = sg;
         int i;
+        struct d40_log_lli *lli = lli_sg;
 
         for_each_sg(sg, current_sg, sg_len, i) {
                 total_size += sg_dma_len(current_sg);
-
-                d40_log_fill_lli(&lli_sg[i],
-                                 sg_phys(current_sg),
-                                 sg_dma_len(current_sg),
-                                 lcsp13, data_width,
-                                 true);
+                lli = d40_log_buf_to_lli(lli,
+                                         sg_phys(current_sg),
+                                         sg_dma_len(current_sg),
+                                         lcsp13,
+                                         data_width1, data_width2, true);
         }
         return total_size;
 }
diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h
index 9e419b90754..9cc43495bea 100644
--- a/drivers/dma/ste_dma40_ll.h
+++ b/drivers/dma/ste_dma40_ll.h
@@ -292,18 +292,20 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
                       struct d40_phy_lli *lli,
                       dma_addr_t lli_phys,
                       u32 reg_cfg,
-                      u32 data_width,
+                      u32 data_width1,
+                      u32 data_width2,
                       int psize);
 
-int d40_phy_fill_lli(struct d40_phy_lli *lli,
-                     dma_addr_t data,
-                     u32 data_size,
-                     int psize,
-                     dma_addr_t next_lli,
-                     u32 reg_cfg,
-                     bool term_int,
-                     u32 data_width,
-                     bool is_device);
+struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
+                                       dma_addr_t data,
+                                       u32 data_size,
+                                       int psize,
+                                       dma_addr_t next_lli,
+                                       u32 reg_cfg,
+                                       bool term_int,
+                                       u32 data_width1,
+                                       u32 data_width2,
+                                       bool is_device);
 
 void d40_phy_lli_write(void __iomem *virtbase,
                        u32 phy_chan_num,
@@ -312,12 +314,12 @@ void d40_phy_lli_write(void __iomem *virtbase,
 
 /* Logical channels */
 
-void d40_log_fill_lli(struct d40_log_lli *lli,
-                      dma_addr_t data,
-                      u32 data_size,
-                      u32 reg_cfg,
-                      u32 data_width,
-                      bool addr_inc);
+struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
+                                       dma_addr_t addr,
+                                       int size,
+                                       u32 lcsp13, /* src or dst*/
+                                       u32 data_width1, u32 data_width2,
+                                       bool addr_inc);
 
 int d40_log_sg_to_dev(struct scatterlist *sg,
                       int sg_len,
@@ -332,7 +334,7 @@ int d40_log_sg_to_lli(struct scatterlist *sg,
                       int sg_len,
                       struct d40_log_lli *lli_sg,
                       u32 lcsp13, /* src or dst*/
-                      u32 data_width);
+                      u32 data_width1, u32 data_width2);
 
 void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
                             struct d40_log_lli *lli_dst,