Merge tag 'dmaengine-4.19-rc1' of git://git.infradead.org/users/vkoul/slave-dma

Pull DMAengine updates from Vinod Koul:
 "This round brings couple of framework changes, a new driver and usual
  driver updates:

   - new managed helper for dmaengine framework registration

   - split dmaengine pause capability to pause and resume and allow
     drivers to report that individually

   - update dma_request_chan_by_mask() to handle deferred probing

   - move imx-sdma to use virt-dma

   - new driver for Actions Semi Owl family S900 controller

   - minor updates to intel, renesas, mv_xor, pl330 etc"

* tag 'dmaengine-4.19-rc1' of git://git.infradead.org/users/vkoul/slave-dma: (46 commits)
  dmaengine: Add Actions Semi Owl family S900 DMA driver
  dt-bindings: dmaengine: Add binding for Actions Semi Owl SoCs
  dmaengine: sh: rcar-dmac: Should not stop the DMAC by rcar_dmac_sync_tcr()
  dmaengine: mic_x100_dma: use the new helper to simplify the code
  dmaengine: add a new helper dmaenginem_async_device_register
  dmaengine: imx-sdma: add memcpy interface
  dmaengine: imx-sdma: add SDMA_BD_MAX_CNT to replace '0xffff'
  dmaengine: dma_request_chan_by_mask() to handle deferred probing
  dmaengine: pl330: fix irq race with terminate_all
  dmaengine: Revert "dmaengine: mv_xor_v2: enable COMPILE_TEST"
  dmaengine: mv_xor_v2: use {lower,upper}_32_bits to configure HW descriptor address
  dmaengine: mv_xor_v2: enable COMPILE_TEST
  dmaengine: mv_xor_v2: move unmap to before callback
  dmaengine: mv_xor_v2: convert callback to helper function
  dmaengine: mv_xor_v2: kill the tasklets upon exit
  dmaengine: mv_xor_v2: explicitly freeup irq
  dmaengine: sh: rcar-dmac: Add dma_pause operation
  dmaengine: sh: rcar-dmac: add a new function to clear CHCR.DE with barrier
  dmaengine: idma64: Support dmaengine_terminate_sync()
  dmaengine: hsu: Support dmaengine_terminate_sync()
  ...

26 files changed, 1600 insertions, 307 deletions

diff --git a/Documentation/devicetree/bindings/dma/owl-dma.txt b/Documentation/devicetree/bindings/dma/owl-dma.txt
new file mode 100644
index 000000000000..03e9bb12b75f
--- /dev/null
+++ b/Documentation/devicetree/bindings/dma/owl-dma.txt
@@ -0,0 +1,47 @@
+* Actions Semi Owl SoCs DMA controller
+
+This binding follows the generic DMA bindings defined in dma.txt.
+
+Required properties:
+- compatible: Should be "actions,s900-dma".
+- reg: Should contain DMA registers location and length.
+- interrupts: Should contain 4 interrupts shared by all channel.
+- #dma-cells: Must be <1>. Used to represent the number of integer
+              cells in the dmas property of client device.
+- dma-channels: Physical channels supported.
+- dma-requests: Number of DMA request signals supported by the controller.
+                Refer to Documentation/devicetree/bindings/dma/dma.txt
+- clocks: Phandle and Specifier of the clock feeding the DMA controller.
+
+Example:
+
+Controller:
+                dma: dma-controller@e0260000 {
+                        compatible = "actions,s900-dma";
+                        reg = <0x0 0xe0260000 0x0 0x1000>;
+                        interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+                                     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+                                     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+                                     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
+                        #dma-cells = <1>;
+                        dma-channels = <12>;
+                        dma-requests = <46>;
+                        clocks = <&clock CLK_DMAC>;
+                };
+
+Client:
+
+DMA clients connected to the Actions Semi Owl SoCs DMA controller must
+use the format described in the dma.txt file, using a two-cell specifier
+for each channel.
+
+The two cells in order are:
+1. A phandle pointing to the DMA controller.
+2. The channel id.
+
+uart5: serial@e012a000 {
+        ...
+        dma-names = "tx", "rx";
+        dmas = <&dma 26>, <&dma 27>;
+        ...
+};
diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
index b1ba639554c0..946229c48657 100644
--- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
+++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
@@ -29,6 +29,7 @@ Required Properties:
                 - "renesas,dmac-r8a77965" (R-Car M3-N)
                 - "renesas,dmac-r8a77970" (R-Car V3M)
                 - "renesas,dmac-r8a77980" (R-Car V3H)
+                - "renesas,dmac-r8a77990" (R-Car E3)
                 - "renesas,dmac-r8a77995" (R-Car D3)
 
 - reg: base address and length of the registers block for the DMAC
diff --git a/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt b/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt
index a2b8bfaec43c..174af2c45e77 100644
--- a/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt
+++ b/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt
@@ -66,6 +66,8 @@ Optional child node properties:
 Optional child node properties for VDMA:
 - xlnx,genlock-mode: Tells Genlock synchronization is
         enabled/disabled in hardware.
+- xlnx,enable-vert-flip: Tells vertical flip is
+        enabled/disabled in hardware(S2MM path).
 Optional child node properties for AXI DMA:
 -dma-channels: Number of dma channels in child node.
 
diff --git a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt
index 7c1bb3d0c222..43681ca0837f 100644
--- a/Documentation/driver-model/devres.txt
+++ b/Documentation/driver-model/devres.txt
@@ -240,6 +240,7 @@ CLOCK
   devm_of_clk_add_hw_provider()
 
 DMA
+  dmaenginem_async_device_register()
   dmam_alloc_coherent()
   dmam_alloc_attrs()
   dmam_declare_coherent_memory()
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index 56bd612927ab..80dc567801ec 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -42,6 +42,8 @@ static struct page *pq_scribble_page;
 #define P(b, d) (b[d-2])
 #define Q(b, d) (b[d-1])
 
+#define MAX_DISKS 255
+
 /**
  * do_async_gen_syndrome - asynchronously calculate P and/or Q
  */
@@ -184,7 +186,7 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
         struct dma_device *device = chan ? chan->device : NULL;
         struct dmaengine_unmap_data *unmap = NULL;
 
-        BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks)));
+        BUG_ON(disks > MAX_DISKS || !(P(blocks, disks) || Q(blocks, disks)));
 
         if (device)
                 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT);
@@ -196,7 +198,7 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
             is_dma_pq_aligned(device, offset, 0, len)) {
                 struct dma_async_tx_descriptor *tx;
                 enum dma_ctrl_flags dma_flags = 0;
-                unsigned char coefs[src_cnt];
+                unsigned char coefs[MAX_DISKS];
                 int i, j;
 
                 /* run the p+q asynchronously */
@@ -299,11 +301,11 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
         struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len);
         struct dma_device *device = chan ? chan->device : NULL;
         struct dma_async_tx_descriptor *tx;
-        unsigned char coefs[disks-2];
+        unsigned char coefs[MAX_DISKS];
         enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;
         struct dmaengine_unmap_data *unmap = NULL;
 
-        BUG_ON(disks < 4);
+        BUG_ON(disks < 4 || disks > MAX_DISKS);
 
         if (device)
                 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT);
diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c
index dad95f45b88f..a5edaabae12a 100644
--- a/crypto/async_tx/raid6test.c
+++ b/crypto/async_tx/raid6test.c
@@ -81,11 +81,13 @@ static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, stru
                         init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
                         tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit);
                 } else {
-                        struct page *blocks[disks];
+                        struct page *blocks[NDISKS];
                         struct page *dest;
                         int count = 0;
                         int i;
 
+                        BUG_ON(disks > NDISKS);
+
                         /* data+Q failure.  Reconstruct data from P,
                          * then rebuild syndrome
                          */
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index ca1680afa20a..dacf3f42426d 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -250,6 +250,7 @@ config IMX_SDMA
         tristate "i.MX SDMA support"
         depends on ARCH_MXC
         select DMA_ENGINE
+        select DMA_VIRTUAL_CHANNELS
         help
           Support the i.MX SDMA engine. This engine is integrated into
           Freescale i.MX25/31/35/51/53/6 chips.
@@ -413,6 +414,14 @@ config NBPFAXI_DMA
         help
           Support for "Type-AXI" NBPF DMA IPs from Renesas
 
+config OWL_DMA
+        tristate "Actions Semi Owl SoCs DMA support"
+        depends on ARCH_ACTIONS
+        select DMA_ENGINE
+        select DMA_VIRTUAL_CHANNELS
+        help
+          Enable support for the Actions Semi Owl SoCs DMA controller.
+
 config PCH_DMA
         tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA"
         depends on PCI && (X86_32 || COMPILE_TEST)
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 203a99d68315..c91702d88b95 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -52,6 +52,7 @@ obj-$(CONFIG_MV_XOR_V2) += mv_xor_v2.o
 obj-$(CONFIG_MXS_DMA) += mxs-dma.o
 obj-$(CONFIG_MX3_IPU) += ipu/
 obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o
+obj-$(CONFIG_OWL_DMA) += owl-dma.o
 obj-$(CONFIG_PCH_DMA) += pch_dma.o
 obj-$(CONFIG_PL330_DMA) += pl330.o
 obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 08ba8473a284..272bed6c8ba7 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -500,12 +500,8 @@ int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
         caps->max_burst = device->max_burst;
         caps->residue_granularity = device->residue_granularity;
         caps->descriptor_reuse = device->descriptor_reuse;
-
-        /*
-         * Some devices implement only pause (e.g. to get residuum) but no
-         * resume. However cmd_pause is advertised as pause AND resume.
-         */
-        caps->cmd_pause = !!(device->device_pause && device->device_resume);
+        caps->cmd_pause = !!device->device_pause;
+        caps->cmd_resume = !!device->device_resume;
         caps->cmd_terminate = !!device->device_terminate_all;
 
         return 0;
@@ -774,8 +770,14 @@ struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
                 return ERR_PTR(-ENODEV);
 
         chan = __dma_request_channel(mask, NULL, NULL);
-        if (!chan)
-                chan = ERR_PTR(-ENODEV);
+        if (!chan) {
+                mutex_lock(&dma_list_mutex);
+                if (list_empty(&dma_device_list))
+                        chan = ERR_PTR(-EPROBE_DEFER);
+                else
+                        chan = ERR_PTR(-ENODEV);
+                mutex_unlock(&dma_list_mutex);
+        }
 
         return chan;
 }
@@ -1139,6 +1141,41 @@ void dma_async_device_unregister(struct dma_device *device)
 }
 EXPORT_SYMBOL(dma_async_device_unregister);
 
+static void dmam_device_release(struct device *dev, void *res)
+{
+        struct dma_device *device;
+
+        device = *(struct dma_device **)res;
+        dma_async_device_unregister(device);
+}
+
+/**
+ * dmaenginem_async_device_register - registers DMA devices found
+ * @device: &dma_device
+ *
+ * The operation is managed and will be undone on driver detach.
+ */
+int dmaenginem_async_device_register(struct dma_device *device)
+{
+        void *p;
+        int ret;
+
+        p = devres_alloc(dmam_device_release, sizeof(void *), GFP_KERNEL);
+        if (!p)
+                return -ENOMEM;
+
+        ret = dma_async_device_register(device);
+        if (!ret) {
+                *(struct dma_device **)p = device;
+                devres_add(device->dev, p);
+        } else {
+                devres_free(p);
+        }
+
+        return ret;
+}
+EXPORT_SYMBOL(dmaenginem_async_device_register);
+
 struct dmaengine_unmap_pool {
         struct kmem_cache *cache;
         const char *name;
diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c
index 29d04ca71d52..202ffa9f7611 100644
--- a/drivers/dma/hsu/hsu.c
+++ b/drivers/dma/hsu/hsu.c
@@ -413,6 +413,13 @@ static void hsu_dma_free_chan_resources(struct dma_chan *chan)
         vchan_free_chan_resources(to_virt_chan(chan));
 }
 
+static void hsu_dma_synchronize(struct dma_chan *chan)
+{
+        struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
+
+        vchan_synchronize(&hsuc->vchan);
+}
+
 int hsu_dma_probe(struct hsu_dma_chip *chip)
 {
         struct hsu_dma *hsu;
@@ -459,6 +466,7 @@ int hsu_dma_probe(struct hsu_dma_chip *chip)
         hsu->dma.device_pause = hsu_dma_pause;
         hsu->dma.device_resume = hsu_dma_resume;
         hsu->dma.device_terminate_all = hsu_dma_terminate_all;
+        hsu->dma.device_synchronize = hsu_dma_synchronize;
 
         hsu->dma.src_addr_widths = HSU_DMA_BUSWIDTHS;
         hsu->dma.dst_addr_widths = HSU_DMA_BUSWIDTHS;
diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c
index e5c911200bdb..1fbf9cb9b742 100644
--- a/drivers/dma/idma64.c
+++ b/drivers/dma/idma64.c
@@ -496,6 +496,13 @@ static int idma64_terminate_all(struct dma_chan *chan)
         return 0;
 }
 
+static void idma64_synchronize(struct dma_chan *chan)
+{
+        struct idma64_chan *idma64c = to_idma64_chan(chan);
+
+        vchan_synchronize(&idma64c->vchan);
+}
+
 static int idma64_alloc_chan_resources(struct dma_chan *chan)
 {
         struct idma64_chan *idma64c = to_idma64_chan(chan);
@@ -583,6 +590,7 @@ static int idma64_probe(struct idma64_chip *chip)
         idma64->dma.device_pause = idma64_pause;
         idma64->dma.device_resume = idma64_resume;
         idma64->dma.device_terminate_all = idma64_terminate_all;
+        idma64->dma.device_synchronize = idma64_synchronize;
 
         idma64->dma.src_addr_widths = IDMA64_BUSWIDTHS;
         idma64->dma.dst_addr_widths = IDMA64_BUSWIDTHS;
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index f077992635c2..b4ec2d20e661 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -24,6 +24,7 @@
 #include <linux/spinlock.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
 #include <linux/firmware.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
@@ -41,6 +42,7 @@
 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
 
 #include "dmaengine.h"
+#include "virt-dma.h"
 
 /* SDMA registers */
 #define SDMA_H_C0PTR            0x000
@@ -183,6 +185,7 @@
  * Mode/Count of data node descriptors - IPCv2
  */
 struct sdma_mode_count {
+#define SDMA_BD_MAX_CNT 0xffff
         u32 count   : 16; /* size of the buffer pointed by this BD */
         u32 status  :  8; /* E,R,I,C,W,D status bits stored here */
         u32 command :  8; /* command mostly used for channel 0 */
@@ -200,9 +203,9 @@ struct sdma_buffer_descriptor {
 /**
  * struct sdma_channel_control - Channel control Block
  *
- * @current_bd_ptr      current buffer descriptor processed
- * @base_bd_ptr         first element of buffer descriptor array
- * @unused              padding. The SDMA engine expects an array of 128 byte
+ * @current_bd_ptr:     current buffer descriptor processed
+ * @base_bd_ptr:        first element of buffer descriptor array
+ * @unused:             padding. The SDMA engine expects an array of 128 byte
  *                      control blocks
  */
 struct sdma_channel_control {
@@ -215,10 +218,13 @@ struct sdma_channel_control {
  * struct sdma_state_registers - SDMA context for a channel
  *
  * @pc:         program counter
+ * @unused1:    unused
  * @t:          test bit: status of arithmetic & test instruction
  * @rpc:        return program counter
+ * @unused0:    unused
  * @sf:         source fault while loading data
  * @spc:        loop start program counter
+ * @unused2:    unused
  * @df:         destination fault while storing data
  * @epc:        loop end program counter
  * @lm:         loop mode
@@ -256,6 +262,14 @@ struct sdma_state_registers {
  * @dsa:                dedicated core source address register
  * @ds:                 dedicated core status register
  * @dd:                 dedicated core data register
+ * @scratch0:           1st word of dedicated ram for context switch
+ * @scratch1:           2nd word of dedicated ram for context switch
+ * @scratch2:           3rd word of dedicated ram for context switch
+ * @scratch3:           4th word of dedicated ram for context switch
+ * @scratch4:           5th word of dedicated ram for context switch
+ * @scratch5:           6th word of dedicated ram for context switch
+ * @scratch6:           7th word of dedicated ram for context switch
+ * @scratch7:           8th word of dedicated ram for context switch
  */
 struct sdma_context_data {
         struct sdma_state_registers  channel_state;
@@ -284,25 +298,67 @@ struct sdma_context_data {
         u32  scratch7;
 } __attribute__ ((packed));
 
-#define NUM_BD (int)(PAGE_SIZE / sizeof(struct sdma_buffer_descriptor))
 
 struct sdma_engine;
 
 /**
+ * struct sdma_desc - descriptor structor for one transfer
+ * @vd:                 descriptor for virt dma
+ * @num_bd:             number of descriptors currently handling
+ * @bd_phys:            physical address of bd
+ * @buf_tail:           ID of the buffer that was processed
+ * @buf_ptail:          ID of the previous buffer that was processed
+ * @period_len:         period length, used in cyclic.
+ * @chn_real_count:     the real count updated from bd->mode.count
+ * @chn_count:          the transfer count set
+ * @sdmac:              sdma_channel pointer
+ * @bd:                 pointer of allocate bd
+ */
+struct sdma_desc {
+        struct virt_dma_desc    vd;
+        unsigned int            num_bd;
+        dma_addr_t              bd_phys;
+        unsigned int            buf_tail;
+        unsigned int            buf_ptail;
+        unsigned int            period_len;
+        unsigned int            chn_real_count;
+        unsigned int            chn_count;
+        struct sdma_channel     *sdmac;
+        struct sdma_buffer_descriptor *bd;
+};
+
+/**
  * struct sdma_channel - housekeeping for a SDMA channel
  *
- * @sdma                pointer to the SDMA engine for this channel
- * @channel             the channel number, matches dmaengine chan_id + 1
- * @direction           transfer type. Needed for setting SDMA script
- * @peripheral_type     Peripheral type. Needed for setting SDMA script
- * @event_id0           aka dma request line
- * @event_id1           for channels that use 2 events
- * @word_size           peripheral access size
- * @buf_tail            ID of the buffer that was processed
- * @buf_ptail           ID of the previous buffer that was processed
- * @num_bd              max NUM_BD. number of descriptors currently handling
+ * @vc:                 virt_dma base structure
+ * @desc:               sdma description including vd and other special member
+ * @sdma:               pointer to the SDMA engine for this channel
+ * @channel:            the channel number, matches dmaengine chan_id + 1
+ * @direction:          transfer type. Needed for setting SDMA script
+ * @peripheral_type:    Peripheral type. Needed for setting SDMA script
+ * @event_id0:          aka dma request line
+ * @event_id1:          for channels that use 2 events
+ * @word_size:          peripheral access size
+ * @pc_from_device:     script address for those device_2_memory
+ * @pc_to_device:       script address for those memory_2_device
+ * @device_to_device:   script address for those device_2_device
+ * @pc_to_pc:           script address for those memory_2_memory
+ * @flags:              loop mode or not
+ * @per_address:        peripheral source or destination address in common case
+ *                      destination address in p_2_p case
+ * @per_address2:       peripheral source address in p_2_p case
+ * @event_mask:         event mask used in p_2_p script
+ * @watermark_level:    value for gReg[7], some script will extend it from
+ *                      basic watermark such as p_2_p
+ * @shp_addr:           value for gReg[6]
+ * @per_addr:           value for gReg[2]
+ * @status:             status of dma channel
+ * @data:               specific sdma interface structure
+ * @bd_pool:            dma_pool for bd
  */
 struct sdma_channel {
+        struct virt_dma_chan            vc;
+        struct sdma_desc                *desc;
         struct sdma_engine              *sdma;
         unsigned int                    channel;
         enum dma_transfer_direction             direction;
@@ -310,28 +366,17 @@ struct sdma_channel {
         unsigned int                    event_id0;
         unsigned int                    event_id1;
         enum dma_slave_buswidth         word_size;
-        unsigned int                    buf_tail;
-        unsigned int                    buf_ptail;
-        unsigned int                    num_bd;
-        unsigned int                    period_len;
-        struct sdma_buffer_descriptor   *bd;
-        dma_addr_t                      bd_phys;
         unsigned int                    pc_from_device, pc_to_device;
         unsigned int                    device_to_device;
+        unsigned int                    pc_to_pc;
         unsigned long                   flags;
         dma_addr_t                      per_address, per_address2;
         unsigned long                   event_mask[2];
         unsigned long                   watermark_level;
         u32                             shp_addr, per_addr;
-        struct dma_chan                 chan;
-        spinlock_t                      lock;
-        struct dma_async_tx_descriptor  desc;
         enum dma_status                 status;
-        unsigned int                    chn_count;
-        unsigned int                    chn_real_count;
-        struct tasklet_struct           tasklet;
         struct imx_dma_data             data;
-        bool                            enabled;
+        struct dma_pool                 *bd_pool;
 };
 
 #define IMX_DMA_SG_LOOP         BIT(0)
@@ -346,15 +391,15 @@ struct sdma_channel {
 /**
  * struct sdma_firmware_header - Layout of the firmware image
  *
- * @magic               "SDMA"
- * @version_major       increased whenever layout of struct sdma_script_start_addrs
- *                      changes.
- * @version_minor       firmware minor version (for binary compatible changes)
- * @script_addrs_start  offset of struct sdma_script_start_addrs in this image
- * @num_script_addrs    Number of script addresses in this image
- * @ram_code_start      offset of SDMA ram image in this firmware image
- * @ram_code_size       size of SDMA ram image
- * @script_addrs        Stores the start address of the SDMA scripts
+ * @magic:              "SDMA"
+ * @version_major:      increased whenever layout of struct
+ *                      sdma_script_start_addrs changes.
+ * @version_minor:      firmware minor version (for binary compatible changes)
+ * @script_addrs_start: offset of struct sdma_script_start_addrs in this image
+ * @num_script_addrs:   Number of script addresses in this image
+ * @ram_code_start:     offset of SDMA ram image in this firmware image
+ * @ram_code_size:      size of SDMA ram image
+ * @script_addrs:       Stores the start address of the SDMA scripts
  *                      (in SDMA memory space)
  */
 struct sdma_firmware_header {
@@ -391,6 +436,8 @@ struct sdma_engine {
         u32                             spba_start_addr;
         u32                             spba_end_addr;
         unsigned int                    irq;
+        dma_addr_t                      bd0_phys;
+        struct sdma_buffer_descriptor   *bd0;
 };
 
 static struct sdma_driver_data sdma_imx31 = {
@@ -590,14 +637,7 @@ static int sdma_config_ownership(struct sdma_channel *sdmac,
 
 static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
 {
-        unsigned long flags;
-        struct sdma_channel *sdmac = &sdma->channel[channel];
-
         writel(BIT(channel), sdma->regs + SDMA_H_START);
-
-        spin_lock_irqsave(&sdmac->lock, flags);
-        sdmac->enabled = true;
-        spin_unlock_irqrestore(&sdmac->lock, flags);
 }
 
 /*
@@ -625,7 +665,7 @@ static int sdma_run_channel0(struct sdma_engine *sdma)
 static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
                 u32 address)
 {
-        struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;
+        struct sdma_buffer_descriptor *bd0 = sdma->bd0;
         void *buf_virt;
         dma_addr_t buf_phys;
         int ret;
@@ -681,26 +721,49 @@ static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
         writel_relaxed(val, sdma->regs + chnenbl);
 }
 
+static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
+{
+        return container_of(t, struct sdma_desc, vd.tx);
+}
+
+static void sdma_start_desc(struct sdma_channel *sdmac)
+{
+        struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
+        struct sdma_desc *desc;
+        struct sdma_engine *sdma = sdmac->sdma;
+        int channel = sdmac->channel;
+
+        if (!vd) {
+                sdmac->desc = NULL;
+                return;
+        }
+        sdmac->desc = desc = to_sdma_desc(&vd->tx);
+        /*
+         * Do not delete the node in desc_issued list in cyclic mode, otherwise
+         * the desc allocated will never be freed in vchan_dma_desc_free_list
+         */
+        if (!(sdmac->flags & IMX_DMA_SG_LOOP))
+                list_del(&vd->node);
+
+        sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
+        sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
+        sdma_enable_channel(sdma, sdmac->channel);
+}
+
 static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 {
         struct sdma_buffer_descriptor *bd;
         int error = 0;
         enum dma_status old_status = sdmac->status;
-        unsigned long flags;
-
-        spin_lock_irqsave(&sdmac->lock, flags);
-        if (!sdmac->enabled) {
-                spin_unlock_irqrestore(&sdmac->lock, flags);
-                return;
-        }
-        spin_unlock_irqrestore(&sdmac->lock, flags);
 
         /*
          * loop mode. Iterate over descriptors, re-setup them and
          * call callback function.
          */
-        while (1) {
-                bd = &sdmac->bd[sdmac->buf_tail];
+        while (sdmac->desc) {
+                struct sdma_desc *desc = sdmac->desc;
+
+                bd = &desc->bd[desc->buf_tail];
 
                 if (bd->mode.status & BD_DONE)
                         break;
@@ -716,11 +779,11 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
                 * the number of bytes present in the current buffer descriptor.
                 */
 
-                sdmac->chn_real_count = bd->mode.count;
+                desc->chn_real_count = bd->mode.count;
                 bd->mode.status |= BD_DONE;
-                bd->mode.count = sdmac->period_len;
-                sdmac->buf_ptail = sdmac->buf_tail;
-                sdmac->buf_tail = (sdmac->buf_tail + 1) % sdmac->num_bd;
+                bd->mode.count = desc->period_len;
+                desc->buf_ptail = desc->buf_tail;
+                desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
 
                 /*
                  * The callback is called from the interrupt context in order
@@ -728,41 +791,38 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
                  * SDMA transaction status by the time the client tasklet is
                  * executed.
                  */
-
-                dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL);
+                spin_unlock(&sdmac->vc.lock);
+                dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
+                spin_lock(&sdmac->vc.lock);
 
                 if (error)
                         sdmac->status = old_status;
         }
 }
 
-static void mxc_sdma_handle_channel_normal(unsigned long data)
+static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
 {
         struct sdma_channel *sdmac = (struct sdma_channel *) data;
         struct sdma_buffer_descriptor *bd;
         int i, error = 0;
 
-        sdmac->chn_real_count = 0;
+        sdmac->desc->chn_real_count = 0;
         /*
          * non loop mode. Iterate over all descriptors, collect
          * errors and call callback function
          */
-        for (i = 0; i < sdmac->num_bd; i++) {
-                bd = &sdmac->bd[i];
+        for (i = 0; i < sdmac->desc->num_bd; i++) {
+                bd = &sdmac->desc->bd[i];
 
                  if (bd->mode.status & (BD_DONE | BD_RROR))
                         error = -EIO;
-                 sdmac->chn_real_count += bd->mode.count;
+                 sdmac->desc->chn_real_count += bd->mode.count;
         }
 
         if (error)
                 sdmac->status = DMA_ERROR;
         else
                 sdmac->status = DMA_COMPLETE;
-
-        dma_cookie_complete(&sdmac->desc);
-
-        dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL);
 }
 
 static irqreturn_t sdma_int_handler(int irq, void *dev_id)
@@ -778,12 +838,21 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id)
         while (stat) {
                 int channel = fls(stat) - 1;
                 struct sdma_channel *sdmac = &sdma->channel[channel];
+                struct sdma_desc *desc;
+
+                spin_lock(&sdmac->vc.lock);
+                desc = sdmac->desc;
+                if (desc) {
+                        if (sdmac->flags & IMX_DMA_SG_LOOP) {
+                                sdma_update_channel_loop(sdmac);
+                        } else {
+                                mxc_sdma_handle_channel_normal(sdmac);
+                                vchan_cookie_complete(&desc->vd);
+                                sdma_start_desc(sdmac);
+                        }
+                }
 
-                if (sdmac->flags & IMX_DMA_SG_LOOP)
-                        sdma_update_channel_loop(sdmac);
-                else
-                        tasklet_schedule(&sdmac->tasklet);
-
+                spin_unlock(&sdmac->vc.lock);
                 __clear_bit(channel, &stat);
         }
 
@@ -802,14 +871,16 @@ static void sdma_get_pc(struct sdma_channel *sdmac,
          * These are needed once we start to support transfers between
          * two peripherals or memory-to-memory transfers
          */
-        int per_2_per = 0;
+        int per_2_per = 0, emi_2_emi = 0;
 
         sdmac->pc_from_device = 0;
         sdmac->pc_to_device = 0;
         sdmac->device_to_device = 0;
+        sdmac->pc_to_pc = 0;
 
         switch (peripheral_type) {
         case IMX_DMATYPE_MEMORY:
+                emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
                 break;
         case IMX_DMATYPE_DSP:
                 emi_2_per = sdma->script_addrs->bp_2_ap_addr;
@@ -882,6 +953,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac,
         sdmac->pc_from_device = per_2_emi;
         sdmac->pc_to_device = emi_2_per;
         sdmac->device_to_device = per_2_per;
+        sdmac->pc_to_pc = emi_2_emi;
 }
 
 static int sdma_load_context(struct sdma_channel *sdmac)
@@ -890,7 +962,7 @@ static int sdma_load_context(struct sdma_channel *sdmac)
         int channel = sdmac->channel;
         int load_address;
         struct sdma_context_data *context = sdma->context;
-        struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;
+        struct sdma_buffer_descriptor *bd0 = sdma->bd0;
         int ret;
         unsigned long flags;
 
@@ -898,6 +970,8 @@ static int sdma_load_context(struct sdma_channel *sdmac)
                 load_address = sdmac->pc_from_device;
         else if (sdmac->direction == DMA_DEV_TO_DEV)
                 load_address = sdmac->device_to_device;
+        else if (sdmac->direction == DMA_MEM_TO_MEM)
+                load_address = sdmac->pc_to_pc;
         else
                 load_address = sdmac->pc_to_device;
 
@@ -939,7 +1013,7 @@ static int sdma_load_context(struct sdma_channel *sdmac)
 
 static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
 {
-        return container_of(chan, struct sdma_channel, chan);
+        return container_of(chan, struct sdma_channel, vc.chan);
 }
 
 static int sdma_disable_channel(struct dma_chan *chan)
@@ -947,21 +1021,25 @@ static int sdma_disable_channel(struct dma_chan *chan)
         struct sdma_channel *sdmac = to_sdma_chan(chan);
         struct sdma_engine *sdma = sdmac->sdma;
         int channel = sdmac->channel;
-        unsigned long flags;
 
         writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
         sdmac->status = DMA_ERROR;
 
-        spin_lock_irqsave(&sdmac->lock, flags);
-        sdmac->enabled = false;
-        spin_unlock_irqrestore(&sdmac->lock, flags);
-
         return 0;
 }
 
 static int sdma_disable_channel_with_delay(struct dma_chan *chan)
 {
+        struct sdma_channel *sdmac = to_sdma_chan(chan);
+        unsigned long flags;
+        LIST_HEAD(head);
+
         sdma_disable_channel(chan);
+        spin_lock_irqsave(&sdmac->vc.lock, flags);
+        vchan_get_all_descriptors(&sdmac->vc, &head);
+        sdmac->desc = NULL;
+        spin_unlock_irqrestore(&sdmac->vc.lock, flags);
+        vchan_dma_desc_free_list(&sdmac->vc, &head);
 
         /*
          * According to NXP R&D team a delay of one BD SDMA cost time
@@ -1090,52 +1168,81 @@ static int sdma_set_channel_priority(struct sdma_channel *sdmac,
         return 0;
 }
 
-static int sdma_request_channel(struct sdma_channel *sdmac)
+static int sdma_request_channel0(struct sdma_engine *sdma)
 {
-        struct sdma_engine *sdma = sdmac->sdma;
-        int channel = sdmac->channel;
         int ret = -EBUSY;
 
-        sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys,
-                                        GFP_KERNEL);
-        if (!sdmac->bd) {
+        sdma->bd0 = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdma->bd0_phys,
+                                        GFP_NOWAIT);
+        if (!sdma->bd0) {
                 ret = -ENOMEM;
                 goto out;
         }
 
-        sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys;
-        sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
+        sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
+        sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
 
-        sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY);
+        sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
         return 0;
 out:
 
         return ret;
 }
 
-static dma_cookie_t sdma_tx_submit(struct dma_async_tx_descriptor *tx)
+
+static int sdma_alloc_bd(struct sdma_desc *desc)
 {
-        unsigned long flags;
-        struct sdma_channel *sdmac = to_sdma_chan(tx->chan);
-        dma_cookie_t cookie;
+        int ret = 0;
 
-        spin_lock_irqsave(&sdmac->lock, flags);
+        desc->bd = dma_pool_alloc(desc->sdmac->bd_pool, GFP_NOWAIT,
+                                  &desc->bd_phys);
+        if (!desc->bd) {
+                ret = -ENOMEM;
+                goto out;
+        }
+out:
+        return ret;
+}
 
-        cookie = dma_cookie_assign(tx);
+static void sdma_free_bd(struct sdma_desc *desc)
+{
+        dma_pool_free(desc->sdmac->bd_pool, desc->bd, desc->bd_phys);
+}
 
-        spin_unlock_irqrestore(&sdmac->lock, flags);
+static void sdma_desc_free(struct virt_dma_desc *vd)
+{
+        struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
 
-        return cookie;
+        sdma_free_bd(desc);
+        kfree(desc);
 }
 
 static int sdma_alloc_chan_resources(struct dma_chan *chan)
 {
         struct sdma_channel *sdmac = to_sdma_chan(chan);
         struct imx_dma_data *data = chan->private;
+        struct imx_dma_data mem_data;
         int prio, ret;
 
-        if (!data)
-                return -EINVAL;
+        /*
+         * MEMCPY may never setup chan->private by filter function such as
+         * dmatest, thus create 'struct imx_dma_data mem_data' for this case.
+         * Please note in any other slave case, you have to setup chan->private
+         * with 'struct imx_dma_data' in your own filter function if you want to
+         * request dma channel by dma_request_channel() rather than
+         * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear
+         * to warn you to correct your filter function.
+         */
+        if (!data) {
+                dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
+                mem_data.priority = 2;
+                mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
+                mem_data.dma_request = 0;
+                mem_data.dma_request2 = 0;
+                data = &mem_data;
+
+                sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
+        }
 
         switch (data->priority) {
         case DMA_PRIO_HIGH:
@@ -1161,18 +1268,13 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan)
         if (ret)
                 goto disable_clk_ipg;
 
-        ret = sdma_request_channel(sdmac);
-        if (ret)
-                goto disable_clk_ahb;
-
         ret = sdma_set_channel_priority(sdmac, prio);
         if (ret)
                 goto disable_clk_ahb;
 
-        dma_async_tx_descriptor_init(&sdmac->desc, chan);
-        sdmac->desc.tx_submit = sdma_tx_submit;
-        /* txd.flags will be overwritten in prep funcs */
-        sdmac->desc.flags = DMA_CTRL_ACK;
+        sdmac->bd_pool = dma_pool_create("bd_pool", chan->device->dev,
+                                sizeof(struct sdma_buffer_descriptor),
+                                32, 0);
 
         return 0;
 
@@ -1188,7 +1290,7 @@ static void sdma_free_chan_resources(struct dma_chan *chan)
         struct sdma_channel *sdmac = to_sdma_chan(chan);
         struct sdma_engine *sdma = sdmac->sdma;
 
-        sdma_disable_channel(chan);
+        sdma_disable_channel_with_delay(chan);
 
         if (sdmac->event_id0)
                 sdma_event_disable(sdmac, sdmac->event_id0);
@@ -1200,10 +1302,105 @@ static void sdma_free_chan_resources(struct dma_chan *chan)
 
         sdma_set_channel_priority(sdmac, 0);
 
-        dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys);
-
         clk_disable(sdma->clk_ipg);
         clk_disable(sdma->clk_ahb);
+
+        dma_pool_destroy(sdmac->bd_pool);
+        sdmac->bd_pool = NULL;
+}
+
+static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
+                                enum dma_transfer_direction direction, u32 bds)
+{
+        struct sdma_desc *desc;
+
+        desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
+        if (!desc)
+                goto err_out;
+
+        sdmac->status = DMA_IN_PROGRESS;
+        sdmac->direction = direction;
+        sdmac->flags = 0;
+
+        desc->chn_count = 0;
+        desc->chn_real_count = 0;
+        desc->buf_tail = 0;
+        desc->buf_ptail = 0;
+        desc->sdmac = sdmac;
+        desc->num_bd = bds;
+
+        if (sdma_alloc_bd(desc))
+                goto err_desc_out;
+
+        /* No slave_config called in MEMCPY case, so do here */
+        if (direction == DMA_MEM_TO_MEM)
+                sdma_config_ownership(sdmac, false, true, false);
+
+        if (sdma_load_context(sdmac))
+                goto err_desc_out;
+
+        return desc;
+
+err_desc_out:
+        kfree(desc);
+err_out:
+        return NULL;
+}
+
+static struct dma_async_tx_descriptor *sdma_prep_memcpy(
+                struct dma_chan *chan, dma_addr_t dma_dst,
+                dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+        struct sdma_channel *sdmac = to_sdma_chan(chan);
+        struct sdma_engine *sdma = sdmac->sdma;
+        int channel = sdmac->channel;
+        size_t count;
+        int i = 0, param;
+        struct sdma_buffer_descriptor *bd;
+        struct sdma_desc *desc;
+
+        if (!chan || !len)
+                return NULL;
+
+        dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
+                &dma_src, &dma_dst, len, channel);
+
+        desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
+                                        len / SDMA_BD_MAX_CNT + 1);
+        if (!desc)
+                return NULL;
+
+        do {
+                count = min_t(size_t, len, SDMA_BD_MAX_CNT);
+                bd = &desc->bd[i];
+                bd->buffer_addr = dma_src;
+                bd->ext_buffer_addr = dma_dst;
+                bd->mode.count = count;
+                desc->chn_count += count;
+                bd->mode.command = 0;
+
+                dma_src += count;
+                dma_dst += count;
+                len -= count;
+                i++;
+
+                param = BD_DONE | BD_EXTD | BD_CONT;
+                /* last bd */
+                if (!len) {
+                        param |= BD_INTR;
+                        param |= BD_LAST;
+                        param &= ~BD_CONT;
+                }
+
+                dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
+                                i, count, bd->buffer_addr,
+                                param & BD_WRAP ? "wrap" : "",
+                                param & BD_INTR ? " intr" : "");
+
+                bd->mode.status = param;
+        } while (len);
+
+        return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
 }
 
 static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
@@ -1213,75 +1410,54 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
 {
         struct sdma_channel *sdmac = to_sdma_chan(chan);
         struct sdma_engine *sdma = sdmac->sdma;
-        int ret, i, count;
+        int i, count;
         int channel = sdmac->channel;
         struct scatterlist *sg;
+        struct sdma_desc *desc;
 
-        if (sdmac->status == DMA_IN_PROGRESS)
-                return NULL;
-        sdmac->status = DMA_IN_PROGRESS;
-
-        sdmac->flags = 0;
-
-        sdmac->buf_tail = 0;
-        sdmac->buf_ptail = 0;
-        sdmac->chn_real_count = 0;
+        desc = sdma_transfer_init(sdmac, direction, sg_len);
+        if (!desc)
+                goto err_out;
 
         dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
                         sg_len, channel);
 
-        sdmac->direction = direction;
-        ret = sdma_load_context(sdmac);
-        if (ret)
-                goto err_out;
-
-        if (sg_len > NUM_BD) {
-                dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",
-                                channel, sg_len, NUM_BD);
-                ret = -EINVAL;
-                goto err_out;
-        }
-
-        sdmac->chn_count = 0;
         for_each_sg(sgl, sg, sg_len, i) {
-                struct sdma_buffer_descriptor *bd = &sdmac->bd[i];
+                struct sdma_buffer_descriptor *bd = &desc->bd[i];
                 int param;
 
                 bd->buffer_addr = sg->dma_address;
 
                 count = sg_dma_len(sg);
 
-                if (count > 0xffff) {
+                if (count > SDMA_BD_MAX_CNT) {
                         dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
-                                        channel, count, 0xffff);
-                        ret = -EINVAL;
-                        goto err_out;
+                                        channel, count, SDMA_BD_MAX_CNT);
+                        goto err_bd_out;
                 }
 
                 bd->mode.count = count;
-                sdmac->chn_count += count;
+                desc->chn_count += count;
 
-                if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) {
-                        ret =  -EINVAL;
-                        goto err_out;
-                }
+                if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
+                        goto err_bd_out;
 
                 switch (sdmac->word_size) {
                 case DMA_SLAVE_BUSWIDTH_4_BYTES:
                         bd->mode.command = 0;
                         if (count & 3 || sg->dma_address & 3)
-                                return NULL;
+                                goto err_bd_out;
                         break;
                 case DMA_SLAVE_BUSWIDTH_2_BYTES:
                         bd->mode.command = 2;
                         if (count & 1 || sg->dma_address & 1)
-                                return NULL;
+                                goto err_bd_out;
                         break;
                 case DMA_SLAVE_BUSWIDTH_1_BYTE:
                         bd->mode.command = 1;
                         break;
                 default:
-                        return NULL;
+                        goto err_bd_out;
                 }
 
                 param = BD_DONE | BD_EXTD | BD_CONT;
@@ -1300,10 +1476,10 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
                 bd->mode.status = param;
         }
 
-        sdmac->num_bd = sg_len;
-        sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
-
-        return &sdmac->desc;
+        return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
+err_bd_out:
+        sdma_free_bd(desc);
+        kfree(desc);
 err_out:
         sdmac->status = DMA_ERROR;
         return NULL;
@@ -1318,40 +1494,27 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
         struct sdma_engine *sdma = sdmac->sdma;
         int num_periods = buf_len / period_len;
         int channel = sdmac->channel;
-        int ret, i = 0, buf = 0;
+        int i = 0, buf = 0;
+        struct sdma_desc *desc;
 
         dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
 
-        if (sdmac->status == DMA_IN_PROGRESS)
-                return NULL;
-
-        sdmac->status = DMA_IN_PROGRESS;
+        desc = sdma_transfer_init(sdmac, direction, num_periods);
+        if (!desc)
+                goto err_out;
 
-        sdmac->buf_tail = 0;
-        sdmac->buf_ptail = 0;
-        sdmac->chn_real_count = 0;
-        sdmac->period_len = period_len;
+        desc->period_len = period_len;
 
         sdmac->flags |= IMX_DMA_SG_LOOP;
-        sdmac->direction = direction;
-        ret = sdma_load_context(sdmac);
-        if (ret)
-                goto err_out;
-
-        if (num_periods > NUM_BD) {
-                dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",
-                                channel, num_periods, NUM_BD);
-                goto err_out;
-        }
 
-        if (period_len > 0xffff) {
+        if (period_len > SDMA_BD_MAX_CNT) {
                 dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
-                                channel, period_len, 0xffff);
-                goto err_out;
+                                channel, period_len, SDMA_BD_MAX_CNT);
+                goto err_bd_out;
         }
 
         while (buf < buf_len) {
-                struct sdma_buffer_descriptor *bd = &sdmac->bd[i];
+                struct sdma_buffer_descriptor *bd = &desc->bd[i];
                 int param;
 
                 bd->buffer_addr = dma_addr;
@@ -1359,7 +1522,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
                 bd->mode.count = period_len;
 
                 if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
-                        goto err_out;
+                        goto err_bd_out;
                 if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
                         bd->mode.command = 0;
                 else
@@ -1382,10 +1545,10 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
                 i++;
         }
 
-        sdmac->num_bd = num_periods;
-        sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
-
-        return &sdmac->desc;
+        return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
+err_bd_out:
+        sdma_free_bd(desc);
+        kfree(desc);
 err_out:
         sdmac->status = DMA_ERROR;
         return NULL;
@@ -1424,13 +1587,31 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
                                       struct dma_tx_state *txstate)
 {
         struct sdma_channel *sdmac = to_sdma_chan(chan);
+        struct sdma_desc *desc;
         u32 residue;
+        struct virt_dma_desc *vd;
+        enum dma_status ret;
+        unsigned long flags;
 
-        if (sdmac->flags & IMX_DMA_SG_LOOP)
-                residue = (sdmac->num_bd - sdmac->buf_ptail) *
-                           sdmac->period_len - sdmac->chn_real_count;
-        else
-                residue = sdmac->chn_count - sdmac->chn_real_count;
+        ret = dma_cookie_status(chan, cookie, txstate);
+        if (ret == DMA_COMPLETE || !txstate)
+                return ret;
+
+        spin_lock_irqsave(&sdmac->vc.lock, flags);
+        vd = vchan_find_desc(&sdmac->vc, cookie);
+        if (vd) {
+                desc = to_sdma_desc(&vd->tx);
+                if (sdmac->flags & IMX_DMA_SG_LOOP)
+                        residue = (desc->num_bd - desc->buf_ptail) *
+                                desc->period_len - desc->chn_real_count;
+                else
+                        residue = desc->chn_count - desc->chn_real_count;
+        } else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) {
+                residue = sdmac->desc->chn_count - sdmac->desc->chn_real_count;
+        } else {
+                residue = 0;
+        }
+        spin_unlock_irqrestore(&sdmac->vc.lock, flags);
 
         dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
                          residue);
@@ -1441,10 +1622,12 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
 static void sdma_issue_pending(struct dma_chan *chan)
 {
         struct sdma_channel *sdmac = to_sdma_chan(chan);
-        struct sdma_engine *sdma = sdmac->sdma;
+        unsigned long flags;
 
-        if (sdmac->status == DMA_IN_PROGRESS)
-                sdma_enable_channel(sdma, sdmac->channel);
+        spin_lock_irqsave(&sdmac->vc.lock, flags);
+        if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
+                sdma_start_desc(sdmac);
+        spin_unlock_irqrestore(&sdmac->vc.lock, flags);
 }
 
 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34
@@ -1650,7 +1833,7 @@ static int sdma_init(struct sdma_engine *sdma)
         for (i = 0; i < MAX_DMA_CHANNELS; i++)
                 writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
 
-        ret = sdma_request_channel(&sdma->channel[0]);
+        ret = sdma_request_channel0(sdma);
         if (ret)
                 goto err_dma_alloc;
 
@@ -1805,6 +1988,7 @@ static int sdma_probe(struct platform_device *pdev)
 
         dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
         dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
+        dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
 
         INIT_LIST_HEAD(&sdma->dma_device.channels);
         /* Initialize channel parameters */
@@ -1812,22 +1996,16 @@ static int sdma_probe(struct platform_device *pdev)
                 struct sdma_channel *sdmac = &sdma->channel[i];
 
                 sdmac->sdma = sdma;
-                spin_lock_init(&sdmac->lock);
 
-                sdmac->chan.device = &sdma->dma_device;
-                dma_cookie_init(&sdmac->chan);
                 sdmac->channel = i;
-
-                tasklet_init(&sdmac->tasklet, mxc_sdma_handle_channel_normal,
-                             (unsigned long) sdmac);
+                sdmac->vc.desc_free = sdma_desc_free;
                 /*
                  * Add the channel to the DMAC list. Do not add channel 0 though
                  * because we need it internally in the SDMA driver. This also means
                  * that channel 0 in dmaengine counting matches sdma channel 1.
                  */
                 if (i)
-                        list_add_tail(&sdmac->chan.device_node,
-                                        &sdma->dma_device.channels);
+                        vchan_init(&sdmac->vc, &sdma->dma_device);
         }
 
         ret = sdma_init(sdma);
@@ -1877,9 +2055,10 @@ static int sdma_probe(struct platform_device *pdev)
         sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
         sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
         sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+        sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
         sdma->dma_device.device_issue_pending = sdma_issue_pending;
         sdma->dma_device.dev->dma_parms = &sdma->dma_parms;
-        dma_set_max_seg_size(sdma->dma_device.dev, 65535);
+        dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
 
         platform_set_drvdata(pdev, sdma);
 
@@ -1932,7 +2111,8 @@ static int sdma_remove(struct platform_device *pdev)
         for (i = 0; i < MAX_DMA_CHANNELS; i++) {
                 struct sdma_channel *sdmac = &sdma->channel[i];
 
-                tasklet_kill(&sdmac->tasklet);
+                tasklet_kill(&sdmac->vc.task);
+                sdma_free_chan_resources(&sdmac->vc.chan);
         }
 
         platform_set_drvdata(pdev, NULL);
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 8b5b23a8ace9..23fb2fa04000 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -688,6 +688,12 @@ static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
 {
         u64 phys_complete;
 
+        /* set the completion address register again */
+        writel(lower_32_bits(ioat_chan->completion_dma),
+               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+        writel(upper_32_bits(ioat_chan->completion_dma),
+               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
         ioat_quiesce(ioat_chan, 0);
         if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
                 __cleanup(ioat_chan, phys_complete);
diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c
index 68dd79783b54..b76cb17d879c 100644
--- a/drivers/dma/mic_x100_dma.c
+++ b/drivers/dma/mic_x100_dma.c
@@ -470,11 +470,6 @@ static void mic_dma_chan_destroy(struct mic_dma_chan *ch)
         mic_dma_chan_mask_intr(ch);
 }
 
-static void mic_dma_unregister_dma_device(struct mic_dma_device *mic_dma_dev)
-{
-        dma_async_device_unregister(&mic_dma_dev->dma_dev);
-}
-
 static int mic_dma_setup_irq(struct mic_dma_chan *ch)
 {
         ch->cookie =
@@ -630,7 +625,7 @@ static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev,
                 list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node,
                               &mic_dma_dev->dma_dev.channels);
         }
-        return dma_async_device_register(&mic_dma_dev->dma_dev);
+        return dmaenginem_async_device_register(&mic_dma_dev->dma_dev);
 }
 
 /*
@@ -678,7 +673,6 @@ alloc_error:
 
 static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev)
 {
-        mic_dma_unregister_dma_device(mic_dma_dev);
         mic_dma_uninit(mic_dma_dev);
         kfree(mic_dma_dev);
 }
diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c
index c6589ccf1b9a..8dc0aa4d73ab 100644
--- a/drivers/dma/mv_xor_v2.c
+++ b/drivers/dma/mv_xor_v2.c
@@ -174,6 +174,7 @@ struct mv_xor_v2_device {
         int desc_size;
         unsigned int npendings;
         unsigned int hw_queue_idx;
+        struct msi_desc *msi_desc;
 };
 
 /**
@@ -588,11 +589,9 @@ static void mv_xor_v2_tasklet(unsigned long data)
                          */
                         dma_cookie_complete(&next_pending_sw_desc->async_tx);
 
-                        if (next_pending_sw_desc->async_tx.callback)
-                                next_pending_sw_desc->async_tx.callback(
-                                next_pending_sw_desc->async_tx.callback_param);
-
                         dma_descriptor_unmap(&next_pending_sw_desc->async_tx);
+                        dmaengine_desc_get_callback_invoke(
+                                        &next_pending_sw_desc->async_tx, NULL);
                 }
 
                 dma_run_dependencies(&next_pending_sw_desc->async_tx);
@@ -643,9 +642,9 @@ static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev)
                xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF);
 
         /* write the DESQ address to the DMA enngine*/
-        writel(xor_dev->hw_desq & 0xFFFFFFFF,
+        writel(lower_32_bits(xor_dev->hw_desq),
                xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF);
-        writel((xor_dev->hw_desq & 0xFFFF00000000) >> 32,
+        writel(upper_32_bits(xor_dev->hw_desq),
                xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF);
 
         /*
@@ -780,6 +779,7 @@ static int mv_xor_v2_probe(struct platform_device *pdev)
         msi_desc = first_msi_entry(&pdev->dev);
         if (!msi_desc)
                 goto free_msi_irqs;
+        xor_dev->msi_desc = msi_desc;
 
         ret = devm_request_irq(&pdev->dev, msi_desc->irq,
                                mv_xor_v2_interrupt_handler, 0,
@@ -897,8 +897,12 @@ static int mv_xor_v2_remove(struct platform_device *pdev)
                           xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
                           xor_dev->hw_desq_virt, xor_dev->hw_desq);
 
+        devm_free_irq(&pdev->dev, xor_dev->msi_desc->irq, xor_dev);
+
         platform_msi_domain_free_irqs(&pdev->dev);
 
+        tasklet_kill(&xor_dev->irq_tasklet);
+
         clk_disable_unprepare(xor_dev->clk);
 
         return 0;
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index 2f9974ddfbb2..8c7b2e8703da 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -479,6 +479,7 @@ static size_t nbpf_xfer_size(struct nbpf_device *nbpf,
 
         default:
                 pr_warn("%s(): invalid bus width %u\n", __func__, width);
+                /* fall through */
         case DMA_SLAVE_BUSWIDTH_1_BYTE:
                 size = burst;
         }
diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c
new file mode 100644
index 000000000000..7812a6338acd
--- /dev/null
+++ b/drivers/dma/owl-dma.c
@@ -0,0 +1,971 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Actions Semi Owl SoCs DMA driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+#include "virt-dma.h"
+
+#define OWL_DMA_FRAME_MAX_LENGTH                0xfffff
+
+/* Global DMA Controller Registers */
+#define OWL_DMA_IRQ_PD0                         0x00
+#define OWL_DMA_IRQ_PD1                         0x04
+#define OWL_DMA_IRQ_PD2                         0x08
+#define OWL_DMA_IRQ_PD3                         0x0C
+#define OWL_DMA_IRQ_EN0                         0x10
+#define OWL_DMA_IRQ_EN1                         0x14
+#define OWL_DMA_IRQ_EN2                         0x18
+#define OWL_DMA_IRQ_EN3                         0x1C
+#define OWL_DMA_SECURE_ACCESS_CTL               0x20
+#define OWL_DMA_NIC_QOS                         0x24
+#define OWL_DMA_DBGSEL                          0x28
+#define OWL_DMA_IDLE_STAT                       0x2C
+
+/* Channel Registers */
+#define OWL_DMA_CHAN_BASE(i)                    (0x100 + (i) * 0x100)
+#define OWL_DMAX_MODE                           0x00
+#define OWL_DMAX_SOURCE                         0x04
+#define OWL_DMAX_DESTINATION                    0x08
+#define OWL_DMAX_FRAME_LEN                      0x0C
+#define OWL_DMAX_FRAME_CNT                      0x10
+#define OWL_DMAX_REMAIN_FRAME_CNT               0x14
+#define OWL_DMAX_REMAIN_CNT                     0x18
+#define OWL_DMAX_SOURCE_STRIDE                  0x1C
+#define OWL_DMAX_DESTINATION_STRIDE             0x20
+#define OWL_DMAX_START                          0x24
+#define OWL_DMAX_PAUSE                          0x28
+#define OWL_DMAX_CHAINED_CTL                    0x2C
+#define OWL_DMAX_CONSTANT                       0x30
+#define OWL_DMAX_LINKLIST_CTL                   0x34
+#define OWL_DMAX_NEXT_DESCRIPTOR                0x38
+#define OWL_DMAX_CURRENT_DESCRIPTOR_NUM         0x3C
+#define OWL_DMAX_INT_CTL                        0x40
+#define OWL_DMAX_INT_STATUS                     0x44
+#define OWL_DMAX_CURRENT_SOURCE_POINTER         0x48
+#define OWL_DMAX_CURRENT_DESTINATION_POINTER    0x4C
+
+/* OWL_DMAX_MODE Bits */
+#define OWL_DMA_MODE_TS(x)                      (((x) & GENMASK(5, 0)) << 0)
+#define OWL_DMA_MODE_ST(x)                      (((x) & GENMASK(1, 0)) << 8)
+#define OWL_DMA_MODE_ST_DEV                     OWL_DMA_MODE_ST(0)
+#define OWL_DMA_MODE_ST_DCU                     OWL_DMA_MODE_ST(2)
+#define OWL_DMA_MODE_ST_SRAM                    OWL_DMA_MODE_ST(3)
+#define OWL_DMA_MODE_DT(x)                      (((x) & GENMASK(1, 0)) << 10)
+#define OWL_DMA_MODE_DT_DEV                     OWL_DMA_MODE_DT(0)
+#define OWL_DMA_MODE_DT_DCU                     OWL_DMA_MODE_DT(2)
+#define OWL_DMA_MODE_DT_SRAM                    OWL_DMA_MODE_DT(3)
+#define OWL_DMA_MODE_SAM(x)                     (((x) & GENMASK(1, 0)) << 16)
+#define OWL_DMA_MODE_SAM_CONST                  OWL_DMA_MODE_SAM(0)
+#define OWL_DMA_MODE_SAM_INC                    OWL_DMA_MODE_SAM(1)
+#define OWL_DMA_MODE_SAM_STRIDE                 OWL_DMA_MODE_SAM(2)
+#define OWL_DMA_MODE_DAM(x)                     (((x) & GENMASK(1, 0)) << 18)
+#define OWL_DMA_MODE_DAM_CONST                  OWL_DMA_MODE_DAM(0)
+#define OWL_DMA_MODE_DAM_INC                    OWL_DMA_MODE_DAM(1)
+#define OWL_DMA_MODE_DAM_STRIDE                 OWL_DMA_MODE_DAM(2)
+#define OWL_DMA_MODE_PW(x)                      (((x) & GENMASK(2, 0)) << 20)
+#define OWL_DMA_MODE_CB                         BIT(23)
+#define OWL_DMA_MODE_NDDBW(x)                   (((x) & 0x1) << 28)
+#define OWL_DMA_MODE_NDDBW_32BIT                OWL_DMA_MODE_NDDBW(0)
+#define OWL_DMA_MODE_NDDBW_8BIT                 OWL_DMA_MODE_NDDBW(1)
+#define OWL_DMA_MODE_CFE                        BIT(29)
+#define OWL_DMA_MODE_LME                        BIT(30)
+#define OWL_DMA_MODE_CME                        BIT(31)
+
+/* OWL_DMAX_LINKLIST_CTL Bits */
+#define OWL_DMA_LLC_SAV(x)                      (((x) & GENMASK(1, 0)) << 8)
+#define OWL_DMA_LLC_SAV_INC                     OWL_DMA_LLC_SAV(0)
+#define OWL_DMA_LLC_SAV_LOAD_NEXT               OWL_DMA_LLC_SAV(1)
+#define OWL_DMA_LLC_SAV_LOAD_PREV               OWL_DMA_LLC_SAV(2)
+#define OWL_DMA_LLC_DAV(x)                      (((x) & GENMASK(1, 0)) << 10)
+#define OWL_DMA_LLC_DAV_INC                     OWL_DMA_LLC_DAV(0)
+#define OWL_DMA_LLC_DAV_LOAD_NEXT               OWL_DMA_LLC_DAV(1)
+#define OWL_DMA_LLC_DAV_LOAD_PREV               OWL_DMA_LLC_DAV(2)
+#define OWL_DMA_LLC_SUSPEND                     BIT(16)
+
+/* OWL_DMAX_INT_CTL Bits */
+#define OWL_DMA_INTCTL_BLOCK                    BIT(0)
+#define OWL_DMA_INTCTL_SUPER_BLOCK              BIT(1)
+#define OWL_DMA_INTCTL_FRAME                    BIT(2)
+#define OWL_DMA_INTCTL_HALF_FRAME               BIT(3)
+#define OWL_DMA_INTCTL_LAST_FRAME               BIT(4)
+
+/* OWL_DMAX_INT_STATUS Bits */
+#define OWL_DMA_INTSTAT_BLOCK                   BIT(0)
+#define OWL_DMA_INTSTAT_SUPER_BLOCK             BIT(1)
+#define OWL_DMA_INTSTAT_FRAME                   BIT(2)
+#define OWL_DMA_INTSTAT_HALF_FRAME              BIT(3)
+#define OWL_DMA_INTSTAT_LAST_FRAME              BIT(4)
+
+/* Pack shift and newshift in a single word */
+#define BIT_FIELD(val, width, shift, newshift)  \
+                ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
+
+/**
+ * struct owl_dma_lli_hw - Hardware link list for dma transfer
+ * @next_lli: physical address of the next link list
+ * @saddr: source physical address
+ * @daddr: destination physical address
+ * @flen: frame length
+ * @fcnt: frame count
+ * @src_stride: source stride
+ * @dst_stride: destination stride
+ * @ctrla: dma_mode and linklist ctrl config
+ * @ctrlb: interrupt config
+ * @const_num: data for constant fill
+ */
+struct owl_dma_lli_hw {
+        u32     next_lli;
+        u32     saddr;
+        u32     daddr;
+        u32     flen:20;
+        u32     fcnt:12;
+        u32     src_stride;
+        u32     dst_stride;
+        u32     ctrla;
+        u32     ctrlb;
+        u32     const_num;
+};
+
+/**
+ * struct owl_dma_lli - Link list for dma transfer
+ * @hw: hardware link list
+ * @phys: physical address of hardware link list
+ * @node: node for txd's lli_list
+ */
+struct owl_dma_lli {
+        struct  owl_dma_lli_hw  hw;
+        dma_addr_t              phys;
+        struct list_head        node;
+};
+
+/**
+ * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
+ * @vd: virtual DMA descriptor
+ * @lli_list: link list of lli nodes
+ */
+struct owl_dma_txd {
+        struct virt_dma_desc    vd;
+        struct list_head        lli_list;
+};
+
+/**
+ * struct owl_dma_pchan - Holder for the physical channels
+ * @id: physical index to this channel
+ * @base: virtual memory base for the dma channel
+ * @vchan: the virtual channel currently being served by this physical channel
+ * @lock: a lock to use when altering an instance of this struct
+ */
+struct owl_dma_pchan {
+        u32                     id;
+        void __iomem            *base;
+        struct owl_dma_vchan    *vchan;
+        spinlock_t              lock;
+};
+
+/**
+ * struct owl_dma_pchan - Wrapper for DMA ENGINE channel
+ * @vc: wrappped virtual channel
+ * @pchan: the physical channel utilized by this channel
+ * @txd: active transaction on this channel
+ */
+struct owl_dma_vchan {
+        struct virt_dma_chan    vc;
+        struct owl_dma_pchan    *pchan;
+        struct owl_dma_txd      *txd;
+};
+
+/**
+ * struct owl_dma - Holder for the Owl DMA controller
+ * @dma: dma engine for this instance
+ * @base: virtual memory base for the DMA controller
+ * @clk: clock for the DMA controller
+ * @lock: a lock to use when change DMA controller global register
+ * @lli_pool: a pool for the LLI descriptors
+ * @nr_pchans: the number of physical channels
+ * @pchans: array of data for the physical channels
+ * @nr_vchans: the number of physical channels
+ * @vchans: array of data for the physical channels
+ */
+struct owl_dma {
+        struct dma_device       dma;
+        void __iomem            *base;
+        struct clk              *clk;
+        spinlock_t              lock;
+        struct dma_pool         *lli_pool;
+        int                     irq;
+
+        unsigned int            nr_pchans;
+        struct owl_dma_pchan    *pchans;
+
+        unsigned int            nr_vchans;
+        struct owl_dma_vchan    *vchans;
+};
+
+static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
+                         u32 val, bool state)
+{
+        u32 regval;
+
+        regval = readl(pchan->base + reg);
+
+        if (state)
+                regval |= val;
+        else
+                regval &= ~val;
+
+        writel(val, pchan->base + reg);
+}
+
+static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
+{
+        writel(data, pchan->base + reg);
+}
+
+static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
+{
+        return readl(pchan->base + reg);
+}
+
+static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
+{
+        u32 regval;
+
+        regval = readl(od->base + reg);
+
+        if (state)
+                regval |= val;
+        else
+                regval &= ~val;
+
+        writel(val, od->base + reg);
+}
+
+static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
+{
+        writel(data, od->base + reg);
+}
+
+static u32 dma_readl(struct owl_dma *od, u32 reg)
+{
+        return readl(od->base + reg);
+}
+
+static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
+{
+        return container_of(dd, struct owl_dma, dma);
+}
+
+static struct device *chan2dev(struct dma_chan *chan)
+{
+        return &chan->dev->device;
+}
+
+static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
+{
+        return container_of(chan, struct owl_dma_vchan, vc.chan);
+}
+
+static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
+{
+        return container_of(tx, struct owl_dma_txd, vd.tx);
+}
+
+static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
+{
+        u32 ctl;
+
+        ctl = BIT_FIELD(mode, 4, 28, 28) |
+              BIT_FIELD(mode, 8, 16, 20) |
+              BIT_FIELD(mode, 4, 8, 16) |
+              BIT_FIELD(mode, 6, 0, 10) |
+              BIT_FIELD(llc_ctl, 2, 10, 8) |
+              BIT_FIELD(llc_ctl, 2, 8, 6);
+
+        return ctl;
+}
+
+static inline u32 llc_hw_ctrlb(u32 int_ctl)
+{
+        u32 ctl;
+
+        ctl = BIT_FIELD(int_ctl, 7, 0, 18);
+
+        return ctl;
+}
+
+static void owl_dma_free_lli(struct owl_dma *od,
+                             struct owl_dma_lli *lli)
+{
+        list_del(&lli->node);
+        dma_pool_free(od->lli_pool, lli, lli->phys);
+}
+
+static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
+{
+        struct owl_dma_lli *lli;
+        dma_addr_t phys;
+
+        lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
+        if (!lli)
+                return NULL;
+
+        INIT_LIST_HEAD(&lli->node);
+        lli->phys = phys;
+
+        return lli;
+}
+
+static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
+                                           struct owl_dma_lli *prev,
+                                           struct owl_dma_lli *next)
+{
+        list_add_tail(&next->node, &txd->lli_list);
+
+        if (prev) {
+                prev->hw.next_lli = next->phys;
+                prev->hw.ctrla |= llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
+        }
+
+        return next;
+}
+
+static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
+                                  struct owl_dma_lli *lli,
+                                  dma_addr_t src, dma_addr_t dst,
+                                  u32 len, enum dma_transfer_direction dir)
+{
+        struct owl_dma_lli_hw *hw = &lli->hw;
+        u32 mode;
+
+        mode = OWL_DMA_MODE_PW(0);
+
+        switch (dir) {
+        case DMA_MEM_TO_MEM:
+                mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
+                        OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
+                        OWL_DMA_MODE_DAM_INC;
+
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        hw->next_lli = 0; /* One link list by default */
+        hw->saddr = src;
+        hw->daddr = dst;
+
+        hw->fcnt = 1; /* Frame count fixed as 1 */
+        hw->flen = len; /* Max frame length is 1MB */
+        hw->src_stride = 0;
+        hw->dst_stride = 0;
+        hw->ctrla = llc_hw_ctrla(mode,
+                                 OWL_DMA_LLC_SAV_LOAD_NEXT |
+                                 OWL_DMA_LLC_DAV_LOAD_NEXT);
+
+        hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
+
+        return 0;
+}
+
+static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
+                                               struct owl_dma_vchan *vchan)
+{
+        struct owl_dma_pchan *pchan = NULL;
+        unsigned long flags;
+        int i;
+
+        for (i = 0; i < od->nr_pchans; i++) {
+                pchan = &od->pchans[i];
+
+                spin_lock_irqsave(&pchan->lock, flags);
+                if (!pchan->vchan) {
+                        pchan->vchan = vchan;
+                        spin_unlock_irqrestore(&pchan->lock, flags);
+                        break;
+                }
+
+                spin_unlock_irqrestore(&pchan->lock, flags);
+        }
+
+        return pchan;
+}
+
+static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
+{
+        unsigned int val;
+
+        val = dma_readl(od, OWL_DMA_IDLE_STAT);
+
+        return !(val & (1 << pchan->id));
+}
+
+static void owl_dma_terminate_pchan(struct owl_dma *od,
+                                    struct owl_dma_pchan *pchan)
+{
+        unsigned long flags;
+        u32 irq_pd;
+
+        pchan_writel(pchan, OWL_DMAX_START, 0);
+        pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
+
+        spin_lock_irqsave(&od->lock, flags);
+        dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
+
+        irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
+        if (irq_pd & (1 << pchan->id)) {
+                dev_warn(od->dma.dev,
+                         "terminating pchan %d that still has pending irq\n",
+                         pchan->id);
+                dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
+        }
+
+        pchan->vchan = NULL;
+
+        spin_unlock_irqrestore(&od->lock, flags);
+}
+
+static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
+{
+        struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
+        struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
+        struct owl_dma_pchan *pchan = vchan->pchan;
+        struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
+        struct owl_dma_lli *lli;
+        unsigned long flags;
+        u32 int_ctl;
+
+        list_del(&vd->node);
+
+        vchan->txd = txd;
+
+        /* Wait for channel inactive */
+        while (owl_dma_pchan_busy(od, pchan))
+                cpu_relax();
+
+        lli = list_first_entry(&txd->lli_list,
+                               struct owl_dma_lli, node);
+
+        int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
+
+        pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
+        pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
+                     OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
+        pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
+        pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
+
+        /* Clear IRQ status for this pchan */
+        pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
+
+        spin_lock_irqsave(&od->lock, flags);
+
+        dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
+
+        spin_unlock_irqrestore(&od->lock, flags);
+
+        dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
+
+        /* Start DMA transfer for this pchan */
+        pchan_writel(pchan, OWL_DMAX_START, 0x1);
+
+        return 0;
+}
+
+static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
+{
+        /* Ensure that the physical channel is stopped */
+        owl_dma_terminate_pchan(od, vchan->pchan);
+
+        vchan->pchan = NULL;
+}
+
+static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
+{
+        struct owl_dma *od = dev_id;
+        struct owl_dma_vchan *vchan;
+        struct owl_dma_pchan *pchan;
+        unsigned long pending;
+        int i;
+        unsigned int global_irq_pending, chan_irq_pending;
+
+        spin_lock(&od->lock);
+
+        pending = dma_readl(od, OWL_DMA_IRQ_PD0);
+
+        /* Clear IRQ status for each pchan */
+        for_each_set_bit(i, &pending, od->nr_pchans) {
+                pchan = &od->pchans[i];
+                pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
+        }
+
+        /* Clear pending IRQ */
+        dma_writel(od, OWL_DMA_IRQ_PD0, pending);
+
+        /* Check missed pending IRQ */
+        for (i = 0; i < od->nr_pchans; i++) {
+                pchan = &od->pchans[i];
+                chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
+                                   pchan_readl(pchan, OWL_DMAX_INT_STATUS);
+
+                /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
+                dma_readl(od, OWL_DMA_IRQ_PD0);
+
+                global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
+
+                if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
+                        dev_dbg(od->dma.dev,
+                                "global and channel IRQ pending match err\n");
+
+                        /* Clear IRQ status for this pchan */
+                        pchan_update(pchan, OWL_DMAX_INT_STATUS,
+                                     0xff, false);
+
+                        /* Update global IRQ pending */
+                        pending |= BIT(i);
+                }
+        }
+
+        spin_unlock(&od->lock);
+
+        for_each_set_bit(i, &pending, od->nr_pchans) {
+                struct owl_dma_txd *txd;
+
+                pchan = &od->pchans[i];
+
+                vchan = pchan->vchan;
+                if (!vchan) {
+                        dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
+                                 pchan->id);
+                        continue;
+                }
+
+                spin_lock(&vchan->vc.lock);
+
+                txd = vchan->txd;
+                if (txd) {
+                        vchan->txd = NULL;
+
+                        vchan_cookie_complete(&txd->vd);
+
+                        /*
+                         * Start the next descriptor (if any),
+                         * otherwise free this channel.
+                         */
+                        if (vchan_next_desc(&vchan->vc))
+                                owl_dma_start_next_txd(vchan);
+                        else
+                                owl_dma_phy_free(od, vchan);
+                }
+
+                spin_unlock(&vchan->vc.lock);
+        }
+
+        return IRQ_HANDLED;
+}
+
+static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
+{
+        struct owl_dma_lli *lli, *_lli;
+
+        if (unlikely(!txd))
+                return;
+
+        list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
+                owl_dma_free_lli(od, lli);
+
+        kfree(txd);
+}
+
+static void owl_dma_desc_free(struct virt_dma_desc *vd)
+{
+        struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
+        struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
+
+        owl_dma_free_txd(od, txd);
+}
+
+static int owl_dma_terminate_all(struct dma_chan *chan)
+{
+        struct owl_dma *od = to_owl_dma(chan->device);
+        struct owl_dma_vchan *vchan = to_owl_vchan(chan);
+        unsigned long flags;
+        LIST_HEAD(head);
+
+        spin_lock_irqsave(&vchan->vc.lock, flags);
+
+        if (vchan->pchan)
+                owl_dma_phy_free(od, vchan);
+
+        if (vchan->txd) {
+                owl_dma_desc_free(&vchan->txd->vd);
+                vchan->txd = NULL;
+        }
+
+        vchan_get_all_descriptors(&vchan->vc, &head);
+        vchan_dma_desc_free_list(&vchan->vc, &head);
+
+        spin_unlock_irqrestore(&vchan->vc.lock, flags);
+
+        return 0;
+}
+
+static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
+{
+        struct owl_dma_pchan *pchan;
+        struct owl_dma_txd *txd;
+        struct owl_dma_lli *lli;
+        unsigned int next_lli_phy;
+        size_t bytes;
+
+        pchan = vchan->pchan;
+        txd = vchan->txd;
+
+        if (!pchan || !txd)
+                return 0;
+
+        /* Get remain count of current node in link list */
+        bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
+
+        /* Loop through the preceding nodes to get total remaining bytes */
+        if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
+                next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
+                list_for_each_entry(lli, &txd->lli_list, node) {
+                        /* Start from the next active node */
+                        if (lli->phys == next_lli_phy) {
+                                list_for_each_entry(lli, &txd->lli_list, node)
+                                        bytes += lli->hw.flen;
+                                break;
+                        }
+                }
+        }
+
+        return bytes;
+}
+
+static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
+                                         dma_cookie_t cookie,
+                                         struct dma_tx_state *state)
+{
+        struct owl_dma_vchan *vchan = to_owl_vchan(chan);
+        struct owl_dma_lli *lli;
+        struct virt_dma_desc *vd;
+        struct owl_dma_txd *txd;
+        enum dma_status ret;
+        unsigned long flags;
+        size_t bytes = 0;
+
+        ret = dma_cookie_status(chan, cookie, state);
+        if (ret == DMA_COMPLETE || !state)
+                return ret;
+
+        spin_lock_irqsave(&vchan->vc.lock, flags);
+
+        vd = vchan_find_desc(&vchan->vc, cookie);
+        if (vd) {
+                txd = to_owl_txd(&vd->tx);
+                list_for_each_entry(lli, &txd->lli_list, node)
+                        bytes += lli->hw.flen;
+        } else {
+                bytes = owl_dma_getbytes_chan(vchan);
+        }
+
+        spin_unlock_irqrestore(&vchan->vc.lock, flags);
+
+        dma_set_residue(state, bytes);
+
+        return ret;
+}
+
+static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
+{
+        struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
+        struct owl_dma_pchan *pchan;
+
+        pchan = owl_dma_get_pchan(od, vchan);
+        if (!pchan)
+                return;
+
+        dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
+
+        vchan->pchan = pchan;
+        owl_dma_start_next_txd(vchan);
+}
+
+static void owl_dma_issue_pending(struct dma_chan *chan)
+{
+        struct owl_dma_vchan *vchan = to_owl_vchan(chan);
+        unsigned long flags;
+
+        spin_lock_irqsave(&vchan->vc.lock, flags);
+        if (vchan_issue_pending(&vchan->vc)) {
+                if (!vchan->pchan)
+                        owl_dma_phy_alloc_and_start(vchan);
+        }
+        spin_unlock_irqrestore(&vchan->vc.lock, flags);
+}
+
+static struct dma_async_tx_descriptor
+                *owl_dma_prep_memcpy(struct dma_chan *chan,
+                                     dma_addr_t dst, dma_addr_t src,
+                                     size_t len, unsigned long flags)
+{
+        struct owl_dma *od = to_owl_dma(chan->device);
+        struct owl_dma_vchan *vchan = to_owl_vchan(chan);
+        struct owl_dma_txd *txd;
+        struct owl_dma_lli *lli, *prev = NULL;
+        size_t offset, bytes;
+        int ret;
+
+        if (!len)
+                return NULL;
+
+        txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
+        if (!txd)
+                return NULL;
+
+        INIT_LIST_HEAD(&txd->lli_list);
+
+        /* Process the transfer as frame by frame */
+        for (offset = 0; offset < len; offset += bytes) {
+                lli = owl_dma_alloc_lli(od);
+                if (!lli) {
+                        dev_warn(chan2dev(chan), "failed to allocate lli\n");
+                        goto err_txd_free;
+                }
+
+                bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
+
+                ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
+                                      bytes, DMA_MEM_TO_MEM);
+                if (ret) {
+                        dev_warn(chan2dev(chan), "failed to config lli\n");
+                        goto err_txd_free;
+                }
+
+                prev = owl_dma_add_lli(txd, prev, lli);
+        }
+
+        return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
+
+err_txd_free:
+        owl_dma_free_txd(od, txd);
+        return NULL;
+}
+
+static void owl_dma_free_chan_resources(struct dma_chan *chan)
+{
+        struct owl_dma_vchan *vchan = to_owl_vchan(chan);
+
+        /* Ensure all queued descriptors are freed */
+        vchan_free_chan_resources(&vchan->vc);
+}
+
+static inline void owl_dma_free(struct owl_dma *od)
+{
+        struct owl_dma_vchan *vchan = NULL;
+        struct owl_dma_vchan *next;
+
+        list_for_each_entry_safe(vchan,
+                                 next, &od->dma.channels, vc.chan.device_node) {
+                list_del(&vchan->vc.chan.device_node);
+                tasklet_kill(&vchan->vc.task);
+        }
+}
+
+static int owl_dma_probe(struct platform_device *pdev)
+{
+        struct device_node *np = pdev->dev.of_node;
+        struct owl_dma *od;
+        struct resource *res;
+        int ret, i, nr_channels, nr_requests;
+
+        od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
+        if (!od)
+                return -ENOMEM;
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        if (!res)
+                return -EINVAL;
+
+        od->base = devm_ioremap_resource(&pdev->dev, res);
+        if (IS_ERR(od->base))
+                return PTR_ERR(od->base);
+
+        ret = of_property_read_u32(np, "dma-channels", &nr_channels);
+        if (ret) {
+                dev_err(&pdev->dev, "can't get dma-channels\n");
+                return ret;
+        }
+
+        ret = of_property_read_u32(np, "dma-requests", &nr_requests);
+        if (ret) {
+                dev_err(&pdev->dev, "can't get dma-requests\n");
+                return ret;
+        }
+
+        dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
+                 nr_channels, nr_requests);
+
+        od->nr_pchans = nr_channels;
+        od->nr_vchans = nr_requests;
+
+        pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
+
+        platform_set_drvdata(pdev, od);
+        spin_lock_init(&od->lock);
+
+        dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
+
+        od->dma.dev = &pdev->dev;
+        od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
+        od->dma.device_tx_status = owl_dma_tx_status;
+        od->dma.device_issue_pending = owl_dma_issue_pending;
+        od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
+        od->dma.device_terminate_all = owl_dma_terminate_all;
+        od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+        od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+        od->dma.directions = BIT(DMA_MEM_TO_MEM);
+        od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+
+        INIT_LIST_HEAD(&od->dma.channels);
+
+        od->clk = devm_clk_get(&pdev->dev, NULL);
+        if (IS_ERR(od->clk)) {
+                dev_err(&pdev->dev, "unable to get clock\n");
+                return PTR_ERR(od->clk);
+        }
+
+        /*
+         * Eventhough the DMA controller is capable of generating 4
+         * IRQ's for DMA priority feature, we only use 1 IRQ for
+         * simplification.
+         */
+        od->irq = platform_get_irq(pdev, 0);
+        ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
+                               dev_name(&pdev->dev), od);
+        if (ret) {
+                dev_err(&pdev->dev, "unable to request IRQ\n");
+                return ret;
+        }
+
+        /* Init physical channel */
+        od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
+                                  sizeof(struct owl_dma_pchan), GFP_KERNEL);
+        if (!od->pchans)
+                return -ENOMEM;
+
+        for (i = 0; i < od->nr_pchans; i++) {
+                struct owl_dma_pchan *pchan = &od->pchans[i];
+
+                pchan->id = i;
+                pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
+        }
+
+        /* Init virtual channel */
+        od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
+                                  sizeof(struct owl_dma_vchan), GFP_KERNEL);
+        if (!od->vchans)
+                return -ENOMEM;
+
+        for (i = 0; i < od->nr_vchans; i++) {
+                struct owl_dma_vchan *vchan = &od->vchans[i];
+
+                vchan->vc.desc_free = owl_dma_desc_free;
+                vchan_init(&vchan->vc, &od->dma);
+        }
+
+        /* Create a pool of consistent memory blocks for hardware descriptors */
+        od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
+                                       sizeof(struct owl_dma_lli),
+                                       __alignof__(struct owl_dma_lli),
+                                       0);
+        if (!od->lli_pool) {
+                dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
+                return -ENOMEM;
+        }
+
+        clk_prepare_enable(od->clk);
+
+        ret = dma_async_device_register(&od->dma);
+        if (ret) {
+                dev_err(&pdev->dev, "failed to register DMA engine device\n");
+                goto err_pool_free;
+        }
+
+        return 0;
+
+err_pool_free:
+        clk_disable_unprepare(od->clk);
+        dma_pool_destroy(od->lli_pool);
+
+        return ret;
+}
+
+static int owl_dma_remove(struct platform_device *pdev)
+{
+        struct owl_dma *od = platform_get_drvdata(pdev);
+
+        dma_async_device_unregister(&od->dma);
+
+        /* Mask all interrupts for this execution environment */
+        dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
+
+        /* Make sure we won't have any further interrupts */
+        devm_free_irq(od->dma.dev, od->irq, od);
+
+        owl_dma_free(od);
+
+        clk_disable_unprepare(od->clk);
+
+        return 0;
+}
+
+static const struct of_device_id owl_dma_match[] = {
+        { .compatible = "actions,s900-dma", },
+        { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, owl_dma_match);
+
+static struct platform_driver owl_dma_driver = {
+        .probe  = owl_dma_probe,
+        .remove = owl_dma_remove,
+        .driver = {
+                .name = "dma-owl",
+                .of_match_table = of_match_ptr(owl_dma_match),
+        },
+};
+
+static int owl_dma_init(void)
+{
+        return platform_driver_register(&owl_dma_driver);
+}
+subsys_initcall(owl_dma_init);
+
+static void __exit owl_dma_exit(void)
+{
+        platform_driver_unregister(&owl_dma_driver);
+}
+module_exit(owl_dma_exit);
+
+MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
+MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
+MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index de0957fe9668..88750a34e859 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -1046,13 +1046,16 @@ static bool _start(struct pl330_thread *thrd)
 
                 if (_state(thrd) == PL330_STATE_KILLING)
                         UNTIL(thrd, PL330_STATE_STOPPED)
+                /* fall through */
 
         case PL330_STATE_FAULTING:
                 _stop(thrd);
+                /* fall through */
 
         case PL330_STATE_KILLING:
         case PL330_STATE_COMPLETING:
                 UNTIL(thrd, PL330_STATE_STOPPED)
+                /* fall through */
 
         case PL330_STATE_STOPPED:
                 return _trigger(thrd);
@@ -1779,8 +1782,6 @@ static inline void _free_event(struct pl330_thread *thrd, int ev)
 
 static void pl330_release_channel(struct pl330_thread *thrd)
 {
-        struct pl330_dmac *pl330;
-
         if (!thrd || thrd->free)
                 return;
 
@@ -1789,8 +1790,6 @@ static void pl330_release_channel(struct pl330_thread *thrd)
         dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);
         dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);
 
-        pl330 = thrd->dmac;
-
         _free_event(thrd, thrd->ev);
         thrd->free = true;
 }
@@ -2257,13 +2256,14 @@ static int pl330_terminate_all(struct dma_chan *chan)
 
         pm_runtime_get_sync(pl330->ddma.dev);
         spin_lock_irqsave(&pch->lock, flags);
+
         spin_lock(&pl330->lock);
         _stop(pch->thread);
-        spin_unlock(&pl330->lock);
-
         pch->thread->req[0].desc = NULL;
         pch->thread->req[1].desc = NULL;
         pch->thread->req_running = -1;
+        spin_unlock(&pl330->lock);
+
         power_down = pch->active;
         pch->active = false;
 
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index 2a2ccd9c78e4..48ee35e2bce6 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Renesas R-Car Gen2 DMA Controller Driver
  *
  * Copyright (C) 2014 Renesas Electronics Inc.
  *
  * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
- *
- * This is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
  */
 
 #include <linux/delay.h>
@@ -431,7 +428,8 @@ static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)
                 chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE;
         }
 
-        rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr | RCAR_DMACHCR_DE);
+        rcar_dmac_chan_write(chan, RCAR_DMACHCR,
+                             chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE);
 }
 
 static int rcar_dmac_init(struct rcar_dmac *dmac)
@@ -761,21 +759,15 @@ static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan)
         dev_err(chan->chan.device->dev, "CHCR DE check error\n");
 }
 
-static void rcar_dmac_sync_tcr(struct rcar_dmac_chan *chan)
+static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan)
 {
         u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
 
-        if (!(chcr & RCAR_DMACHCR_DE))
-                return;
-
         /* set DE=0 and flush remaining data */
         rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE));
 
         /* make sure all remaining data was flushed */
         rcar_dmac_chcr_de_barrier(chan);
-
-        /* back DE */
-        rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
 }
 
 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
@@ -783,7 +775,8 @@ static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
         u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
 
         chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE |
-                  RCAR_DMACHCR_TE | RCAR_DMACHCR_DE);
+                  RCAR_DMACHCR_TE | RCAR_DMACHCR_DE |
+                  RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE);
         rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
         rcar_dmac_chcr_de_barrier(chan);
 }
@@ -812,12 +805,7 @@ static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)
         }
 }
 
-static void rcar_dmac_stop(struct rcar_dmac *dmac)
-{
-        rcar_dmac_write(dmac, RCAR_DMAOR, 0);
-}
-
-static void rcar_dmac_abort(struct rcar_dmac *dmac)
+static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac)
 {
         unsigned int i;
 
@@ -826,14 +814,24 @@ static void rcar_dmac_abort(struct rcar_dmac *dmac)
                 struct rcar_dmac_chan *chan = &dmac->channels[i];
 
                 /* Stop and reinitialize the channel. */
-                spin_lock(&chan->lock);
+                spin_lock_irq(&chan->lock);
                 rcar_dmac_chan_halt(chan);
-                spin_unlock(&chan->lock);
-
-                rcar_dmac_chan_reinit(chan);
+                spin_unlock_irq(&chan->lock);
         }
 }
 
+static int rcar_dmac_chan_pause(struct dma_chan *chan)
+{
+        unsigned long flags;
+        struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
+
+        spin_lock_irqsave(&rchan->lock, flags);
+        rcar_dmac_clear_chcr_de(rchan);
+        spin_unlock_irqrestore(&rchan->lock, flags);
+
+        return 0;
+}
+
 /* -----------------------------------------------------------------------------
  * Descriptors preparation
  */
@@ -1355,9 +1353,6 @@ static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,
                 residue += chunk->size;
         }
 
-        if (desc->direction == DMA_DEV_TO_MEM)
-                rcar_dmac_sync_tcr(chan);
-
         /* Add the residue for the current chunk. */
         residue += rcar_dmac_chan_read(chan, RCAR_DMATCRB) << desc->xfer_shift;
 
@@ -1522,11 +1517,26 @@ static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev)
         u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE;
         struct rcar_dmac_chan *chan = dev;
         irqreturn_t ret = IRQ_NONE;
+        bool reinit = false;
         u32 chcr;
 
         spin_lock(&chan->lock);
 
         chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
+        if (chcr & RCAR_DMACHCR_CAE) {
+                struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device);
+
+                /*
+                 * We don't need to call rcar_dmac_chan_halt()
+                 * because channel is already stopped in error case.
+                 * We need to clear register and check DE bit as recovery.
+                 */
+                rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index);
+                rcar_dmac_chcr_de_barrier(chan);
+                reinit = true;
+                goto spin_lock_end;
+        }
+
         if (chcr & RCAR_DMACHCR_TE)
                 mask |= RCAR_DMACHCR_DE;
         rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);
@@ -1539,8 +1549,16 @@ static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev)
         if (chcr & RCAR_DMACHCR_TE)
                 ret |= rcar_dmac_isr_transfer_end(chan);
 
+spin_lock_end:
         spin_unlock(&chan->lock);
 
+        if (reinit) {
+                dev_err(chan->chan.device->dev, "Channel Address Error\n");
+
+                rcar_dmac_chan_reinit(chan);
+                ret = IRQ_HANDLED;
+        }
+
         return ret;
 }
 
@@ -1597,24 +1615,6 @@ static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev)
         return IRQ_HANDLED;
 }
 
-static irqreturn_t rcar_dmac_isr_error(int irq, void *data)
-{
-        struct rcar_dmac *dmac = data;
-
-        if (!(rcar_dmac_read(dmac, RCAR_DMAOR) & RCAR_DMAOR_AE))
-                return IRQ_NONE;
-
-        /*
-         * An unrecoverable error occurred on an unknown channel. Halt the DMAC,
-         * abort transfers on all channels, and reinitialize the DMAC.
-         */
-        rcar_dmac_stop(dmac);
-        rcar_dmac_abort(dmac);
-        rcar_dmac_init(dmac);
-
-        return IRQ_HANDLED;
-}
-
 /* -----------------------------------------------------------------------------
  * OF xlate and channel filter
  */
@@ -1784,8 +1784,6 @@ static int rcar_dmac_probe(struct platform_device *pdev)
         struct rcar_dmac *dmac;
         struct resource *mem;
         unsigned int i;
-        char *irqname;
-        int irq;
         int ret;
 
         dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
@@ -1824,17 +1822,6 @@ static int rcar_dmac_probe(struct platform_device *pdev)
         if (IS_ERR(dmac->iomem))
                 return PTR_ERR(dmac->iomem);
 
-        irq = platform_get_irq_byname(pdev, "error");
-        if (irq < 0) {
-                dev_err(&pdev->dev, "no error IRQ specified\n");
-                return -ENODEV;
-        }
-
-        irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:error",
-                                 dev_name(dmac->dev));
-        if (!irqname)
-                return -ENOMEM;
-
         /* Enable runtime PM and initialize the device. */
         pm_runtime_enable(&pdev->dev);
         ret = pm_runtime_get_sync(&pdev->dev);
@@ -1871,6 +1858,7 @@ static int rcar_dmac_probe(struct platform_device *pdev)
         engine->device_prep_slave_sg            = rcar_dmac_prep_slave_sg;
         engine->device_prep_dma_cyclic          = rcar_dmac_prep_dma_cyclic;
         engine->device_config                   = rcar_dmac_device_config;
+        engine->device_pause                    = rcar_dmac_chan_pause;
         engine->device_terminate_all            = rcar_dmac_chan_terminate_all;
         engine->device_tx_status                = rcar_dmac_tx_status;
         engine->device_issue_pending            = rcar_dmac_issue_pending;
@@ -1885,14 +1873,6 @@ static int rcar_dmac_probe(struct platform_device *pdev)
                         goto error;
         }
 
-        ret = devm_request_irq(&pdev->dev, irq, rcar_dmac_isr_error, 0,
-                               irqname, dmac);
-        if (ret) {
-                dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
-                        irq, ret);
-                return ret;
-        }
-
         /* Register the DMAC as a DMA provider for DT. */
         ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate,
                                          NULL);
@@ -1932,7 +1912,7 @@ static void rcar_dmac_shutdown(struct platform_device *pdev)
 {
         struct rcar_dmac *dmac = platform_get_drvdata(pdev);
 
-        rcar_dmac_stop(dmac);
+        rcar_dmac_stop_all_chan(dmac);
 }
 
 static const struct of_device_id rcar_dmac_of_ids[] = {
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 1bc149af990e..f4edfc56f34e 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -555,6 +555,7 @@ struct d40_gen_dmac {
  * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
  * later
  * @reg_val_backup_chan: Backup data for standard channel parameter registers.
+ * @regs_interrupt: Scratch space for registers during interrupt.
  * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
  * @gen_dmac: the struct for generic registers values to represent u8500/8540
  * DMA controller
@@ -592,6 +593,7 @@ struct d40_base {
         u32                               reg_val_backup[BACKUP_REGS_SZ];
         u32                               reg_val_backup_v4[BACKUP_REGS_SZ_MAX];
         u32                              *reg_val_backup_chan;
+        u32                              *regs_interrupt;
         u16                               gcc_pwr_off_mask;
         struct d40_gen_dmac               gen_dmac;
 };
@@ -1637,7 +1639,7 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
         struct d40_chan *d40c;
         unsigned long flags;
         struct d40_base *base = data;
-        u32 regs[base->gen_dmac.il_size];
+        u32 *regs = base->regs_interrupt;
         struct d40_interrupt_lookup *il = base->gen_dmac.il;
         u32 il_size = base->gen_dmac.il_size;
 
@@ -3258,13 +3260,22 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
         if (!base->lcla_pool.alloc_map)
                 goto free_backup_chan;
 
+        base->regs_interrupt = kmalloc_array(base->gen_dmac.il_size,
+                                             sizeof(*base->regs_interrupt),
+                                             GFP_KERNEL);
+        if (!base->regs_interrupt)
+                goto free_map;
+
         base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
                                             0, SLAB_HWCACHE_ALIGN,
                                             NULL);
         if (base->desc_slab == NULL)
-                goto free_map;
+                goto free_regs;
+
 
         return base;
+ free_regs:
+        kfree(base->regs_interrupt);
  free_map:
         kfree(base->lcla_pool.alloc_map);
  free_backup_chan:
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index 8c5807362a25..379e8d534e61 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -594,7 +594,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan)
 
         chan->busy = true;
 
-        dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan);
+        dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
 }
 
 static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan)
@@ -693,7 +693,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c)
 
         spin_lock_irqsave(&chan->vchan.lock, flags);
         if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) {
-                dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan);
+                dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
                 stm32_dma_start_transfer(chan);
 
         }
diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c
index 9dc450b7ace6..06dd1725375e 100644
--- a/drivers/dma/stm32-mdma.c
+++ b/drivers/dma/stm32-mdma.c
@@ -1170,7 +1170,7 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
 
         chan->busy = true;
 
-        dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan);
+        dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
 }
 
 static void stm32_mdma_issue_pending(struct dma_chan *c)
@@ -1183,7 +1183,7 @@ static void stm32_mdma_issue_pending(struct dma_chan *c)
         if (!vchan_issue_pending(&chan->vchan))
                 goto end;
 
-        dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan);
+        dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
 
         if (!chan->desc && !chan->busy)
                 stm32_mdma_start_transfer(chan);
@@ -1203,7 +1203,7 @@ static int stm32_mdma_pause(struct dma_chan *c)
         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 
         if (!ret)
-                dev_dbg(chan2dev(chan), "vchan %p: pause\n", &chan->vchan);
+                dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan);
 
         return ret;
 }
@@ -1240,7 +1240,7 @@ static int stm32_mdma_resume(struct dma_chan *c)
 
         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 
-        dev_dbg(chan2dev(chan), "vchan %p: resume\n", &chan->vchan);
+        dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan);
 
         return 0;
 }
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 27b523530c4a..c12442312595 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -115,6 +115,9 @@
 #define XILINX_VDMA_REG_START_ADDRESS(n)        (0x000c + 4 * (n))
 #define XILINX_VDMA_REG_START_ADDRESS_64(n)     (0x000c + 8 * (n))
 
+#define XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP    0x00ec
+#define XILINX_VDMA_ENABLE_VERTICAL_FLIP        BIT(0)
+
 /* HW specific definitions */
 #define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x20
 
@@ -340,6 +343,7 @@ struct xilinx_dma_tx_descriptor {
  * @start_transfer: Differentiate b/w DMA IP's transfer
  * @stop_transfer: Differentiate b/w DMA IP's quiesce
  * @tdest: TDEST value for mcdma
+ * @has_vflip: S2MM vertical flip
  */
 struct xilinx_dma_chan {
         struct xilinx_dma_device *xdev;
@@ -376,6 +380,7 @@ struct xilinx_dma_chan {
         void (*start_transfer)(struct xilinx_dma_chan *chan);
         int (*stop_transfer)(struct xilinx_dma_chan *chan);
         u16 tdest;
+        bool has_vflip;
 };
 
 /**
@@ -1092,6 +1097,14 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
                                 desc->async_tx.phys);
 
         /* Configure the hardware using info in the config structure */
+        if (chan->has_vflip) {
+                reg = dma_read(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP);
+                reg &= ~XILINX_VDMA_ENABLE_VERTICAL_FLIP;
+                reg |= config->vflip_en;
+                dma_write(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP,
+                          reg);
+        }
+
         reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
 
         if (config->frm_cnt_en)
@@ -2105,6 +2118,8 @@ int xilinx_vdma_channel_set_config(struct dma_chan *dchan,
         }
 
         chan->config.frm_cnt_en = cfg->frm_cnt_en;
+        chan->config.vflip_en = cfg->vflip_en;
+
         if (cfg->park)
                 chan->config.park_frm = cfg->park_frm;
         else
@@ -2428,6 +2443,13 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
                 chan->direction = DMA_DEV_TO_MEM;
                 chan->id = chan_id;
                 chan->tdest = chan_id - xdev->nr_channels;
+                chan->has_vflip = of_property_read_bool(node,
+                                        "xlnx,enable-vert-flip");
+                if (chan->has_vflip) {
+                        chan->config.vflip_en = dma_read(chan,
+                                XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP) &
+                                XILINX_VDMA_ENABLE_VERTICAL_FLIP;
+                }
 
                 chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;
                 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
diff --git a/include/linux/dma/xilinx_dma.h b/include/linux/dma/xilinx_dma.h
index 34b98f276ed0..5b6e61e4b3aa 100644
--- a/include/linux/dma/xilinx_dma.h
+++ b/include/linux/dma/xilinx_dma.h
@@ -27,6 +27,7 @@
  * @delay: Delay counter
  * @reset: Reset Channel
  * @ext_fsync: External Frame Sync source
+ * @vflip_en:  Vertical Flip enable
  */
 struct xilinx_vdma_config {
         int frm_dly;
@@ -39,6 +40,7 @@ struct xilinx_vdma_config {
         int delay;
         int reset;
         int ext_fsync;
+        bool vflip_en;
 };
 
 int xilinx_vdma_channel_set_config(struct dma_chan *dchan,
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 861be5cab1df..d49ec5c31944 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -415,7 +415,9 @@ enum dma_residue_granularity {
  *      each type, the dma controller should set BIT(<TYPE>) and same
  *      should be checked by controller as well
  * @max_burst: max burst capability per-transfer
- * @cmd_pause: true, if pause and thereby resume is supported
+ * @cmd_pause: true, if pause is supported (i.e. for reading residue or
+ *             for resume later)
+ * @cmd_resume: true, if resume is supported
  * @cmd_terminate: true, if terminate cmd is supported
  * @residue_granularity: granularity of the reported transfer residue
  * @descriptor_reuse: if a descriptor can be reused by client and
@@ -427,6 +429,7 @@ struct dma_slave_caps {
         u32 directions;
         u32 max_burst;
         bool cmd_pause;
+        bool cmd_resume;
         bool cmd_terminate;
         enum dma_residue_granularity residue_granularity;
         bool descriptor_reuse;
@@ -1403,6 +1406,7 @@ static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc)
 /* --- DMA device --- */
 
 int dma_async_device_register(struct dma_device *device);
+int dmaenginem_async_device_register(struct dma_device *device);
 void dma_async_device_unregister(struct dma_device *device);
 void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan);
diff --git a/sound/soc/soc-generic-dmaengine-pcm.c b/sound/soc/soc-generic-dmaengine-pcm.c
index 52fd7af952a5..30e791a53352 100644
--- a/sound/soc/soc-generic-dmaengine-pcm.c
+++ b/sound/soc/soc-generic-dmaengine-pcm.c
@@ -147,7 +147,7 @@ static int dmaengine_pcm_set_runtime_hwparams(struct snd_pcm_substream *substrea
 
         ret = dma_get_slave_caps(chan, &dma_caps);
         if (ret == 0) {
-                if (dma_caps.cmd_pause)
+                if (dma_caps.cmd_pause && dma_caps.cmd_resume)
                         hw.info |= SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME;
                 if (dma_caps.residue_granularity <= DMA_RESIDUE_GRANULARITY_SEGMENT)
                         hw.info |= SNDRV_PCM_INFO_BATCH;