2 files changed, 259 insertions, 19 deletions

diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 15b311d54b74..c2cf8cfaf7d4 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -14,6 +14,8 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
 #include <linux/err.h>
 #include <linux/amba/bus.h>
 
@@ -32,6 +34,9 @@
 /* Maximum iterations taken before giving up suspending a channel */
 #define D40_SUSPEND_MAX_IT 500
 
+/* Milliseconds */
+#define DMA40_AUTOSUSPEND_DELAY 100
+
 /* Hardware requirement on LCLA alignment */
 #define LCLA_ALIGNMENT 0x40000
 
@@ -62,6 +67,55 @@ enum d40_command {
         D40_DMA_SUSPENDED       = 3
 };
 
+/*
+ * These are the registers that has to be saved and later restored
+ * when the DMA hw is powered off.
+ * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
+ */
+static u32 d40_backup_regs[] = {
+        D40_DREG_LCPA,
+        D40_DREG_LCLA,
+        D40_DREG_PRMSE,
+        D40_DREG_PRMSO,
+        D40_DREG_PRMOE,
+        D40_DREG_PRMOO,
+};
+
+#define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
+
+/* TODO: Check if all these registers have to be saved/restored on dma40 v3 */
+static u32 d40_backup_regs_v3[] = {
+        D40_DREG_PSEG1,
+        D40_DREG_PSEG2,
+        D40_DREG_PSEG3,
+        D40_DREG_PSEG4,
+        D40_DREG_PCEG1,
+        D40_DREG_PCEG2,
+        D40_DREG_PCEG3,
+        D40_DREG_PCEG4,
+        D40_DREG_RSEG1,
+        D40_DREG_RSEG2,
+        D40_DREG_RSEG3,
+        D40_DREG_RSEG4,
+        D40_DREG_RCEG1,
+        D40_DREG_RCEG2,
+        D40_DREG_RCEG3,
+        D40_DREG_RCEG4,
+};
+
+#define BACKUP_REGS_SZ_V3 ARRAY_SIZE(d40_backup_regs_v3)
+
+static u32 d40_backup_regs_chan[] = {
+        D40_CHAN_REG_SSCFG,
+        D40_CHAN_REG_SSELT,
+        D40_CHAN_REG_SSPTR,
+        D40_CHAN_REG_SSLNK,
+        D40_CHAN_REG_SDCFG,
+        D40_CHAN_REG_SDELT,
+        D40_CHAN_REG_SDPTR,
+        D40_CHAN_REG_SDLNK,
+};
+
 /**
  * struct d40_lli_pool - Structure for keeping LLIs in memory
  *
@@ -96,7 +150,7 @@ struct d40_lli_pool {
  * during a transfer.
  * @node: List entry.
  * @is_in_client_list: true if the client owns this descriptor.
- * the previous one.
+ * @cyclic: true if this is a cyclic job
  *
  * This descriptor is used for both logical and physical transfers.
  */
@@ -143,6 +197,7 @@ struct d40_lcla_pool {
  * channels.
  *
  * @lock: A lock protection this entity.
+ * @reserved: True if used by secure world or otherwise.
  * @num: The physical channel number of this entity.
  * @allocated_src: Bit mapped to show which src event line's are mapped to
  * this physical channel. Can also be free or physically allocated.
@@ -152,6 +207,7 @@ struct d40_lcla_pool {
  */
 struct d40_phy_res {
         spinlock_t lock;
+        bool       reserved;
         int        num;
         u32        allocated_src;
         u32        allocated_dst;
@@ -185,7 +241,6 @@ struct d40_base;
  * @src_def_cfg: Default cfg register setting for src.
  * @dst_def_cfg: Default cfg register setting for dst.
  * @log_def: Default logical channel settings.
- * @lcla: Space for one dst src pair for logical channel transfers.
  * @lcpa: Pointer to dst and src lcpa settings.
  * @runtime_addr: runtime configured address.
  * @runtime_direction: runtime configured direction.
@@ -241,6 +296,7 @@ struct d40_chan {
  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
  * @dma_slave: dma_device channels that can do only do slave transfers.
  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
+ * @phy_chans: Room for all possible physical channels in system.
  * @log_chans: Room for all possible logical channels in system.
  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
  * to log_chans entries.
@@ -254,6 +310,13 @@ struct d40_chan {
  * @phy_lcpa: The physical address of the LCPA.
  * @lcpa_size: The size of the LCPA area.
  * @desc_slab: cache for descriptors.
+ * @reg_val_backup: Here the values of some hardware registers are stored
+ * before the DMA is powered off. They are restored when the power is back on.
+ * @reg_val_backup_v3: Backup of registers that only exits on dma40 v3 and
+ * later.
+ * @reg_val_backup_chan: Backup data for standard channel parameter registers.
+ * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
+ * @initialized: true if the dma has been initialized
  */
 struct d40_base {
         spinlock_t                       interrupt_lock;
@@ -282,6 +345,11 @@ struct d40_base {
         dma_addr_t                        phy_lcpa;
         resource_size_t                   lcpa_size;
         struct kmem_cache                *desc_slab;
+        u32                               reg_val_backup[BACKUP_REGS_SZ];
+        u32                               reg_val_backup_v3[BACKUP_REGS_SZ_V3];
+        u32                              *reg_val_backup_chan;
+        u16                               gcc_pwr_off_mask;
+        bool                              initialized;
 };
 
 /**
@@ -479,13 +547,14 @@ static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
                 struct d40_desc *d;
                 struct d40_desc *_d;
 
-                list_for_each_entry_safe(d, _d, &d40c->client, node)
+                list_for_each_entry_safe(d, _d, &d40c->client, node) {
                         if (async_tx_test_ack(&d->txd)) {
                                 d40_desc_remove(d);
                                 desc = d;
                                 memset(desc, 0, sizeof(*desc));
                                 break;
                         }
+                }
         }
 
         if (!desc)
@@ -740,7 +809,61 @@ static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
         return len;
 }
 
-/* Support functions for logical channels */
+
+#ifdef CONFIG_PM
+static void dma40_backup(void __iomem *baseaddr, u32 *backup,
+                         u32 *regaddr, int num, bool save)
+{
+        int i;
+
+        for (i = 0; i < num; i++) {
+                void __iomem *addr = baseaddr + regaddr[i];
+
+                if (save)
+                        backup[i] = readl_relaxed(addr);
+                else
+                        writel_relaxed(backup[i], addr);
+        }
+}
+
+static void d40_save_restore_registers(struct d40_base *base, bool save)
+{
+        int i;
+
+        /* Save/Restore channel specific registers */
+        for (i = 0; i < base->num_phy_chans; i++) {
+                void __iomem *addr;
+                int idx;
+
+                if (base->phy_res[i].reserved)
+                        continue;
+
+                addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
+                idx = i * ARRAY_SIZE(d40_backup_regs_chan);
+
+                dma40_backup(addr, &base->reg_val_backup_chan[idx],
+                             d40_backup_regs_chan,
+                             ARRAY_SIZE(d40_backup_regs_chan),
+                             save);
+        }
+
+        /* Save/Restore global registers */
+        dma40_backup(base->virtbase, base->reg_val_backup,
+                     d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
+                     save);
+
+        /* Save/Restore registers only existing on dma40 v3 and later */
+        if (base->rev >= 3)
+                dma40_backup(base->virtbase, base->reg_val_backup_v3,
+                             d40_backup_regs_v3,
+                             ARRAY_SIZE(d40_backup_regs_v3),
+                             save);
+}
+#else
+static void d40_save_restore_registers(struct d40_base *base, bool save)
+{
+}
+#endif
 
 static int d40_channel_execute_command(struct d40_chan *d40c,
                                        enum d40_command command)
@@ -1013,6 +1136,7 @@ static int d40_pause(struct d40_chan *d40c)
         if (!d40c->busy)
                 return 0;
 
+        pm_runtime_get_sync(d40c->base->dev);
         spin_lock_irqsave(&d40c->lock, flags);
 
         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
@@ -1025,7 +1149,8 @@ static int d40_pause(struct d40_chan *d40c)
                                                                   D40_DMA_RUN);
                 }
         }
-
+        pm_runtime_mark_last_busy(d40c->base->dev);
+        pm_runtime_put_autosuspend(d40c->base->dev);
         spin_unlock_irqrestore(&d40c->lock, flags);
         return res;
 }
@@ -1039,7 +1164,7 @@ static int d40_resume(struct d40_chan *d40c)
                 return 0;
 
         spin_lock_irqsave(&d40c->lock, flags);
-
+        pm_runtime_get_sync(d40c->base->dev);
         if (d40c->base->rev == 0)
                 if (chan_is_logical(d40c)) {
                         res = d40_channel_execute_command(d40c,
@@ -1057,6 +1182,8 @@ static int d40_resume(struct d40_chan *d40c)
         }
 
 no_suspend:
+        pm_runtime_mark_last_busy(d40c->base->dev);
+        pm_runtime_put_autosuspend(d40c->base->dev);
         spin_unlock_irqrestore(&d40c->lock, flags);
         return res;
 }
@@ -1129,7 +1256,10 @@ static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
         d40d = d40_first_queued(d40c);
 
         if (d40d != NULL) {
-                d40c->busy = true;
+                if (!d40c->busy)
+                        d40c->busy = true;
+
+                pm_runtime_get_sync(d40c->base->dev);
 
                 /* Remove from queue */
                 d40_desc_remove(d40d);
@@ -1190,6 +1320,8 @@ static void dma_tc_handle(struct d40_chan *d40c)
 
                 if (d40_queue_start(d40c) == NULL)
                         d40c->busy = false;
+                pm_runtime_mark_last_busy(d40c->base->dev);
+                pm_runtime_put_autosuspend(d40c->base->dev);
         }
 
         d40c->pending_tx++;
@@ -1643,10 +1775,11 @@ static int d40_free_dma(struct d40_chan *d40c)
                 return -EINVAL;
         }
 
+        pm_runtime_get_sync(d40c->base->dev);
         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
         if (res) {
                 chan_err(d40c, "suspend failed\n");
-                return res;
+                goto out;
         }
 
         if (chan_is_logical(d40c)) {
@@ -1664,13 +1797,11 @@ static int d40_free_dma(struct d40_chan *d40c)
                         if (d40_chan_has_events(d40c)) {
                                 res = d40_channel_execute_command(d40c,
                                                                   D40_DMA_RUN);
-                                if (res) {
+                                if (res)
                                         chan_err(d40c,
                                                 "Executing RUN command\n");
-                                        return res;
-                                }
                         }
-                        return 0;
+                        goto out;
                 }
         } else {
                 (void) d40_alloc_mask_free(phy, is_src, 0);
@@ -1680,13 +1811,23 @@ static int d40_free_dma(struct d40_chan *d40c)
         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
         if (res) {
                 chan_err(d40c, "Failed to stop channel\n");
-                return res;
+                goto out;
         }
+
+        if (d40c->busy) {
+                pm_runtime_mark_last_busy(d40c->base->dev);
+                pm_runtime_put_autosuspend(d40c->base->dev);
+        }
+
+        d40c->busy = false;
         d40c->phy_chan = NULL;
         d40c->configured = false;
         d40c->base->lookup_phy_chans[phy->num] = NULL;
+out:
 
-        return 0;
+        pm_runtime_mark_last_busy(d40c->base->dev);
+        pm_runtime_put_autosuspend(d40c->base->dev);
+        return res;
 }
 
 static bool d40_is_paused(struct d40_chan *d40c)
@@ -2016,9 +2157,11 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
         err = d40_allocate_channel(d40c);
         if (err) {
                 chan_err(d40c, "Failed to allocate channel\n");
+                d40c->configured = false;
                 goto fail;
         }
 
+        pm_runtime_get_sync(d40c->base->dev);
         /* Fill in basic CFG register values */
         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
                     &d40c->dst_def_cfg, chan_is_logical(d40c));
@@ -2046,6 +2189,8 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
         if (is_free_phy)
                 d40_config_write(d40c);
 fail:
+        pm_runtime_mark_last_busy(d40c->base->dev);
+        pm_runtime_put_autosuspend(d40c->base->dev);
         spin_unlock_irqrestore(&d40c->lock, flags);
         return err;
 }
@@ -2519,6 +2664,55 @@ failure1:
         return err;
 }
 
+/* Suspend resume functionality */
+#ifdef CONFIG_PM
+static int dma40_pm_suspend(struct device *dev)
+{
+        if (!pm_runtime_suspended(dev))
+                return -EBUSY;
+
+        return 0;
+}
+
+static int dma40_runtime_suspend(struct device *dev)
+{
+        struct platform_device *pdev = to_platform_device(dev);
+        struct d40_base *base = platform_get_drvdata(pdev);
+
+        d40_save_restore_registers(base, true);
+
+        /* Don't disable/enable clocks for v1 due to HW bugs */
+        if (base->rev != 1)
+                writel_relaxed(base->gcc_pwr_off_mask,
+                               base->virtbase + D40_DREG_GCC);
+
+        return 0;
+}
+
+static int dma40_runtime_resume(struct device *dev)
+{
+        struct platform_device *pdev = to_platform_device(dev);
+        struct d40_base *base = platform_get_drvdata(pdev);
+
+        if (base->initialized)
+                d40_save_restore_registers(base, false);
+
+        writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
+                       base->virtbase + D40_DREG_GCC);
+        return 0;
+}
+
+
+static const struct dev_pm_ops dma40_pm_ops = {
+        .suspend                = dma40_pm_suspend,
+        .runtime_suspend        = dma40_runtime_suspend,
+        .runtime_resume         = dma40_runtime_resume,
+};
+#define DMA40_PM_OPS    (&dma40_pm_ops)
+#else
+#define DMA40_PM_OPS    NULL
+#endif
+
 /* Initialization functions. */
 
 static int __init d40_phy_res_init(struct d40_base *base)
@@ -2527,6 +2721,7 @@ static int __init d40_phy_res_init(struct d40_base *base)
         int num_phy_chans_avail = 0;
         u32 val[2];
         int odd_even_bit = -2;
+        int gcc = D40_DREG_GCC_ENA;
 
         val[0] = readl(base->virtbase + D40_DREG_PRSME);
         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
@@ -2538,9 +2733,17 @@ static int __init d40_phy_res_init(struct d40_base *base)
                         /* Mark security only channels as occupied */
                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
+                        base->phy_res[i].reserved = true;
+                        gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
+                                                       D40_DREG_GCC_SRC);
+                        gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
+                                                       D40_DREG_GCC_DST);
+
+
                 } else {
                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
+                        base->phy_res[i].reserved = false;
                         num_phy_chans_avail++;
                 }
                 spin_lock_init(&base->phy_res[i].lock);
@@ -2552,6 +2755,11 @@ static int __init d40_phy_res_init(struct d40_base *base)
 
                 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
                 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
+                base->phy_res[chan].reserved = true;
+                gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
+                                               D40_DREG_GCC_SRC);
+                gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
+                                               D40_DREG_GCC_DST);
                 num_phy_chans_avail--;
         }
 
@@ -2572,6 +2780,15 @@ static int __init d40_phy_res_init(struct d40_base *base)
                 val[0] = val[0] >> 2;
         }
 
+        /*
+         * To keep things simple, Enable all clocks initially.
+         * The clocks will get managed later post channel allocation.
+         * The clocks for the event lines on which reserved channels exists
+         * are not managed here.
+         */
+        writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
+        base->gcc_pwr_off_mask = gcc;
+
         return num_phy_chans_avail;
 }
 
@@ -2699,10 +2916,15 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
                         goto failure;
         }
 
-        base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
-                                            sizeof(struct d40_desc *) *
-                                            D40_LCLA_LINK_PER_EVENT_GRP,
+        base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
+                                            sizeof(d40_backup_regs_chan),
                                             GFP_KERNEL);
+        if (!base->reg_val_backup_chan)
+                goto failure;
+
+        base->lcla_pool.alloc_map =
+                kzalloc(num_phy_chans * sizeof(struct d40_desc *)
+                        * D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
         if (!base->lcla_pool.alloc_map)
                 goto failure;
 
@@ -2741,9 +2963,9 @@ failure:
 static void __init d40_hw_init(struct d40_base *base)
 {
 
-        static const struct d40_reg_val dma_init_reg[] = {
+        static struct d40_reg_val dma_init_reg[] = {
                 /* Clock every part of the DMA block from start */
-                { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
+                { .reg = D40_DREG_GCC,    .val = D40_DREG_GCC_ENABLE_ALL},
 
                 /* Interrupts on all logical channels */
                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
@@ -2960,6 +3182,12 @@ static int __init d40_probe(struct platform_device *pdev)
                 goto failure;
         }
 
+        pm_runtime_irq_safe(base->dev);
+        pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);
+        pm_runtime_use_autosuspend(base->dev);
+        pm_runtime_enable(base->dev);
+        pm_runtime_resume(base->dev);
+        base->initialized = true;
         err = d40_dmaengine_init(base, num_reserved_chans);
         if (err)
                 goto failure;
@@ -3013,6 +3241,7 @@ static struct platform_driver d40_driver = {
         .driver = {
                 .owner = THIS_MODULE,
                 .name  = D40_NAME,
+                .pm = DMA40_PM_OPS,
         },
 };
 
diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h
index b44c455158de..8d3d490968a3 100644
--- a/drivers/dma/ste_dma40_ll.h
+++ b/drivers/dma/ste_dma40_ll.h
@@ -16,6 +16,8 @@
 
 #define D40_TYPE_TO_GROUP(type) (type / 16)
 #define D40_TYPE_TO_EVENT(type) (type % 16)
+#define D40_GROUP_SIZE 8
+#define D40_PHYS_TO_GROUP(phys) ((phys & (D40_GROUP_SIZE - 1)) / 2)
 
 /* Most bits of the CFG register are the same in log as in phy mode */
 #define D40_SREG_CFG_MST_POS            15
@@ -123,6 +125,15 @@
 
 /* DMA Register Offsets */
 #define D40_DREG_GCC            0x000
+#define D40_DREG_GCC_ENA        0x1
+/* This assumes that there are only 4 event groups */
+#define D40_DREG_GCC_ENABLE_ALL 0xff01
+#define D40_DREG_GCC_EVTGRP_POS 8
+#define D40_DREG_GCC_SRC 0
+#define D40_DREG_GCC_DST 1
+#define D40_DREG_GCC_EVTGRP_ENA(x, y) \
+        (1 << (D40_DREG_GCC_EVTGRP_POS + 2 * x + y))
+
 #define D40_DREG_PRTYP          0x004
 #define D40_DREG_PRSME          0x008
 #define D40_DREG_PRSMO          0x00C