1 files changed, 279 insertions, 53 deletions

diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c
index eb6b54dbb806..028330044201 100644
--- a/drivers/dma/shdma.c
+++ b/drivers/dma/shdma.c
@@ -27,7 +27,10 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/sh_dma.h>
-
+#include <linux/notifier.h>
+#include <linux/kdebug.h>
+#include <linux/spinlock.h>
+#include <linux/rculist.h>
 #include "shdma.h"
 
 /* DMA descriptor control */
@@ -43,6 +46,13 @@ enum sh_dmae_desc_status {
 /* Default MEMCPY transfer size = 2^2 = 4 bytes */
 #define LOG2_DEFAULT_XFER_SIZE  2
 
+/*
+ * Used for write-side mutual exclusion for the global device list,
+ * read-side synchronization by way of RCU, and per-controller data.
+ */
+static DEFINE_SPINLOCK(sh_dmae_lock);
+static LIST_HEAD(sh_dmae_devices);
+
 /* A bitmask with bits enough for enum sh_dmae_slave_chan_id */
 static unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SH_DMA_SLAVE_NUMBER)];
 
@@ -75,22 +85,35 @@ static void dmaor_write(struct sh_dmae_device *shdev, u16 data)
  */
 static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev)
 {
-        unsigned short dmaor = dmaor_read(shdev);
+        unsigned short dmaor;
+        unsigned long flags;
 
+        spin_lock_irqsave(&sh_dmae_lock, flags);
+
+        dmaor = dmaor_read(shdev);
         dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME));
+
+        spin_unlock_irqrestore(&sh_dmae_lock, flags);
 }
 
 static int sh_dmae_rst(struct sh_dmae_device *shdev)
 {
         unsigned short dmaor;
+        unsigned long flags;
 
-        sh_dmae_ctl_stop(shdev);
-        dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init;
+        spin_lock_irqsave(&sh_dmae_lock, flags);
 
-        dmaor_write(shdev, dmaor);
-        if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) {
-                pr_warning("dma-sh: Can't initialize DMAOR.\n");
-                return -EINVAL;
+        dmaor = dmaor_read(shdev) & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME);
+
+        dmaor_write(shdev, dmaor | shdev->pdata->dmaor_init);
+
+        dmaor = dmaor_read(shdev);
+
+        spin_unlock_irqrestore(&sh_dmae_lock, flags);
+
+        if (dmaor & (DMAOR_AE | DMAOR_NMIF)) {
+                dev_warn(shdev->common.dev, "Can't initialize DMAOR.\n");
+                return -EIO;
         }
         return 0;
 }
@@ -174,7 +197,7 @@ static void dmae_init(struct sh_dmae_chan *sh_chan)
 
 static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
 {
-        /* When DMA was working, can not set data to CHCR */
+        /* If DMA is active, cannot set CHCR. TODO: remove this superfluous check */
         if (dmae_is_busy(sh_chan))
                 return -EBUSY;
 
@@ -190,12 +213,17 @@ static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
                                                 struct sh_dmae_device, common);
         struct sh_dmae_pdata *pdata = shdev->pdata;
         const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id];
-        u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16);
+        u16 __iomem *addr = shdev->dmars;
         int shift = chan_pdata->dmars_bit;
 
         if (dmae_is_busy(sh_chan))
                 return -EBUSY;
 
+        /* in the case of a missing DMARS resource use first memory window */
+        if (!addr)
+                addr = (u16 __iomem *)shdev->chan_reg;
+        addr += chan_pdata->dmars / sizeof(u16);
+
         __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift),
                      addr);
 
@@ -315,7 +343,7 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
 
                 dmae_set_dmars(sh_chan, cfg->mid_rid);
                 dmae_set_chcr(sh_chan, cfg->chcr);
-        } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) {
+        } else {
                 dmae_init(sh_chan);
         }
 
@@ -364,7 +392,12 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan)
         LIST_HEAD(list);
         int descs = sh_chan->descs_allocated;
 
+        /* Protect against ISR */
+        spin_lock_irq(&sh_chan->desc_lock);
         dmae_halt(sh_chan);
+        spin_unlock_irq(&sh_chan->desc_lock);
+
+        /* Now no new interrupts will occur */
 
         /* Prepared and not submitted descriptors can still be on the queue */
         if (!list_empty(&sh_chan->ld_queue))
@@ -374,6 +407,7 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan)
                 /* The caller is holding dma_list_mutex */
                 struct sh_dmae_slave *param = chan->private;
                 clear_bit(param->slave_id, sh_dmae_slave_used);
+                chan->private = NULL;
         }
 
         spin_lock_bh(&sh_chan->desc_lock);
@@ -553,8 +587,6 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
         if (!chan || !len)
                 return NULL;
 
-        chan->private = NULL;
-
         sh_chan = to_sh_chan(chan);
 
         sg_init_table(&sg, 1);
@@ -610,9 +642,9 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
         if (!chan)
                 return -EINVAL;
 
+        spin_lock_bh(&sh_chan->desc_lock);
         dmae_halt(sh_chan);
 
-        spin_lock_bh(&sh_chan->desc_lock);
         if (!list_empty(&sh_chan->ld_queue)) {
                 /* Record partial transfer */
                 struct sh_desc *desc = list_entry(sh_chan->ld_queue.next,
@@ -706,6 +738,14 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all
                         list_move(&desc->node, &sh_chan->ld_free);
                 }
         }
+
+        if (all && !callback)
+                /*
+                 * Terminating and the loop completed normally: forgive
+                 * uncompleted cookies
+                 */
+                sh_chan->completed_cookie = sh_chan->common.cookie;
+
         spin_unlock_bh(&sh_chan->desc_lock);
 
         if (callback)
@@ -723,10 +763,6 @@ static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
 {
         while (__ld_cleanup(sh_chan, all))
                 ;
-
-        if (all)
-                /* Terminating - forgive uncompleted cookies */
-                sh_chan->completed_cookie = sh_chan->common.cookie;
 }
 
 static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
@@ -740,7 +776,7 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
                 return;
         }
 
-        /* Find the first not transferred desciptor */
+        /* Find the first not transferred descriptor */
         list_for_each_entry(desc, &sh_chan->ld_queue, node)
                 if (desc->mark == DESC_SUBMITTED) {
                         dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n",
@@ -772,8 +808,10 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
 
         sh_dmae_chan_ld_cleanup(sh_chan, false);
 
-        last_used = chan->cookie;
+        /* First read completed cookie to avoid a skew */
         last_complete = sh_chan->completed_cookie;
+        rmb();
+        last_used = chan->cookie;
         BUG_ON(last_complete < 0);
         dma_set_tx_state(txstate, last_complete, last_used, 0);
 
@@ -803,8 +841,12 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
 static irqreturn_t sh_dmae_interrupt(int irq, void *data)
 {
         irqreturn_t ret = IRQ_NONE;
-        struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
-        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+        struct sh_dmae_chan *sh_chan = data;
+        u32 chcr;
+
+        spin_lock(&sh_chan->desc_lock);
+
+        chcr = sh_dmae_readl(sh_chan, CHCR);
 
         if (chcr & CHCR_TE) {
                 /* DMA stop */
@@ -814,13 +856,15 @@ static irqreturn_t sh_dmae_interrupt(int irq, void *data)
                 tasklet_schedule(&sh_chan->tasklet);
         }
 
+        spin_unlock(&sh_chan->desc_lock);
+
         return ret;
 }
 
-#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
-static irqreturn_t sh_dmae_err(int irq, void *data)
+/* Called from error IRQ or NMI */
+static bool sh_dmae_reset(struct sh_dmae_device *shdev)
 {
-        struct sh_dmae_device *shdev = (struct sh_dmae_device *)data;
+        unsigned int handled = 0;
         int i;
 
         /* halt the dma controller */
@@ -829,25 +873,51 @@ static irqreturn_t sh_dmae_err(int irq, void *data)
         /* We cannot detect, which channel caused the error, have to reset all */
         for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) {
                 struct sh_dmae_chan *sh_chan = shdev->chan[i];
-                if (sh_chan) {
-                        struct sh_desc *desc;
-                        /* Stop the channel */
-                        dmae_halt(sh_chan);
-                        /* Complete all  */
-                        list_for_each_entry(desc, &sh_chan->ld_queue, node) {
-                                struct dma_async_tx_descriptor *tx = &desc->async_tx;
-                                desc->mark = DESC_IDLE;
-                                if (tx->callback)
-                                        tx->callback(tx->callback_param);
-                        }
-                        list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free);
+                struct sh_desc *desc;
+                LIST_HEAD(dl);
+
+                if (!sh_chan)
+                        continue;
+
+                spin_lock(&sh_chan->desc_lock);
+
+                /* Stop the channel */
+                dmae_halt(sh_chan);
+
+                list_splice_init(&sh_chan->ld_queue, &dl);
+
+                spin_unlock(&sh_chan->desc_lock);
+
+                /* Complete all  */
+                list_for_each_entry(desc, &dl, node) {
+                        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+                        desc->mark = DESC_IDLE;
+                        if (tx->callback)
+                                tx->callback(tx->callback_param);
                 }
+
+                spin_lock(&sh_chan->desc_lock);
+                list_splice(&dl, &sh_chan->ld_free);
+                spin_unlock(&sh_chan->desc_lock);
+
+                handled++;
         }
+
         sh_dmae_rst(shdev);
 
+        return !!handled;
+}
+
+static irqreturn_t sh_dmae_err(int irq, void *data)
+{
+        struct sh_dmae_device *shdev = data;
+
+        if (!(dmaor_read(shdev) & DMAOR_AE))
+                return IRQ_NONE;
+
+        sh_dmae_reset(data);
         return IRQ_HANDLED;
 }
-#endif
 
 static void dmae_do_tasklet(unsigned long data)
 {
@@ -876,6 +946,54 @@ static void dmae_do_tasklet(unsigned long data)
         sh_dmae_chan_ld_cleanup(sh_chan, false);
 }
 
+static bool sh_dmae_nmi_notify(struct sh_dmae_device *shdev)
+{
+        /* Fast path out if NMIF is not asserted for this controller */
+        if ((dmaor_read(shdev) & DMAOR_NMIF) == 0)
+                return false;
+
+        return sh_dmae_reset(shdev);
+}
+
+static int sh_dmae_nmi_handler(struct notifier_block *self,
+                               unsigned long cmd, void *data)
+{
+        struct sh_dmae_device *shdev;
+        int ret = NOTIFY_DONE;
+        bool triggered;
+
+        /*
+         * Only concern ourselves with NMI events.
+         *
+         * Normally we would check the die chain value, but as this needs
+         * to be architecture independent, check for NMI context instead.
+         */
+        if (!in_nmi())
+                return NOTIFY_DONE;
+
+        rcu_read_lock();
+        list_for_each_entry_rcu(shdev, &sh_dmae_devices, node) {
+                /*
+                 * Only stop if one of the controllers has NMIF asserted,
+                 * we do not want to interfere with regular address error
+                 * handling or NMI events that don't concern the DMACs.
+                 */
+                triggered = sh_dmae_nmi_notify(shdev);
+                if (triggered == true)
+                        ret = NOTIFY_OK;
+        }
+        rcu_read_unlock();
+
+        return ret;
+}
+
+static struct notifier_block sh_dmae_nmi_notifier __read_mostly = {
+        .notifier_call  = sh_dmae_nmi_handler,
+
+        /* Run before NMI debug handler and KGDB */
+        .priority       = 1,
+};
+
 static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id,
                                         int irq, unsigned long flags)
 {
@@ -904,9 +1022,6 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id,
         tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet,
                         (unsigned long)new_sh_chan);
 
-        /* Init the channel */
-        dmae_init(new_sh_chan);
-
         spin_lock_init(&new_sh_chan->desc_lock);
 
         /* Init descripter manage list */
@@ -968,7 +1083,7 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
         unsigned long irqflags = IRQF_DISABLED,
                 chan_flag[SH_DMAC_MAX_CHANNELS] = {};
         int errirq, chan_irq[SH_DMAC_MAX_CHANNELS];
-        int err, i, irq_cnt = 0, irqres = 0;
+        int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0;
         struct sh_dmae_device *shdev;
         struct resource *chan, *dmars, *errirq_res, *chanirq_res;
 
@@ -977,7 +1092,7 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
                 return -ENODEV;
 
         chan = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-        /* DMARS area is optional, if absent, this controller cannot do slave DMA */
+        /* DMARS area is optional */
         dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1);
         /*
          * IRQ resources:
@@ -1029,10 +1144,16 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
         /* platform data */
         shdev->pdata = pdata;
 
+        platform_set_drvdata(pdev, shdev);
+
         pm_runtime_enable(&pdev->dev);
         pm_runtime_get_sync(&pdev->dev);
 
-        /* reset dma controller */
+        spin_lock_irq(&sh_dmae_lock);
+        list_add_tail_rcu(&shdev->node, &sh_dmae_devices);
+        spin_unlock_irq(&sh_dmae_lock);
+
+        /* reset dma controller - only needed as a test */
         err = sh_dmae_rst(shdev);
         if (err)
                 goto rst_err;
@@ -1040,7 +1161,7 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
         INIT_LIST_HEAD(&shdev->common.channels);
 
         dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask);
-        if (dmars)
+        if (pdata->slave && pdata->slave_num)
                 dma_cap_set(DMA_SLAVE, shdev->common.cap_mask);
 
         shdev->common.device_alloc_chan_resources
@@ -1089,12 +1210,22 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
             !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) {
                 /* Special case - all multiplexed */
                 for (; irq_cnt < pdata->channel_num; irq_cnt++) {
-                        chan_irq[irq_cnt] = chanirq_res->start;
-                        chan_flag[irq_cnt] = IRQF_SHARED;
+                        if (irq_cnt < SH_DMAC_MAX_CHANNELS) {
+                                chan_irq[irq_cnt] = chanirq_res->start;
+                                chan_flag[irq_cnt] = IRQF_SHARED;
+                        } else {
+                                irq_cap = 1;
+                                break;
+                        }
                 }
         } else {
                 do {
                         for (i = chanirq_res->start; i <= chanirq_res->end; i++) {
+                                if (irq_cnt >= SH_DMAC_MAX_CHANNELS) {
+                                        irq_cap = 1;
+                                        break;
+                                }
+
                                 if ((errirq_res->flags & IORESOURCE_BITS) ==
                                     IORESOURCE_IRQ_SHAREABLE)
                                         chan_flag[irq_cnt] = IRQF_SHARED;
@@ -1105,41 +1236,55 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
                                         i, irq_cnt);
                                 chan_irq[irq_cnt++] = i;
                         }
+
+                        if (irq_cnt >= SH_DMAC_MAX_CHANNELS)
+                                break;
+
                         chanirq_res = platform_get_resource(pdev,
                                                 IORESOURCE_IRQ, ++irqres);
                 } while (irq_cnt < pdata->channel_num && chanirq_res);
         }
 
-        if (irq_cnt < pdata->channel_num)
-                goto eirqres;
-
         /* Create DMA Channel */
-        for (i = 0; i < pdata->channel_num; i++) {
+        for (i = 0; i < irq_cnt; i++) {
                 err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]);
                 if (err)
                         goto chan_probe_err;
         }
 
+        if (irq_cap)
+                dev_notice(&pdev->dev, "Attempting to register %d DMA "
+                           "channels when a maximum of %d are supported.\n",
+                           pdata->channel_num, SH_DMAC_MAX_CHANNELS);
+
         pm_runtime_put(&pdev->dev);
 
-        platform_set_drvdata(pdev, shdev);
         dma_async_device_register(&shdev->common);
 
         return err;
 
 chan_probe_err:
         sh_dmae_chan_remove(shdev);
-eirqres:
+
 #if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
         free_irq(errirq, shdev);
 eirq_err:
 #endif
 rst_err:
+        spin_lock_irq(&sh_dmae_lock);
+        list_del_rcu(&shdev->node);
+        spin_unlock_irq(&sh_dmae_lock);
+
         pm_runtime_put(&pdev->dev);
+        pm_runtime_disable(&pdev->dev);
+
         if (dmars)
                 iounmap(shdev->dmars);
+
+        platform_set_drvdata(pdev, NULL);
 emapdmars:
         iounmap(shdev->chan_reg);
+        synchronize_rcu();
 emapchan:
         kfree(shdev);
 ealloc:
@@ -1162,6 +1307,10 @@ static int __exit sh_dmae_remove(struct platform_device *pdev)
         if (errirq > 0)
                 free_irq(errirq, shdev);
 
+        spin_lock_irq(&sh_dmae_lock);
+        list_del_rcu(&shdev->node);
+        spin_unlock_irq(&sh_dmae_lock);
+
         /* channel data remove */
         sh_dmae_chan_remove(shdev);
 
@@ -1171,6 +1320,9 @@ static int __exit sh_dmae_remove(struct platform_device *pdev)
                 iounmap(shdev->dmars);
         iounmap(shdev->chan_reg);
 
+        platform_set_drvdata(pdev, NULL);
+
+        synchronize_rcu();
         kfree(shdev);
 
         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -1189,17 +1341,88 @@ static void sh_dmae_shutdown(struct platform_device *pdev)
         sh_dmae_ctl_stop(shdev);
 }
 
+static int sh_dmae_runtime_suspend(struct device *dev)
+{
+        return 0;
+}
+
+static int sh_dmae_runtime_resume(struct device *dev)
+{
+        struct sh_dmae_device *shdev = dev_get_drvdata(dev);
+
+        return sh_dmae_rst(shdev);
+}
+
+#ifdef CONFIG_PM
+static int sh_dmae_suspend(struct device *dev)
+{
+        struct sh_dmae_device *shdev = dev_get_drvdata(dev);
+        int i;
+
+        for (i = 0; i < shdev->pdata->channel_num; i++) {
+                struct sh_dmae_chan *sh_chan = shdev->chan[i];
+                if (sh_chan->descs_allocated)
+                        sh_chan->pm_error = pm_runtime_put_sync(dev);
+        }
+
+        return 0;
+}
+
+static int sh_dmae_resume(struct device *dev)
+{
+        struct sh_dmae_device *shdev = dev_get_drvdata(dev);
+        int i;
+
+        for (i = 0; i < shdev->pdata->channel_num; i++) {
+                struct sh_dmae_chan *sh_chan = shdev->chan[i];
+                struct sh_dmae_slave *param = sh_chan->common.private;
+
+                if (!sh_chan->descs_allocated)
+                        continue;
+
+                if (!sh_chan->pm_error)
+                        pm_runtime_get_sync(dev);
+
+                if (param) {
+                        const struct sh_dmae_slave_config *cfg = param->config;
+                        dmae_set_dmars(sh_chan, cfg->mid_rid);
+                        dmae_set_chcr(sh_chan, cfg->chcr);
+                } else {
+                        dmae_init(sh_chan);
+                }
+        }
+
+        return 0;
+}
+#else
+#define sh_dmae_suspend NULL
+#define sh_dmae_resume NULL
+#endif
+
+const struct dev_pm_ops sh_dmae_pm = {
+        .suspend                = sh_dmae_suspend,
+        .resume                 = sh_dmae_resume,
+        .runtime_suspend        = sh_dmae_runtime_suspend,
+        .runtime_resume         = sh_dmae_runtime_resume,
+};
+
 static struct platform_driver sh_dmae_driver = {
         .remove         = __exit_p(sh_dmae_remove),
         .shutdown       = sh_dmae_shutdown,
         .driver = {
                 .owner  = THIS_MODULE,
                 .name   = "sh-dma-engine",
+                .pm     = &sh_dmae_pm,
         },
 };
 
 static int __init sh_dmae_init(void)
 {
+        /* Wire up NMI handling */
+        int err = register_die_notifier(&sh_dmae_nmi_notifier);
+        if (err)
+                return err;
+
         return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe);
 }
 module_init(sh_dmae_init);
@@ -1207,9 +1430,12 @@ module_init(sh_dmae_init);
 static void __exit sh_dmae_exit(void)
 {
         platform_driver_unregister(&sh_dmae_driver);
+
+        unregister_die_notifier(&sh_dmae_nmi_notifier);
 }
 module_exit(sh_dmae_exit);
 
 MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
 MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
 MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:sh-dma-engine");