Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

1 files changed, 196 insertions, 104 deletions

diff --git a/arch/arm/common/gic.c b/arch/arm/common/gic.c
index 7dfa9a85bc0c..4ddd0a6ac7ff 100644
--- a/arch/arm/common/gic.c
+++ b/arch/arm/common/gic.c
@@ -35,83 +35,92 @@
 
 static DEFINE_SPINLOCK(irq_controller_lock);
 
+/* Address of GIC 0 CPU interface */
+void __iomem *gic_cpu_base_addr __read_mostly;
+
 struct gic_chip_data {
         unsigned int irq_offset;
         void __iomem *dist_base;
         void __iomem *cpu_base;
 };
 
+/*
+ * Supported arch specific GIC irq extension.
+ * Default make them NULL.
+ */
+struct irq_chip gic_arch_extn = {
+        .irq_eoi        = NULL,
+        .irq_mask       = NULL,
+        .irq_unmask     = NULL,
+        .irq_retrigger  = NULL,
+        .irq_set_type   = NULL,
+        .irq_set_wake   = NULL,
+};
+
 #ifndef MAX_GIC_NR
 #define MAX_GIC_NR      1
 #endif
 
-static struct gic_chip_data gic_data[MAX_GIC_NR];
+static struct gic_chip_data gic_data[MAX_GIC_NR] __read_mostly;
 
-static inline void __iomem *gic_dist_base(unsigned int irq)
+static inline void __iomem *gic_dist_base(struct irq_data *d)
 {
-        struct gic_chip_data *gic_data = get_irq_chip_data(irq);
+        struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
         return gic_data->dist_base;
 }
 
-static inline void __iomem *gic_cpu_base(unsigned int irq)
+static inline void __iomem *gic_cpu_base(struct irq_data *d)
 {
-        struct gic_chip_data *gic_data = get_irq_chip_data(irq);
+        struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
         return gic_data->cpu_base;
 }
 
-static inline unsigned int gic_irq(unsigned int irq)
+static inline unsigned int gic_irq(struct irq_data *d)
 {
-        struct gic_chip_data *gic_data = get_irq_chip_data(irq);
-        return irq - gic_data->irq_offset;
+        struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
+        return d->irq - gic_data->irq_offset;
 }
 
 /*
  * Routines to acknowledge, disable and enable interrupts
- *
- * Linux assumes that when we're done with an interrupt we need to
- * unmask it, in the same way we need to unmask an interrupt when
- * we first enable it.
- *
- * The GIC has a separate notion of "end of interrupt" to re-enable
- * an interrupt after handling, in order to support hardware
- * prioritisation.
- *
- * We can make the GIC behave in the way that Linux expects by making
- * our "acknowledge" routine disable the interrupt, then mark it as
- * complete.
  */
-static void gic_ack_irq(unsigned int irq)
+static void gic_mask_irq(struct irq_data *d)
 {
-        u32 mask = 1 << (irq % 32);
+        u32 mask = 1 << (d->irq % 32);
 
         spin_lock(&irq_controller_lock);
-        writel(mask, gic_dist_base(irq) + GIC_DIST_ENABLE_CLEAR + (gic_irq(irq) / 32) * 4);
-        writel(gic_irq(irq), gic_cpu_base(irq) + GIC_CPU_EOI);
+        writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_CLEAR + (gic_irq(d) / 32) * 4);
+        if (gic_arch_extn.irq_mask)
+                gic_arch_extn.irq_mask(d);
         spin_unlock(&irq_controller_lock);
 }
 
-static void gic_mask_irq(unsigned int irq)
+static void gic_unmask_irq(struct irq_data *d)
 {
-        u32 mask = 1 << (irq % 32);
+        u32 mask = 1 << (d->irq % 32);
 
         spin_lock(&irq_controller_lock);
-        writel(mask, gic_dist_base(irq) + GIC_DIST_ENABLE_CLEAR + (gic_irq(irq) / 32) * 4);
+        if (gic_arch_extn.irq_unmask)
+                gic_arch_extn.irq_unmask(d);
+        writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_SET + (gic_irq(d) / 32) * 4);
         spin_unlock(&irq_controller_lock);
 }
 
-static void gic_unmask_irq(unsigned int irq)
+static void gic_eoi_irq(struct irq_data *d)
 {
-        u32 mask = 1 << (irq % 32);
+        if (gic_arch_extn.irq_eoi) {
+                spin_lock(&irq_controller_lock);
+                gic_arch_extn.irq_eoi(d);
+                spin_unlock(&irq_controller_lock);
+        }
 
-        spin_lock(&irq_controller_lock);
-        writel(mask, gic_dist_base(irq) + GIC_DIST_ENABLE_SET + (gic_irq(irq) / 32) * 4);
-        spin_unlock(&irq_controller_lock);
+        writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
 }
 
-static int gic_set_type(unsigned int irq, unsigned int type)
+static int gic_set_type(struct irq_data *d, unsigned int type)
 {
-        void __iomem *base = gic_dist_base(irq);
-        unsigned int gicirq = gic_irq(irq);
+        void __iomem *base = gic_dist_base(d);
+        unsigned int gicirq = gic_irq(d);
         u32 enablemask = 1 << (gicirq % 32);
         u32 enableoff = (gicirq / 32) * 4;
         u32 confmask = 0x2 << ((gicirq % 16) * 2);
@@ -128,7 +137,10 @@ static int gic_set_type(unsigned int irq, unsigned int type)
 
         spin_lock(&irq_controller_lock);
 
-        val = readl(base + GIC_DIST_CONFIG + confoff);
+        if (gic_arch_extn.irq_set_type)
+                gic_arch_extn.irq_set_type(d, type);
+
+        val = readl_relaxed(base + GIC_DIST_CONFIG + confoff);
         if (type == IRQ_TYPE_LEVEL_HIGH)
                 val &= ~confmask;
         else if (type == IRQ_TYPE_EDGE_RISING)
@@ -138,52 +150,80 @@ static int gic_set_type(unsigned int irq, unsigned int type)
          * As recommended by the spec, disable the interrupt before changing
          * the configuration
          */
-        if (readl(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) {
-                writel(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff);
+        if (readl_relaxed(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) {
+                writel_relaxed(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff);
                 enabled = true;
         }
 
-        writel(val, base + GIC_DIST_CONFIG + confoff);
+        writel_relaxed(val, base + GIC_DIST_CONFIG + confoff);
 
         if (enabled)
-                writel(enablemask, base + GIC_DIST_ENABLE_SET + enableoff);
+                writel_relaxed(enablemask, base + GIC_DIST_ENABLE_SET + enableoff);
 
         spin_unlock(&irq_controller_lock);
 
         return 0;
 }
 
+static int gic_retrigger(struct irq_data *d)
+{
+        if (gic_arch_extn.irq_retrigger)
+                return gic_arch_extn.irq_retrigger(d);
+
+        return -ENXIO;
+}
+
 #ifdef CONFIG_SMP
-static int gic_set_cpu(unsigned int irq, const struct cpumask *mask_val)
+static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
+                            bool force)
 {
-        void __iomem *reg = gic_dist_base(irq) + GIC_DIST_TARGET + (gic_irq(irq) & ~3);
-        unsigned int shift = (irq % 4) * 8;
+        void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3);
+        unsigned int shift = (d->irq % 4) * 8;
         unsigned int cpu = cpumask_first(mask_val);
-        u32 val;
+        u32 val, mask, bit;
+
+        if (cpu >= 8)
+                return -EINVAL;
+
+        mask = 0xff << shift;
+        bit = 1 << (cpu + shift);
 
         spin_lock(&irq_controller_lock);
-        irq_desc[irq].node = cpu;
-        val = readl(reg) & ~(0xff << shift);
-        val |= 1 << (cpu + shift);
-        writel(val, reg);
+        d->node = cpu;
+        val = readl_relaxed(reg) & ~mask;
+        writel_relaxed(val | bit, reg);
         spin_unlock(&irq_controller_lock);
 
         return 0;
 }
 #endif
 
+#ifdef CONFIG_PM
+static int gic_set_wake(struct irq_data *d, unsigned int on)
+{
+        int ret = -ENXIO;
+
+        if (gic_arch_extn.irq_set_wake)
+                ret = gic_arch_extn.irq_set_wake(d, on);
+
+        return ret;
+}
+
+#else
+#define gic_set_wake    NULL
+#endif
+
 static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc)
 {
-        struct gic_chip_data *chip_data = get_irq_data(irq);
-        struct irq_chip *chip = get_irq_chip(irq);
+        struct gic_chip_data *chip_data = irq_get_handler_data(irq);
+        struct irq_chip *chip = irq_get_chip(irq);
         unsigned int cascade_irq, gic_irq;
         unsigned long status;
 
-        /* primary controller ack'ing */
-        chip->ack(irq);
+        chained_irq_enter(chip, desc);
 
         spin_lock(&irq_controller_lock);
-        status = readl(chip_data->cpu_base + GIC_CPU_INTACK);
+        status = readl_relaxed(chip_data->cpu_base + GIC_CPU_INTACK);
         spin_unlock(&irq_controller_lock);
 
         gic_irq = (status & 0x3ff);
@@ -197,107 +237,153 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc)
                 generic_handle_irq(cascade_irq);
 
  out:
-        /* primary controller unmasking */
-        chip->unmask(irq);
+        chained_irq_exit(chip, desc);
 }
 
 static struct irq_chip gic_chip = {
-        .name           = "GIC",
-        .ack            = gic_ack_irq,
-        .mask           = gic_mask_irq,
-        .unmask         = gic_unmask_irq,
-        .set_type       = gic_set_type,
+        .name                   = "GIC",
+        .irq_mask               = gic_mask_irq,
+        .irq_unmask             = gic_unmask_irq,
+        .irq_eoi                = gic_eoi_irq,
+        .irq_set_type           = gic_set_type,
+        .irq_retrigger          = gic_retrigger,
 #ifdef CONFIG_SMP
-        .set_affinity   = gic_set_cpu,
+        .irq_set_affinity       = gic_set_affinity,
 #endif
+        .irq_set_wake           = gic_set_wake,
 };
 
 void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
 {
         if (gic_nr >= MAX_GIC_NR)
                 BUG();
-        if (set_irq_data(irq, &gic_data[gic_nr]) != 0)
+        if (irq_set_handler_data(irq, &gic_data[gic_nr]) != 0)
                 BUG();
-        set_irq_chained_handler(irq, gic_handle_cascade_irq);
+        irq_set_chained_handler(irq, gic_handle_cascade_irq);
 }
 
-void __init gic_dist_init(unsigned int gic_nr, void __iomem *base,
-                          unsigned int irq_start)
+static void __init gic_dist_init(struct gic_chip_data *gic,
+        unsigned int irq_start)
 {
-        unsigned int max_irq, i;
+        unsigned int gic_irqs, irq_limit, i;
+        void __iomem *base = gic->dist_base;
         u32 cpumask = 1 << smp_processor_id();
 
-        if (gic_nr >= MAX_GIC_NR)
-                BUG();
-
         cpumask |= cpumask << 8;
         cpumask |= cpumask << 16;
 
-        gic_data[gic_nr].dist_base = base;
-        gic_data[gic_nr].irq_offset = (irq_start - 1) & ~31;
-
-        writel(0, base + GIC_DIST_CTRL);
+        writel_relaxed(0, base + GIC_DIST_CTRL);
 
         /*
          * Find out how many interrupts are supported.
-         */
-        max_irq = readl(base + GIC_DIST_CTR) & 0x1f;
-        max_irq = (max_irq + 1) * 32;
-
-        /*
          * The GIC only supports up to 1020 interrupt sources.
-         * Limit this to either the architected maximum, or the
-         * platform maximum.
          */
-        if (max_irq > max(1020, NR_IRQS))
-                max_irq = max(1020, NR_IRQS);
+        gic_irqs = readl_relaxed(base + GIC_DIST_CTR) & 0x1f;
+        gic_irqs = (gic_irqs + 1) * 32;
+        if (gic_irqs > 1020)
+                gic_irqs = 1020;
 
         /*
          * Set all global interrupts to be level triggered, active low.
          */
-        for (i = 32; i < max_irq; i += 16)
-                writel(0, base + GIC_DIST_CONFIG + i * 4 / 16);
+        for (i = 32; i < gic_irqs; i += 16)
+                writel_relaxed(0, base + GIC_DIST_CONFIG + i * 4 / 16);
 
         /*
          * Set all global interrupts to this CPU only.
          */
-        for (i = 32; i < max_irq; i += 4)
-                writel(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
+        for (i = 32; i < gic_irqs; i += 4)
+                writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
+
+        /*
+         * Set priority on all global interrupts.
+         */
+        for (i = 32; i < gic_irqs; i += 4)
+                writel_relaxed(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4);
 
         /*
-         * Set priority on all interrupts.
+         * Disable all interrupts.  Leave the PPI and SGIs alone
+         * as these enables are banked registers.
          */
-        for (i = 0; i < max_irq; i += 4)
-                writel(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4);
+        for (i = 32; i < gic_irqs; i += 32)
+                writel_relaxed(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32);
 
         /*
-         * Disable all interrupts.
+         * Limit number of interrupts registered to the platform maximum
          */
-        for (i = 0; i < max_irq; i += 32)
-                writel(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32);
+        irq_limit = gic->irq_offset + gic_irqs;
+        if (WARN_ON(irq_limit > NR_IRQS))
+                irq_limit = NR_IRQS;
 
         /*
          * Setup the Linux IRQ subsystem.
          */
-        for (i = irq_start; i < gic_data[gic_nr].irq_offset + max_irq; i++) {
-                set_irq_chip(i, &gic_chip);
-                set_irq_chip_data(i, &gic_data[gic_nr]);
-                set_irq_handler(i, handle_level_irq);
+        for (i = irq_start; i < irq_limit; i++) {
+                irq_set_chip_and_handler(i, &gic_chip, handle_fasteoi_irq);
+                irq_set_chip_data(i, gic);
                 set_irq_flags(i, IRQF_VALID | IRQF_PROBE);
         }
 
-        writel(1, base + GIC_DIST_CTRL);
+        writel_relaxed(1, base + GIC_DIST_CTRL);
 }
 
-void __cpuinit gic_cpu_init(unsigned int gic_nr, void __iomem *base)
+static void __cpuinit gic_cpu_init(struct gic_chip_data *gic)
 {
-        if (gic_nr >= MAX_GIC_NR)
-                BUG();
+        void __iomem *dist_base = gic->dist_base;
+        void __iomem *base = gic->cpu_base;
+        int i;
 
-        gic_data[gic_nr].cpu_base = base;
+        /*
+         * Deal with the banked PPI and SGI interrupts - disable all
+         * PPI interrupts, ensure all SGI interrupts are enabled.
+         */
+        writel_relaxed(0xffff0000, dist_base + GIC_DIST_ENABLE_CLEAR);
+        writel_relaxed(0x0000ffff, dist_base + GIC_DIST_ENABLE_SET);
+
+        /*
+         * Set priority on PPI and SGI interrupts
+         */
+        for (i = 0; i < 32; i += 4)
+                writel_relaxed(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4 / 4);
 
-        writel(0xf0, base + GIC_CPU_PRIMASK);
-        writel(1, base + GIC_CPU_CTRL);
+        writel_relaxed(0xf0, base + GIC_CPU_PRIMASK);
+        writel_relaxed(1, base + GIC_CPU_CTRL);
+}
+
+void __init gic_init(unsigned int gic_nr, unsigned int irq_start,
+        void __iomem *dist_base, void __iomem *cpu_base)
+{
+        struct gic_chip_data *gic;
+
+        BUG_ON(gic_nr >= MAX_GIC_NR);
+
+        gic = &gic_data[gic_nr];
+        gic->dist_base = dist_base;
+        gic->cpu_base = cpu_base;
+        gic->irq_offset = (irq_start - 1) & ~31;
+
+        if (gic_nr == 0)
+                gic_cpu_base_addr = cpu_base;
+
+        gic_dist_init(gic, irq_start);
+        gic_cpu_init(gic);
+}
+
+void __cpuinit gic_secondary_init(unsigned int gic_nr)
+{
+        BUG_ON(gic_nr >= MAX_GIC_NR);
+
+        gic_cpu_init(&gic_data[gic_nr]);
+}
+
+void __cpuinit gic_enable_ppi(unsigned int irq)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        irq_set_status_flags(irq, IRQ_NOPROBE);
+        gic_unmask_irq(irq_get_irq_data(irq));
+        local_irq_restore(flags);
 }
 
 #ifdef CONFIG_SMP
@@ -305,7 +391,13 @@ void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 {
         unsigned long map = *cpus_addr(*mask);
 
+        /*
+         * Ensure that stores to Normal memory are visible to the
+         * other CPUs before issuing the IPI.
+         */
+        dsb();
+
         /* this always happens on GIC0 */
-        writel(map << 16 | irq, gic_data[0].dist_base + GIC_DIST_SOFTINT);
+        writel_relaxed(map << 16 | irq, gic_data[0].dist_base + GIC_DIST_SOFTINT);
 }
 #endif