6 files changed, 102 insertions, 1 deletions

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 12ea3b3d49a9..1cacda426a0e 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1439,6 +1439,16 @@ config ARM_ERRATA_775420
          to deadlock. This workaround puts DSB before executing ISB if
          an abort may occur on cache maintenance.
 
+config ARM_ERRATA_798181
+        bool "ARM errata: TLBI/DSB failure on Cortex-A15"
+        depends on CPU_V7 && SMP
+        help
+          On Cortex-A15 (r0p0..r3p2) the TLBI*IS/DSB operations are not
+          adequately shooting down all use of the old entries. This
+          option enables the Linux kernel workaround for this erratum
+          which sends an IPI to the CPUs that are running the same ASID
+          as the one being invalidated.
+
 endmenu
 
 source "arch/arm/common/Kconfig"
diff --git a/arch/arm/include/asm/highmem.h b/arch/arm/include/asm/highmem.h
index 8c5e828f484d..91b99abe7a95 100644
--- a/arch/arm/include/asm/highmem.h
+++ b/arch/arm/include/asm/highmem.h
@@ -41,6 +41,13 @@ extern void kunmap_high(struct page *page);
 #endif
 #endif
 
+/*
+ * Needed to be able to broadcast the TLB invalidation for kmap.
+ */
+#ifdef CONFIG_ARM_ERRATA_798181
+#undef ARCH_NEEDS_KMAP_HIGH_GET
+#endif
+
 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
 extern void *kmap_high_get(struct page *page);
 #else
diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h
index 863a6611323c..a7b85e0d0cc1 100644
--- a/arch/arm/include/asm/mmu_context.h
+++ b/arch/arm/include/asm/mmu_context.h
@@ -27,6 +27,8 @@ void __check_vmalloc_seq(struct mm_struct *mm);
 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk);
 #define init_new_context(tsk,mm)        ({ atomic64_set(&mm->context.id, 0); 0; })
 
+DECLARE_PER_CPU(atomic64_t, active_asids);
+
 #else   /* !CONFIG_CPU_HAS_ASID */
 
 #ifdef CONFIG_MMU
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index 4db8c8820f0d..9e9c041358ca 100644
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -450,6 +450,21 @@ static inline void local_flush_bp_all(void)
                 isb();
 }
 
+#ifdef CONFIG_ARM_ERRATA_798181
+static inline void dummy_flush_tlb_a15_erratum(void)
+{
+        /*
+         * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
+         */
+        asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
+        dsb();
+}
+#else
+static inline void dummy_flush_tlb_a15_erratum(void)
+{
+}
+#endif
+
 /*
  *      flush_pmd_entry
  *
diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c
index bd0300531399..e82e1d248772 100644
--- a/arch/arm/kernel/smp_tlb.c
+++ b/arch/arm/kernel/smp_tlb.c
@@ -12,6 +12,7 @@
 
 #include <asm/smp_plat.h>
 #include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
 
 /**********************************************************************/
 
@@ -69,12 +70,72 @@ static inline void ipi_flush_bp_all(void *ignored)
         local_flush_bp_all();
 }
 
+#ifdef CONFIG_ARM_ERRATA_798181
+static int erratum_a15_798181(void)
+{
+        unsigned int midr = read_cpuid_id();
+
+        /* Cortex-A15 r0p0..r3p2 affected */
+        if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
+                return 0;
+        return 1;
+}
+#else
+static int erratum_a15_798181(void)
+{
+        return 0;
+}
+#endif
+
+static void ipi_flush_tlb_a15_erratum(void *arg)
+{
+        dmb();
+}
+
+static void broadcast_tlb_a15_erratum(void)
+{
+        if (!erratum_a15_798181())
+                return;
+
+        dummy_flush_tlb_a15_erratum();
+        smp_call_function_many(cpu_online_mask, ipi_flush_tlb_a15_erratum,
+                               NULL, 1);
+}
+
+static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
+{
+        int cpu;
+        cpumask_t mask = { CPU_BITS_NONE };
+
+        if (!erratum_a15_798181())
+                return;
+
+        dummy_flush_tlb_a15_erratum();
+        for_each_online_cpu(cpu) {
+                if (cpu == smp_processor_id())
+                        continue;
+                /*
+                 * We only need to send an IPI if the other CPUs are running
+                 * the same ASID as the one being invalidated. There is no
+                 * need for locking around the active_asids check since the
+                 * switch_mm() function has at least one dmb() (as required by
+                 * this workaround) in case a context switch happens on
+                 * another CPU after the condition below.
+                 */
+                if (atomic64_read(&mm->context.id) ==
+                    atomic64_read(&per_cpu(active_asids, cpu)))
+                        cpumask_set_cpu(cpu, &mask);
+        }
+        smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
+}
+
 void flush_tlb_all(void)
 {
         if (tlb_ops_need_broadcast())
                 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
         else
                 local_flush_tlb_all();
+        broadcast_tlb_a15_erratum();
 }
 
 void flush_tlb_mm(struct mm_struct *mm)
@@ -83,6 +144,7 @@ void flush_tlb_mm(struct mm_struct *mm)
                 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
         else
                 local_flush_tlb_mm(mm);
+        broadcast_tlb_mm_a15_erratum(mm);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
@@ -95,6 +157,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
                                         &ta, 1);
         } else
                 local_flush_tlb_page(vma, uaddr);
+        broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 }
 
 void flush_tlb_kernel_page(unsigned long kaddr)
@@ -105,6 +168,7 @@ void flush_tlb_kernel_page(unsigned long kaddr)
                 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
         } else
                 local_flush_tlb_kernel_page(kaddr);
+        broadcast_tlb_a15_erratum();
 }
 
 void flush_tlb_range(struct vm_area_struct *vma,
@@ -119,6 +183,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
                                         &ta, 1);
         } else
                 local_flush_tlb_range(vma, start, end);
+        broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -130,6 +195,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
                 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
         } else
                 local_flush_tlb_kernel_range(start, end);
+        broadcast_tlb_a15_erratum();
 }
 
 void flush_bp_all(void)
diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index a5a4b2bc42ba..2ac37372ef52 100644
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -48,7 +48,7 @@ static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
 static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
 static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
 
-static DEFINE_PER_CPU(atomic64_t, active_asids);
+DEFINE_PER_CPU(atomic64_t, active_asids);
 static DEFINE_PER_CPU(u64, reserved_asids);
 static cpumask_t tlb_flush_pending;
 
@@ -215,6 +215,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
         if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
                 local_flush_bp_all();
                 local_flush_tlb_all();
+                dummy_flush_tlb_a15_erratum();
         }
 
         atomic64_set(&per_cpu(active_asids, cpu), asid);