5 files changed, 105 insertions, 36 deletions

diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h
index 7b9d27e749b8..b3e656c6fb78 100644
--- a/arch/arm/include/asm/cputype.h
+++ b/arch/arm/include/asm/cputype.h
@@ -8,6 +8,21 @@
 #define CPUID_TCM       2
 #define CPUID_TLBTYPE   3
 
+#define CPUID_EXT_PFR0  "c1, 0"
+#define CPUID_EXT_PFR1  "c1, 1"
+#define CPUID_EXT_DFR0  "c1, 2"
+#define CPUID_EXT_AFR0  "c1, 3"
+#define CPUID_EXT_MMFR0 "c1, 4"
+#define CPUID_EXT_MMFR1 "c1, 5"
+#define CPUID_EXT_MMFR2 "c1, 6"
+#define CPUID_EXT_MMFR3 "c1, 7"
+#define CPUID_EXT_ISAR0 "c2, 0"
+#define CPUID_EXT_ISAR1 "c2, 1"
+#define CPUID_EXT_ISAR2 "c2, 2"
+#define CPUID_EXT_ISAR3 "c2, 3"
+#define CPUID_EXT_ISAR4 "c2, 4"
+#define CPUID_EXT_ISAR5 "c2, 5"
+
 #ifdef CONFIG_CPU_CP15
 #define read_cpuid(reg)                                                 \
         ({                                                              \
@@ -18,9 +33,19 @@
                     : "cc");                                            \
                 __val;                                                  \
         })
+#define read_cpuid_ext(ext_reg)                                         \
+        ({                                                              \
+                unsigned int __val;                                     \
+                asm("mrc        p15, 0, %0, c0, " ext_reg               \
+                    : "=r" (__val)                                      \
+                    :                                                   \
+                    : "cc");                                            \
+                __val;                                                  \
+        })
 #else
 extern unsigned int processor_id;
 #define read_cpuid(reg) (processor_id)
+#define read_cpuid_ext(reg) 0
 #endif
 
 /*
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index a62218013c78..c964f3fc3bc5 100644
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -40,6 +40,12 @@
 #define TLB_V6_I_ASID   (1 << 18)
 
 #define TLB_BTB         (1 << 28)
+
+/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
+#define TLB_V7_UIS_PAGE (1 << 19)
+#define TLB_V7_UIS_FULL (1 << 20)
+#define TLB_V7_UIS_ASID (1 << 21)
+
 #define TLB_L2CLEAN_FR  (1 << 29)               /* Feroceon */
 #define TLB_DCLEAN      (1 << 30)
 #define TLB_WB          (1 << 31)
@@ -176,9 +182,17 @@
 # define v6wbi_always_flags     (-1UL)
 #endif
 
+#ifdef CONFIG_SMP
+#define v7wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \
+                         TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
+#else
+#define v7wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \
+                         TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
+#endif
+
 #ifdef CONFIG_CPU_TLB_V7
-# define v7wbi_possible_flags   v6wbi_tlb_flags
-# define v7wbi_always_flags     v6wbi_tlb_flags
+# define v7wbi_possible_flags   v7wbi_tlb_flags
+# define v7wbi_always_flags     v7wbi_tlb_flags
 # ifdef _TLB
 #  define MULTI_TLB 1
 # else
@@ -316,6 +330,8 @@ static inline void local_flush_tlb_all(void)
                 asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc");
         if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL))
                 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
+        if (tlb_flag(TLB_V7_UIS_FULL))
+                asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc");
 
         if (tlb_flag(TLB_BTB)) {
                 /* flush the branch target cache */
@@ -351,6 +367,8 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm)
                 asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc");
         if (tlb_flag(TLB_V6_I_ASID))
                 asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc");
+        if (tlb_flag(TLB_V7_UIS_ASID))
+                asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc");
 
         if (tlb_flag(TLB_BTB)) {
                 /* flush the branch target cache */
@@ -389,6 +407,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
                 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc");
         if (tlb_flag(TLB_V6_I_PAGE))
                 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc");
+        if (tlb_flag(TLB_V7_UIS_PAGE))
+                asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc");
 
         if (tlb_flag(TLB_BTB)) {
                 /* flush the branch target cache */
@@ -424,6 +444,8 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
                 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc");
         if (tlb_flag(TLB_V6_I_PAGE))
                 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc");
+        if (tlb_flag(TLB_V7_UIS_PAGE))
+                asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc");
 
         if (tlb_flag(TLB_BTB)) {
                 /* flush the branch target cache */
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 6014dfd22af4..ece658e773a6 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -32,6 +32,7 @@
 #include <asm/processor.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
+#include <asm/cputype.h>
 
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
@@ -545,6 +546,12 @@ struct tlb_args {
         unsigned long ta_end;
 };
 
+/* all SMP configurations have the extended CPUID registers */
+static inline int tlb_ops_need_broadcast(void)
+{
+        return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2;
+}
+
 static inline void ipi_flush_tlb_all(void *ignored)
 {
         local_flush_tlb_all();
@@ -587,51 +594,61 @@ static inline void ipi_flush_tlb_kernel_range(void *arg)
 
 void flush_tlb_all(void)
 {
-        on_each_cpu(ipi_flush_tlb_all, NULL, 1);
+        if (tlb_ops_need_broadcast())
+                on_each_cpu(ipi_flush_tlb_all, NULL, 1);
+        else
+                local_flush_tlb_all();
 }
 
 void flush_tlb_mm(struct mm_struct *mm)
 {
-        on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask);
+        if (tlb_ops_need_broadcast())
+                on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask);
+        else
+                local_flush_tlb_mm(mm);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 {
-        struct tlb_args ta;
-
-        ta.ta_vma = vma;
-        ta.ta_start = uaddr;
-
-        on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_vma = vma;
+                ta.ta_start = uaddr;
+                on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+        } else
+                local_flush_tlb_page(vma, uaddr);
 }
 
 void flush_tlb_kernel_page(unsigned long kaddr)
 {
-        struct tlb_args ta;
-
-        ta.ta_start = kaddr;
-
-        on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_start = kaddr;
+                on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
+        } else
+                local_flush_tlb_kernel_page(kaddr);
 }
 
 void flush_tlb_range(struct vm_area_struct *vma,
                      unsigned long start, unsigned long end)
 {
-        struct tlb_args ta;
-
-        ta.ta_vma = vma;
-        ta.ta_start = start;
-        ta.ta_end = end;
-
-        on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_vma = vma;
+                ta.ta_start = start;
+                ta.ta_end = end;
+                on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+        } else
+                local_flush_tlb_range(vma, start, end);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
-        struct tlb_args ta;
-
-        ta.ta_start = start;
-        ta.ta_end = end;
-
-        on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_start = start;
+                ta.ta_end = end;
+                on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
+        } else
+                local_flush_tlb_kernel_range(start, end);
 }
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 3397f1e64d76..c3f737516836 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -176,8 +176,8 @@ cpu_v7_name:
  */
 __v7_setup:
 #ifdef CONFIG_SMP
-        mrc     p15, 0, r0, c1, c0, 1           @ Enable SMP/nAMP mode
-        orr     r0, r0, #(0x1 << 6)
+        mrc     p15, 0, r0, c1, c0, 1           @ Enable SMP/nAMP mode and
+        orr     r0, r0, #(1 << 6) | (1 << 0)    @ TLB ops broadcasting
         mcr     p15, 0, r0, c1, c0, 1
 #endif
         adr     r12, __v7_setup_stack           @ the local stack
diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S
index b637e7380ab7..a26a605b73bd 100644
--- a/arch/arm/mm/tlb-v7.S
+++ b/arch/arm/mm/tlb-v7.S
@@ -42,9 +42,11 @@ ENTRY(v7wbi_flush_user_tlb_range)
         mov     r1, r1, lsl #PAGE_SHIFT
         vma_vm_flags r2, r2                     @ get vma->vm_flags
 1:
-        mcr     p15, 0, r0, c8, c6, 1           @ TLB invalidate D MVA (was 1)
-        tst     r2, #VM_EXEC                    @ Executable area ?
-        mcrne   p15, 0, r0, c8, c5, 1           @ TLB invalidate I MVA (was 1)
+#ifdef CONFIG_SMP
+        mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable) 
+#else
+        mcr     p15, 0, r0, c8, c7, 1           @ TLB invalidate U MVA
+#endif
         add     r0, r0, #PAGE_SZ
         cmp     r0, r1
         blo     1b
@@ -69,8 +71,11 @@ ENTRY(v7wbi_flush_kern_tlb_range)
         mov     r0, r0, lsl #PAGE_SHIFT
         mov     r1, r1, lsl #PAGE_SHIFT
 1:
-        mcr     p15, 0, r0, c8, c6, 1           @ TLB invalidate D MVA
-        mcr     p15, 0, r0, c8, c5, 1           @ TLB invalidate I MVA
+#ifdef CONFIG_SMP
+        mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable)
+#else
+        mcr     p15, 0, r0, c8, c7, 1           @ TLB invalidate U MVA
+#endif
         add     r0, r0, #PAGE_SZ
         cmp     r0, r1
         blo     1b
@@ -87,5 +92,5 @@ ENDPROC(v7wbi_flush_kern_tlb_range)
 ENTRY(v7wbi_tlb_fns)
         .long   v7wbi_flush_user_tlb_range
         .long   v7wbi_flush_kern_tlb_range
-        .long   v6wbi_tlb_flags
+        .long   v7wbi_tlb_flags
         .size   v7wbi_tlb_fns, . - v7wbi_tlb_fns