aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/arm/include/asm/tlbflush.h58
-rw-r--r--arch/arm/mm/tlb-fa.S4
-rw-r--r--arch/arm/mm/tlb-v6.S4
-rw-r--r--arch/arm/mm/tlb-v7.S6
4 files changed, 12 insertions, 60 deletions
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index d2005de383b8..8077145698ff 100644
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -34,16 +34,12 @@
34#define TLB_V6_D_ASID (1 << 17) 34#define TLB_V6_D_ASID (1 << 17)
35#define TLB_V6_I_ASID (1 << 18) 35#define TLB_V6_I_ASID (1 << 18)
36 36
37#define TLB_BTB (1 << 28)
38
39/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ 37/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
40#define TLB_V7_UIS_PAGE (1 << 19) 38#define TLB_V7_UIS_PAGE (1 << 19)
41#define TLB_V7_UIS_FULL (1 << 20) 39#define TLB_V7_UIS_FULL (1 << 20)
42#define TLB_V7_UIS_ASID (1 << 21) 40#define TLB_V7_UIS_ASID (1 << 21)
43 41
44/* Inner Shareable BTB operation (ARMv7 MP extensions) */ 42#define TLB_BARRIER (1 << 28)
45#define TLB_V7_IS_BTB (1 << 22)
46
47#define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ 43#define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
48#define TLB_DCLEAN (1 << 30) 44#define TLB_DCLEAN (1 << 30)
49#define TLB_WB (1 << 31) 45#define TLB_WB (1 << 31)
@@ -58,7 +54,7 @@
58 * v4wb - ARMv4 with write buffer without I TLB flush entry instruction 54 * v4wb - ARMv4 with write buffer without I TLB flush entry instruction
59 * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction 55 * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
60 * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) 56 * fr - Feroceon (v4wbi with non-outer-cacheable page table walks)
61 * fa - Faraday (v4 with write buffer with UTLB and branch target buffer (BTB)) 57 * fa - Faraday (v4 with write buffer with UTLB)
62 * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction 58 * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction
63 * v7wbi - identical to v6wbi 59 * v7wbi - identical to v6wbi
64 */ 60 */
@@ -99,7 +95,7 @@
99# define v4_always_flags (-1UL) 95# define v4_always_flags (-1UL)
100#endif 96#endif
101 97
102#define fa_tlb_flags (TLB_WB | TLB_BTB | TLB_DCLEAN | \ 98#define fa_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
103 TLB_V4_U_FULL | TLB_V4_U_PAGE) 99 TLB_V4_U_FULL | TLB_V4_U_PAGE)
104 100
105#ifdef CONFIG_CPU_TLB_FA 101#ifdef CONFIG_CPU_TLB_FA
@@ -166,7 +162,7 @@
166# define v4wb_always_flags (-1UL) 162# define v4wb_always_flags (-1UL)
167#endif 163#endif
168 164
169#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ 165#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
170 TLB_V6_I_FULL | TLB_V6_D_FULL | \ 166 TLB_V6_I_FULL | TLB_V6_D_FULL | \
171 TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ 167 TLB_V6_I_PAGE | TLB_V6_D_PAGE | \
172 TLB_V6_I_ASID | TLB_V6_D_ASID) 168 TLB_V6_I_ASID | TLB_V6_D_ASID)
@@ -184,9 +180,9 @@
184# define v6wbi_always_flags (-1UL) 180# define v6wbi_always_flags (-1UL)
185#endif 181#endif
186 182
187#define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_V7_IS_BTB | \ 183#define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
188 TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) 184 TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
189#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BTB | \ 185#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
190 TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) 186 TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
191 187
192#ifdef CONFIG_CPU_TLB_V7 188#ifdef CONFIG_CPU_TLB_V7
@@ -341,15 +337,7 @@ static inline void local_flush_tlb_all(void)
341 if (tlb_flag(TLB_V7_UIS_FULL)) 337 if (tlb_flag(TLB_V7_UIS_FULL))
342 asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); 338 asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc");
343 339
344 if (tlb_flag(TLB_BTB)) { 340 if (tlb_flag(TLB_BARRIER)) {
345 /* flush the branch target cache */
346 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc");
347 dsb();
348 isb();
349 }
350 if (tlb_flag(TLB_V7_IS_BTB)) {
351 /* flush the branch target cache */
352 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc");
353 dsb(); 341 dsb();
354 isb(); 342 isb();
355 } 343 }
@@ -389,17 +377,8 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm)
389 asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); 377 asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc");
390#endif 378#endif
391 379
392 if (tlb_flag(TLB_BTB)) { 380 if (tlb_flag(TLB_BARRIER))
393 /* flush the branch target cache */
394 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc");
395 dsb();
396 }
397 if (tlb_flag(TLB_V7_IS_BTB)) {
398 /* flush the branch target cache */
399 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc");
400 dsb(); 381 dsb();
401 isb();
402 }
403} 382}
404 383
405static inline void 384static inline void
@@ -439,17 +418,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
439 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); 418 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc");
440#endif 419#endif
441 420
442 if (tlb_flag(TLB_BTB)) { 421 if (tlb_flag(TLB_BARRIER))
443 /* flush the branch target cache */
444 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc");
445 dsb();
446 }
447 if (tlb_flag(TLB_V7_IS_BTB)) {
448 /* flush the branch target cache */
449 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc");
450 dsb(); 422 dsb();
451 isb();
452 }
453} 423}
454 424
455static inline void local_flush_tlb_kernel_page(unsigned long kaddr) 425static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
@@ -482,15 +452,7 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
482 if (tlb_flag(TLB_V7_UIS_PAGE)) 452 if (tlb_flag(TLB_V7_UIS_PAGE))
483 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); 453 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc");
484 454
485 if (tlb_flag(TLB_BTB)) { 455 if (tlb_flag(TLB_BARRIER)) {
486 /* flush the branch target cache */
487 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc");
488 dsb();
489 isb();
490 }
491 if (tlb_flag(TLB_V7_IS_BTB)) {
492 /* flush the branch target cache */
493 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc");
494 dsb(); 456 dsb();
495 isb(); 457 isb();
496 } 458 }
diff --git a/arch/arm/mm/tlb-fa.S b/arch/arm/mm/tlb-fa.S
index 9694f1f6f485..d887a31faaae 100644
--- a/arch/arm/mm/tlb-fa.S
+++ b/arch/arm/mm/tlb-fa.S
@@ -46,7 +46,6 @@ ENTRY(fa_flush_user_tlb_range)
46 add r0, r0, #PAGE_SZ 46 add r0, r0, #PAGE_SZ
47 cmp r0, r1 47 cmp r0, r1
48 blo 1b 48 blo 1b
49 mcr p15, 0, r3, c7, c5, 6 @ invalidate BTB
50 mcr p15, 0, r3, c7, c10, 4 @ data write barrier 49 mcr p15, 0, r3, c7, c10, 4 @ data write barrier
51 mov pc, lr 50 mov pc, lr
52 51
@@ -60,9 +59,8 @@ ENTRY(fa_flush_kern_tlb_range)
60 add r0, r0, #PAGE_SZ 59 add r0, r0, #PAGE_SZ
61 cmp r0, r1 60 cmp r0, r1
62 blo 1b 61 blo 1b
63 mcr p15, 0, r3, c7, c5, 6 @ invalidate BTB
64 mcr p15, 0, r3, c7, c10, 4 @ data write barrier 62 mcr p15, 0, r3, c7, c10, 4 @ data write barrier
65 mcr p15, 0, r3, c7, c5, 4 @ prefetch flush 63 mcr p15, 0, r3, c7, c5, 4 @ prefetch flush (isb)
66 mov pc, lr 64 mov pc, lr
67 65
68 __INITDATA 66 __INITDATA
diff --git a/arch/arm/mm/tlb-v6.S b/arch/arm/mm/tlb-v6.S
index 73d7d89b04c4..ffe06a69a6e5 100644
--- a/arch/arm/mm/tlb-v6.S
+++ b/arch/arm/mm/tlb-v6.S
@@ -54,7 +54,6 @@ ENTRY(v6wbi_flush_user_tlb_range)
54 add r0, r0, #PAGE_SZ 54 add r0, r0, #PAGE_SZ
55 cmp r0, r1 55 cmp r0, r1
56 blo 1b 56 blo 1b
57 mcr p15, 0, ip, c7, c5, 6 @ flush BTAC/BTB
58 mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier 57 mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier
59 mov pc, lr 58 mov pc, lr
60 59
@@ -83,9 +82,8 @@ ENTRY(v6wbi_flush_kern_tlb_range)
83 add r0, r0, #PAGE_SZ 82 add r0, r0, #PAGE_SZ
84 cmp r0, r1 83 cmp r0, r1
85 blo 1b 84 blo 1b
86 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
87 mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier 85 mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier
88 mcr p15, 0, r2, c7, c5, 4 @ prefetch flush 86 mcr p15, 0, r2, c7, c5, 4 @ prefetch flush (isb)
89 mov pc, lr 87 mov pc, lr
90 88
91 __INIT 89 __INIT
diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S
index 53cd5b454673..86bb71664508 100644
--- a/arch/arm/mm/tlb-v7.S
+++ b/arch/arm/mm/tlb-v7.S
@@ -48,9 +48,6 @@ ENTRY(v7wbi_flush_user_tlb_range)
48 add r0, r0, #PAGE_SZ 48 add r0, r0, #PAGE_SZ
49 cmp r0, r1 49 cmp r0, r1
50 blo 1b 50 blo 1b
51 mov ip, #0
52 ALT_SMP(mcr p15, 0, ip, c7, c1, 6) @ flush BTAC/BTB Inner Shareable
53 ALT_UP(mcr p15, 0, ip, c7, c5, 6) @ flush BTAC/BTB
54 dsb 51 dsb
55 mov pc, lr 52 mov pc, lr
56ENDPROC(v7wbi_flush_user_tlb_range) 53ENDPROC(v7wbi_flush_user_tlb_range)
@@ -75,9 +72,6 @@ ENTRY(v7wbi_flush_kern_tlb_range)
75 add r0, r0, #PAGE_SZ 72 add r0, r0, #PAGE_SZ
76 cmp r0, r1 73 cmp r0, r1
77 blo 1b 74 blo 1b
78 mov r2, #0
79 ALT_SMP(mcr p15, 0, r2, c7, c1, 6) @ flush BTAC/BTB Inner Shareable
80 ALT_UP(mcr p15, 0, r2, c7, c5, 6) @ flush BTAC/BTB
81 dsb 75 dsb
82 isb 76 isb
83 mov pc, lr 77 mov pc, lr