diff options
-rw-r--r-- | arch/arm/include/asm/tlbflush.h | 58 | ||||
-rw-r--r-- | arch/arm/mm/tlb-fa.S | 4 | ||||
-rw-r--r-- | arch/arm/mm/tlb-v6.S | 4 | ||||
-rw-r--r-- | arch/arm/mm/tlb-v7.S | 6 |
4 files changed, 12 insertions, 60 deletions
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index d2005de383b8..8077145698ff 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h | |||
@@ -34,16 +34,12 @@ | |||
34 | #define TLB_V6_D_ASID (1 << 17) | 34 | #define TLB_V6_D_ASID (1 << 17) |
35 | #define TLB_V6_I_ASID (1 << 18) | 35 | #define TLB_V6_I_ASID (1 << 18) |
36 | 36 | ||
37 | #define TLB_BTB (1 << 28) | ||
38 | |||
39 | /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ | 37 | /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ |
40 | #define TLB_V7_UIS_PAGE (1 << 19) | 38 | #define TLB_V7_UIS_PAGE (1 << 19) |
41 | #define TLB_V7_UIS_FULL (1 << 20) | 39 | #define TLB_V7_UIS_FULL (1 << 20) |
42 | #define TLB_V7_UIS_ASID (1 << 21) | 40 | #define TLB_V7_UIS_ASID (1 << 21) |
43 | 41 | ||
44 | /* Inner Shareable BTB operation (ARMv7 MP extensions) */ | 42 | #define TLB_BARRIER (1 << 28) |
45 | #define TLB_V7_IS_BTB (1 << 22) | ||
46 | |||
47 | #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ | 43 | #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ |
48 | #define TLB_DCLEAN (1 << 30) | 44 | #define TLB_DCLEAN (1 << 30) |
49 | #define TLB_WB (1 << 31) | 45 | #define TLB_WB (1 << 31) |
@@ -58,7 +54,7 @@ | |||
58 | * v4wb - ARMv4 with write buffer without I TLB flush entry instruction | 54 | * v4wb - ARMv4 with write buffer without I TLB flush entry instruction |
59 | * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction | 55 | * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction |
60 | * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) | 56 | * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) |
61 | * fa - Faraday (v4 with write buffer with UTLB and branch target buffer (BTB)) | 57 | * fa - Faraday (v4 with write buffer with UTLB) |
62 | * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction | 58 | * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction |
63 | * v7wbi - identical to v6wbi | 59 | * v7wbi - identical to v6wbi |
64 | */ | 60 | */ |
@@ -99,7 +95,7 @@ | |||
99 | # define v4_always_flags (-1UL) | 95 | # define v4_always_flags (-1UL) |
100 | #endif | 96 | #endif |
101 | 97 | ||
102 | #define fa_tlb_flags (TLB_WB | TLB_BTB | TLB_DCLEAN | \ | 98 | #define fa_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
103 | TLB_V4_U_FULL | TLB_V4_U_PAGE) | 99 | TLB_V4_U_FULL | TLB_V4_U_PAGE) |
104 | 100 | ||
105 | #ifdef CONFIG_CPU_TLB_FA | 101 | #ifdef CONFIG_CPU_TLB_FA |
@@ -166,7 +162,7 @@ | |||
166 | # define v4wb_always_flags (-1UL) | 162 | # define v4wb_always_flags (-1UL) |
167 | #endif | 163 | #endif |
168 | 164 | ||
169 | #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ | 165 | #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
170 | TLB_V6_I_FULL | TLB_V6_D_FULL | \ | 166 | TLB_V6_I_FULL | TLB_V6_D_FULL | \ |
171 | TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ | 167 | TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ |
172 | TLB_V6_I_ASID | TLB_V6_D_ASID) | 168 | TLB_V6_I_ASID | TLB_V6_D_ASID) |
@@ -184,9 +180,9 @@ | |||
184 | # define v6wbi_always_flags (-1UL) | 180 | # define v6wbi_always_flags (-1UL) |
185 | #endif | 181 | #endif |
186 | 182 | ||
187 | #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_V7_IS_BTB | \ | 183 | #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
188 | TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) | 184 | TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) |
189 | #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BTB | \ | 185 | #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ |
190 | TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) | 186 | TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) |
191 | 187 | ||
192 | #ifdef CONFIG_CPU_TLB_V7 | 188 | #ifdef CONFIG_CPU_TLB_V7 |
@@ -341,15 +337,7 @@ static inline void local_flush_tlb_all(void) | |||
341 | if (tlb_flag(TLB_V7_UIS_FULL)) | 337 | if (tlb_flag(TLB_V7_UIS_FULL)) |
342 | asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); | 338 | asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); |
343 | 339 | ||
344 | if (tlb_flag(TLB_BTB)) { | 340 | if (tlb_flag(TLB_BARRIER)) { |
345 | /* flush the branch target cache */ | ||
346 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | ||
347 | dsb(); | ||
348 | isb(); | ||
349 | } | ||
350 | if (tlb_flag(TLB_V7_IS_BTB)) { | ||
351 | /* flush the branch target cache */ | ||
352 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | ||
353 | dsb(); | 341 | dsb(); |
354 | isb(); | 342 | isb(); |
355 | } | 343 | } |
@@ -389,17 +377,8 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) | |||
389 | asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); | 377 | asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); |
390 | #endif | 378 | #endif |
391 | 379 | ||
392 | if (tlb_flag(TLB_BTB)) { | 380 | if (tlb_flag(TLB_BARRIER)) |
393 | /* flush the branch target cache */ | ||
394 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | ||
395 | dsb(); | ||
396 | } | ||
397 | if (tlb_flag(TLB_V7_IS_BTB)) { | ||
398 | /* flush the branch target cache */ | ||
399 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | ||
400 | dsb(); | 381 | dsb(); |
401 | isb(); | ||
402 | } | ||
403 | } | 382 | } |
404 | 383 | ||
405 | static inline void | 384 | static inline void |
@@ -439,17 +418,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) | |||
439 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); | 418 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); |
440 | #endif | 419 | #endif |
441 | 420 | ||
442 | if (tlb_flag(TLB_BTB)) { | 421 | if (tlb_flag(TLB_BARRIER)) |
443 | /* flush the branch target cache */ | ||
444 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | ||
445 | dsb(); | ||
446 | } | ||
447 | if (tlb_flag(TLB_V7_IS_BTB)) { | ||
448 | /* flush the branch target cache */ | ||
449 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | ||
450 | dsb(); | 422 | dsb(); |
451 | isb(); | ||
452 | } | ||
453 | } | 423 | } |
454 | 424 | ||
455 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) | 425 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) |
@@ -482,15 +452,7 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) | |||
482 | if (tlb_flag(TLB_V7_UIS_PAGE)) | 452 | if (tlb_flag(TLB_V7_UIS_PAGE)) |
483 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); | 453 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); |
484 | 454 | ||
485 | if (tlb_flag(TLB_BTB)) { | 455 | if (tlb_flag(TLB_BARRIER)) { |
486 | /* flush the branch target cache */ | ||
487 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); | ||
488 | dsb(); | ||
489 | isb(); | ||
490 | } | ||
491 | if (tlb_flag(TLB_V7_IS_BTB)) { | ||
492 | /* flush the branch target cache */ | ||
493 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); | ||
494 | dsb(); | 456 | dsb(); |
495 | isb(); | 457 | isb(); |
496 | } | 458 | } |
diff --git a/arch/arm/mm/tlb-fa.S b/arch/arm/mm/tlb-fa.S index 9694f1f6f485..d887a31faaae 100644 --- a/arch/arm/mm/tlb-fa.S +++ b/arch/arm/mm/tlb-fa.S | |||
@@ -46,7 +46,6 @@ ENTRY(fa_flush_user_tlb_range) | |||
46 | add r0, r0, #PAGE_SZ | 46 | add r0, r0, #PAGE_SZ |
47 | cmp r0, r1 | 47 | cmp r0, r1 |
48 | blo 1b | 48 | blo 1b |
49 | mcr p15, 0, r3, c7, c5, 6 @ invalidate BTB | ||
50 | mcr p15, 0, r3, c7, c10, 4 @ data write barrier | 49 | mcr p15, 0, r3, c7, c10, 4 @ data write barrier |
51 | mov pc, lr | 50 | mov pc, lr |
52 | 51 | ||
@@ -60,9 +59,8 @@ ENTRY(fa_flush_kern_tlb_range) | |||
60 | add r0, r0, #PAGE_SZ | 59 | add r0, r0, #PAGE_SZ |
61 | cmp r0, r1 | 60 | cmp r0, r1 |
62 | blo 1b | 61 | blo 1b |
63 | mcr p15, 0, r3, c7, c5, 6 @ invalidate BTB | ||
64 | mcr p15, 0, r3, c7, c10, 4 @ data write barrier | 62 | mcr p15, 0, r3, c7, c10, 4 @ data write barrier |
65 | mcr p15, 0, r3, c7, c5, 4 @ prefetch flush | 63 | mcr p15, 0, r3, c7, c5, 4 @ prefetch flush (isb) |
66 | mov pc, lr | 64 | mov pc, lr |
67 | 65 | ||
68 | __INITDATA | 66 | __INITDATA |
diff --git a/arch/arm/mm/tlb-v6.S b/arch/arm/mm/tlb-v6.S index 73d7d89b04c4..ffe06a69a6e5 100644 --- a/arch/arm/mm/tlb-v6.S +++ b/arch/arm/mm/tlb-v6.S | |||
@@ -54,7 +54,6 @@ ENTRY(v6wbi_flush_user_tlb_range) | |||
54 | add r0, r0, #PAGE_SZ | 54 | add r0, r0, #PAGE_SZ |
55 | cmp r0, r1 | 55 | cmp r0, r1 |
56 | blo 1b | 56 | blo 1b |
57 | mcr p15, 0, ip, c7, c5, 6 @ flush BTAC/BTB | ||
58 | mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier | 57 | mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier |
59 | mov pc, lr | 58 | mov pc, lr |
60 | 59 | ||
@@ -83,9 +82,8 @@ ENTRY(v6wbi_flush_kern_tlb_range) | |||
83 | add r0, r0, #PAGE_SZ | 82 | add r0, r0, #PAGE_SZ |
84 | cmp r0, r1 | 83 | cmp r0, r1 |
85 | blo 1b | 84 | blo 1b |
86 | mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB | ||
87 | mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier | 85 | mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier |
88 | mcr p15, 0, r2, c7, c5, 4 @ prefetch flush | 86 | mcr p15, 0, r2, c7, c5, 4 @ prefetch flush (isb) |
89 | mov pc, lr | 87 | mov pc, lr |
90 | 88 | ||
91 | __INIT | 89 | __INIT |
diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index 53cd5b454673..86bb71664508 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S | |||
@@ -48,9 +48,6 @@ ENTRY(v7wbi_flush_user_tlb_range) | |||
48 | add r0, r0, #PAGE_SZ | 48 | add r0, r0, #PAGE_SZ |
49 | cmp r0, r1 | 49 | cmp r0, r1 |
50 | blo 1b | 50 | blo 1b |
51 | mov ip, #0 | ||
52 | ALT_SMP(mcr p15, 0, ip, c7, c1, 6) @ flush BTAC/BTB Inner Shareable | ||
53 | ALT_UP(mcr p15, 0, ip, c7, c5, 6) @ flush BTAC/BTB | ||
54 | dsb | 51 | dsb |
55 | mov pc, lr | 52 | mov pc, lr |
56 | ENDPROC(v7wbi_flush_user_tlb_range) | 53 | ENDPROC(v7wbi_flush_user_tlb_range) |
@@ -75,9 +72,6 @@ ENTRY(v7wbi_flush_kern_tlb_range) | |||
75 | add r0, r0, #PAGE_SZ | 72 | add r0, r0, #PAGE_SZ |
76 | cmp r0, r1 | 73 | cmp r0, r1 |
77 | blo 1b | 74 | blo 1b |
78 | mov r2, #0 | ||
79 | ALT_SMP(mcr p15, 0, r2, c7, c1, 6) @ flush BTAC/BTB Inner Shareable | ||
80 | ALT_UP(mcr p15, 0, r2, c7, c5, 6) @ flush BTAC/BTB | ||
81 | dsb | 75 | dsb |
82 | isb | 76 | isb |
83 | mov pc, lr | 77 | mov pc, lr |