aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSuzuki K Poulose <suzuki.poulose@arm.com>2018-10-09 09:47:06 -0400
committerCatalin Marinas <catalin.marinas@arm.com>2018-10-16 06:53:31 -0400
commit1602df02f33f61fe0de1bbfeba0d1c97c14bff19 (patch)
tree97caef6e1301b3566fa5e33ab142f6912f623441
parent8ab66cbe63aeaf9e5970fb4aaef1c660fca59321 (diff)
arm64: cpufeature: Fix handling of CTR_EL0.IDC field
CTR_EL0.IDC reports the data cache clean requirements for instruction to data coherence. However, if the field is 0, we need to check the CLIDR_EL1 fields to detect the status of the feature. Currently we don't do this and generate a warning with tainting the kernel, when there is a mismatch in the field among the CPUs. Also the userspace doesn't have a reliable way to check the CLIDR_EL1 register to check the status. This patch fixes the problem by checking the CLIDR_EL1 fields, when (CTR_EL0.IDC == 0) and updates the kernel's copy of the CTR_EL0 for the CPU with the actual status of the feature. This would allow the sanity check infrastructure to do the proper checking of the fields and also allow the CTR_EL0 emulation code to supply the real status of the feature. Now, if a CPU has raw CTR_EL0.IDC == 0 and effective IDC == 1 (with overall system wide IDC == 1), we need to expose the real value to the user. So, we trap CTR_EL0 access on the CPU which reports incorrect CTR_EL0.IDC. Fixes: commit 6ae4b6e057888 ("arm64: Add support for new control bits CTR_EL0.DIC and CTR_EL0.IDC") Cc: Shanker Donthineni <shankerd@codeaurora.org> Cc: Philip Elcan <pelcan@codeaurora.org> Cc: Will Deacon <will.deacon@arm.com> Cc: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Suzuki K Poulose <suzuki.poulose@arm.com> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
-rw-r--r--arch/arm64/include/asm/cache.h40
-rw-r--r--arch/arm64/kernel/cpu_errata.c27
-rw-r--r--arch/arm64/kernel/cpufeature.c15
-rw-r--r--arch/arm64/kernel/cpuinfo.c10
4 files changed, 87 insertions, 5 deletions
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index 5ee5bca8c24b..13dd42c3ad4e 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -40,6 +40,15 @@
40#define L1_CACHE_SHIFT (6) 40#define L1_CACHE_SHIFT (6)
41#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) 41#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
42 42
43
44#define CLIDR_LOUU_SHIFT 27
45#define CLIDR_LOC_SHIFT 24
46#define CLIDR_LOUIS_SHIFT 21
47
48#define CLIDR_LOUU(clidr) (((clidr) >> CLIDR_LOUU_SHIFT) & 0x7)
49#define CLIDR_LOC(clidr) (((clidr) >> CLIDR_LOC_SHIFT) & 0x7)
50#define CLIDR_LOUIS(clidr) (((clidr) >> CLIDR_LOUIS_SHIFT) & 0x7)
51
43/* 52/*
44 * Memory returned by kmalloc() may be used for DMA, so we must make 53 * Memory returned by kmalloc() may be used for DMA, so we must make
45 * sure that all such allocations are cache aligned. Otherwise, 54 * sure that all such allocations are cache aligned. Otherwise,
@@ -84,6 +93,37 @@ static inline int cache_line_size(void)
84 return cwg ? 4 << cwg : ARCH_DMA_MINALIGN; 93 return cwg ? 4 << cwg : ARCH_DMA_MINALIGN;
85} 94}
86 95
96/*
97 * Read the effective value of CTR_EL0.
98 *
99 * According to ARM ARM for ARMv8-A (ARM DDI 0487C.a),
100 * section D10.2.33 "CTR_EL0, Cache Type Register" :
101 *
102 * CTR_EL0.IDC reports the data cache clean requirements for
103 * instruction to data coherence.
104 *
105 * 0 - dcache clean to PoU is required unless :
106 * (CLIDR_EL1.LoC == 0) || (CLIDR_EL1.LoUIS == 0 && CLIDR_EL1.LoUU == 0)
107 * 1 - dcache clean to PoU is not required for i-to-d coherence.
108 *
109 * This routine provides the CTR_EL0 with the IDC field updated to the
110 * effective state.
111 */
112static inline u32 __attribute_const__ read_cpuid_effective_cachetype(void)
113{
114 u32 ctr = read_cpuid_cachetype();
115
116 if (!(ctr & BIT(CTR_IDC_SHIFT))) {
117 u64 clidr = read_sysreg(clidr_el1);
118
119 if (CLIDR_LOC(clidr) == 0 ||
120 (CLIDR_LOUIS(clidr) == 0 && CLIDR_LOUU(clidr) == 0))
121 ctr |= BIT(CTR_IDC_SHIFT);
122 }
123
124 return ctr;
125}
126
87#endif /* __ASSEMBLY__ */ 127#endif /* __ASSEMBLY__ */
88 128
89#endif 129#endif
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index cde948991d68..3deb01c6ed49 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -68,11 +68,32 @@ static bool
68has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, 68has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
69 int scope) 69 int scope)
70{ 70{
71 u64 mask = arm64_ftr_reg_ctrel0.strict_mask;; 71 u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
72 u64 sys = arm64_ftr_reg_ctrel0.sys_val & mask;
73 u64 ctr_raw, ctr_real;
72 74
73 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); 75 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
74 return (read_cpuid_cachetype() & mask) != 76
75 (arm64_ftr_reg_ctrel0.sys_val & mask); 77 /*
78 * We want to make sure that all the CPUs in the system expose
79 * a consistent CTR_EL0 to make sure that applications behaves
80 * correctly with migration.
81 *
82 * If a CPU has CTR_EL0.IDC but does not advertise it via CTR_EL0 :
83 *
84 * 1) It is safe if the system doesn't support IDC, as CPU anyway
85 * reports IDC = 0, consistent with the rest.
86 *
87 * 2) If the system has IDC, it is still safe as we trap CTR_EL0
88 * access on this CPU via the ARM64_HAS_CACHE_IDC capability.
89 *
90 * So, we need to make sure either the raw CTR_EL0 or the effective
91 * CTR_EL0 matches the system's copy to allow a secondary CPU to boot.
92 */
93 ctr_raw = read_cpuid_cachetype() & mask;
94 ctr_real = read_cpuid_effective_cachetype() & mask;
95
96 return (ctr_real != sys) && (ctr_raw != sys);
76} 97}
77 98
78static void 99static void
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index ba16bb7762ca..af50064dea51 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -861,11 +861,23 @@ static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
861 if (scope == SCOPE_SYSTEM) 861 if (scope == SCOPE_SYSTEM)
862 ctr = arm64_ftr_reg_ctrel0.sys_val; 862 ctr = arm64_ftr_reg_ctrel0.sys_val;
863 else 863 else
864 ctr = read_cpuid_cachetype(); 864 ctr = read_cpuid_effective_cachetype();
865 865
866 return ctr & BIT(CTR_IDC_SHIFT); 866 return ctr & BIT(CTR_IDC_SHIFT);
867} 867}
868 868
869static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused)
870{
871 /*
872 * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively
873 * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses
874 * to the CTR_EL0 on this CPU and emulate it with the real/safe
875 * value.
876 */
877 if (!(read_cpuid_cachetype() & BIT(CTR_IDC_SHIFT)))
878 sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
879}
880
869static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, 881static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
870 int scope) 882 int scope)
871{ 883{
@@ -1282,6 +1294,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
1282 .capability = ARM64_HAS_CACHE_IDC, 1294 .capability = ARM64_HAS_CACHE_IDC,
1283 .type = ARM64_CPUCAP_SYSTEM_FEATURE, 1295 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1284 .matches = has_cache_idc, 1296 .matches = has_cache_idc,
1297 .cpu_enable = cpu_emulate_effective_ctr,
1285 }, 1298 },
1286 { 1299 {
1287 .desc = "Instruction cache invalidation not required for I/D coherence", 1300 .desc = "Instruction cache invalidation not required for I/D coherence",
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index dce971f2c167..bcc2831399cb 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -325,7 +325,15 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
325static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) 325static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
326{ 326{
327 info->reg_cntfrq = arch_timer_get_cntfrq(); 327 info->reg_cntfrq = arch_timer_get_cntfrq();
328 info->reg_ctr = read_cpuid_cachetype(); 328 /*
329 * Use the effective value of the CTR_EL0 than the raw value
330 * exposed by the CPU. CTR_E0.IDC field value must be interpreted
331 * with the CLIDR_EL1 fields to avoid triggering false warnings
332 * when there is a mismatch across the CPUs. Keep track of the
333 * effective value of the CTR_EL0 in our internal records for
334 * acurate sanity check and feature enablement.
335 */
336 info->reg_ctr = read_cpuid_effective_cachetype();
329 info->reg_dczid = read_cpuid(DCZID_EL0); 337 info->reg_dczid = read_cpuid(DCZID_EL0);
330 info->reg_midr = read_cpuid_id(); 338 info->reg_midr = read_cpuid_id();
331 info->reg_revidr = read_cpuid(REVIDR_EL1); 339 info->reg_revidr = read_cpuid(REVIDR_EL1);