aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorMarc Zyngier <marc.zyngier@arm.com>2014-12-19 11:05:31 -0500
committerChristoffer Dall <christoffer.dall@linaro.org>2015-01-29 17:24:56 -0500
commit3c1e716508335eb132c9349cb1a1716c8f7e3d2e (patch)
tree39aa2fe691caa61a34cef1ac7d09f3348249ddb5 /arch
parentf3747379accba8e95d70cec0eae0582c8c182050 (diff)
arm/arm64: KVM: Use set/way op trapping to track the state of the caches
Trying to emulate the behaviour of set/way cache ops is fairly pointless, as there are too many ways we can end-up missing stuff. Also, there is some system caches out there that simply ignore set/way operations. So instead of trying to implement them, let's convert it to VA ops, and use them as a way to re-enable the trapping of VM ops. That way, we can detect the point when the MMU/caches are turned off, and do a full VM flush (which is what the guest was trying to do anyway). This allows a 32bit zImage to boot on the APM thingy, and will probably help bootloaders in general. Signed-off-by: Marc Zyngier <marc.zyngier@arm.com> Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/include/asm/kvm_emulate.h10
-rw-r--r--arch/arm/include/asm/kvm_host.h3
-rw-r--r--arch/arm/include/asm/kvm_mmu.h3
-rw-r--r--arch/arm/kvm/arm.c10
-rw-r--r--arch/arm/kvm/coproc.c70
-rw-r--r--arch/arm/kvm/coproc.h6
-rw-r--r--arch/arm/kvm/coproc_a15.c2
-rw-r--r--arch/arm/kvm/coproc_a7.c2
-rw-r--r--arch/arm/kvm/mmu.c70
-rw-r--r--arch/arm/kvm/trace.h39
-rw-r--r--arch/arm64/include/asm/kvm_emulate.h10
-rw-r--r--arch/arm64/include/asm/kvm_host.h3
-rw-r--r--arch/arm64/include/asm/kvm_mmu.h3
-rw-r--r--arch/arm64/kvm/sys_regs.c75
14 files changed, 161 insertions, 145 deletions
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 66ce17655bb9..7b0152321b20 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -38,6 +38,16 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
38 vcpu->arch.hcr = HCR_GUEST_MASK; 38 vcpu->arch.hcr = HCR_GUEST_MASK;
39} 39}
40 40
41static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu)
42{
43 return vcpu->arch.hcr;
44}
45
46static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
47{
48 vcpu->arch.hcr = hcr;
49}
50
41static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu) 51static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu)
42{ 52{
43 return 1; 53 return 1;
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 254e0650e48b..04b4ea0b550a 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -125,9 +125,6 @@ struct kvm_vcpu_arch {
125 * Anything that is not used directly from assembly code goes 125 * Anything that is not used directly from assembly code goes
126 * here. 126 * here.
127 */ 127 */
128 /* dcache set/way operation pending */
129 int last_pcpu;
130 cpumask_t require_dcache_flush;
131 128
132 /* Don't run the guest on this vcpu */ 129 /* Don't run the guest on this vcpu */
133 bool pause; 130 bool pause;
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 63e0ecc04901..286644c729ba 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -190,7 +190,8 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
190 190
191#define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x)) 191#define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x))
192 192
193void stage2_flush_vm(struct kvm *kvm); 193void kvm_set_way_flush(struct kvm_vcpu *vcpu);
194void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
194 195
195#endif /* !__ASSEMBLY__ */ 196#endif /* !__ASSEMBLY__ */
196 197
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 2d6d91001062..0b0d58a905c4 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -281,15 +281,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
281 vcpu->cpu = cpu; 281 vcpu->cpu = cpu;
282 vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state); 282 vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
283 283
284 /*
285 * Check whether this vcpu requires the cache to be flushed on
286 * this physical CPU. This is a consequence of doing dcache
287 * operations by set/way on this vcpu. We do it here to be in
288 * a non-preemptible section.
289 */
290 if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush))
291 flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
292
293 kvm_arm_set_running_vcpu(vcpu); 284 kvm_arm_set_running_vcpu(vcpu);
294} 285}
295 286
@@ -541,7 +532,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
541 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu); 532 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
542 533
543 vcpu->mode = OUTSIDE_GUEST_MODE; 534 vcpu->mode = OUTSIDE_GUEST_MODE;
544 vcpu->arch.last_pcpu = smp_processor_id();
545 kvm_guest_exit(); 535 kvm_guest_exit();
546 trace_kvm_exit(*vcpu_pc(vcpu)); 536 trace_kvm_exit(*vcpu_pc(vcpu));
547 /* 537 /*
diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c
index 7928dbdf2102..f3d88dc388bc 100644
--- a/arch/arm/kvm/coproc.c
+++ b/arch/arm/kvm/coproc.c
@@ -189,82 +189,40 @@ static bool access_l2ectlr(struct kvm_vcpu *vcpu,
189 return true; 189 return true;
190} 190}
191 191
192/* See note at ARM ARM B1.14.4 */ 192/*
193 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
194 */
193static bool access_dcsw(struct kvm_vcpu *vcpu, 195static bool access_dcsw(struct kvm_vcpu *vcpu,
194 const struct coproc_params *p, 196 const struct coproc_params *p,
195 const struct coproc_reg *r) 197 const struct coproc_reg *r)
196{ 198{
197 unsigned long val;
198 int cpu;
199
200 if (!p->is_write) 199 if (!p->is_write)
201 return read_from_write_only(vcpu, p); 200 return read_from_write_only(vcpu, p);
202 201
203 cpu = get_cpu(); 202 kvm_set_way_flush(vcpu);
204
205 cpumask_setall(&vcpu->arch.require_dcache_flush);
206 cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
207
208 /* If we were already preempted, take the long way around */
209 if (cpu != vcpu->arch.last_pcpu) {
210 flush_cache_all();
211 goto done;
212 }
213
214 val = *vcpu_reg(vcpu, p->Rt1);
215
216 switch (p->CRm) {
217 case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */
218 case 14: /* DCCISW */
219 asm volatile("mcr p15, 0, %0, c7, c14, 2" : : "r" (val));
220 break;
221
222 case 10: /* DCCSW */
223 asm volatile("mcr p15, 0, %0, c7, c10, 2" : : "r" (val));
224 break;
225 }
226
227done:
228 put_cpu();
229
230 return true; 203 return true;
231} 204}
232 205
233/* 206/*
234 * Generic accessor for VM registers. Only called as long as HCR_TVM 207 * Generic accessor for VM registers. Only called as long as HCR_TVM
235 * is set. 208 * is set. If the guest enables the MMU, we stop trapping the VM
209 * sys_regs and leave it in complete control of the caches.
210 *
211 * Used by the cpu-specific code.
236 */ 212 */
237static bool access_vm_reg(struct kvm_vcpu *vcpu, 213bool access_vm_reg(struct kvm_vcpu *vcpu,
238 const struct coproc_params *p, 214 const struct coproc_params *p,
239 const struct coproc_reg *r) 215 const struct coproc_reg *r)
240{ 216{
217 bool was_enabled = vcpu_has_cache_enabled(vcpu);
218
241 BUG_ON(!p->is_write); 219 BUG_ON(!p->is_write);
242 220
243 vcpu->arch.cp15[r->reg] = *vcpu_reg(vcpu, p->Rt1); 221 vcpu->arch.cp15[r->reg] = *vcpu_reg(vcpu, p->Rt1);
244 if (p->is_64bit) 222 if (p->is_64bit)
245 vcpu->arch.cp15[r->reg + 1] = *vcpu_reg(vcpu, p->Rt2); 223 vcpu->arch.cp15[r->reg + 1] = *vcpu_reg(vcpu, p->Rt2);
246 224
247 return true; 225 kvm_toggle_cache(vcpu, was_enabled);
248}
249
250/*
251 * SCTLR accessor. Only called as long as HCR_TVM is set. If the
252 * guest enables the MMU, we stop trapping the VM sys_regs and leave
253 * it in complete control of the caches.
254 *
255 * Used by the cpu-specific code.
256 */
257bool access_sctlr(struct kvm_vcpu *vcpu,
258 const struct coproc_params *p,
259 const struct coproc_reg *r)
260{
261 access_vm_reg(vcpu, p, r);
262
263 if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */
264 vcpu->arch.hcr &= ~HCR_TVM;
265 stage2_flush_vm(vcpu->kvm);
266 }
267
268 return true; 226 return true;
269} 227}
270 228
diff --git a/arch/arm/kvm/coproc.h b/arch/arm/kvm/coproc.h
index 1a44bbe39643..88d24a3a9778 100644
--- a/arch/arm/kvm/coproc.h
+++ b/arch/arm/kvm/coproc.h
@@ -153,8 +153,8 @@ static inline int cmp_reg(const struct coproc_reg *i1,
153#define is64 .is_64 = true 153#define is64 .is_64 = true
154#define is32 .is_64 = false 154#define is32 .is_64 = false
155 155
156bool access_sctlr(struct kvm_vcpu *vcpu, 156bool access_vm_reg(struct kvm_vcpu *vcpu,
157 const struct coproc_params *p, 157 const struct coproc_params *p,
158 const struct coproc_reg *r); 158 const struct coproc_reg *r);
159 159
160#endif /* __ARM_KVM_COPROC_LOCAL_H__ */ 160#endif /* __ARM_KVM_COPROC_LOCAL_H__ */
diff --git a/arch/arm/kvm/coproc_a15.c b/arch/arm/kvm/coproc_a15.c
index e6f4ae48bda9..a7136757d373 100644
--- a/arch/arm/kvm/coproc_a15.c
+++ b/arch/arm/kvm/coproc_a15.c
@@ -34,7 +34,7 @@
34static const struct coproc_reg a15_regs[] = { 34static const struct coproc_reg a15_regs[] = {
35 /* SCTLR: swapped by interrupt.S. */ 35 /* SCTLR: swapped by interrupt.S. */
36 { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, 36 { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,
37 access_sctlr, reset_val, c1_SCTLR, 0x00C50078 }, 37 access_vm_reg, reset_val, c1_SCTLR, 0x00C50078 },
38}; 38};
39 39
40static struct kvm_coproc_target_table a15_target_table = { 40static struct kvm_coproc_target_table a15_target_table = {
diff --git a/arch/arm/kvm/coproc_a7.c b/arch/arm/kvm/coproc_a7.c
index 17fc7cd479d3..b19e46d1b2c0 100644
--- a/arch/arm/kvm/coproc_a7.c
+++ b/arch/arm/kvm/coproc_a7.c
@@ -37,7 +37,7 @@
37static const struct coproc_reg a7_regs[] = { 37static const struct coproc_reg a7_regs[] = {
38 /* SCTLR: swapped by interrupt.S. */ 38 /* SCTLR: swapped by interrupt.S. */
39 { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, 39 { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,
40 access_sctlr, reset_val, c1_SCTLR, 0x00C50878 }, 40 access_vm_reg, reset_val, c1_SCTLR, 0x00C50878 },
41}; 41};
42 42
43static struct kvm_coproc_target_table a7_target_table = { 43static struct kvm_coproc_target_table a7_target_table = {
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 1dc9778a00af..106737e309b1 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -278,7 +278,7 @@ static void stage2_flush_memslot(struct kvm *kvm,
278 * Go through the stage 2 page tables and invalidate any cache lines 278 * Go through the stage 2 page tables and invalidate any cache lines
279 * backing memory already mapped to the VM. 279 * backing memory already mapped to the VM.
280 */ 280 */
281void stage2_flush_vm(struct kvm *kvm) 281static void stage2_flush_vm(struct kvm *kvm)
282{ 282{
283 struct kvm_memslots *slots; 283 struct kvm_memslots *slots;
284 struct kvm_memory_slot *memslot; 284 struct kvm_memory_slot *memslot;
@@ -1411,3 +1411,71 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1411 unmap_stage2_range(kvm, gpa, size); 1411 unmap_stage2_range(kvm, gpa, size);
1412 spin_unlock(&kvm->mmu_lock); 1412 spin_unlock(&kvm->mmu_lock);
1413} 1413}
1414
1415/*
1416 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
1417 *
1418 * Main problems:
1419 * - S/W ops are local to a CPU (not broadcast)
1420 * - We have line migration behind our back (speculation)
1421 * - System caches don't support S/W at all (damn!)
1422 *
1423 * In the face of the above, the best we can do is to try and convert
1424 * S/W ops to VA ops. Because the guest is not allowed to infer the
1425 * S/W to PA mapping, it can only use S/W to nuke the whole cache,
1426 * which is a rather good thing for us.
1427 *
1428 * Also, it is only used when turning caches on/off ("The expected
1429 * usage of the cache maintenance instructions that operate by set/way
1430 * is associated with the cache maintenance instructions associated
1431 * with the powerdown and powerup of caches, if this is required by
1432 * the implementation.").
1433 *
1434 * We use the following policy:
1435 *
1436 * - If we trap a S/W operation, we enable VM trapping to detect
1437 * caches being turned on/off, and do a full clean.
1438 *
1439 * - We flush the caches on both caches being turned on and off.
1440 *
1441 * - Once the caches are enabled, we stop trapping VM ops.
1442 */
1443void kvm_set_way_flush(struct kvm_vcpu *vcpu)
1444{
1445 unsigned long hcr = vcpu_get_hcr(vcpu);
1446
1447 /*
1448 * If this is the first time we do a S/W operation
1449 * (i.e. HCR_TVM not set) flush the whole memory, and set the
1450 * VM trapping.
1451 *
1452 * Otherwise, rely on the VM trapping to wait for the MMU +
1453 * Caches to be turned off. At that point, we'll be able to
1454 * clean the caches again.
1455 */
1456 if (!(hcr & HCR_TVM)) {
1457 trace_kvm_set_way_flush(*vcpu_pc(vcpu),
1458 vcpu_has_cache_enabled(vcpu));
1459 stage2_flush_vm(vcpu->kvm);
1460 vcpu_set_hcr(vcpu, hcr | HCR_TVM);
1461 }
1462}
1463
1464void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
1465{
1466 bool now_enabled = vcpu_has_cache_enabled(vcpu);
1467
1468 /*
1469 * If switching the MMU+caches on, need to invalidate the caches.
1470 * If switching it off, need to clean the caches.
1471 * Clean + invalidate does the trick always.
1472 */
1473 if (now_enabled != was_enabled)
1474 stage2_flush_vm(vcpu->kvm);
1475
1476 /* Caches are now on, stop trapping VM ops (until a S/W op) */
1477 if (now_enabled)
1478 vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) & ~HCR_TVM);
1479
1480 trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
1481}
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h
index b1d640f78623..b6a6e7102201 100644
--- a/arch/arm/kvm/trace.h
+++ b/arch/arm/kvm/trace.h
@@ -223,6 +223,45 @@ TRACE_EVENT(kvm_hvc,
223 __entry->vcpu_pc, __entry->r0, __entry->imm) 223 __entry->vcpu_pc, __entry->r0, __entry->imm)
224); 224);
225 225
226TRACE_EVENT(kvm_set_way_flush,
227 TP_PROTO(unsigned long vcpu_pc, bool cache),
228 TP_ARGS(vcpu_pc, cache),
229
230 TP_STRUCT__entry(
231 __field( unsigned long, vcpu_pc )
232 __field( bool, cache )
233 ),
234
235 TP_fast_assign(
236 __entry->vcpu_pc = vcpu_pc;
237 __entry->cache = cache;
238 ),
239
240 TP_printk("S/W flush at 0x%016lx (cache %s)",
241 __entry->vcpu_pc, __entry->cache ? "on" : "off")
242);
243
244TRACE_EVENT(kvm_toggle_cache,
245 TP_PROTO(unsigned long vcpu_pc, bool was, bool now),
246 TP_ARGS(vcpu_pc, was, now),
247
248 TP_STRUCT__entry(
249 __field( unsigned long, vcpu_pc )
250 __field( bool, was )
251 __field( bool, now )
252 ),
253
254 TP_fast_assign(
255 __entry->vcpu_pc = vcpu_pc;
256 __entry->was = was;
257 __entry->now = now;
258 ),
259
260 TP_printk("VM op at 0x%016lx (cache was %s, now %s)",
261 __entry->vcpu_pc, __entry->was ? "on" : "off",
262 __entry->now ? "on" : "off")
263);
264
226#endif /* _TRACE_KVM_H */ 265#endif /* _TRACE_KVM_H */
227 266
228#undef TRACE_INCLUDE_PATH 267#undef TRACE_INCLUDE_PATH
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 865a7e28ea2d..3cb4c856b10d 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -45,6 +45,16 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
45 vcpu->arch.hcr_el2 &= ~HCR_RW; 45 vcpu->arch.hcr_el2 &= ~HCR_RW;
46} 46}
47 47
48static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu)
49{
50 return vcpu->arch.hcr_el2;
51}
52
53static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
54{
55 vcpu->arch.hcr_el2 = hcr;
56}
57
48static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) 58static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
49{ 59{
50 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc; 60 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 0b7dfdb931df..acd101a9014d 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -116,9 +116,6 @@ struct kvm_vcpu_arch {
116 * Anything that is not used directly from assembly code goes 116 * Anything that is not used directly from assembly code goes
117 * here. 117 * here.
118 */ 118 */
119 /* dcache set/way operation pending */
120 int last_pcpu;
121 cpumask_t require_dcache_flush;
122 119
123 /* Don't run the guest */ 120 /* Don't run the guest */
124 bool pause; 121 bool pause;
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 14a74f136272..92d22e94a79b 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -260,7 +260,8 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
260 260
261#define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x)) 261#define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x))
262 262
263void stage2_flush_vm(struct kvm *kvm); 263void kvm_set_way_flush(struct kvm_vcpu *vcpu);
264void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
264 265
265#endif /* __ASSEMBLY__ */ 266#endif /* __ASSEMBLY__ */
266#endif /* __ARM64_KVM_MMU_H__ */ 267#endif /* __ARM64_KVM_MMU_H__ */
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 3d7c2df89946..f31e8bb2bc5b 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -69,68 +69,31 @@ static u32 get_ccsidr(u32 csselr)
69 return ccsidr; 69 return ccsidr;
70} 70}
71 71
72static void do_dc_cisw(u32 val) 72/*
73{ 73 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
74 asm volatile("dc cisw, %x0" : : "r" (val)); 74 */
75 dsb(ish);
76}
77
78static void do_dc_csw(u32 val)
79{
80 asm volatile("dc csw, %x0" : : "r" (val));
81 dsb(ish);
82}
83
84/* See note at ARM ARM B1.14.4 */
85static bool access_dcsw(struct kvm_vcpu *vcpu, 75static bool access_dcsw(struct kvm_vcpu *vcpu,
86 const struct sys_reg_params *p, 76 const struct sys_reg_params *p,
87 const struct sys_reg_desc *r) 77 const struct sys_reg_desc *r)
88{ 78{
89 unsigned long val;
90 int cpu;
91
92 if (!p->is_write) 79 if (!p->is_write)
93 return read_from_write_only(vcpu, p); 80 return read_from_write_only(vcpu, p);
94 81
95 cpu = get_cpu(); 82 kvm_set_way_flush(vcpu);
96
97 cpumask_setall(&vcpu->arch.require_dcache_flush);
98 cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
99
100 /* If we were already preempted, take the long way around */
101 if (cpu != vcpu->arch.last_pcpu) {
102 flush_cache_all();
103 goto done;
104 }
105
106 val = *vcpu_reg(vcpu, p->Rt);
107
108 switch (p->CRm) {
109 case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */
110 case 14: /* DCCISW */
111 do_dc_cisw(val);
112 break;
113
114 case 10: /* DCCSW */
115 do_dc_csw(val);
116 break;
117 }
118
119done:
120 put_cpu();
121
122 return true; 83 return true;
123} 84}
124 85
125/* 86/*
126 * Generic accessor for VM registers. Only called as long as HCR_TVM 87 * Generic accessor for VM registers. Only called as long as HCR_TVM
127 * is set. 88 * is set. If the guest enables the MMU, we stop trapping the VM
89 * sys_regs and leave it in complete control of the caches.
128 */ 90 */
129static bool access_vm_reg(struct kvm_vcpu *vcpu, 91static bool access_vm_reg(struct kvm_vcpu *vcpu,
130 const struct sys_reg_params *p, 92 const struct sys_reg_params *p,
131 const struct sys_reg_desc *r) 93 const struct sys_reg_desc *r)
132{ 94{
133 unsigned long val; 95 unsigned long val;
96 bool was_enabled = vcpu_has_cache_enabled(vcpu);
134 97
135 BUG_ON(!p->is_write); 98 BUG_ON(!p->is_write);
136 99
@@ -143,25 +106,7 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu,
143 vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL; 106 vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL;
144 } 107 }
145 108
146 return true; 109 kvm_toggle_cache(vcpu, was_enabled);
147}
148
149/*
150 * SCTLR_EL1 accessor. Only called as long as HCR_TVM is set. If the
151 * guest enables the MMU, we stop trapping the VM sys_regs and leave
152 * it in complete control of the caches.
153 */
154static bool access_sctlr(struct kvm_vcpu *vcpu,
155 const struct sys_reg_params *p,
156 const struct sys_reg_desc *r)
157{
158 access_vm_reg(vcpu, p, r);
159
160 if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */
161 vcpu->arch.hcr_el2 &= ~HCR_TVM;
162 stage2_flush_vm(vcpu->kvm);
163 }
164
165 return true; 110 return true;
166} 111}
167 112
@@ -377,7 +322,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
377 NULL, reset_mpidr, MPIDR_EL1 }, 322 NULL, reset_mpidr, MPIDR_EL1 },
378 /* SCTLR_EL1 */ 323 /* SCTLR_EL1 */
379 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000), 324 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
380 access_sctlr, reset_val, SCTLR_EL1, 0x00C50078 }, 325 access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
381 /* CPACR_EL1 */ 326 /* CPACR_EL1 */
382 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010), 327 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
383 NULL, reset_val, CPACR_EL1, 0 }, 328 NULL, reset_val, CPACR_EL1, 0 },
@@ -657,7 +602,7 @@ static const struct sys_reg_desc cp14_64_regs[] = {
657 * register). 602 * register).
658 */ 603 */
659static const struct sys_reg_desc cp15_regs[] = { 604static const struct sys_reg_desc cp15_regs[] = {
660 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR }, 605 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
661 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, 606 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
662 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, 607 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
663 { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR }, 608 { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },