diff options
author | Marc Zyngier <marc.zyngier@arm.com> | 2014-12-19 11:05:31 -0500 |
---|---|---|
committer | Christoffer Dall <christoffer.dall@linaro.org> | 2015-01-29 17:24:56 -0500 |
commit | 3c1e716508335eb132c9349cb1a1716c8f7e3d2e (patch) | |
tree | 39aa2fe691caa61a34cef1ac7d09f3348249ddb5 /arch | |
parent | f3747379accba8e95d70cec0eae0582c8c182050 (diff) |
arm/arm64: KVM: Use set/way op trapping to track the state of the caches
Trying to emulate the behaviour of set/way cache ops is fairly
pointless, as there are too many ways we can end-up missing stuff.
Also, there is some system caches out there that simply ignore
set/way operations.
So instead of trying to implement them, let's convert it to VA ops,
and use them as a way to re-enable the trapping of VM ops. That way,
we can detect the point when the MMU/caches are turned off, and do
a full VM flush (which is what the guest was trying to do anyway).
This allows a 32bit zImage to boot on the APM thingy, and will
probably help bootloaders in general.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Diffstat (limited to 'arch')
-rw-r--r-- | arch/arm/include/asm/kvm_emulate.h | 10 | ||||
-rw-r--r-- | arch/arm/include/asm/kvm_host.h | 3 | ||||
-rw-r--r-- | arch/arm/include/asm/kvm_mmu.h | 3 | ||||
-rw-r--r-- | arch/arm/kvm/arm.c | 10 | ||||
-rw-r--r-- | arch/arm/kvm/coproc.c | 70 | ||||
-rw-r--r-- | arch/arm/kvm/coproc.h | 6 | ||||
-rw-r--r-- | arch/arm/kvm/coproc_a15.c | 2 | ||||
-rw-r--r-- | arch/arm/kvm/coproc_a7.c | 2 | ||||
-rw-r--r-- | arch/arm/kvm/mmu.c | 70 | ||||
-rw-r--r-- | arch/arm/kvm/trace.h | 39 | ||||
-rw-r--r-- | arch/arm64/include/asm/kvm_emulate.h | 10 | ||||
-rw-r--r-- | arch/arm64/include/asm/kvm_host.h | 3 | ||||
-rw-r--r-- | arch/arm64/include/asm/kvm_mmu.h | 3 | ||||
-rw-r--r-- | arch/arm64/kvm/sys_regs.c | 75 |
14 files changed, 161 insertions, 145 deletions
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 66ce17655bb9..7b0152321b20 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h | |||
@@ -38,6 +38,16 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) | |||
38 | vcpu->arch.hcr = HCR_GUEST_MASK; | 38 | vcpu->arch.hcr = HCR_GUEST_MASK; |
39 | } | 39 | } |
40 | 40 | ||
41 | static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu) | ||
42 | { | ||
43 | return vcpu->arch.hcr; | ||
44 | } | ||
45 | |||
46 | static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr) | ||
47 | { | ||
48 | vcpu->arch.hcr = hcr; | ||
49 | } | ||
50 | |||
41 | static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu) | 51 | static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu) |
42 | { | 52 | { |
43 | return 1; | 53 | return 1; |
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index 254e0650e48b..04b4ea0b550a 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h | |||
@@ -125,9 +125,6 @@ struct kvm_vcpu_arch { | |||
125 | * Anything that is not used directly from assembly code goes | 125 | * Anything that is not used directly from assembly code goes |
126 | * here. | 126 | * here. |
127 | */ | 127 | */ |
128 | /* dcache set/way operation pending */ | ||
129 | int last_pcpu; | ||
130 | cpumask_t require_dcache_flush; | ||
131 | 128 | ||
132 | /* Don't run the guest on this vcpu */ | 129 | /* Don't run the guest on this vcpu */ |
133 | bool pause; | 130 | bool pause; |
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index 63e0ecc04901..286644c729ba 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h | |||
@@ -190,7 +190,8 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, | |||
190 | 190 | ||
191 | #define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x)) | 191 | #define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x)) |
192 | 192 | ||
193 | void stage2_flush_vm(struct kvm *kvm); | 193 | void kvm_set_way_flush(struct kvm_vcpu *vcpu); |
194 | void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); | ||
194 | 195 | ||
195 | #endif /* !__ASSEMBLY__ */ | 196 | #endif /* !__ASSEMBLY__ */ |
196 | 197 | ||
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 2d6d91001062..0b0d58a905c4 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c | |||
@@ -281,15 +281,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) | |||
281 | vcpu->cpu = cpu; | 281 | vcpu->cpu = cpu; |
282 | vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state); | 282 | vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state); |
283 | 283 | ||
284 | /* | ||
285 | * Check whether this vcpu requires the cache to be flushed on | ||
286 | * this physical CPU. This is a consequence of doing dcache | ||
287 | * operations by set/way on this vcpu. We do it here to be in | ||
288 | * a non-preemptible section. | ||
289 | */ | ||
290 | if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush)) | ||
291 | flush_cache_all(); /* We'd really want v7_flush_dcache_all() */ | ||
292 | |||
293 | kvm_arm_set_running_vcpu(vcpu); | 284 | kvm_arm_set_running_vcpu(vcpu); |
294 | } | 285 | } |
295 | 286 | ||
@@ -541,7 +532,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) | |||
541 | ret = kvm_call_hyp(__kvm_vcpu_run, vcpu); | 532 | ret = kvm_call_hyp(__kvm_vcpu_run, vcpu); |
542 | 533 | ||
543 | vcpu->mode = OUTSIDE_GUEST_MODE; | 534 | vcpu->mode = OUTSIDE_GUEST_MODE; |
544 | vcpu->arch.last_pcpu = smp_processor_id(); | ||
545 | kvm_guest_exit(); | 535 | kvm_guest_exit(); |
546 | trace_kvm_exit(*vcpu_pc(vcpu)); | 536 | trace_kvm_exit(*vcpu_pc(vcpu)); |
547 | /* | 537 | /* |
diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c index 7928dbdf2102..f3d88dc388bc 100644 --- a/arch/arm/kvm/coproc.c +++ b/arch/arm/kvm/coproc.c | |||
@@ -189,82 +189,40 @@ static bool access_l2ectlr(struct kvm_vcpu *vcpu, | |||
189 | return true; | 189 | return true; |
190 | } | 190 | } |
191 | 191 | ||
192 | /* See note at ARM ARM B1.14.4 */ | 192 | /* |
193 | * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). | ||
194 | */ | ||
193 | static bool access_dcsw(struct kvm_vcpu *vcpu, | 195 | static bool access_dcsw(struct kvm_vcpu *vcpu, |
194 | const struct coproc_params *p, | 196 | const struct coproc_params *p, |
195 | const struct coproc_reg *r) | 197 | const struct coproc_reg *r) |
196 | { | 198 | { |
197 | unsigned long val; | ||
198 | int cpu; | ||
199 | |||
200 | if (!p->is_write) | 199 | if (!p->is_write) |
201 | return read_from_write_only(vcpu, p); | 200 | return read_from_write_only(vcpu, p); |
202 | 201 | ||
203 | cpu = get_cpu(); | 202 | kvm_set_way_flush(vcpu); |
204 | |||
205 | cpumask_setall(&vcpu->arch.require_dcache_flush); | ||
206 | cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush); | ||
207 | |||
208 | /* If we were already preempted, take the long way around */ | ||
209 | if (cpu != vcpu->arch.last_pcpu) { | ||
210 | flush_cache_all(); | ||
211 | goto done; | ||
212 | } | ||
213 | |||
214 | val = *vcpu_reg(vcpu, p->Rt1); | ||
215 | |||
216 | switch (p->CRm) { | ||
217 | case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */ | ||
218 | case 14: /* DCCISW */ | ||
219 | asm volatile("mcr p15, 0, %0, c7, c14, 2" : : "r" (val)); | ||
220 | break; | ||
221 | |||
222 | case 10: /* DCCSW */ | ||
223 | asm volatile("mcr p15, 0, %0, c7, c10, 2" : : "r" (val)); | ||
224 | break; | ||
225 | } | ||
226 | |||
227 | done: | ||
228 | put_cpu(); | ||
229 | |||
230 | return true; | 203 | return true; |
231 | } | 204 | } |
232 | 205 | ||
233 | /* | 206 | /* |
234 | * Generic accessor for VM registers. Only called as long as HCR_TVM | 207 | * Generic accessor for VM registers. Only called as long as HCR_TVM |
235 | * is set. | 208 | * is set. If the guest enables the MMU, we stop trapping the VM |
209 | * sys_regs and leave it in complete control of the caches. | ||
210 | * | ||
211 | * Used by the cpu-specific code. | ||
236 | */ | 212 | */ |
237 | static bool access_vm_reg(struct kvm_vcpu *vcpu, | 213 | bool access_vm_reg(struct kvm_vcpu *vcpu, |
238 | const struct coproc_params *p, | 214 | const struct coproc_params *p, |
239 | const struct coproc_reg *r) | 215 | const struct coproc_reg *r) |
240 | { | 216 | { |
217 | bool was_enabled = vcpu_has_cache_enabled(vcpu); | ||
218 | |||
241 | BUG_ON(!p->is_write); | 219 | BUG_ON(!p->is_write); |
242 | 220 | ||
243 | vcpu->arch.cp15[r->reg] = *vcpu_reg(vcpu, p->Rt1); | 221 | vcpu->arch.cp15[r->reg] = *vcpu_reg(vcpu, p->Rt1); |
244 | if (p->is_64bit) | 222 | if (p->is_64bit) |
245 | vcpu->arch.cp15[r->reg + 1] = *vcpu_reg(vcpu, p->Rt2); | 223 | vcpu->arch.cp15[r->reg + 1] = *vcpu_reg(vcpu, p->Rt2); |
246 | 224 | ||
247 | return true; | 225 | kvm_toggle_cache(vcpu, was_enabled); |
248 | } | ||
249 | |||
250 | /* | ||
251 | * SCTLR accessor. Only called as long as HCR_TVM is set. If the | ||
252 | * guest enables the MMU, we stop trapping the VM sys_regs and leave | ||
253 | * it in complete control of the caches. | ||
254 | * | ||
255 | * Used by the cpu-specific code. | ||
256 | */ | ||
257 | bool access_sctlr(struct kvm_vcpu *vcpu, | ||
258 | const struct coproc_params *p, | ||
259 | const struct coproc_reg *r) | ||
260 | { | ||
261 | access_vm_reg(vcpu, p, r); | ||
262 | |||
263 | if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */ | ||
264 | vcpu->arch.hcr &= ~HCR_TVM; | ||
265 | stage2_flush_vm(vcpu->kvm); | ||
266 | } | ||
267 | |||
268 | return true; | 226 | return true; |
269 | } | 227 | } |
270 | 228 | ||
diff --git a/arch/arm/kvm/coproc.h b/arch/arm/kvm/coproc.h index 1a44bbe39643..88d24a3a9778 100644 --- a/arch/arm/kvm/coproc.h +++ b/arch/arm/kvm/coproc.h | |||
@@ -153,8 +153,8 @@ static inline int cmp_reg(const struct coproc_reg *i1, | |||
153 | #define is64 .is_64 = true | 153 | #define is64 .is_64 = true |
154 | #define is32 .is_64 = false | 154 | #define is32 .is_64 = false |
155 | 155 | ||
156 | bool access_sctlr(struct kvm_vcpu *vcpu, | 156 | bool access_vm_reg(struct kvm_vcpu *vcpu, |
157 | const struct coproc_params *p, | 157 | const struct coproc_params *p, |
158 | const struct coproc_reg *r); | 158 | const struct coproc_reg *r); |
159 | 159 | ||
160 | #endif /* __ARM_KVM_COPROC_LOCAL_H__ */ | 160 | #endif /* __ARM_KVM_COPROC_LOCAL_H__ */ |
diff --git a/arch/arm/kvm/coproc_a15.c b/arch/arm/kvm/coproc_a15.c index e6f4ae48bda9..a7136757d373 100644 --- a/arch/arm/kvm/coproc_a15.c +++ b/arch/arm/kvm/coproc_a15.c | |||
@@ -34,7 +34,7 @@ | |||
34 | static const struct coproc_reg a15_regs[] = { | 34 | static const struct coproc_reg a15_regs[] = { |
35 | /* SCTLR: swapped by interrupt.S. */ | 35 | /* SCTLR: swapped by interrupt.S. */ |
36 | { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, | 36 | { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, |
37 | access_sctlr, reset_val, c1_SCTLR, 0x00C50078 }, | 37 | access_vm_reg, reset_val, c1_SCTLR, 0x00C50078 }, |
38 | }; | 38 | }; |
39 | 39 | ||
40 | static struct kvm_coproc_target_table a15_target_table = { | 40 | static struct kvm_coproc_target_table a15_target_table = { |
diff --git a/arch/arm/kvm/coproc_a7.c b/arch/arm/kvm/coproc_a7.c index 17fc7cd479d3..b19e46d1b2c0 100644 --- a/arch/arm/kvm/coproc_a7.c +++ b/arch/arm/kvm/coproc_a7.c | |||
@@ -37,7 +37,7 @@ | |||
37 | static const struct coproc_reg a7_regs[] = { | 37 | static const struct coproc_reg a7_regs[] = { |
38 | /* SCTLR: swapped by interrupt.S. */ | 38 | /* SCTLR: swapped by interrupt.S. */ |
39 | { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, | 39 | { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, |
40 | access_sctlr, reset_val, c1_SCTLR, 0x00C50878 }, | 40 | access_vm_reg, reset_val, c1_SCTLR, 0x00C50878 }, |
41 | }; | 41 | }; |
42 | 42 | ||
43 | static struct kvm_coproc_target_table a7_target_table = { | 43 | static struct kvm_coproc_target_table a7_target_table = { |
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 1dc9778a00af..106737e309b1 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c | |||
@@ -278,7 +278,7 @@ static void stage2_flush_memslot(struct kvm *kvm, | |||
278 | * Go through the stage 2 page tables and invalidate any cache lines | 278 | * Go through the stage 2 page tables and invalidate any cache lines |
279 | * backing memory already mapped to the VM. | 279 | * backing memory already mapped to the VM. |
280 | */ | 280 | */ |
281 | void stage2_flush_vm(struct kvm *kvm) | 281 | static void stage2_flush_vm(struct kvm *kvm) |
282 | { | 282 | { |
283 | struct kvm_memslots *slots; | 283 | struct kvm_memslots *slots; |
284 | struct kvm_memory_slot *memslot; | 284 | struct kvm_memory_slot *memslot; |
@@ -1411,3 +1411,71 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, | |||
1411 | unmap_stage2_range(kvm, gpa, size); | 1411 | unmap_stage2_range(kvm, gpa, size); |
1412 | spin_unlock(&kvm->mmu_lock); | 1412 | spin_unlock(&kvm->mmu_lock); |
1413 | } | 1413 | } |
1414 | |||
1415 | /* | ||
1416 | * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). | ||
1417 | * | ||
1418 | * Main problems: | ||
1419 | * - S/W ops are local to a CPU (not broadcast) | ||
1420 | * - We have line migration behind our back (speculation) | ||
1421 | * - System caches don't support S/W at all (damn!) | ||
1422 | * | ||
1423 | * In the face of the above, the best we can do is to try and convert | ||
1424 | * S/W ops to VA ops. Because the guest is not allowed to infer the | ||
1425 | * S/W to PA mapping, it can only use S/W to nuke the whole cache, | ||
1426 | * which is a rather good thing for us. | ||
1427 | * | ||
1428 | * Also, it is only used when turning caches on/off ("The expected | ||
1429 | * usage of the cache maintenance instructions that operate by set/way | ||
1430 | * is associated with the cache maintenance instructions associated | ||
1431 | * with the powerdown and powerup of caches, if this is required by | ||
1432 | * the implementation."). | ||
1433 | * | ||
1434 | * We use the following policy: | ||
1435 | * | ||
1436 | * - If we trap a S/W operation, we enable VM trapping to detect | ||
1437 | * caches being turned on/off, and do a full clean. | ||
1438 | * | ||
1439 | * - We flush the caches on both caches being turned on and off. | ||
1440 | * | ||
1441 | * - Once the caches are enabled, we stop trapping VM ops. | ||
1442 | */ | ||
1443 | void kvm_set_way_flush(struct kvm_vcpu *vcpu) | ||
1444 | { | ||
1445 | unsigned long hcr = vcpu_get_hcr(vcpu); | ||
1446 | |||
1447 | /* | ||
1448 | * If this is the first time we do a S/W operation | ||
1449 | * (i.e. HCR_TVM not set) flush the whole memory, and set the | ||
1450 | * VM trapping. | ||
1451 | * | ||
1452 | * Otherwise, rely on the VM trapping to wait for the MMU + | ||
1453 | * Caches to be turned off. At that point, we'll be able to | ||
1454 | * clean the caches again. | ||
1455 | */ | ||
1456 | if (!(hcr & HCR_TVM)) { | ||
1457 | trace_kvm_set_way_flush(*vcpu_pc(vcpu), | ||
1458 | vcpu_has_cache_enabled(vcpu)); | ||
1459 | stage2_flush_vm(vcpu->kvm); | ||
1460 | vcpu_set_hcr(vcpu, hcr | HCR_TVM); | ||
1461 | } | ||
1462 | } | ||
1463 | |||
1464 | void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) | ||
1465 | { | ||
1466 | bool now_enabled = vcpu_has_cache_enabled(vcpu); | ||
1467 | |||
1468 | /* | ||
1469 | * If switching the MMU+caches on, need to invalidate the caches. | ||
1470 | * If switching it off, need to clean the caches. | ||
1471 | * Clean + invalidate does the trick always. | ||
1472 | */ | ||
1473 | if (now_enabled != was_enabled) | ||
1474 | stage2_flush_vm(vcpu->kvm); | ||
1475 | |||
1476 | /* Caches are now on, stop trapping VM ops (until a S/W op) */ | ||
1477 | if (now_enabled) | ||
1478 | vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) & ~HCR_TVM); | ||
1479 | |||
1480 | trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); | ||
1481 | } | ||
diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h index b1d640f78623..b6a6e7102201 100644 --- a/arch/arm/kvm/trace.h +++ b/arch/arm/kvm/trace.h | |||
@@ -223,6 +223,45 @@ TRACE_EVENT(kvm_hvc, | |||
223 | __entry->vcpu_pc, __entry->r0, __entry->imm) | 223 | __entry->vcpu_pc, __entry->r0, __entry->imm) |
224 | ); | 224 | ); |
225 | 225 | ||
226 | TRACE_EVENT(kvm_set_way_flush, | ||
227 | TP_PROTO(unsigned long vcpu_pc, bool cache), | ||
228 | TP_ARGS(vcpu_pc, cache), | ||
229 | |||
230 | TP_STRUCT__entry( | ||
231 | __field( unsigned long, vcpu_pc ) | ||
232 | __field( bool, cache ) | ||
233 | ), | ||
234 | |||
235 | TP_fast_assign( | ||
236 | __entry->vcpu_pc = vcpu_pc; | ||
237 | __entry->cache = cache; | ||
238 | ), | ||
239 | |||
240 | TP_printk("S/W flush at 0x%016lx (cache %s)", | ||
241 | __entry->vcpu_pc, __entry->cache ? "on" : "off") | ||
242 | ); | ||
243 | |||
244 | TRACE_EVENT(kvm_toggle_cache, | ||
245 | TP_PROTO(unsigned long vcpu_pc, bool was, bool now), | ||
246 | TP_ARGS(vcpu_pc, was, now), | ||
247 | |||
248 | TP_STRUCT__entry( | ||
249 | __field( unsigned long, vcpu_pc ) | ||
250 | __field( bool, was ) | ||
251 | __field( bool, now ) | ||
252 | ), | ||
253 | |||
254 | TP_fast_assign( | ||
255 | __entry->vcpu_pc = vcpu_pc; | ||
256 | __entry->was = was; | ||
257 | __entry->now = now; | ||
258 | ), | ||
259 | |||
260 | TP_printk("VM op at 0x%016lx (cache was %s, now %s)", | ||
261 | __entry->vcpu_pc, __entry->was ? "on" : "off", | ||
262 | __entry->now ? "on" : "off") | ||
263 | ); | ||
264 | |||
226 | #endif /* _TRACE_KVM_H */ | 265 | #endif /* _TRACE_KVM_H */ |
227 | 266 | ||
228 | #undef TRACE_INCLUDE_PATH | 267 | #undef TRACE_INCLUDE_PATH |
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 865a7e28ea2d..3cb4c856b10d 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h | |||
@@ -45,6 +45,16 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) | |||
45 | vcpu->arch.hcr_el2 &= ~HCR_RW; | 45 | vcpu->arch.hcr_el2 &= ~HCR_RW; |
46 | } | 46 | } |
47 | 47 | ||
48 | static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu) | ||
49 | { | ||
50 | return vcpu->arch.hcr_el2; | ||
51 | } | ||
52 | |||
53 | static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr) | ||
54 | { | ||
55 | vcpu->arch.hcr_el2 = hcr; | ||
56 | } | ||
57 | |||
48 | static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) | 58 | static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) |
49 | { | 59 | { |
50 | return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc; | 60 | return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc; |
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 0b7dfdb931df..acd101a9014d 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h | |||
@@ -116,9 +116,6 @@ struct kvm_vcpu_arch { | |||
116 | * Anything that is not used directly from assembly code goes | 116 | * Anything that is not used directly from assembly code goes |
117 | * here. | 117 | * here. |
118 | */ | 118 | */ |
119 | /* dcache set/way operation pending */ | ||
120 | int last_pcpu; | ||
121 | cpumask_t require_dcache_flush; | ||
122 | 119 | ||
123 | /* Don't run the guest */ | 120 | /* Don't run the guest */ |
124 | bool pause; | 121 | bool pause; |
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 14a74f136272..92d22e94a79b 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h | |||
@@ -260,7 +260,8 @@ static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, | |||
260 | 260 | ||
261 | #define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x)) | 261 | #define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x)) |
262 | 262 | ||
263 | void stage2_flush_vm(struct kvm *kvm); | 263 | void kvm_set_way_flush(struct kvm_vcpu *vcpu); |
264 | void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); | ||
264 | 265 | ||
265 | #endif /* __ASSEMBLY__ */ | 266 | #endif /* __ASSEMBLY__ */ |
266 | #endif /* __ARM64_KVM_MMU_H__ */ | 267 | #endif /* __ARM64_KVM_MMU_H__ */ |
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 3d7c2df89946..f31e8bb2bc5b 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c | |||
@@ -69,68 +69,31 @@ static u32 get_ccsidr(u32 csselr) | |||
69 | return ccsidr; | 69 | return ccsidr; |
70 | } | 70 | } |
71 | 71 | ||
72 | static void do_dc_cisw(u32 val) | 72 | /* |
73 | { | 73 | * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). |
74 | asm volatile("dc cisw, %x0" : : "r" (val)); | 74 | */ |
75 | dsb(ish); | ||
76 | } | ||
77 | |||
78 | static void do_dc_csw(u32 val) | ||
79 | { | ||
80 | asm volatile("dc csw, %x0" : : "r" (val)); | ||
81 | dsb(ish); | ||
82 | } | ||
83 | |||
84 | /* See note at ARM ARM B1.14.4 */ | ||
85 | static bool access_dcsw(struct kvm_vcpu *vcpu, | 75 | static bool access_dcsw(struct kvm_vcpu *vcpu, |
86 | const struct sys_reg_params *p, | 76 | const struct sys_reg_params *p, |
87 | const struct sys_reg_desc *r) | 77 | const struct sys_reg_desc *r) |
88 | { | 78 | { |
89 | unsigned long val; | ||
90 | int cpu; | ||
91 | |||
92 | if (!p->is_write) | 79 | if (!p->is_write) |
93 | return read_from_write_only(vcpu, p); | 80 | return read_from_write_only(vcpu, p); |
94 | 81 | ||
95 | cpu = get_cpu(); | 82 | kvm_set_way_flush(vcpu); |
96 | |||
97 | cpumask_setall(&vcpu->arch.require_dcache_flush); | ||
98 | cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush); | ||
99 | |||
100 | /* If we were already preempted, take the long way around */ | ||
101 | if (cpu != vcpu->arch.last_pcpu) { | ||
102 | flush_cache_all(); | ||
103 | goto done; | ||
104 | } | ||
105 | |||
106 | val = *vcpu_reg(vcpu, p->Rt); | ||
107 | |||
108 | switch (p->CRm) { | ||
109 | case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */ | ||
110 | case 14: /* DCCISW */ | ||
111 | do_dc_cisw(val); | ||
112 | break; | ||
113 | |||
114 | case 10: /* DCCSW */ | ||
115 | do_dc_csw(val); | ||
116 | break; | ||
117 | } | ||
118 | |||
119 | done: | ||
120 | put_cpu(); | ||
121 | |||
122 | return true; | 83 | return true; |
123 | } | 84 | } |
124 | 85 | ||
125 | /* | 86 | /* |
126 | * Generic accessor for VM registers. Only called as long as HCR_TVM | 87 | * Generic accessor for VM registers. Only called as long as HCR_TVM |
127 | * is set. | 88 | * is set. If the guest enables the MMU, we stop trapping the VM |
89 | * sys_regs and leave it in complete control of the caches. | ||
128 | */ | 90 | */ |
129 | static bool access_vm_reg(struct kvm_vcpu *vcpu, | 91 | static bool access_vm_reg(struct kvm_vcpu *vcpu, |
130 | const struct sys_reg_params *p, | 92 | const struct sys_reg_params *p, |
131 | const struct sys_reg_desc *r) | 93 | const struct sys_reg_desc *r) |
132 | { | 94 | { |
133 | unsigned long val; | 95 | unsigned long val; |
96 | bool was_enabled = vcpu_has_cache_enabled(vcpu); | ||
134 | 97 | ||
135 | BUG_ON(!p->is_write); | 98 | BUG_ON(!p->is_write); |
136 | 99 | ||
@@ -143,25 +106,7 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu, | |||
143 | vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL; | 106 | vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL; |
144 | } | 107 | } |
145 | 108 | ||
146 | return true; | 109 | kvm_toggle_cache(vcpu, was_enabled); |
147 | } | ||
148 | |||
149 | /* | ||
150 | * SCTLR_EL1 accessor. Only called as long as HCR_TVM is set. If the | ||
151 | * guest enables the MMU, we stop trapping the VM sys_regs and leave | ||
152 | * it in complete control of the caches. | ||
153 | */ | ||
154 | static bool access_sctlr(struct kvm_vcpu *vcpu, | ||
155 | const struct sys_reg_params *p, | ||
156 | const struct sys_reg_desc *r) | ||
157 | { | ||
158 | access_vm_reg(vcpu, p, r); | ||
159 | |||
160 | if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */ | ||
161 | vcpu->arch.hcr_el2 &= ~HCR_TVM; | ||
162 | stage2_flush_vm(vcpu->kvm); | ||
163 | } | ||
164 | |||
165 | return true; | 110 | return true; |
166 | } | 111 | } |
167 | 112 | ||
@@ -377,7 +322,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { | |||
377 | NULL, reset_mpidr, MPIDR_EL1 }, | 322 | NULL, reset_mpidr, MPIDR_EL1 }, |
378 | /* SCTLR_EL1 */ | 323 | /* SCTLR_EL1 */ |
379 | { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000), | 324 | { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000), |
380 | access_sctlr, reset_val, SCTLR_EL1, 0x00C50078 }, | 325 | access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 }, |
381 | /* CPACR_EL1 */ | 326 | /* CPACR_EL1 */ |
382 | { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010), | 327 | { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010), |
383 | NULL, reset_val, CPACR_EL1, 0 }, | 328 | NULL, reset_val, CPACR_EL1, 0 }, |
@@ -657,7 +602,7 @@ static const struct sys_reg_desc cp14_64_regs[] = { | |||
657 | * register). | 602 | * register). |
658 | */ | 603 | */ |
659 | static const struct sys_reg_desc cp15_regs[] = { | 604 | static const struct sys_reg_desc cp15_regs[] = { |
660 | { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR }, | 605 | { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR }, |
661 | { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, | 606 | { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, |
662 | { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, | 607 | { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, |
663 | { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR }, | 608 | { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR }, |