aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/virtual/kvm/api.txt17
-rw-r--r--arch/arm/include/uapi/asm/kvm.h8
-rw-r--r--arch/arm/kvm/arm.c3
-rw-r--r--arch/arm64/include/uapi/asm/kvm.h8
-rw-r--r--arch/powerpc/include/asm/archrandom.h11
-rw-r--r--arch/powerpc/include/asm/kvm_book3s.h3
-rw-r--r--arch/powerpc/include/asm/kvm_book3s_64.h18
-rw-r--r--arch/powerpc/include/asm/kvm_host.h47
-rw-r--r--arch/powerpc/include/asm/kvm_ppc.h2
-rw-r--r--arch/powerpc/include/asm/time.h3
-rw-r--r--arch/powerpc/kernel/asm-offsets.c20
-rw-r--r--arch/powerpc/kernel/time.c6
-rw-r--r--arch/powerpc/kvm/Kconfig14
-rw-r--r--arch/powerpc/kvm/book3s.c76
-rw-r--r--arch/powerpc/kvm/book3s_64_mmu_hv.c189
-rw-r--r--arch/powerpc/kvm/book3s_hv.c435
-rw-r--r--arch/powerpc/kvm/book3s_hv_builtin.c100
-rw-r--r--arch/powerpc/kvm/book3s_hv_rm_mmu.c25
-rw-r--r--arch/powerpc/kvm/book3s_hv_rm_xics.c238
-rw-r--r--arch/powerpc/kvm/book3s_hv_rmhandlers.S559
-rw-r--r--arch/powerpc/kvm/book3s_pr_papr.c28
-rw-r--r--arch/powerpc/kvm/book3s_xics.c105
-rw-r--r--arch/powerpc/kvm/book3s_xics.h13
-rw-r--r--arch/powerpc/kvm/powerpc.c3
-rw-r--r--arch/powerpc/platforms/powernv/rng.c29
-rw-r--r--arch/s390/kvm/kvm-s390.c2
-rw-r--r--arch/x86/kvm/lapic.c11
-rw-r--r--arch/x86/kvm/mmu.c20
-rw-r--r--arch/x86/kvm/vmx.c12
-rw-r--r--arch/x86/kvm/x86.c10
-rw-r--r--include/uapi/linux/kvm.h1
-rw-r--r--virt/kvm/arm/vgic.c5
-rw-r--r--virt/kvm/kvm_main.c1
33 files changed, 1630 insertions, 392 deletions
diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index bc9f6fe44e27..9fa2bf8c3f6f 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -3573,3 +3573,20 @@ struct {
3573@ar - access register number 3573@ar - access register number
3574 3574
3575KVM handlers should exit to userspace with rc = -EREMOTE. 3575KVM handlers should exit to userspace with rc = -EREMOTE.
3576
3577
35788. Other capabilities.
3579----------------------
3580
3581This section lists capabilities that give information about other
3582features of the KVM implementation.
3583
35848.1 KVM_CAP_PPC_HWRNG
3585
3586Architectures: ppc
3587
3588This capability, if KVM_CHECK_EXTENSION indicates that it is
3589available, means that that the kernel has an implementation of the
3590H_RANDOM hypercall backed by a hardware random-number generator.
3591If present, the kernel H_RANDOM handler can be enabled for guest use
3592with the KVM_CAP_PPC_ENABLE_HCALL capability.
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index 2499867dd0d8..df3f60cb1168 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -195,8 +195,14 @@ struct kvm_arch_memory_slot {
195#define KVM_ARM_IRQ_CPU_IRQ 0 195#define KVM_ARM_IRQ_CPU_IRQ 0
196#define KVM_ARM_IRQ_CPU_FIQ 1 196#define KVM_ARM_IRQ_CPU_FIQ 1
197 197
198/* Highest supported SPI, from VGIC_NR_IRQS */ 198/*
199 * This used to hold the highest supported SPI, but it is now obsolete
200 * and only here to provide source code level compatibility with older
201 * userland. The highest SPI number can be set via KVM_DEV_ARM_VGIC_GRP_NR_IRQS.
202 */
203#ifndef __KERNEL__
199#define KVM_ARM_IRQ_GIC_MAX 127 204#define KVM_ARM_IRQ_GIC_MAX 127
205#endif
200 206
201/* One single KVM irqchip, ie. the VGIC */ 207/* One single KVM irqchip, ie. the VGIC */
202#define KVM_NR_IRQCHIPS 1 208#define KVM_NR_IRQCHIPS 1
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 6f536451ab78..d9631ecddd56 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -671,8 +671,7 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
671 if (!irqchip_in_kernel(kvm)) 671 if (!irqchip_in_kernel(kvm))
672 return -ENXIO; 672 return -ENXIO;
673 673
674 if (irq_num < VGIC_NR_PRIVATE_IRQS || 674 if (irq_num < VGIC_NR_PRIVATE_IRQS)
675 irq_num > KVM_ARM_IRQ_GIC_MAX)
676 return -EINVAL; 675 return -EINVAL;
677 676
678 return kvm_vgic_inject_irq(kvm, 0, irq_num, level); 677 return kvm_vgic_inject_irq(kvm, 0, irq_num, level);
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index c154c0b7eb60..d26832022127 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -188,8 +188,14 @@ struct kvm_arch_memory_slot {
188#define KVM_ARM_IRQ_CPU_IRQ 0 188#define KVM_ARM_IRQ_CPU_IRQ 0
189#define KVM_ARM_IRQ_CPU_FIQ 1 189#define KVM_ARM_IRQ_CPU_FIQ 1
190 190
191/* Highest supported SPI, from VGIC_NR_IRQS */ 191/*
192 * This used to hold the highest supported SPI, but it is now obsolete
193 * and only here to provide source code level compatibility with older
194 * userland. The highest SPI number can be set via KVM_DEV_ARM_VGIC_GRP_NR_IRQS.
195 */
196#ifndef __KERNEL__
192#define KVM_ARM_IRQ_GIC_MAX 127 197#define KVM_ARM_IRQ_GIC_MAX 127
198#endif
193 199
194/* One single KVM irqchip, ie. the VGIC */ 200/* One single KVM irqchip, ie. the VGIC */
195#define KVM_NR_IRQCHIPS 1 201#define KVM_NR_IRQCHIPS 1
diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h
index bde531103638..0cc6eedc4780 100644
--- a/arch/powerpc/include/asm/archrandom.h
+++ b/arch/powerpc/include/asm/archrandom.h
@@ -30,8 +30,6 @@ static inline int arch_has_random(void)
30 return !!ppc_md.get_random_long; 30 return !!ppc_md.get_random_long;
31} 31}
32 32
33int powernv_get_random_long(unsigned long *v);
34
35static inline int arch_get_random_seed_long(unsigned long *v) 33static inline int arch_get_random_seed_long(unsigned long *v)
36{ 34{
37 return 0; 35 return 0;
@@ -47,4 +45,13 @@ static inline int arch_has_random_seed(void)
47 45
48#endif /* CONFIG_ARCH_RANDOM */ 46#endif /* CONFIG_ARCH_RANDOM */
49 47
48#ifdef CONFIG_PPC_POWERNV
49int powernv_hwrng_present(void);
50int powernv_get_random_long(unsigned long *v);
51int powernv_get_random_real_mode(unsigned long *v);
52#else
53static inline int powernv_hwrng_present(void) { return 0; }
54static inline int powernv_get_random_real_mode(unsigned long *v) { return 0; }
55#endif
56
50#endif /* _ASM_POWERPC_ARCHRANDOM_H */ 57#endif /* _ASM_POWERPC_ARCHRANDOM_H */
diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index 993090422690..b91e74a817d8 100644
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -288,6 +288,9 @@ static inline bool kvmppc_supports_magic_page(struct kvm_vcpu *vcpu)
288 return !is_kvmppc_hv_enabled(vcpu->kvm); 288 return !is_kvmppc_hv_enabled(vcpu->kvm);
289} 289}
290 290
291extern int kvmppc_h_logical_ci_load(struct kvm_vcpu *vcpu);
292extern int kvmppc_h_logical_ci_store(struct kvm_vcpu *vcpu);
293
291/* Magic register values loaded into r3 and r4 before the 'sc' assembly 294/* Magic register values loaded into r3 and r4 before the 'sc' assembly
292 * instruction for the OSI hypercalls */ 295 * instruction for the OSI hypercalls */
293#define OSI_SC_MAGIC_R3 0x113724FA 296#define OSI_SC_MAGIC_R3 0x113724FA
diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h
index 14619a59ec09..7ae407941be2 100644
--- a/arch/powerpc/include/asm/kvm_book3s_64.h
+++ b/arch/powerpc/include/asm/kvm_book3s_64.h
@@ -85,6 +85,20 @@ static inline long try_lock_hpte(__be64 *hpte, unsigned long bits)
85 return old == 0; 85 return old == 0;
86} 86}
87 87
88static inline void unlock_hpte(__be64 *hpte, unsigned long hpte_v)
89{
90 hpte_v &= ~HPTE_V_HVLOCK;
91 asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
92 hpte[0] = cpu_to_be64(hpte_v);
93}
94
95/* Without barrier */
96static inline void __unlock_hpte(__be64 *hpte, unsigned long hpte_v)
97{
98 hpte_v &= ~HPTE_V_HVLOCK;
99 hpte[0] = cpu_to_be64(hpte_v);
100}
101
88static inline int __hpte_actual_psize(unsigned int lp, int psize) 102static inline int __hpte_actual_psize(unsigned int lp, int psize)
89{ 103{
90 int i, shift; 104 int i, shift;
@@ -424,6 +438,10 @@ static inline struct kvm_memslots *kvm_memslots_raw(struct kvm *kvm)
424 return rcu_dereference_raw_notrace(kvm->memslots); 438 return rcu_dereference_raw_notrace(kvm->memslots);
425} 439}
426 440
441extern void kvmppc_mmu_debugfs_init(struct kvm *kvm);
442
443extern void kvmhv_rm_send_ipi(int cpu);
444
427#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 445#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
428 446
429#endif /* __ASM_KVM_BOOK3S_64_H__ */ 447#endif /* __ASM_KVM_BOOK3S_64_H__ */
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index c610961720c7..a193a13cf08b 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -227,10 +227,8 @@ struct kvm_arch {
227 unsigned long host_sdr1; 227 unsigned long host_sdr1;
228 int tlbie_lock; 228 int tlbie_lock;
229 unsigned long lpcr; 229 unsigned long lpcr;
230 unsigned long rmor;
231 struct kvm_rma_info *rma;
232 unsigned long vrma_slb_v; 230 unsigned long vrma_slb_v;
233 int rma_setup_done; 231 int hpte_setup_done;
234 u32 hpt_order; 232 u32 hpt_order;
235 atomic_t vcpus_running; 233 atomic_t vcpus_running;
236 u32 online_vcores; 234 u32 online_vcores;
@@ -239,6 +237,8 @@ struct kvm_arch {
239 atomic_t hpte_mod_interest; 237 atomic_t hpte_mod_interest;
240 cpumask_t need_tlb_flush; 238 cpumask_t need_tlb_flush;
241 int hpt_cma_alloc; 239 int hpt_cma_alloc;
240 struct dentry *debugfs_dir;
241 struct dentry *htab_dentry;
242#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 242#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
243#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 243#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
244 struct mutex hpt_mutex; 244 struct mutex hpt_mutex;
@@ -263,18 +263,15 @@ struct kvm_arch {
263 263
264/* 264/*
265 * Struct for a virtual core. 265 * Struct for a virtual core.
266 * Note: entry_exit_count combines an entry count in the bottom 8 bits 266 * Note: entry_exit_map combines a bitmap of threads that have entered
267 * and an exit count in the next 8 bits. This is so that we can 267 * in the bottom 8 bits and a bitmap of threads that have exited in the
268 * atomically increment the entry count iff the exit count is 0 268 * next 8 bits. This is so that we can atomically set the entry bit
269 * without taking the lock. 269 * iff the exit map is 0 without taking a lock.
270 */ 270 */
271struct kvmppc_vcore { 271struct kvmppc_vcore {
272 int n_runnable; 272 int n_runnable;
273 int n_busy;
274 int num_threads; 273 int num_threads;
275 int entry_exit_count; 274 int entry_exit_map;
276 int n_woken;
277 int nap_count;
278 int napping_threads; 275 int napping_threads;
279 int first_vcpuid; 276 int first_vcpuid;
280 u16 pcpu; 277 u16 pcpu;
@@ -299,13 +296,14 @@ struct kvmppc_vcore {
299 ulong conferring_threads; 296 ulong conferring_threads;
300}; 297};
301 298
302#define VCORE_ENTRY_COUNT(vc) ((vc)->entry_exit_count & 0xff) 299#define VCORE_ENTRY_MAP(vc) ((vc)->entry_exit_map & 0xff)
303#define VCORE_EXIT_COUNT(vc) ((vc)->entry_exit_count >> 8) 300#define VCORE_EXIT_MAP(vc) ((vc)->entry_exit_map >> 8)
301#define VCORE_IS_EXITING(vc) (VCORE_EXIT_MAP(vc) != 0)
304 302
305/* Values for vcore_state */ 303/* Values for vcore_state */
306#define VCORE_INACTIVE 0 304#define VCORE_INACTIVE 0
307#define VCORE_SLEEPING 1 305#define VCORE_SLEEPING 1
308#define VCORE_STARTING 2 306#define VCORE_PREEMPT 2
309#define VCORE_RUNNING 3 307#define VCORE_RUNNING 3
310#define VCORE_EXITING 4 308#define VCORE_EXITING 4
311 309
@@ -368,6 +366,14 @@ struct kvmppc_slb {
368 u8 base_page_size; /* MMU_PAGE_xxx */ 366 u8 base_page_size; /* MMU_PAGE_xxx */
369}; 367};
370 368
369/* Struct used to accumulate timing information in HV real mode code */
370struct kvmhv_tb_accumulator {
371 u64 seqcount; /* used to synchronize access, also count * 2 */
372 u64 tb_total; /* total time in timebase ticks */
373 u64 tb_min; /* min time */
374 u64 tb_max; /* max time */
375};
376
371# ifdef CONFIG_PPC_FSL_BOOK3E 377# ifdef CONFIG_PPC_FSL_BOOK3E
372#define KVMPPC_BOOKE_IAC_NUM 2 378#define KVMPPC_BOOKE_IAC_NUM 2
373#define KVMPPC_BOOKE_DAC_NUM 2 379#define KVMPPC_BOOKE_DAC_NUM 2
@@ -656,6 +662,19 @@ struct kvm_vcpu_arch {
656 662
657 u32 emul_inst; 663 u32 emul_inst;
658#endif 664#endif
665
666#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
667 struct kvmhv_tb_accumulator *cur_activity; /* What we're timing */
668 u64 cur_tb_start; /* when it started */
669 struct kvmhv_tb_accumulator rm_entry; /* real-mode entry code */
670 struct kvmhv_tb_accumulator rm_intr; /* real-mode intr handling */
671 struct kvmhv_tb_accumulator rm_exit; /* real-mode exit code */
672 struct kvmhv_tb_accumulator guest_time; /* guest execution */
673 struct kvmhv_tb_accumulator cede_time; /* time napping inside guest */
674
675 struct dentry *debugfs_dir;
676 struct dentry *debugfs_timings;
677#endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
659}; 678};
660 679
661#define VCPU_FPR(vcpu, i) (vcpu)->arch.fp.fpr[i][TS_FPROFFSET] 680#define VCPU_FPR(vcpu, i) (vcpu)->arch.fp.fpr[i][TS_FPROFFSET]
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index 46bf652c9169..b8475daad884 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -302,6 +302,8 @@ static inline bool is_kvmppc_hv_enabled(struct kvm *kvm)
302 return kvm->arch.kvm_ops == kvmppc_hv_ops; 302 return kvm->arch.kvm_ops == kvmppc_hv_ops;
303} 303}
304 304
305extern int kvmppc_hwrng_present(void);
306
305/* 307/*
306 * Cuts out inst bits with ordering according to spec. 308 * Cuts out inst bits with ordering according to spec.
307 * That means the leftmost bit is zero. All given bits are included. 309 * That means the leftmost bit is zero. All given bits are included.
diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h
index 03cbada59d3a..10fc784a2ad4 100644
--- a/arch/powerpc/include/asm/time.h
+++ b/arch/powerpc/include/asm/time.h
@@ -211,5 +211,8 @@ extern void secondary_cpu_time_init(void);
211 211
212DECLARE_PER_CPU(u64, decrementers_next_tb); 212DECLARE_PER_CPU(u64, decrementers_next_tb);
213 213
214/* Convert timebase ticks to nanoseconds */
215unsigned long long tb_to_ns(unsigned long long tb_ticks);
216
214#endif /* __KERNEL__ */ 217#endif /* __KERNEL__ */
215#endif /* __POWERPC_TIME_H */ 218#endif /* __POWERPC_TIME_H */
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 4717859fdd04..0034b6b3556a 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -37,6 +37,7 @@
37#include <asm/thread_info.h> 37#include <asm/thread_info.h>
38#include <asm/rtas.h> 38#include <asm/rtas.h>
39#include <asm/vdso_datapage.h> 39#include <asm/vdso_datapage.h>
40#include <asm/dbell.h>
40#ifdef CONFIG_PPC64 41#ifdef CONFIG_PPC64
41#include <asm/paca.h> 42#include <asm/paca.h>
42#include <asm/lppaca.h> 43#include <asm/lppaca.h>
@@ -459,6 +460,19 @@ int main(void)
459 DEFINE(VCPU_SPRG2, offsetof(struct kvm_vcpu, arch.shregs.sprg2)); 460 DEFINE(VCPU_SPRG2, offsetof(struct kvm_vcpu, arch.shregs.sprg2));
460 DEFINE(VCPU_SPRG3, offsetof(struct kvm_vcpu, arch.shregs.sprg3)); 461 DEFINE(VCPU_SPRG3, offsetof(struct kvm_vcpu, arch.shregs.sprg3));
461#endif 462#endif
463#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
464 DEFINE(VCPU_TB_RMENTRY, offsetof(struct kvm_vcpu, arch.rm_entry));
465 DEFINE(VCPU_TB_RMINTR, offsetof(struct kvm_vcpu, arch.rm_intr));
466 DEFINE(VCPU_TB_RMEXIT, offsetof(struct kvm_vcpu, arch.rm_exit));
467 DEFINE(VCPU_TB_GUEST, offsetof(struct kvm_vcpu, arch.guest_time));
468 DEFINE(VCPU_TB_CEDE, offsetof(struct kvm_vcpu, arch.cede_time));
469 DEFINE(VCPU_CUR_ACTIVITY, offsetof(struct kvm_vcpu, arch.cur_activity));
470 DEFINE(VCPU_ACTIVITY_START, offsetof(struct kvm_vcpu, arch.cur_tb_start));
471 DEFINE(TAS_SEQCOUNT, offsetof(struct kvmhv_tb_accumulator, seqcount));
472 DEFINE(TAS_TOTAL, offsetof(struct kvmhv_tb_accumulator, tb_total));
473 DEFINE(TAS_MIN, offsetof(struct kvmhv_tb_accumulator, tb_min));
474 DEFINE(TAS_MAX, offsetof(struct kvmhv_tb_accumulator, tb_max));
475#endif
462 DEFINE(VCPU_SHARED_SPRG3, offsetof(struct kvm_vcpu_arch_shared, sprg3)); 476 DEFINE(VCPU_SHARED_SPRG3, offsetof(struct kvm_vcpu_arch_shared, sprg3));
463 DEFINE(VCPU_SHARED_SPRG4, offsetof(struct kvm_vcpu_arch_shared, sprg4)); 477 DEFINE(VCPU_SHARED_SPRG4, offsetof(struct kvm_vcpu_arch_shared, sprg4));
464 DEFINE(VCPU_SHARED_SPRG5, offsetof(struct kvm_vcpu_arch_shared, sprg5)); 478 DEFINE(VCPU_SHARED_SPRG5, offsetof(struct kvm_vcpu_arch_shared, sprg5));
@@ -492,7 +506,6 @@ int main(void)
492 DEFINE(KVM_NEED_FLUSH, offsetof(struct kvm, arch.need_tlb_flush.bits)); 506 DEFINE(KVM_NEED_FLUSH, offsetof(struct kvm, arch.need_tlb_flush.bits));
493 DEFINE(KVM_ENABLED_HCALLS, offsetof(struct kvm, arch.enabled_hcalls)); 507 DEFINE(KVM_ENABLED_HCALLS, offsetof(struct kvm, arch.enabled_hcalls));
494 DEFINE(KVM_LPCR, offsetof(struct kvm, arch.lpcr)); 508 DEFINE(KVM_LPCR, offsetof(struct kvm, arch.lpcr));
495 DEFINE(KVM_RMOR, offsetof(struct kvm, arch.rmor));
496 DEFINE(KVM_VRMA_SLB_V, offsetof(struct kvm, arch.vrma_slb_v)); 509 DEFINE(KVM_VRMA_SLB_V, offsetof(struct kvm, arch.vrma_slb_v));
497 DEFINE(VCPU_DSISR, offsetof(struct kvm_vcpu, arch.shregs.dsisr)); 510 DEFINE(VCPU_DSISR, offsetof(struct kvm_vcpu, arch.shregs.dsisr));
498 DEFINE(VCPU_DAR, offsetof(struct kvm_vcpu, arch.shregs.dar)); 511 DEFINE(VCPU_DAR, offsetof(struct kvm_vcpu, arch.shregs.dar));
@@ -550,8 +563,7 @@ int main(void)
550 DEFINE(VCPU_ACOP, offsetof(struct kvm_vcpu, arch.acop)); 563 DEFINE(VCPU_ACOP, offsetof(struct kvm_vcpu, arch.acop));
551 DEFINE(VCPU_WORT, offsetof(struct kvm_vcpu, arch.wort)); 564 DEFINE(VCPU_WORT, offsetof(struct kvm_vcpu, arch.wort));
552 DEFINE(VCPU_SHADOW_SRR1, offsetof(struct kvm_vcpu, arch.shadow_srr1)); 565 DEFINE(VCPU_SHADOW_SRR1, offsetof(struct kvm_vcpu, arch.shadow_srr1));
553 DEFINE(VCORE_ENTRY_EXIT, offsetof(struct kvmppc_vcore, entry_exit_count)); 566 DEFINE(VCORE_ENTRY_EXIT, offsetof(struct kvmppc_vcore, entry_exit_map));
554 DEFINE(VCORE_NAP_COUNT, offsetof(struct kvmppc_vcore, nap_count));
555 DEFINE(VCORE_IN_GUEST, offsetof(struct kvmppc_vcore, in_guest)); 567 DEFINE(VCORE_IN_GUEST, offsetof(struct kvmppc_vcore, in_guest));
556 DEFINE(VCORE_NAPPING_THREADS, offsetof(struct kvmppc_vcore, napping_threads)); 568 DEFINE(VCORE_NAPPING_THREADS, offsetof(struct kvmppc_vcore, napping_threads));
557 DEFINE(VCORE_KVM, offsetof(struct kvmppc_vcore, kvm)); 569 DEFINE(VCORE_KVM, offsetof(struct kvmppc_vcore, kvm));
@@ -748,5 +760,7 @@ int main(void)
748 offsetof(struct paca_struct, subcore_sibling_mask)); 760 offsetof(struct paca_struct, subcore_sibling_mask));
749#endif 761#endif
750 762
763 DEFINE(PPC_DBELL_SERVER, PPC_DBELL_SERVER);
764
751 return 0; 765 return 0;
752} 766}
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 2d7b33fab953..56f44848b044 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -608,6 +608,12 @@ void arch_suspend_enable_irqs(void)
608} 608}
609#endif 609#endif
610 610
611unsigned long long tb_to_ns(unsigned long long ticks)
612{
613 return mulhdu(ticks, tb_to_ns_scale) << tb_to_ns_shift;
614}
615EXPORT_SYMBOL_GPL(tb_to_ns);
616
611/* 617/*
612 * Scheduler clock - returns current time in nanosec units. 618 * Scheduler clock - returns current time in nanosec units.
613 * 619 *
diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig
index 11850f310fb4..2963e4dd0b80 100644
--- a/arch/powerpc/kvm/Kconfig
+++ b/arch/powerpc/kvm/Kconfig
@@ -110,6 +110,20 @@ config KVM_BOOK3S_64_PR
110 processor, including emulating 32-bit processors on a 64-bit 110 processor, including emulating 32-bit processors on a 64-bit
111 host. 111 host.
112 112
113config KVM_BOOK3S_HV_EXIT_TIMING
114 bool "Detailed timing for hypervisor real-mode code"
115 depends on KVM_BOOK3S_HV_POSSIBLE && DEBUG_FS
116 ---help---
117 Calculate time taken for each vcpu in the real-mode guest entry,
118 exit, and interrupt handling code, plus time spent in the guest
119 and in nap mode due to idle (cede) while other threads are still
120 in the guest. The total, minimum and maximum times in nanoseconds
121 together with the number of executions are reported in debugfs in
122 kvm/vm#/vcpu#/timings. The overhead is of the order of 30 - 40
123 ns per exit on POWER8.
124
125 If unsure, say N.
126
113config KVM_BOOKE_HV 127config KVM_BOOKE_HV
114 bool 128 bool
115 129
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index cfbcdc654201..453a8a47a467 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -821,6 +821,82 @@ void kvmppc_core_destroy_vm(struct kvm *kvm)
821#endif 821#endif
822} 822}
823 823
824int kvmppc_h_logical_ci_load(struct kvm_vcpu *vcpu)
825{
826 unsigned long size = kvmppc_get_gpr(vcpu, 4);
827 unsigned long addr = kvmppc_get_gpr(vcpu, 5);
828 u64 buf;
829 int ret;
830
831 if (!is_power_of_2(size) || (size > sizeof(buf)))
832 return H_TOO_HARD;
833
834 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, size, &buf);
835 if (ret != 0)
836 return H_TOO_HARD;
837
838 switch (size) {
839 case 1:
840 kvmppc_set_gpr(vcpu, 4, *(u8 *)&buf);
841 break;
842
843 case 2:
844 kvmppc_set_gpr(vcpu, 4, be16_to_cpu(*(__be16 *)&buf));
845 break;
846
847 case 4:
848 kvmppc_set_gpr(vcpu, 4, be32_to_cpu(*(__be32 *)&buf));
849 break;
850
851 case 8:
852 kvmppc_set_gpr(vcpu, 4, be64_to_cpu(*(__be64 *)&buf));
853 break;
854
855 default:
856 BUG();
857 }
858
859 return H_SUCCESS;
860}
861EXPORT_SYMBOL_GPL(kvmppc_h_logical_ci_load);
862
863int kvmppc_h_logical_ci_store(struct kvm_vcpu *vcpu)
864{
865 unsigned long size = kvmppc_get_gpr(vcpu, 4);
866 unsigned long addr = kvmppc_get_gpr(vcpu, 5);
867 unsigned long val = kvmppc_get_gpr(vcpu, 6);
868 u64 buf;
869 int ret;
870
871 switch (size) {
872 case 1:
873 *(u8 *)&buf = val;
874 break;
875
876 case 2:
877 *(__be16 *)&buf = cpu_to_be16(val);
878 break;
879
880 case 4:
881 *(__be32 *)&buf = cpu_to_be32(val);
882 break;
883
884 case 8:
885 *(__be64 *)&buf = cpu_to_be64(val);
886 break;
887
888 default:
889 return H_TOO_HARD;
890 }
891
892 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, size, &buf);
893 if (ret != 0)
894 return H_TOO_HARD;
895
896 return H_SUCCESS;
897}
898EXPORT_SYMBOL_GPL(kvmppc_h_logical_ci_store);
899
824int kvmppc_core_check_processor_compat(void) 900int kvmppc_core_check_processor_compat(void)
825{ 901{
826 /* 902 /*
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index 534acb3c6c3d..d6fe30835c58 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -27,6 +27,7 @@
27#include <linux/srcu.h> 27#include <linux/srcu.h>
28#include <linux/anon_inodes.h> 28#include <linux/anon_inodes.h>
29#include <linux/file.h> 29#include <linux/file.h>
30#include <linux/debugfs.h>
30 31
31#include <asm/tlbflush.h> 32#include <asm/tlbflush.h>
32#include <asm/kvm_ppc.h> 33#include <asm/kvm_ppc.h>
@@ -116,12 +117,12 @@ long kvmppc_alloc_reset_hpt(struct kvm *kvm, u32 *htab_orderp)
116 long order; 117 long order;
117 118
118 mutex_lock(&kvm->lock); 119 mutex_lock(&kvm->lock);
119 if (kvm->arch.rma_setup_done) { 120 if (kvm->arch.hpte_setup_done) {
120 kvm->arch.rma_setup_done = 0; 121 kvm->arch.hpte_setup_done = 0;
121 /* order rma_setup_done vs. vcpus_running */ 122 /* order hpte_setup_done vs. vcpus_running */
122 smp_mb(); 123 smp_mb();
123 if (atomic_read(&kvm->arch.vcpus_running)) { 124 if (atomic_read(&kvm->arch.vcpus_running)) {
124 kvm->arch.rma_setup_done = 1; 125 kvm->arch.hpte_setup_done = 1;
125 goto out; 126 goto out;
126 } 127 }
127 } 128 }
@@ -338,9 +339,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
338 v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK; 339 v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
339 gr = kvm->arch.revmap[index].guest_rpte; 340 gr = kvm->arch.revmap[index].guest_rpte;
340 341
341 /* Unlock the HPTE */ 342 unlock_hpte(hptep, v);
342 asm volatile("lwsync" : : : "memory");
343 hptep[0] = cpu_to_be64(v);
344 preempt_enable(); 343 preempt_enable();
345 344
346 gpte->eaddr = eaddr; 345 gpte->eaddr = eaddr;
@@ -469,8 +468,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
469 hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK; 468 hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
470 hpte[1] = be64_to_cpu(hptep[1]); 469 hpte[1] = be64_to_cpu(hptep[1]);
471 hpte[2] = r = rev->guest_rpte; 470 hpte[2] = r = rev->guest_rpte;
472 asm volatile("lwsync" : : : "memory"); 471 unlock_hpte(hptep, hpte[0]);
473 hptep[0] = cpu_to_be64(hpte[0]);
474 preempt_enable(); 472 preempt_enable();
475 473
476 if (hpte[0] != vcpu->arch.pgfault_hpte[0] || 474 if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
@@ -621,7 +619,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
621 619
622 hptep[1] = cpu_to_be64(r); 620 hptep[1] = cpu_to_be64(r);
623 eieio(); 621 eieio();
624 hptep[0] = cpu_to_be64(hpte[0]); 622 __unlock_hpte(hptep, hpte[0]);
625 asm volatile("ptesync" : : : "memory"); 623 asm volatile("ptesync" : : : "memory");
626 preempt_enable(); 624 preempt_enable();
627 if (page && hpte_is_writable(r)) 625 if (page && hpte_is_writable(r))
@@ -642,7 +640,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
642 return ret; 640 return ret;
643 641
644 out_unlock: 642 out_unlock:
645 hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK); 643 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
646 preempt_enable(); 644 preempt_enable();
647 goto out_put; 645 goto out_put;
648} 646}
@@ -771,7 +769,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
771 } 769 }
772 } 770 }
773 unlock_rmap(rmapp); 771 unlock_rmap(rmapp);
774 hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK); 772 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
775 } 773 }
776 return 0; 774 return 0;
777} 775}
@@ -857,7 +855,7 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
857 } 855 }
858 ret = 1; 856 ret = 1;
859 } 857 }
860 hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK); 858 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
861 } while ((i = j) != head); 859 } while ((i = j) != head);
862 860
863 unlock_rmap(rmapp); 861 unlock_rmap(rmapp);
@@ -974,8 +972,7 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
974 972
975 /* Now check and modify the HPTE */ 973 /* Now check and modify the HPTE */
976 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) { 974 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
977 /* unlock and continue */ 975 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
978 hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
979 continue; 976 continue;
980 } 977 }
981 978
@@ -996,9 +993,9 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
996 npages_dirty = n; 993 npages_dirty = n;
997 eieio(); 994 eieio();
998 } 995 }
999 v &= ~(HPTE_V_ABSENT | HPTE_V_HVLOCK); 996 v &= ~HPTE_V_ABSENT;
1000 v |= HPTE_V_VALID; 997 v |= HPTE_V_VALID;
1001 hptep[0] = cpu_to_be64(v); 998 __unlock_hpte(hptep, v);
1002 } while ((i = j) != head); 999 } while ((i = j) != head);
1003 1000
1004 unlock_rmap(rmapp); 1001 unlock_rmap(rmapp);
@@ -1218,8 +1215,7 @@ static long record_hpte(unsigned long flags, __be64 *hptp,
1218 r &= ~HPTE_GR_MODIFIED; 1215 r &= ~HPTE_GR_MODIFIED;
1219 revp->guest_rpte = r; 1216 revp->guest_rpte = r;
1220 } 1217 }
1221 asm volatile(PPC_RELEASE_BARRIER "" : : : "memory"); 1218 unlock_hpte(hptp, be64_to_cpu(hptp[0]));
1222 hptp[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
1223 preempt_enable(); 1219 preempt_enable();
1224 if (!(valid == want_valid && (first_pass || dirty))) 1220 if (!(valid == want_valid && (first_pass || dirty)))
1225 ok = 0; 1221 ok = 0;
@@ -1339,20 +1335,20 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1339 unsigned long tmp[2]; 1335 unsigned long tmp[2];
1340 ssize_t nb; 1336 ssize_t nb;
1341 long int err, ret; 1337 long int err, ret;
1342 int rma_setup; 1338 int hpte_setup;
1343 1339
1344 if (!access_ok(VERIFY_READ, buf, count)) 1340 if (!access_ok(VERIFY_READ, buf, count))
1345 return -EFAULT; 1341 return -EFAULT;
1346 1342
1347 /* lock out vcpus from running while we're doing this */ 1343 /* lock out vcpus from running while we're doing this */
1348 mutex_lock(&kvm->lock); 1344 mutex_lock(&kvm->lock);
1349 rma_setup = kvm->arch.rma_setup_done; 1345 hpte_setup = kvm->arch.hpte_setup_done;
1350 if (rma_setup) { 1346 if (hpte_setup) {
1351 kvm->arch.rma_setup_done = 0; /* temporarily */ 1347 kvm->arch.hpte_setup_done = 0; /* temporarily */
1352 /* order rma_setup_done vs. vcpus_running */ 1348 /* order hpte_setup_done vs. vcpus_running */
1353 smp_mb(); 1349 smp_mb();
1354 if (atomic_read(&kvm->arch.vcpus_running)) { 1350 if (atomic_read(&kvm->arch.vcpus_running)) {
1355 kvm->arch.rma_setup_done = 1; 1351 kvm->arch.hpte_setup_done = 1;
1356 mutex_unlock(&kvm->lock); 1352 mutex_unlock(&kvm->lock);
1357 return -EBUSY; 1353 return -EBUSY;
1358 } 1354 }
@@ -1405,7 +1401,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1405 "r=%lx\n", ret, i, v, r); 1401 "r=%lx\n", ret, i, v, r);
1406 goto out; 1402 goto out;
1407 } 1403 }
1408 if (!rma_setup && is_vrma_hpte(v)) { 1404 if (!hpte_setup && is_vrma_hpte(v)) {
1409 unsigned long psize = hpte_base_page_size(v, r); 1405 unsigned long psize = hpte_base_page_size(v, r);
1410 unsigned long senc = slb_pgsize_encoding(psize); 1406 unsigned long senc = slb_pgsize_encoding(psize);
1411 unsigned long lpcr; 1407 unsigned long lpcr;
@@ -1414,7 +1410,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1414 (VRMA_VSID << SLB_VSID_SHIFT_1T); 1410 (VRMA_VSID << SLB_VSID_SHIFT_1T);
1415 lpcr = senc << (LPCR_VRMASD_SH - 4); 1411 lpcr = senc << (LPCR_VRMASD_SH - 4);
1416 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); 1412 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
1417 rma_setup = 1; 1413 hpte_setup = 1;
1418 } 1414 }
1419 ++i; 1415 ++i;
1420 hptp += 2; 1416 hptp += 2;
@@ -1430,9 +1426,9 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1430 } 1426 }
1431 1427
1432 out: 1428 out:
1433 /* Order HPTE updates vs. rma_setup_done */ 1429 /* Order HPTE updates vs. hpte_setup_done */
1434 smp_wmb(); 1430 smp_wmb();
1435 kvm->arch.rma_setup_done = rma_setup; 1431 kvm->arch.hpte_setup_done = hpte_setup;
1436 mutex_unlock(&kvm->lock); 1432 mutex_unlock(&kvm->lock);
1437 1433
1438 if (err) 1434 if (err)
@@ -1495,6 +1491,141 @@ int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
1495 return ret; 1491 return ret;
1496} 1492}
1497 1493
1494struct debugfs_htab_state {
1495 struct kvm *kvm;
1496 struct mutex mutex;
1497 unsigned long hpt_index;
1498 int chars_left;
1499 int buf_index;
1500 char buf[64];
1501};
1502
1503static int debugfs_htab_open(struct inode *inode, struct file *file)
1504{
1505 struct kvm *kvm = inode->i_private;
1506 struct debugfs_htab_state *p;
1507
1508 p = kzalloc(sizeof(*p), GFP_KERNEL);
1509 if (!p)
1510 return -ENOMEM;
1511
1512 kvm_get_kvm(kvm);
1513 p->kvm = kvm;
1514 mutex_init(&p->mutex);
1515 file->private_data = p;
1516
1517 return nonseekable_open(inode, file);
1518}
1519
1520static int debugfs_htab_release(struct inode *inode, struct file *file)
1521{
1522 struct debugfs_htab_state *p = file->private_data;
1523
1524 kvm_put_kvm(p->kvm);
1525 kfree(p);
1526 return 0;
1527}
1528
1529static ssize_t debugfs_htab_read(struct file *file, char __user *buf,
1530 size_t len, loff_t *ppos)
1531{
1532 struct debugfs_htab_state *p = file->private_data;
1533 ssize_t ret, r;
1534 unsigned long i, n;
1535 unsigned long v, hr, gr;
1536 struct kvm *kvm;
1537 __be64 *hptp;
1538
1539 ret = mutex_lock_interruptible(&p->mutex);
1540 if (ret)
1541 return ret;
1542
1543 if (p->chars_left) {
1544 n = p->chars_left;
1545 if (n > len)
1546 n = len;
1547 r = copy_to_user(buf, p->buf + p->buf_index, n);
1548 n -= r;
1549 p->chars_left -= n;
1550 p->buf_index += n;
1551 buf += n;
1552 len -= n;
1553 ret = n;
1554 if (r) {
1555 if (!n)
1556 ret = -EFAULT;
1557 goto out;
1558 }
1559 }
1560
1561 kvm = p->kvm;
1562 i = p->hpt_index;
1563 hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
1564 for (; len != 0 && i < kvm->arch.hpt_npte; ++i, hptp += 2) {
1565 if (!(be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)))
1566 continue;
1567
1568 /* lock the HPTE so it's stable and read it */
1569 preempt_disable();
1570 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1571 cpu_relax();
1572 v = be64_to_cpu(hptp[0]) & ~HPTE_V_HVLOCK;
1573 hr = be64_to_cpu(hptp[1]);
1574 gr = kvm->arch.revmap[i].guest_rpte;
1575 unlock_hpte(hptp, v);
1576 preempt_enable();
1577
1578 if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT)))
1579 continue;
1580
1581 n = scnprintf(p->buf, sizeof(p->buf),
1582 "%6lx %.16lx %.16lx %.16lx\n",
1583 i, v, hr, gr);
1584 p->chars_left = n;
1585 if (n > len)
1586 n = len;
1587 r = copy_to_user(buf, p->buf, n);
1588 n -= r;
1589 p->chars_left -= n;
1590 p->buf_index = n;
1591 buf += n;
1592 len -= n;
1593 ret += n;
1594 if (r) {
1595 if (!ret)
1596 ret = -EFAULT;
1597 goto out;
1598 }
1599 }
1600 p->hpt_index = i;
1601
1602 out:
1603 mutex_unlock(&p->mutex);
1604 return ret;
1605}
1606
1607ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
1608 size_t len, loff_t *ppos)
1609{
1610 return -EACCES;
1611}
1612
1613static const struct file_operations debugfs_htab_fops = {
1614 .owner = THIS_MODULE,
1615 .open = debugfs_htab_open,
1616 .release = debugfs_htab_release,
1617 .read = debugfs_htab_read,
1618 .write = debugfs_htab_write,
1619 .llseek = generic_file_llseek,
1620};
1621
1622void kvmppc_mmu_debugfs_init(struct kvm *kvm)
1623{
1624 kvm->arch.htab_dentry = debugfs_create_file("htab", 0400,
1625 kvm->arch.debugfs_dir, kvm,
1626 &debugfs_htab_fops);
1627}
1628
1498void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu) 1629void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
1499{ 1630{
1500 struct kvmppc_mmu *mmu = &vcpu->arch.mmu; 1631 struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index de747563d29d..48d3c5d2ecc9 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -32,6 +32,7 @@
32#include <linux/page-flags.h> 32#include <linux/page-flags.h>
33#include <linux/srcu.h> 33#include <linux/srcu.h>
34#include <linux/miscdevice.h> 34#include <linux/miscdevice.h>
35#include <linux/debugfs.h>
35 36
36#include <asm/reg.h> 37#include <asm/reg.h>
37#include <asm/cputable.h> 38#include <asm/cputable.h>
@@ -50,6 +51,7 @@
50#include <asm/hvcall.h> 51#include <asm/hvcall.h>
51#include <asm/switch_to.h> 52#include <asm/switch_to.h>
52#include <asm/smp.h> 53#include <asm/smp.h>
54#include <asm/dbell.h>
53#include <linux/gfp.h> 55#include <linux/gfp.h>
54#include <linux/vmalloc.h> 56#include <linux/vmalloc.h>
55#include <linux/highmem.h> 57#include <linux/highmem.h>
@@ -83,9 +85,35 @@ static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
83static void kvmppc_end_cede(struct kvm_vcpu *vcpu); 85static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
84static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); 86static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
85 87
88static bool kvmppc_ipi_thread(int cpu)
89{
90 /* On POWER8 for IPIs to threads in the same core, use msgsnd */
91 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
92 preempt_disable();
93 if (cpu_first_thread_sibling(cpu) ==
94 cpu_first_thread_sibling(smp_processor_id())) {
95 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
96 msg |= cpu_thread_in_core(cpu);
97 smp_mb();
98 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
99 preempt_enable();
100 return true;
101 }
102 preempt_enable();
103 }
104
105#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
106 if (cpu >= 0 && cpu < nr_cpu_ids && paca[cpu].kvm_hstate.xics_phys) {
107 xics_wake_cpu(cpu);
108 return true;
109 }
110#endif
111
112 return false;
113}
114
86static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) 115static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
87{ 116{
88 int me;
89 int cpu = vcpu->cpu; 117 int cpu = vcpu->cpu;
90 wait_queue_head_t *wqp; 118 wait_queue_head_t *wqp;
91 119
@@ -95,20 +123,12 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
95 ++vcpu->stat.halt_wakeup; 123 ++vcpu->stat.halt_wakeup;
96 } 124 }
97 125
98 me = get_cpu(); 126 if (kvmppc_ipi_thread(cpu + vcpu->arch.ptid))
127 return;
99 128
100 /* CPU points to the first thread of the core */ 129 /* CPU points to the first thread of the core */
101 if (cpu != me && cpu >= 0 && cpu < nr_cpu_ids) { 130 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
102#ifdef CONFIG_PPC_ICP_NATIVE 131 smp_send_reschedule(cpu);
103 int real_cpu = cpu + vcpu->arch.ptid;
104 if (paca[real_cpu].kvm_hstate.xics_phys)
105 xics_wake_cpu(real_cpu);
106 else
107#endif
108 if (cpu_online(cpu))
109 smp_send_reschedule(cpu);
110 }
111 put_cpu();
112} 132}
113 133
114/* 134/*
@@ -706,6 +726,16 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
706 726
707 /* Send the error out to userspace via KVM_RUN */ 727 /* Send the error out to userspace via KVM_RUN */
708 return rc; 728 return rc;
729 case H_LOGICAL_CI_LOAD:
730 ret = kvmppc_h_logical_ci_load(vcpu);
731 if (ret == H_TOO_HARD)
732 return RESUME_HOST;
733 break;
734 case H_LOGICAL_CI_STORE:
735 ret = kvmppc_h_logical_ci_store(vcpu);
736 if (ret == H_TOO_HARD)
737 return RESUME_HOST;
738 break;
709 case H_SET_MODE: 739 case H_SET_MODE:
710 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4), 740 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
711 kvmppc_get_gpr(vcpu, 5), 741 kvmppc_get_gpr(vcpu, 5),
@@ -740,6 +770,8 @@ static int kvmppc_hcall_impl_hv(unsigned long cmd)
740 case H_CONFER: 770 case H_CONFER:
741 case H_REGISTER_VPA: 771 case H_REGISTER_VPA:
742 case H_SET_MODE: 772 case H_SET_MODE:
773 case H_LOGICAL_CI_LOAD:
774 case H_LOGICAL_CI_STORE:
743#ifdef CONFIG_KVM_XICS 775#ifdef CONFIG_KVM_XICS
744 case H_XIRR: 776 case H_XIRR:
745 case H_CPPR: 777 case H_CPPR:
@@ -1410,6 +1442,154 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
1410 return vcore; 1442 return vcore;
1411} 1443}
1412 1444
1445#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1446static struct debugfs_timings_element {
1447 const char *name;
1448 size_t offset;
1449} timings[] = {
1450 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)},
1451 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)},
1452 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)},
1453 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)},
1454 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)},
1455};
1456
1457#define N_TIMINGS (sizeof(timings) / sizeof(timings[0]))
1458
1459struct debugfs_timings_state {
1460 struct kvm_vcpu *vcpu;
1461 unsigned int buflen;
1462 char buf[N_TIMINGS * 100];
1463};
1464
1465static int debugfs_timings_open(struct inode *inode, struct file *file)
1466{
1467 struct kvm_vcpu *vcpu = inode->i_private;
1468 struct debugfs_timings_state *p;
1469
1470 p = kzalloc(sizeof(*p), GFP_KERNEL);
1471 if (!p)
1472 return -ENOMEM;
1473
1474 kvm_get_kvm(vcpu->kvm);
1475 p->vcpu = vcpu;
1476 file->private_data = p;
1477
1478 return nonseekable_open(inode, file);
1479}
1480
1481static int debugfs_timings_release(struct inode *inode, struct file *file)
1482{
1483 struct debugfs_timings_state *p = file->private_data;
1484
1485 kvm_put_kvm(p->vcpu->kvm);
1486 kfree(p);
1487 return 0;
1488}
1489
1490static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
1491 size_t len, loff_t *ppos)
1492{
1493 struct debugfs_timings_state *p = file->private_data;
1494 struct kvm_vcpu *vcpu = p->vcpu;
1495 char *s, *buf_end;
1496 struct kvmhv_tb_accumulator tb;
1497 u64 count;
1498 loff_t pos;
1499 ssize_t n;
1500 int i, loops;
1501 bool ok;
1502
1503 if (!p->buflen) {
1504 s = p->buf;
1505 buf_end = s + sizeof(p->buf);
1506 for (i = 0; i < N_TIMINGS; ++i) {
1507 struct kvmhv_tb_accumulator *acc;
1508
1509 acc = (struct kvmhv_tb_accumulator *)
1510 ((unsigned long)vcpu + timings[i].offset);
1511 ok = false;
1512 for (loops = 0; loops < 1000; ++loops) {
1513 count = acc->seqcount;
1514 if (!(count & 1)) {
1515 smp_rmb();
1516 tb = *acc;
1517 smp_rmb();
1518 if (count == acc->seqcount) {
1519 ok = true;
1520 break;
1521 }
1522 }
1523 udelay(1);
1524 }
1525 if (!ok)
1526 snprintf(s, buf_end - s, "%s: stuck\n",
1527 timings[i].name);
1528 else
1529 snprintf(s, buf_end - s,
1530 "%s: %llu %llu %llu %llu\n",
1531 timings[i].name, count / 2,
1532 tb_to_ns(tb.tb_total),
1533 tb_to_ns(tb.tb_min),
1534 tb_to_ns(tb.tb_max));
1535 s += strlen(s);
1536 }
1537 p->buflen = s - p->buf;
1538 }
1539
1540 pos = *ppos;
1541 if (pos >= p->buflen)
1542 return 0;
1543 if (len > p->buflen - pos)
1544 len = p->buflen - pos;
1545 n = copy_to_user(buf, p->buf + pos, len);
1546 if (n) {
1547 if (n == len)
1548 return -EFAULT;
1549 len -= n;
1550 }
1551 *ppos = pos + len;
1552 return len;
1553}
1554
1555static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
1556 size_t len, loff_t *ppos)
1557{
1558 return -EACCES;
1559}
1560
1561static const struct file_operations debugfs_timings_ops = {
1562 .owner = THIS_MODULE,
1563 .open = debugfs_timings_open,
1564 .release = debugfs_timings_release,
1565 .read = debugfs_timings_read,
1566 .write = debugfs_timings_write,
1567 .llseek = generic_file_llseek,
1568};
1569
1570/* Create a debugfs directory for the vcpu */
1571static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
1572{
1573 char buf[16];
1574 struct kvm *kvm = vcpu->kvm;
1575
1576 snprintf(buf, sizeof(buf), "vcpu%u", id);
1577 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
1578 return;
1579 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
1580 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir))
1581 return;
1582 vcpu->arch.debugfs_timings =
1583 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir,
1584 vcpu, &debugfs_timings_ops);
1585}
1586
1587#else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
1588static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
1589{
1590}
1591#endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
1592
1413static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, 1593static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
1414 unsigned int id) 1594 unsigned int id)
1415{ 1595{
@@ -1479,6 +1659,8 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
1479 vcpu->arch.cpu_type = KVM_CPU_3S_64; 1659 vcpu->arch.cpu_type = KVM_CPU_3S_64;
1480 kvmppc_sanity_check(vcpu); 1660 kvmppc_sanity_check(vcpu);
1481 1661
1662 debugfs_vcpu_init(vcpu, id);
1663
1482 return vcpu; 1664 return vcpu;
1483 1665
1484free_vcpu: 1666free_vcpu:
@@ -1566,8 +1748,10 @@ static int kvmppc_grab_hwthread(int cpu)
1566 tpaca = &paca[cpu]; 1748 tpaca = &paca[cpu];
1567 1749
1568 /* Ensure the thread won't go into the kernel if it wakes */ 1750 /* Ensure the thread won't go into the kernel if it wakes */
1569 tpaca->kvm_hstate.hwthread_req = 1;
1570 tpaca->kvm_hstate.kvm_vcpu = NULL; 1751 tpaca->kvm_hstate.kvm_vcpu = NULL;
1752 tpaca->kvm_hstate.napping = 0;
1753 smp_wmb();
1754 tpaca->kvm_hstate.hwthread_req = 1;
1571 1755
1572 /* 1756 /*
1573 * If the thread is already executing in the kernel (e.g. handling 1757 * If the thread is already executing in the kernel (e.g. handling
@@ -1610,35 +1794,41 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
1610 } 1794 }
1611 cpu = vc->pcpu + vcpu->arch.ptid; 1795 cpu = vc->pcpu + vcpu->arch.ptid;
1612 tpaca = &paca[cpu]; 1796 tpaca = &paca[cpu];
1613 tpaca->kvm_hstate.kvm_vcpu = vcpu;
1614 tpaca->kvm_hstate.kvm_vcore = vc; 1797 tpaca->kvm_hstate.kvm_vcore = vc;
1615 tpaca->kvm_hstate.ptid = vcpu->arch.ptid; 1798 tpaca->kvm_hstate.ptid = vcpu->arch.ptid;
1616 vcpu->cpu = vc->pcpu; 1799 vcpu->cpu = vc->pcpu;
1800 /* Order stores to hstate.kvm_vcore etc. before store to kvm_vcpu */
1617 smp_wmb(); 1801 smp_wmb();
1618#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) 1802 tpaca->kvm_hstate.kvm_vcpu = vcpu;
1619 if (cpu != smp_processor_id()) { 1803 if (cpu != smp_processor_id())
1620 xics_wake_cpu(cpu); 1804 kvmppc_ipi_thread(cpu);
1621 if (vcpu->arch.ptid)
1622 ++vc->n_woken;
1623 }
1624#endif
1625} 1805}
1626 1806
1627static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc) 1807static void kvmppc_wait_for_nap(void)
1628{ 1808{
1629 int i; 1809 int cpu = smp_processor_id();
1810 int i, loops;
1630 1811
1631 HMT_low(); 1812 for (loops = 0; loops < 1000000; ++loops) {
1632 i = 0; 1813 /*
1633 while (vc->nap_count < vc->n_woken) { 1814 * Check if all threads are finished.
1634 if (++i >= 1000000) { 1815 * We set the vcpu pointer when starting a thread
1635 pr_err("kvmppc_wait_for_nap timeout %d %d\n", 1816 * and the thread clears it when finished, so we look
1636 vc->nap_count, vc->n_woken); 1817 * for any threads that still have a non-NULL vcpu ptr.
1637 break; 1818 */
1819 for (i = 1; i < threads_per_subcore; ++i)
1820 if (paca[cpu + i].kvm_hstate.kvm_vcpu)
1821 break;
1822 if (i == threads_per_subcore) {
1823 HMT_medium();
1824 return;
1638 } 1825 }
1639 cpu_relax(); 1826 HMT_low();
1640 } 1827 }
1641 HMT_medium(); 1828 HMT_medium();
1829 for (i = 1; i < threads_per_subcore; ++i)
1830 if (paca[cpu + i].kvm_hstate.kvm_vcpu)
1831 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
1642} 1832}
1643 1833
1644/* 1834/*
@@ -1700,63 +1890,103 @@ static void kvmppc_start_restoring_l2_cache(const struct kvmppc_vcore *vc)
1700 mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_WHOLE_TABLE); 1890 mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_WHOLE_TABLE);
1701} 1891}
1702 1892
1893static void prepare_threads(struct kvmppc_vcore *vc)
1894{
1895 struct kvm_vcpu *vcpu, *vnext;
1896
1897 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1898 arch.run_list) {
1899 if (signal_pending(vcpu->arch.run_task))
1900 vcpu->arch.ret = -EINTR;
1901 else if (vcpu->arch.vpa.update_pending ||
1902 vcpu->arch.slb_shadow.update_pending ||
1903 vcpu->arch.dtl.update_pending)
1904 vcpu->arch.ret = RESUME_GUEST;
1905 else
1906 continue;
1907 kvmppc_remove_runnable(vc, vcpu);
1908 wake_up(&vcpu->arch.cpu_run);
1909 }
1910}
1911
1912static void post_guest_process(struct kvmppc_vcore *vc)
1913{
1914 u64 now;
1915 long ret;
1916 struct kvm_vcpu *vcpu, *vnext;
1917
1918 now = get_tb();
1919 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1920 arch.run_list) {
1921 /* cancel pending dec exception if dec is positive */
1922 if (now < vcpu->arch.dec_expires &&
1923 kvmppc_core_pending_dec(vcpu))
1924 kvmppc_core_dequeue_dec(vcpu);
1925
1926 trace_kvm_guest_exit(vcpu);
1927
1928 ret = RESUME_GUEST;
1929 if (vcpu->arch.trap)
1930 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
1931 vcpu->arch.run_task);
1932
1933 vcpu->arch.ret = ret;
1934 vcpu->arch.trap = 0;
1935
1936 if (vcpu->arch.ceded) {
1937 if (!is_kvmppc_resume_guest(ret))
1938 kvmppc_end_cede(vcpu);
1939 else
1940 kvmppc_set_timer(vcpu);
1941 }
1942 if (!is_kvmppc_resume_guest(vcpu->arch.ret)) {
1943 kvmppc_remove_runnable(vc, vcpu);
1944 wake_up(&vcpu->arch.cpu_run);
1945 }
1946 }
1947}
1948
1703/* 1949/*
1704 * Run a set of guest threads on a physical core. 1950 * Run a set of guest threads on a physical core.
1705 * Called with vc->lock held. 1951 * Called with vc->lock held.
1706 */ 1952 */
1707static void kvmppc_run_core(struct kvmppc_vcore *vc) 1953static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
1708{ 1954{
1709 struct kvm_vcpu *vcpu, *vnext; 1955 struct kvm_vcpu *vcpu;
1710 long ret; 1956 int i;
1711 u64 now;
1712 int i, need_vpa_update;
1713 int srcu_idx; 1957 int srcu_idx;
1714 struct kvm_vcpu *vcpus_to_update[threads_per_core];
1715 1958
1716 /* don't start if any threads have a signal pending */ 1959 /*
1717 need_vpa_update = 0; 1960 * Remove from the list any threads that have a signal pending
1718 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) { 1961 * or need a VPA update done
1719 if (signal_pending(vcpu->arch.run_task)) 1962 */
1720 return; 1963 prepare_threads(vc);
1721 if (vcpu->arch.vpa.update_pending || 1964
1722 vcpu->arch.slb_shadow.update_pending || 1965 /* if the runner is no longer runnable, let the caller pick a new one */
1723 vcpu->arch.dtl.update_pending) 1966 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
1724 vcpus_to_update[need_vpa_update++] = vcpu; 1967 return;
1725 }
1726 1968
1727 /* 1969 /*
1728 * Initialize *vc, in particular vc->vcore_state, so we can 1970 * Initialize *vc.
1729 * drop the vcore lock if necessary.
1730 */ 1971 */
1731 vc->n_woken = 0; 1972 vc->entry_exit_map = 0;
1732 vc->nap_count = 0;
1733 vc->entry_exit_count = 0;
1734 vc->preempt_tb = TB_NIL; 1973 vc->preempt_tb = TB_NIL;
1735 vc->vcore_state = VCORE_STARTING;
1736 vc->in_guest = 0; 1974 vc->in_guest = 0;
1737 vc->napping_threads = 0; 1975 vc->napping_threads = 0;
1738 vc->conferring_threads = 0; 1976 vc->conferring_threads = 0;
1739 1977
1740 /* 1978 /*
1741 * Updating any of the vpas requires calling kvmppc_pin_guest_page,
1742 * which can't be called with any spinlocks held.
1743 */
1744 if (need_vpa_update) {
1745 spin_unlock(&vc->lock);
1746 for (i = 0; i < need_vpa_update; ++i)
1747 kvmppc_update_vpas(vcpus_to_update[i]);
1748 spin_lock(&vc->lock);
1749 }
1750
1751 /*
1752 * Make sure we are running on primary threads, and that secondary 1979 * Make sure we are running on primary threads, and that secondary
1753 * threads are offline. Also check if the number of threads in this 1980 * threads are offline. Also check if the number of threads in this
1754 * guest are greater than the current system threads per guest. 1981 * guest are greater than the current system threads per guest.
1755 */ 1982 */
1756 if ((threads_per_core > 1) && 1983 if ((threads_per_core > 1) &&
1757 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { 1984 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
1758 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) 1985 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1759 vcpu->arch.ret = -EBUSY; 1986 vcpu->arch.ret = -EBUSY;
1987 kvmppc_remove_runnable(vc, vcpu);
1988 wake_up(&vcpu->arch.cpu_run);
1989 }
1760 goto out; 1990 goto out;
1761 } 1991 }
1762 1992
@@ -1797,8 +2027,7 @@ static void kvmppc_run_core(struct kvmppc_vcore *vc)
1797 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) 2027 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1798 vcpu->cpu = -1; 2028 vcpu->cpu = -1;
1799 /* wait for secondary threads to finish writing their state to memory */ 2029 /* wait for secondary threads to finish writing their state to memory */
1800 if (vc->nap_count < vc->n_woken) 2030 kvmppc_wait_for_nap();
1801 kvmppc_wait_for_nap(vc);
1802 for (i = 0; i < threads_per_subcore; ++i) 2031 for (i = 0; i < threads_per_subcore; ++i)
1803 kvmppc_release_hwthread(vc->pcpu + i); 2032 kvmppc_release_hwthread(vc->pcpu + i);
1804 /* prevent other vcpu threads from doing kvmppc_start_thread() now */ 2033 /* prevent other vcpu threads from doing kvmppc_start_thread() now */
@@ -1812,44 +2041,12 @@ static void kvmppc_run_core(struct kvmppc_vcore *vc)
1812 kvm_guest_exit(); 2041 kvm_guest_exit();
1813 2042
1814 preempt_enable(); 2043 preempt_enable();
1815 cond_resched();
1816 2044
1817 spin_lock(&vc->lock); 2045 spin_lock(&vc->lock);
1818 now = get_tb(); 2046 post_guest_process(vc);
1819 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1820 /* cancel pending dec exception if dec is positive */
1821 if (now < vcpu->arch.dec_expires &&
1822 kvmppc_core_pending_dec(vcpu))
1823 kvmppc_core_dequeue_dec(vcpu);
1824
1825 trace_kvm_guest_exit(vcpu);
1826
1827 ret = RESUME_GUEST;
1828 if (vcpu->arch.trap)
1829 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
1830 vcpu->arch.run_task);
1831
1832 vcpu->arch.ret = ret;
1833 vcpu->arch.trap = 0;
1834
1835 if (vcpu->arch.ceded) {
1836 if (!is_kvmppc_resume_guest(ret))
1837 kvmppc_end_cede(vcpu);
1838 else
1839 kvmppc_set_timer(vcpu);
1840 }
1841 }
1842 2047
1843 out: 2048 out:
1844 vc->vcore_state = VCORE_INACTIVE; 2049 vc->vcore_state = VCORE_INACTIVE;
1845 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1846 arch.run_list) {
1847 if (!is_kvmppc_resume_guest(vcpu->arch.ret)) {
1848 kvmppc_remove_runnable(vc, vcpu);
1849 wake_up(&vcpu->arch.cpu_run);
1850 }
1851 }
1852
1853 trace_kvmppc_run_core(vc, 1); 2050 trace_kvmppc_run_core(vc, 1);
1854} 2051}
1855 2052
@@ -1939,8 +2136,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1939 * this thread straight away and have it join in. 2136 * this thread straight away and have it join in.
1940 */ 2137 */
1941 if (!signal_pending(current)) { 2138 if (!signal_pending(current)) {
1942 if (vc->vcore_state == VCORE_RUNNING && 2139 if (vc->vcore_state == VCORE_RUNNING && !VCORE_IS_EXITING(vc)) {
1943 VCORE_EXIT_COUNT(vc) == 0) {
1944 kvmppc_create_dtl_entry(vcpu, vc); 2140 kvmppc_create_dtl_entry(vcpu, vc);
1945 kvmppc_start_thread(vcpu); 2141 kvmppc_start_thread(vcpu);
1946 trace_kvm_guest_enter(vcpu); 2142 trace_kvm_guest_enter(vcpu);
@@ -1971,7 +2167,6 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1971 } 2167 }
1972 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 2168 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
1973 break; 2169 break;
1974 vc->runner = vcpu;
1975 n_ceded = 0; 2170 n_ceded = 0;
1976 list_for_each_entry(v, &vc->runnable_threads, arch.run_list) { 2171 list_for_each_entry(v, &vc->runnable_threads, arch.run_list) {
1977 if (!v->arch.pending_exceptions) 2172 if (!v->arch.pending_exceptions)
@@ -1979,10 +2174,17 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1979 else 2174 else
1980 v->arch.ceded = 0; 2175 v->arch.ceded = 0;
1981 } 2176 }
1982 if (n_ceded == vc->n_runnable) 2177 vc->runner = vcpu;
2178 if (n_ceded == vc->n_runnable) {
1983 kvmppc_vcore_blocked(vc); 2179 kvmppc_vcore_blocked(vc);
1984 else 2180 } else if (should_resched()) {
2181 vc->vcore_state = VCORE_PREEMPT;
2182 /* Let something else run */
2183 cond_resched_lock(&vc->lock);
2184 vc->vcore_state = VCORE_INACTIVE;
2185 } else {
1985 kvmppc_run_core(vc); 2186 kvmppc_run_core(vc);
2187 }
1986 vc->runner = NULL; 2188 vc->runner = NULL;
1987 } 2189 }
1988 2190
@@ -2032,11 +2234,11 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
2032 } 2234 }
2033 2235
2034 atomic_inc(&vcpu->kvm->arch.vcpus_running); 2236 atomic_inc(&vcpu->kvm->arch.vcpus_running);
2035 /* Order vcpus_running vs. rma_setup_done, see kvmppc_alloc_reset_hpt */ 2237 /* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */
2036 smp_mb(); 2238 smp_mb();
2037 2239
2038 /* On the first time here, set up HTAB and VRMA */ 2240 /* On the first time here, set up HTAB and VRMA */
2039 if (!vcpu->kvm->arch.rma_setup_done) { 2241 if (!vcpu->kvm->arch.hpte_setup_done) {
2040 r = kvmppc_hv_setup_htab_rma(vcpu); 2242 r = kvmppc_hv_setup_htab_rma(vcpu);
2041 if (r) 2243 if (r)
2042 goto out; 2244 goto out;
@@ -2238,7 +2440,7 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
2238 int srcu_idx; 2440 int srcu_idx;
2239 2441
2240 mutex_lock(&kvm->lock); 2442 mutex_lock(&kvm->lock);
2241 if (kvm->arch.rma_setup_done) 2443 if (kvm->arch.hpte_setup_done)
2242 goto out; /* another vcpu beat us to it */ 2444 goto out; /* another vcpu beat us to it */
2243 2445
2244 /* Allocate hashed page table (if not done already) and reset it */ 2446 /* Allocate hashed page table (if not done already) and reset it */
@@ -2289,9 +2491,9 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
2289 2491
2290 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); 2492 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
2291 2493
2292 /* Order updates to kvm->arch.lpcr etc. vs. rma_setup_done */ 2494 /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */
2293 smp_wmb(); 2495 smp_wmb();
2294 kvm->arch.rma_setup_done = 1; 2496 kvm->arch.hpte_setup_done = 1;
2295 err = 0; 2497 err = 0;
2296 out_srcu: 2498 out_srcu:
2297 srcu_read_unlock(&kvm->srcu, srcu_idx); 2499 srcu_read_unlock(&kvm->srcu, srcu_idx);
@@ -2307,6 +2509,7 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
2307static int kvmppc_core_init_vm_hv(struct kvm *kvm) 2509static int kvmppc_core_init_vm_hv(struct kvm *kvm)
2308{ 2510{
2309 unsigned long lpcr, lpid; 2511 unsigned long lpcr, lpid;
2512 char buf[32];
2310 2513
2311 /* Allocate the guest's logical partition ID */ 2514 /* Allocate the guest's logical partition ID */
2312 2515
@@ -2347,6 +2550,14 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm)
2347 */ 2550 */
2348 kvm_hv_vm_activated(); 2551 kvm_hv_vm_activated();
2349 2552
2553 /*
2554 * Create a debugfs directory for the VM
2555 */
2556 snprintf(buf, sizeof(buf), "vm%d", current->pid);
2557 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
2558 if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
2559 kvmppc_mmu_debugfs_init(kvm);
2560
2350 return 0; 2561 return 0;
2351} 2562}
2352 2563
@@ -2367,6 +2578,8 @@ static void kvmppc_free_vcores(struct kvm *kvm)
2367 2578
2368static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) 2579static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
2369{ 2580{
2581 debugfs_remove_recursive(kvm->arch.debugfs_dir);
2582
2370 kvm_hv_vm_deactivated(); 2583 kvm_hv_vm_deactivated();
2371 2584
2372 kvmppc_free_vcores(kvm); 2585 kvmppc_free_vcores(kvm);
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index 1f083ff8a61a..ed2589d4593f 100644
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -21,6 +21,10 @@
21#include <asm/cputable.h> 21#include <asm/cputable.h>
22#include <asm/kvm_ppc.h> 22#include <asm/kvm_ppc.h>
23#include <asm/kvm_book3s.h> 23#include <asm/kvm_book3s.h>
24#include <asm/archrandom.h>
25#include <asm/xics.h>
26#include <asm/dbell.h>
27#include <asm/cputhreads.h>
24 28
25#define KVM_CMA_CHUNK_ORDER 18 29#define KVM_CMA_CHUNK_ORDER 18
26 30
@@ -114,11 +118,11 @@ long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target,
114 int rv = H_SUCCESS; /* => don't yield */ 118 int rv = H_SUCCESS; /* => don't yield */
115 119
116 set_bit(vcpu->arch.ptid, &vc->conferring_threads); 120 set_bit(vcpu->arch.ptid, &vc->conferring_threads);
117 while ((get_tb() < stop) && (VCORE_EXIT_COUNT(vc) == 0)) { 121 while ((get_tb() < stop) && !VCORE_IS_EXITING(vc)) {
118 threads_running = VCORE_ENTRY_COUNT(vc); 122 threads_running = VCORE_ENTRY_MAP(vc);
119 threads_ceded = hweight32(vc->napping_threads); 123 threads_ceded = vc->napping_threads;
120 threads_conferring = hweight32(vc->conferring_threads); 124 threads_conferring = vc->conferring_threads;
121 if (threads_ceded + threads_conferring >= threads_running) { 125 if ((threads_ceded | threads_conferring) == threads_running) {
122 rv = H_TOO_HARD; /* => do yield */ 126 rv = H_TOO_HARD; /* => do yield */
123 break; 127 break;
124 } 128 }
@@ -169,3 +173,89 @@ int kvmppc_hcall_impl_hv_realmode(unsigned long cmd)
169 return 0; 173 return 0;
170} 174}
171EXPORT_SYMBOL_GPL(kvmppc_hcall_impl_hv_realmode); 175EXPORT_SYMBOL_GPL(kvmppc_hcall_impl_hv_realmode);
176
177int kvmppc_hwrng_present(void)
178{
179 return powernv_hwrng_present();
180}
181EXPORT_SYMBOL_GPL(kvmppc_hwrng_present);
182
183long kvmppc_h_random(struct kvm_vcpu *vcpu)
184{
185 if (powernv_get_random_real_mode(&vcpu->arch.gpr[4]))
186 return H_SUCCESS;
187
188 return H_HARDWARE;
189}
190
191static inline void rm_writeb(unsigned long paddr, u8 val)
192{
193 __asm__ __volatile__("stbcix %0,0,%1"
194 : : "r" (val), "r" (paddr) : "memory");
195}
196
197/*
198 * Send an interrupt or message to another CPU.
199 * This can only be called in real mode.
200 * The caller needs to include any barrier needed to order writes
201 * to memory vs. the IPI/message.
202 */
203void kvmhv_rm_send_ipi(int cpu)
204{
205 unsigned long xics_phys;
206
207 /* On POWER8 for IPIs to threads in the same core, use msgsnd */
208 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
209 cpu_first_thread_sibling(cpu) ==
210 cpu_first_thread_sibling(raw_smp_processor_id())) {
211 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
212 msg |= cpu_thread_in_core(cpu);
213 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
214 return;
215 }
216
217 /* Else poke the target with an IPI */
218 xics_phys = paca[cpu].kvm_hstate.xics_phys;
219 rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY);
220}
221
222/*
223 * The following functions are called from the assembly code
224 * in book3s_hv_rmhandlers.S.
225 */
226static void kvmhv_interrupt_vcore(struct kvmppc_vcore *vc, int active)
227{
228 int cpu = vc->pcpu;
229
230 /* Order setting of exit map vs. msgsnd/IPI */
231 smp_mb();
232 for (; active; active >>= 1, ++cpu)
233 if (active & 1)
234 kvmhv_rm_send_ipi(cpu);
235}
236
237void kvmhv_commence_exit(int trap)
238{
239 struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
240 int ptid = local_paca->kvm_hstate.ptid;
241 int me, ee;
242
243 /* Set our bit in the threads-exiting-guest map in the 0xff00
244 bits of vcore->entry_exit_map */
245 me = 0x100 << ptid;
246 do {
247 ee = vc->entry_exit_map;
248 } while (cmpxchg(&vc->entry_exit_map, ee, ee | me) != ee);
249
250 /* Are we the first here? */
251 if ((ee >> 8) != 0)
252 return;
253
254 /*
255 * Trigger the other threads in this vcore to exit the guest.
256 * If this is a hypervisor decrementer interrupt then they
257 * will be already on their way out of the guest.
258 */
259 if (trap != BOOK3S_INTERRUPT_HV_DECREMENTER)
260 kvmhv_interrupt_vcore(vc, ee & ~(1 << ptid));
261}
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 625407e4d3b0..f6bf0b1de6d7 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -150,12 +150,6 @@ static pte_t lookup_linux_pte_and_update(pgd_t *pgdir, unsigned long hva,
150 return kvmppc_read_update_linux_pte(ptep, writing, hugepage_shift); 150 return kvmppc_read_update_linux_pte(ptep, writing, hugepage_shift);
151} 151}
152 152
153static inline void unlock_hpte(__be64 *hpte, unsigned long hpte_v)
154{
155 asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
156 hpte[0] = cpu_to_be64(hpte_v);
157}
158
159long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, 153long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
160 long pte_index, unsigned long pteh, unsigned long ptel, 154 long pte_index, unsigned long pteh, unsigned long ptel,
161 pgd_t *pgdir, bool realmode, unsigned long *pte_idx_ret) 155 pgd_t *pgdir, bool realmode, unsigned long *pte_idx_ret)
@@ -271,10 +265,10 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
271 u64 pte; 265 u64 pte;
272 while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) 266 while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
273 cpu_relax(); 267 cpu_relax();
274 pte = be64_to_cpu(*hpte); 268 pte = be64_to_cpu(hpte[0]);
275 if (!(pte & (HPTE_V_VALID | HPTE_V_ABSENT))) 269 if (!(pte & (HPTE_V_VALID | HPTE_V_ABSENT)))
276 break; 270 break;
277 *hpte &= ~cpu_to_be64(HPTE_V_HVLOCK); 271 __unlock_hpte(hpte, pte);
278 hpte += 2; 272 hpte += 2;
279 } 273 }
280 if (i == 8) 274 if (i == 8)
@@ -290,9 +284,9 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
290 284
291 while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) 285 while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
292 cpu_relax(); 286 cpu_relax();
293 pte = be64_to_cpu(*hpte); 287 pte = be64_to_cpu(hpte[0]);
294 if (pte & (HPTE_V_VALID | HPTE_V_ABSENT)) { 288 if (pte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
295 *hpte &= ~cpu_to_be64(HPTE_V_HVLOCK); 289 __unlock_hpte(hpte, pte);
296 return H_PTEG_FULL; 290 return H_PTEG_FULL;
297 } 291 }
298 } 292 }
@@ -331,7 +325,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
331 325
332 /* Write the first HPTE dword, unlocking the HPTE and making it valid */ 326 /* Write the first HPTE dword, unlocking the HPTE and making it valid */
333 eieio(); 327 eieio();
334 hpte[0] = cpu_to_be64(pteh); 328 __unlock_hpte(hpte, pteh);
335 asm volatile("ptesync" : : : "memory"); 329 asm volatile("ptesync" : : : "memory");
336 330
337 *pte_idx_ret = pte_index; 331 *pte_idx_ret = pte_index;
@@ -412,7 +406,7 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
412 if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || 406 if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
413 ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn) || 407 ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn) ||
414 ((flags & H_ANDCOND) && (pte & avpn) != 0)) { 408 ((flags & H_ANDCOND) && (pte & avpn) != 0)) {
415 hpte[0] &= ~cpu_to_be64(HPTE_V_HVLOCK); 409 __unlock_hpte(hpte, pte);
416 return H_NOT_FOUND; 410 return H_NOT_FOUND;
417 } 411 }
418 412
@@ -548,7 +542,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
548 be64_to_cpu(hp[0]), be64_to_cpu(hp[1])); 542 be64_to_cpu(hp[0]), be64_to_cpu(hp[1]));
549 rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C); 543 rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C);
550 args[j] |= rcbits << (56 - 5); 544 args[j] |= rcbits << (56 - 5);
551 hp[0] = 0; 545 __unlock_hpte(hp, 0);
552 } 546 }
553 } 547 }
554 548
@@ -574,7 +568,7 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
574 pte = be64_to_cpu(hpte[0]); 568 pte = be64_to_cpu(hpte[0]);
575 if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || 569 if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
576 ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn)) { 570 ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn)) {
577 hpte[0] &= ~cpu_to_be64(HPTE_V_HVLOCK); 571 __unlock_hpte(hpte, pte);
578 return H_NOT_FOUND; 572 return H_NOT_FOUND;
579 } 573 }
580 574
@@ -755,8 +749,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
755 /* Return with the HPTE still locked */ 749 /* Return with the HPTE still locked */
756 return (hash << 3) + (i >> 1); 750 return (hash << 3) + (i >> 1);
757 751
758 /* Unlock and move on */ 752 __unlock_hpte(&hpte[i], v);
759 hpte[i] = cpu_to_be64(v);
760 } 753 }
761 754
762 if (val & HPTE_V_SECONDARY) 755 if (val & HPTE_V_SECONDARY)
diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c
index 7c22997de906..00e45b6d4f24 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_xics.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c
@@ -23,17 +23,37 @@
23 23
24#define DEBUG_PASSUP 24#define DEBUG_PASSUP
25 25
26static inline void rm_writeb(unsigned long paddr, u8 val) 26static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
27 u32 new_irq);
28
29/* -- ICS routines -- */
30static void ics_rm_check_resend(struct kvmppc_xics *xics,
31 struct kvmppc_ics *ics, struct kvmppc_icp *icp)
27{ 32{
28 __asm__ __volatile__("sync; stbcix %0,0,%1" 33 int i;
29 : : "r" (val), "r" (paddr) : "memory"); 34
35 arch_spin_lock(&ics->lock);
36
37 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
38 struct ics_irq_state *state = &ics->irq_state[i];
39
40 if (!state->resend)
41 continue;
42
43 arch_spin_unlock(&ics->lock);
44 icp_rm_deliver_irq(xics, icp, state->number);
45 arch_spin_lock(&ics->lock);
46 }
47
48 arch_spin_unlock(&ics->lock);
30} 49}
31 50
51/* -- ICP routines -- */
52
32static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, 53static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu,
33 struct kvm_vcpu *this_vcpu) 54 struct kvm_vcpu *this_vcpu)
34{ 55{
35 struct kvmppc_icp *this_icp = this_vcpu->arch.icp; 56 struct kvmppc_icp *this_icp = this_vcpu->arch.icp;
36 unsigned long xics_phys;
37 int cpu; 57 int cpu;
38 58
39 /* Mark the target VCPU as having an interrupt pending */ 59 /* Mark the target VCPU as having an interrupt pending */
@@ -56,9 +76,8 @@ static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu,
56 /* In SMT cpu will always point to thread 0, we adjust it */ 76 /* In SMT cpu will always point to thread 0, we adjust it */
57 cpu += vcpu->arch.ptid; 77 cpu += vcpu->arch.ptid;
58 78
59 /* Not too hard, then poke the target */ 79 smp_mb();
60 xics_phys = paca[cpu].kvm_hstate.xics_phys; 80 kvmhv_rm_send_ipi(cpu);
61 rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY);
62} 81}
63 82
64static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu) 83static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu)
@@ -116,6 +135,180 @@ static inline int check_too_hard(struct kvmppc_xics *xics,
116 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS; 135 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS;
117} 136}
118 137
138static void icp_rm_check_resend(struct kvmppc_xics *xics,
139 struct kvmppc_icp *icp)
140{
141 u32 icsid;
142
143 /* Order this load with the test for need_resend in the caller */
144 smp_rmb();
145 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {
146 struct kvmppc_ics *ics = xics->ics[icsid];
147
148 if (!test_and_clear_bit(icsid, icp->resend_map))
149 continue;
150 if (!ics)
151 continue;
152 ics_rm_check_resend(xics, ics, icp);
153 }
154}
155
156static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,
157 u32 *reject)
158{
159 union kvmppc_icp_state old_state, new_state;
160 bool success;
161
162 do {
163 old_state = new_state = READ_ONCE(icp->state);
164
165 *reject = 0;
166
167 /* See if we can deliver */
168 success = new_state.cppr > priority &&
169 new_state.mfrr > priority &&
170 new_state.pending_pri > priority;
171
172 /*
173 * If we can, check for a rejection and perform the
174 * delivery
175 */
176 if (success) {
177 *reject = new_state.xisr;
178 new_state.xisr = irq;
179 new_state.pending_pri = priority;
180 } else {
181 /*
182 * If we failed to deliver we set need_resend
183 * so a subsequent CPPR state change causes us
184 * to try a new delivery.
185 */
186 new_state.need_resend = true;
187 }
188
189 } while (!icp_rm_try_update(icp, old_state, new_state));
190
191 return success;
192}
193
194static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
195 u32 new_irq)
196{
197 struct ics_irq_state *state;
198 struct kvmppc_ics *ics;
199 u32 reject;
200 u16 src;
201
202 /*
203 * This is used both for initial delivery of an interrupt and
204 * for subsequent rejection.
205 *
206 * Rejection can be racy vs. resends. We have evaluated the
207 * rejection in an atomic ICP transaction which is now complete,
208 * so potentially the ICP can already accept the interrupt again.
209 *
210 * So we need to retry the delivery. Essentially the reject path
211 * boils down to a failed delivery. Always.
212 *
213 * Now the interrupt could also have moved to a different target,
214 * thus we may need to re-do the ICP lookup as well
215 */
216
217 again:
218 /* Get the ICS state and lock it */
219 ics = kvmppc_xics_find_ics(xics, new_irq, &src);
220 if (!ics) {
221 /* Unsafe increment, but this does not need to be accurate */
222 xics->err_noics++;
223 return;
224 }
225 state = &ics->irq_state[src];
226
227 /* Get a lock on the ICS */
228 arch_spin_lock(&ics->lock);
229
230 /* Get our server */
231 if (!icp || state->server != icp->server_num) {
232 icp = kvmppc_xics_find_server(xics->kvm, state->server);
233 if (!icp) {
234 /* Unsafe increment again*/
235 xics->err_noicp++;
236 goto out;
237 }
238 }
239
240 /* Clear the resend bit of that interrupt */
241 state->resend = 0;
242
243 /*
244 * If masked, bail out
245 *
246 * Note: PAPR doesn't mention anything about masked pending
247 * when doing a resend, only when doing a delivery.
248 *
249 * However that would have the effect of losing a masked
250 * interrupt that was rejected and isn't consistent with
251 * the whole masked_pending business which is about not
252 * losing interrupts that occur while masked.
253 *
254 * I don't differentiate normal deliveries and resends, this
255 * implementation will differ from PAPR and not lose such
256 * interrupts.
257 */
258 if (state->priority == MASKED) {
259 state->masked_pending = 1;
260 goto out;
261 }
262
263 /*
264 * Try the delivery, this will set the need_resend flag
265 * in the ICP as part of the atomic transaction if the
266 * delivery is not possible.
267 *
268 * Note that if successful, the new delivery might have itself
269 * rejected an interrupt that was "delivered" before we took the
270 * ics spin lock.
271 *
272 * In this case we do the whole sequence all over again for the
273 * new guy. We cannot assume that the rejected interrupt is less
274 * favored than the new one, and thus doesn't need to be delivered,
275 * because by the time we exit icp_rm_try_to_deliver() the target
276 * processor may well have already consumed & completed it, and thus
277 * the rejected interrupt might actually be already acceptable.
278 */
279 if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) {
280 /*
281 * Delivery was successful, did we reject somebody else ?
282 */
283 if (reject && reject != XICS_IPI) {
284 arch_spin_unlock(&ics->lock);
285 new_irq = reject;
286 goto again;
287 }
288 } else {
289 /*
290 * We failed to deliver the interrupt we need to set the
291 * resend map bit and mark the ICS state as needing a resend
292 */
293 set_bit(ics->icsid, icp->resend_map);
294 state->resend = 1;
295
296 /*
297 * If the need_resend flag got cleared in the ICP some time
298 * between icp_rm_try_to_deliver() atomic update and now, then
299 * we know it might have missed the resend_map bit. So we
300 * retry
301 */
302 smp_mb();
303 if (!icp->state.need_resend) {
304 arch_spin_unlock(&ics->lock);
305 goto again;
306 }
307 }
308 out:
309 arch_spin_unlock(&ics->lock);
310}
311
119static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 312static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
120 u8 new_cppr) 313 u8 new_cppr)
121{ 314{
@@ -184,8 +377,8 @@ static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
184 * separately here as well. 377 * separately here as well.
185 */ 378 */
186 if (resend) { 379 if (resend) {
187 icp->rm_action |= XICS_RM_CHECK_RESEND; 380 icp->n_check_resend++;
188 icp->rm_resend_icp = icp; 381 icp_rm_check_resend(xics, icp);
189 } 382 }
190} 383}
191 384
@@ -300,16 +493,16 @@ int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
300 } 493 }
301 } while (!icp_rm_try_update(icp, old_state, new_state)); 494 } while (!icp_rm_try_update(icp, old_state, new_state));
302 495
303 /* Pass rejects to virtual mode */ 496 /* Handle reject in real mode */
304 if (reject && reject != XICS_IPI) { 497 if (reject && reject != XICS_IPI) {
305 this_icp->rm_action |= XICS_RM_REJECT; 498 this_icp->n_reject++;
306 this_icp->rm_reject = reject; 499 icp_rm_deliver_irq(xics, icp, reject);
307 } 500 }
308 501
309 /* Pass resends to virtual mode */ 502 /* Handle resends in real mode */
310 if (resend) { 503 if (resend) {
311 this_icp->rm_action |= XICS_RM_CHECK_RESEND; 504 this_icp->n_check_resend++;
312 this_icp->rm_resend_icp = icp; 505 icp_rm_check_resend(xics, icp);
313 } 506 }
314 507
315 return check_too_hard(xics, this_icp); 508 return check_too_hard(xics, this_icp);
@@ -365,10 +558,13 @@ int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
365 558
366 } while (!icp_rm_try_update(icp, old_state, new_state)); 559 } while (!icp_rm_try_update(icp, old_state, new_state));
367 560
368 /* Pass rejects to virtual mode */ 561 /*
562 * Check for rejects. They are handled by doing a new delivery
563 * attempt (see comments in icp_rm_deliver_irq).
564 */
369 if (reject && reject != XICS_IPI) { 565 if (reject && reject != XICS_IPI) {
370 icp->rm_action |= XICS_RM_REJECT; 566 icp->n_reject++;
371 icp->rm_reject = reject; 567 icp_rm_deliver_irq(xics, icp, reject);
372 } 568 }
373 bail: 569 bail:
374 return check_too_hard(xics, icp); 570 return check_too_hard(xics, icp);
@@ -416,10 +612,10 @@ int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
416 goto bail; 612 goto bail;
417 state = &ics->irq_state[src]; 613 state = &ics->irq_state[src];
418 614
419 /* Still asserted, resend it, we make it look like a reject */ 615 /* Still asserted, resend it */
420 if (state->asserted) { 616 if (state->asserted) {
421 icp->rm_action |= XICS_RM_REJECT; 617 icp->n_reject++;
422 icp->rm_reject = irq; 618 icp_rm_deliver_irq(xics, icp, irq);
423 } 619 }
424 620
425 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) { 621 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) {
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 6cbf1630cb70..4d70df26c402 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -172,6 +172,22 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
172 172
173kvmppc_primary_no_guest: 173kvmppc_primary_no_guest:
174 /* We handle this much like a ceded vcpu */ 174 /* We handle this much like a ceded vcpu */
175 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
176 mfspr r3, SPRN_HDEC
177 mtspr SPRN_DEC, r3
178 /*
179 * Make sure the primary has finished the MMU switch.
180 * We should never get here on a secondary thread, but
181 * check it for robustness' sake.
182 */
183 ld r5, HSTATE_KVM_VCORE(r13)
18465: lbz r0, VCORE_IN_GUEST(r5)
185 cmpwi r0, 0
186 beq 65b
187 /* Set LPCR. */
188 ld r8,VCORE_LPCR(r5)
189 mtspr SPRN_LPCR,r8
190 isync
175 /* set our bit in napping_threads */ 191 /* set our bit in napping_threads */
176 ld r5, HSTATE_KVM_VCORE(r13) 192 ld r5, HSTATE_KVM_VCORE(r13)
177 lbz r7, HSTATE_PTID(r13) 193 lbz r7, HSTATE_PTID(r13)
@@ -182,7 +198,7 @@ kvmppc_primary_no_guest:
182 or r3, r3, r0 198 or r3, r3, r0
183 stwcx. r3, 0, r6 199 stwcx. r3, 0, r6
184 bne 1b 200 bne 1b
185 /* order napping_threads update vs testing entry_exit_count */ 201 /* order napping_threads update vs testing entry_exit_map */
186 isync 202 isync
187 li r12, 0 203 li r12, 0
188 lwz r7, VCORE_ENTRY_EXIT(r5) 204 lwz r7, VCORE_ENTRY_EXIT(r5)
@@ -191,6 +207,7 @@ kvmppc_primary_no_guest:
191 li r3, NAPPING_NOVCPU 207 li r3, NAPPING_NOVCPU
192 stb r3, HSTATE_NAPPING(r13) 208 stb r3, HSTATE_NAPPING(r13)
193 209
210 li r3, 0 /* Don't wake on privileged (OS) doorbell */
194 b kvm_do_nap 211 b kvm_do_nap
195 212
196kvm_novcpu_wakeup: 213kvm_novcpu_wakeup:
@@ -202,7 +219,7 @@ kvm_novcpu_wakeup:
202 219
203 /* check the wake reason */ 220 /* check the wake reason */
204 bl kvmppc_check_wake_reason 221 bl kvmppc_check_wake_reason
205 222
206 /* see if any other thread is already exiting */ 223 /* see if any other thread is already exiting */
207 lwz r0, VCORE_ENTRY_EXIT(r5) 224 lwz r0, VCORE_ENTRY_EXIT(r5)
208 cmpwi r0, 0x100 225 cmpwi r0, 0x100
@@ -222,13 +239,37 @@ kvm_novcpu_wakeup:
222 cmpdi r3, 0 239 cmpdi r3, 0
223 bge kvm_novcpu_exit 240 bge kvm_novcpu_exit
224 241
242 /* See if our timeslice has expired (HDEC is negative) */
243 mfspr r0, SPRN_HDEC
244 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
245 cmpwi r0, 0
246 blt kvm_novcpu_exit
247
225 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 248 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
226 ld r4, HSTATE_KVM_VCPU(r13) 249 ld r4, HSTATE_KVM_VCPU(r13)
227 cmpdi r4, 0 250 cmpdi r4, 0
228 bne kvmppc_got_guest 251 beq kvmppc_primary_no_guest
252
253#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
254 addi r3, r4, VCPU_TB_RMENTRY
255 bl kvmhv_start_timing
256#endif
257 b kvmppc_got_guest
229 258
230kvm_novcpu_exit: 259kvm_novcpu_exit:
231 b hdec_soon 260#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
261 ld r4, HSTATE_KVM_VCPU(r13)
262 cmpdi r4, 0
263 beq 13f
264 addi r3, r4, VCPU_TB_RMEXIT
265 bl kvmhv_accumulate_time
266#endif
26713: mr r3, r12
268 stw r12, 112-4(r1)
269 bl kvmhv_commence_exit
270 nop
271 lwz r12, 112-4(r1)
272 b kvmhv_switch_to_host
232 273
233/* 274/*
234 * We come in here when wakened from nap mode. 275 * We come in here when wakened from nap mode.
@@ -239,9 +280,9 @@ kvm_novcpu_exit:
239kvm_start_guest: 280kvm_start_guest:
240 281
241 /* Set runlatch bit the minute you wake up from nap */ 282 /* Set runlatch bit the minute you wake up from nap */
242 mfspr r1, SPRN_CTRLF 283 mfspr r0, SPRN_CTRLF
243 ori r1, r1, 1 284 ori r0, r0, 1
244 mtspr SPRN_CTRLT, r1 285 mtspr SPRN_CTRLT, r0
245 286
246 ld r2,PACATOC(r13) 287 ld r2,PACATOC(r13)
247 288
@@ -286,26 +327,21 @@ kvm_secondary_got_guest:
286 ld r6, PACA_DSCR(r13) 327 ld r6, PACA_DSCR(r13)
287 std r6, HSTATE_DSCR(r13) 328 std r6, HSTATE_DSCR(r13)
288 329
330 /* Order load of vcore, ptid etc. after load of vcpu */
331 lwsync
289 bl kvmppc_hv_entry 332 bl kvmppc_hv_entry
290 333
291 /* Back from the guest, go back to nap */ 334 /* Back from the guest, go back to nap */
292 /* Clear our vcpu pointer so we don't come back in early */ 335 /* Clear our vcpu pointer so we don't come back in early */
293 li r0, 0 336 li r0, 0
294 std r0, HSTATE_KVM_VCPU(r13)
295 /* 337 /*
296 * Make sure we clear HSTATE_KVM_VCPU(r13) before incrementing 338 * Once we clear HSTATE_KVM_VCPU(r13), the code in
297 * the nap_count, because once the increment to nap_count is 339 * kvmppc_run_core() is going to assume that all our vcpu
298 * visible we could be given another vcpu. 340 * state is visible in memory. This lwsync makes sure
341 * that that is true.
299 */ 342 */
300 lwsync 343 lwsync
301 344 std r0, HSTATE_KVM_VCPU(r13)
302 /* increment the nap count and then go to nap mode */
303 ld r4, HSTATE_KVM_VCORE(r13)
304 addi r4, r4, VCORE_NAP_COUNT
30551: lwarx r3, 0, r4
306 addi r3, r3, 1
307 stwcx. r3, 0, r4
308 bne 51b
309 345
310/* 346/*
311 * At this point we have finished executing in the guest. 347 * At this point we have finished executing in the guest.
@@ -376,6 +412,14 @@ kvmppc_hv_entry:
376 li r6, KVM_GUEST_MODE_HOST_HV 412 li r6, KVM_GUEST_MODE_HOST_HV
377 stb r6, HSTATE_IN_GUEST(r13) 413 stb r6, HSTATE_IN_GUEST(r13)
378 414
415#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
416 /* Store initial timestamp */
417 cmpdi r4, 0
418 beq 1f
419 addi r3, r4, VCPU_TB_RMENTRY
420 bl kvmhv_start_timing
4211:
422#endif
379 /* Clear out SLB */ 423 /* Clear out SLB */
380 li r6,0 424 li r6,0
381 slbmte r6,r6 425 slbmte r6,r6
@@ -387,21 +431,23 @@ kvmppc_hv_entry:
387 * We don't have to lock against concurrent tlbies, 431 * We don't have to lock against concurrent tlbies,
388 * but we do have to coordinate across hardware threads. 432 * but we do have to coordinate across hardware threads.
389 */ 433 */
390 /* Increment entry count iff exit count is zero. */ 434 /* Set bit in entry map iff exit map is zero. */
391 ld r5,HSTATE_KVM_VCORE(r13) 435 ld r5, HSTATE_KVM_VCORE(r13)
392 addi r9,r5,VCORE_ENTRY_EXIT 436 li r7, 1
39321: lwarx r3,0,r9 437 lbz r6, HSTATE_PTID(r13)
394 cmpwi r3,0x100 /* any threads starting to exit? */ 438 sld r7, r7, r6
439 addi r9, r5, VCORE_ENTRY_EXIT
44021: lwarx r3, 0, r9
441 cmpwi r3, 0x100 /* any threads starting to exit? */
395 bge secondary_too_late /* if so we're too late to the party */ 442 bge secondary_too_late /* if so we're too late to the party */
396 addi r3,r3,1 443 or r3, r3, r7
397 stwcx. r3,0,r9 444 stwcx. r3, 0, r9
398 bne 21b 445 bne 21b
399 446
400 /* Primary thread switches to guest partition. */ 447 /* Primary thread switches to guest partition. */
401 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */ 448 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
402 lbz r6,HSTATE_PTID(r13)
403 cmpwi r6,0 449 cmpwi r6,0
404 bne 20f 450 bne 10f
405 ld r6,KVM_SDR1(r9) 451 ld r6,KVM_SDR1(r9)
406 lwz r7,KVM_LPID(r9) 452 lwz r7,KVM_LPID(r9)
407 li r0,LPID_RSVD /* switch to reserved LPID */ 453 li r0,LPID_RSVD /* switch to reserved LPID */
@@ -472,28 +518,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
472 518
473 li r0,1 519 li r0,1
474 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 520 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
475 b 10f
476
477 /* Secondary threads wait for primary to have done partition switch */
47820: lbz r0,VCORE_IN_GUEST(r5)
479 cmpwi r0,0
480 beq 20b
481
482 /* Set LPCR and RMOR. */
48310: ld r8,VCORE_LPCR(r5)
484 mtspr SPRN_LPCR,r8
485 ld r8,KVM_RMOR(r9)
486 mtspr SPRN_RMOR,r8
487 isync
488
489 /* Check if HDEC expires soon */
490 mfspr r3,SPRN_HDEC
491 cmpwi r3,512 /* 1 microsecond */
492 li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
493 blt hdec_soon
494 521
495 /* Do we have a guest vcpu to run? */ 522 /* Do we have a guest vcpu to run? */
496 cmpdi r4, 0 52310: cmpdi r4, 0
497 beq kvmppc_primary_no_guest 524 beq kvmppc_primary_no_guest
498kvmppc_got_guest: 525kvmppc_got_guest:
499 526
@@ -818,6 +845,30 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
818 clrrdi r6,r6,1 845 clrrdi r6,r6,1
819 mtspr SPRN_CTRLT,r6 846 mtspr SPRN_CTRLT,r6
8204: 8474:
848 /* Secondary threads wait for primary to have done partition switch */
849 ld r5, HSTATE_KVM_VCORE(r13)
850 lbz r6, HSTATE_PTID(r13)
851 cmpwi r6, 0
852 beq 21f
853 lbz r0, VCORE_IN_GUEST(r5)
854 cmpwi r0, 0
855 bne 21f
856 HMT_LOW
85720: lbz r0, VCORE_IN_GUEST(r5)
858 cmpwi r0, 0
859 beq 20b
860 HMT_MEDIUM
86121:
862 /* Set LPCR. */
863 ld r8,VCORE_LPCR(r5)
864 mtspr SPRN_LPCR,r8
865 isync
866
867 /* Check if HDEC expires soon */
868 mfspr r3, SPRN_HDEC
869 cmpwi r3, 512 /* 1 microsecond */
870 blt hdec_soon
871
821 ld r6, VCPU_CTR(r4) 872 ld r6, VCPU_CTR(r4)
822 lwz r7, VCPU_XER(r4) 873 lwz r7, VCPU_XER(r4)
823 874
@@ -880,6 +931,12 @@ fast_guest_return:
880 li r9, KVM_GUEST_MODE_GUEST_HV 931 li r9, KVM_GUEST_MODE_GUEST_HV
881 stb r9, HSTATE_IN_GUEST(r13) 932 stb r9, HSTATE_IN_GUEST(r13)
882 933
934#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
935 /* Accumulate timing */
936 addi r3, r4, VCPU_TB_GUEST
937 bl kvmhv_accumulate_time
938#endif
939
883 /* Enter guest */ 940 /* Enter guest */
884 941
885BEGIN_FTR_SECTION 942BEGIN_FTR_SECTION
@@ -917,6 +974,27 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
917 hrfid 974 hrfid
918 b . 975 b .
919 976
977secondary_too_late:
978 li r12, 0
979 cmpdi r4, 0
980 beq 11f
981 stw r12, VCPU_TRAP(r4)
982#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
983 addi r3, r4, VCPU_TB_RMEXIT
984 bl kvmhv_accumulate_time
985#endif
98611: b kvmhv_switch_to_host
987
988hdec_soon:
989 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
990 stw r12, VCPU_TRAP(r4)
991 mr r9, r4
992#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
993 addi r3, r4, VCPU_TB_RMEXIT
994 bl kvmhv_accumulate_time
995#endif
996 b guest_exit_cont
997
920/****************************************************************************** 998/******************************************************************************
921 * * 999 * *
922 * Exit code * 1000 * Exit code *
@@ -1002,6 +1080,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1002 1080
1003 stw r12,VCPU_TRAP(r9) 1081 stw r12,VCPU_TRAP(r9)
1004 1082
1083#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1084 addi r3, r9, VCPU_TB_RMINTR
1085 mr r4, r9
1086 bl kvmhv_accumulate_time
1087 ld r5, VCPU_GPR(R5)(r9)
1088 ld r6, VCPU_GPR(R6)(r9)
1089 ld r7, VCPU_GPR(R7)(r9)
1090 ld r8, VCPU_GPR(R8)(r9)
1091#endif
1092
1005 /* Save HEIR (HV emulation assist reg) in emul_inst 1093 /* Save HEIR (HV emulation assist reg) in emul_inst
1006 if this is an HEI (HV emulation interrupt, e40) */ 1094 if this is an HEI (HV emulation interrupt, e40) */
1007 li r3,KVM_INST_FETCH_FAILED 1095 li r3,KVM_INST_FETCH_FAILED
@@ -1028,34 +1116,37 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1028 bne 2f 1116 bne 2f
1029 mfspr r3,SPRN_HDEC 1117 mfspr r3,SPRN_HDEC
1030 cmpwi r3,0 1118 cmpwi r3,0
1031 bge ignore_hdec 1119 mr r4,r9
1120 bge fast_guest_return
10322: 11212:
1033 /* See if this is an hcall we can handle in real mode */ 1122 /* See if this is an hcall we can handle in real mode */
1034 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1123 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1035 beq hcall_try_real_mode 1124 beq hcall_try_real_mode
1036 1125
1126 /* Hypervisor doorbell - exit only if host IPI flag set */
1127 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1128 bne 3f
1129 lbz r0, HSTATE_HOST_IPI(r13)
1130 beq 4f
1131 b guest_exit_cont
11323:
1037 /* External interrupt ? */ 1133 /* External interrupt ? */
1038 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1134 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1039 bne+ ext_interrupt_to_host 1135 bne+ guest_exit_cont
1040 1136
1041 /* External interrupt, first check for host_ipi. If this is 1137 /* External interrupt, first check for host_ipi. If this is
1042 * set, we know the host wants us out so let's do it now 1138 * set, we know the host wants us out so let's do it now
1043 */ 1139 */
1044 bl kvmppc_read_intr 1140 bl kvmppc_read_intr
1045 cmpdi r3, 0 1141 cmpdi r3, 0
1046 bgt ext_interrupt_to_host 1142 bgt guest_exit_cont
1047 1143
1048 /* Check if any CPU is heading out to the host, if so head out too */ 1144 /* Check if any CPU is heading out to the host, if so head out too */
1049 ld r5, HSTATE_KVM_VCORE(r13) 11454: ld r5, HSTATE_KVM_VCORE(r13)
1050 lwz r0, VCORE_ENTRY_EXIT(r5) 1146 lwz r0, VCORE_ENTRY_EXIT(r5)
1051 cmpwi r0, 0x100 1147 cmpwi r0, 0x100
1052 bge ext_interrupt_to_host
1053
1054 /* Return to guest after delivering any pending interrupt */
1055 mr r4, r9 1148 mr r4, r9
1056 b deliver_guest_interrupt 1149 blt deliver_guest_interrupt
1057
1058ext_interrupt_to_host:
1059 1150
1060guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1151guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1061 /* Save more register state */ 1152 /* Save more register state */
@@ -1065,7 +1156,7 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1065 stw r7, VCPU_DSISR(r9) 1156 stw r7, VCPU_DSISR(r9)
1066 /* don't overwrite fault_dar/fault_dsisr if HDSI */ 1157 /* don't overwrite fault_dar/fault_dsisr if HDSI */
1067 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE 1158 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1068 beq 6f 1159 beq mc_cont
1069 std r6, VCPU_FAULT_DAR(r9) 1160 std r6, VCPU_FAULT_DAR(r9)
1070 stw r7, VCPU_FAULT_DSISR(r9) 1161 stw r7, VCPU_FAULT_DSISR(r9)
1071 1162
@@ -1073,9 +1164,20 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1073 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1164 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1074 beq machine_check_realmode 1165 beq machine_check_realmode
1075mc_cont: 1166mc_cont:
1167#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1168 addi r3, r9, VCPU_TB_RMEXIT
1169 mr r4, r9
1170 bl kvmhv_accumulate_time
1171#endif
1172
1173 /* Increment exit count, poke other threads to exit */
1174 bl kvmhv_commence_exit
1175 nop
1176 ld r9, HSTATE_KVM_VCPU(r13)
1177 lwz r12, VCPU_TRAP(r9)
1076 1178
1077 /* Save guest CTRL register, set runlatch to 1 */ 1179 /* Save guest CTRL register, set runlatch to 1 */
10786: mfspr r6,SPRN_CTRLF 1180 mfspr r6,SPRN_CTRLF
1079 stw r6,VCPU_CTRL(r9) 1181 stw r6,VCPU_CTRL(r9)
1080 andi. r0,r6,1 1182 andi. r0,r6,1
1081 bne 4f 1183 bne 4f
@@ -1417,68 +1519,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1417 slbia 1519 slbia
1418 ptesync 1520 ptesync
1419 1521
1420hdec_soon: /* r12 = trap, r13 = paca */
1421 /* 1522 /*
1422 * POWER7/POWER8 guest -> host partition switch code. 1523 * POWER7/POWER8 guest -> host partition switch code.
1423 * We don't have to lock against tlbies but we do 1524 * We don't have to lock against tlbies but we do
1424 * have to coordinate the hardware threads. 1525 * have to coordinate the hardware threads.
1425 */ 1526 */
1426 /* Increment the threads-exiting-guest count in the 0xff00 1527kvmhv_switch_to_host:
1427 bits of vcore->entry_exit_count */
1428 ld r5,HSTATE_KVM_VCORE(r13)
1429 addi r6,r5,VCORE_ENTRY_EXIT
143041: lwarx r3,0,r6
1431 addi r0,r3,0x100
1432 stwcx. r0,0,r6
1433 bne 41b
1434 isync /* order stwcx. vs. reading napping_threads */
1435
1436 /*
1437 * At this point we have an interrupt that we have to pass
1438 * up to the kernel or qemu; we can't handle it in real mode.
1439 * Thus we have to do a partition switch, so we have to
1440 * collect the other threads, if we are the first thread
1441 * to take an interrupt. To do this, we set the HDEC to 0,
1442 * which causes an HDEC interrupt in all threads within 2ns
1443 * because the HDEC register is shared between all 4 threads.
1444 * However, we don't need to bother if this is an HDEC
1445 * interrupt, since the other threads will already be on their
1446 * way here in that case.
1447 */
1448 cmpwi r3,0x100 /* Are we the first here? */
1449 bge 43f
1450 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1451 beq 40f
1452 li r0,0
1453 mtspr SPRN_HDEC,r0
145440:
1455 /*
1456 * Send an IPI to any napping threads, since an HDEC interrupt
1457 * doesn't wake CPUs up from nap.
1458 */
1459 lwz r3,VCORE_NAPPING_THREADS(r5)
1460 lbz r4,HSTATE_PTID(r13)
1461 li r0,1
1462 sld r0,r0,r4
1463 andc. r3,r3,r0 /* no sense IPI'ing ourselves */
1464 beq 43f
1465 /* Order entry/exit update vs. IPIs */
1466 sync
1467 mulli r4,r4,PACA_SIZE /* get paca for thread 0 */
1468 subf r6,r4,r13
146942: andi. r0,r3,1
1470 beq 44f
1471 ld r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */
1472 li r0,IPI_PRIORITY
1473 li r7,XICS_MFRR
1474 stbcix r0,r7,r8 /* trigger the IPI */
147544: srdi. r3,r3,1
1476 addi r6,r6,PACA_SIZE
1477 bne 42b
1478
1479secondary_too_late:
1480 /* Secondary threads wait for primary to do partition switch */ 1528 /* Secondary threads wait for primary to do partition switch */
148143: ld r5,HSTATE_KVM_VCORE(r13) 1529 ld r5,HSTATE_KVM_VCORE(r13)
1482 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1530 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1483 lbz r3,HSTATE_PTID(r13) 1531 lbz r3,HSTATE_PTID(r13)
1484 cmpwi r3,0 1532 cmpwi r3,0
@@ -1562,6 +1610,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
15621: addi r8,r8,16 16101: addi r8,r8,16
1563 .endr 1611 .endr
1564 1612
1613#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1614 /* Finish timing, if we have a vcpu */
1615 ld r4, HSTATE_KVM_VCPU(r13)
1616 cmpdi r4, 0
1617 li r3, 0
1618 beq 2f
1619 bl kvmhv_accumulate_time
16202:
1621#endif
1565 /* Unset guest mode */ 1622 /* Unset guest mode */
1566 li r0, KVM_GUEST_MODE_NONE 1623 li r0, KVM_GUEST_MODE_NONE
1567 stb r0, HSTATE_IN_GUEST(r13) 1624 stb r0, HSTATE_IN_GUEST(r13)
@@ -1696,8 +1753,10 @@ kvmppc_hisi:
1696 * Returns to the guest if we handle it, or continues on up to 1753 * Returns to the guest if we handle it, or continues on up to
1697 * the kernel if we can't (i.e. if we don't have a handler for 1754 * the kernel if we can't (i.e. if we don't have a handler for
1698 * it, or if the handler returns H_TOO_HARD). 1755 * it, or if the handler returns H_TOO_HARD).
1756 *
1757 * r5 - r8 contain hcall args,
1758 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1699 */ 1759 */
1700 .globl hcall_try_real_mode
1701hcall_try_real_mode: 1760hcall_try_real_mode:
1702 ld r3,VCPU_GPR(R3)(r9) 1761 ld r3,VCPU_GPR(R3)(r9)
1703 andi. r0,r11,MSR_PR 1762 andi. r0,r11,MSR_PR
@@ -1839,13 +1898,124 @@ hcall_real_table:
1839 .long 0 /* 0x12c */ 1898 .long 0 /* 0x12c */
1840 .long 0 /* 0x130 */ 1899 .long 0 /* 0x130 */
1841 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 1900 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1901 .long 0 /* 0x138 */
1902 .long 0 /* 0x13c */
1903 .long 0 /* 0x140 */
1904 .long 0 /* 0x144 */
1905 .long 0 /* 0x148 */
1906 .long 0 /* 0x14c */
1907 .long 0 /* 0x150 */
1908 .long 0 /* 0x154 */
1909 .long 0 /* 0x158 */
1910 .long 0 /* 0x15c */
1911 .long 0 /* 0x160 */
1912 .long 0 /* 0x164 */
1913 .long 0 /* 0x168 */
1914 .long 0 /* 0x16c */
1915 .long 0 /* 0x170 */
1916 .long 0 /* 0x174 */
1917 .long 0 /* 0x178 */
1918 .long 0 /* 0x17c */
1919 .long 0 /* 0x180 */
1920 .long 0 /* 0x184 */
1921 .long 0 /* 0x188 */
1922 .long 0 /* 0x18c */
1923 .long 0 /* 0x190 */
1924 .long 0 /* 0x194 */
1925 .long 0 /* 0x198 */
1926 .long 0 /* 0x19c */
1927 .long 0 /* 0x1a0 */
1928 .long 0 /* 0x1a4 */
1929 .long 0 /* 0x1a8 */
1930 .long 0 /* 0x1ac */
1931 .long 0 /* 0x1b0 */
1932 .long 0 /* 0x1b4 */
1933 .long 0 /* 0x1b8 */
1934 .long 0 /* 0x1bc */
1935 .long 0 /* 0x1c0 */
1936 .long 0 /* 0x1c4 */
1937 .long 0 /* 0x1c8 */
1938 .long 0 /* 0x1cc */
1939 .long 0 /* 0x1d0 */
1940 .long 0 /* 0x1d4 */
1941 .long 0 /* 0x1d8 */
1942 .long 0 /* 0x1dc */
1943 .long 0 /* 0x1e0 */
1944 .long 0 /* 0x1e4 */
1945 .long 0 /* 0x1e8 */
1946 .long 0 /* 0x1ec */
1947 .long 0 /* 0x1f0 */
1948 .long 0 /* 0x1f4 */
1949 .long 0 /* 0x1f8 */
1950 .long 0 /* 0x1fc */
1951 .long 0 /* 0x200 */
1952 .long 0 /* 0x204 */
1953 .long 0 /* 0x208 */
1954 .long 0 /* 0x20c */
1955 .long 0 /* 0x210 */
1956 .long 0 /* 0x214 */
1957 .long 0 /* 0x218 */
1958 .long 0 /* 0x21c */
1959 .long 0 /* 0x220 */
1960 .long 0 /* 0x224 */
1961 .long 0 /* 0x228 */
1962 .long 0 /* 0x22c */
1963 .long 0 /* 0x230 */
1964 .long 0 /* 0x234 */
1965 .long 0 /* 0x238 */
1966 .long 0 /* 0x23c */
1967 .long 0 /* 0x240 */
1968 .long 0 /* 0x244 */
1969 .long 0 /* 0x248 */
1970 .long 0 /* 0x24c */
1971 .long 0 /* 0x250 */
1972 .long 0 /* 0x254 */
1973 .long 0 /* 0x258 */
1974 .long 0 /* 0x25c */
1975 .long 0 /* 0x260 */
1976 .long 0 /* 0x264 */
1977 .long 0 /* 0x268 */
1978 .long 0 /* 0x26c */
1979 .long 0 /* 0x270 */
1980 .long 0 /* 0x274 */
1981 .long 0 /* 0x278 */
1982 .long 0 /* 0x27c */
1983 .long 0 /* 0x280 */
1984 .long 0 /* 0x284 */
1985 .long 0 /* 0x288 */
1986 .long 0 /* 0x28c */
1987 .long 0 /* 0x290 */
1988 .long 0 /* 0x294 */
1989 .long 0 /* 0x298 */
1990 .long 0 /* 0x29c */
1991 .long 0 /* 0x2a0 */
1992 .long 0 /* 0x2a4 */
1993 .long 0 /* 0x2a8 */
1994 .long 0 /* 0x2ac */
1995 .long 0 /* 0x2b0 */
1996 .long 0 /* 0x2b4 */
1997 .long 0 /* 0x2b8 */
1998 .long 0 /* 0x2bc */
1999 .long 0 /* 0x2c0 */
2000 .long 0 /* 0x2c4 */
2001 .long 0 /* 0x2c8 */
2002 .long 0 /* 0x2cc */
2003 .long 0 /* 0x2d0 */
2004 .long 0 /* 0x2d4 */
2005 .long 0 /* 0x2d8 */
2006 .long 0 /* 0x2dc */
2007 .long 0 /* 0x2e0 */
2008 .long 0 /* 0x2e4 */
2009 .long 0 /* 0x2e8 */
2010 .long 0 /* 0x2ec */
2011 .long 0 /* 0x2f0 */
2012 .long 0 /* 0x2f4 */
2013 .long 0 /* 0x2f8 */
2014 .long 0 /* 0x2fc */
2015 .long DOTSYM(kvmppc_h_random) - hcall_real_table
1842 .globl hcall_real_table_end 2016 .globl hcall_real_table_end
1843hcall_real_table_end: 2017hcall_real_table_end:
1844 2018
1845ignore_hdec:
1846 mr r4,r9
1847 b fast_guest_return
1848
1849_GLOBAL(kvmppc_h_set_xdabr) 2019_GLOBAL(kvmppc_h_set_xdabr)
1850 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2020 andi. r0, r5, DABRX_USER | DABRX_KERNEL
1851 beq 6f 2021 beq 6f
@@ -1884,7 +2054,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1884 li r3, 0 2054 li r3, 0
1885 blr 2055 blr
1886 2056
1887_GLOBAL(kvmppc_h_cede) 2057_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
1888 ori r11,r11,MSR_EE 2058 ori r11,r11,MSR_EE
1889 std r11,VCPU_MSR(r3) 2059 std r11,VCPU_MSR(r3)
1890 li r0,1 2060 li r0,1
@@ -1893,8 +2063,8 @@ _GLOBAL(kvmppc_h_cede)
1893 lbz r5,VCPU_PRODDED(r3) 2063 lbz r5,VCPU_PRODDED(r3)
1894 cmpwi r5,0 2064 cmpwi r5,0
1895 bne kvm_cede_prodded 2065 bne kvm_cede_prodded
1896 li r0,0 /* set trap to 0 to say hcall is handled */ 2066 li r12,0 /* set trap to 0 to say hcall is handled */
1897 stw r0,VCPU_TRAP(r3) 2067 stw r12,VCPU_TRAP(r3)
1898 li r0,H_SUCCESS 2068 li r0,H_SUCCESS
1899 std r0,VCPU_GPR(R3)(r3) 2069 std r0,VCPU_GPR(R3)(r3)
1900 2070
@@ -1912,12 +2082,11 @@ _GLOBAL(kvmppc_h_cede)
1912 addi r6,r5,VCORE_NAPPING_THREADS 2082 addi r6,r5,VCORE_NAPPING_THREADS
191331: lwarx r4,0,r6 208331: lwarx r4,0,r6
1914 or r4,r4,r0 2084 or r4,r4,r0
1915 PPC_POPCNTW(R7,R4) 2085 cmpw r4,r8
1916 cmpw r7,r8 2086 beq kvm_cede_exit
1917 bge kvm_cede_exit
1918 stwcx. r4,0,r6 2087 stwcx. r4,0,r6
1919 bne 31b 2088 bne 31b
1920 /* order napping_threads update vs testing entry_exit_count */ 2089 /* order napping_threads update vs testing entry_exit_map */
1921 isync 2090 isync
1922 li r0,NAPPING_CEDE 2091 li r0,NAPPING_CEDE
1923 stb r0,HSTATE_NAPPING(r13) 2092 stb r0,HSTATE_NAPPING(r13)
@@ -1955,21 +2124,52 @@ _GLOBAL(kvmppc_h_cede)
1955 bl kvmppc_save_fp 2124 bl kvmppc_save_fp
1956 2125
1957 /* 2126 /*
2127 * Set DEC to the smaller of DEC and HDEC, so that we wake
2128 * no later than the end of our timeslice (HDEC interrupts
2129 * don't wake us from nap).
2130 */
2131 mfspr r3, SPRN_DEC
2132 mfspr r4, SPRN_HDEC
2133 mftb r5
2134 cmpw r3, r4
2135 ble 67f
2136 mtspr SPRN_DEC, r4
213767:
2138 /* save expiry time of guest decrementer */
2139 extsw r3, r3
2140 add r3, r3, r5
2141 ld r4, HSTATE_KVM_VCPU(r13)
2142 ld r5, HSTATE_KVM_VCORE(r13)
2143 ld r6, VCORE_TB_OFFSET(r5)
2144 subf r3, r6, r3 /* convert to host TB value */
2145 std r3, VCPU_DEC_EXPIRES(r4)
2146
2147#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2148 ld r4, HSTATE_KVM_VCPU(r13)
2149 addi r3, r4, VCPU_TB_CEDE
2150 bl kvmhv_accumulate_time
2151#endif
2152
2153 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2154
2155 /*
1958 * Take a nap until a decrementer or external or doobell interrupt 2156 * Take a nap until a decrementer or external or doobell interrupt
1959 * occurs, with PECE1, PECE0 and PECEDP set in LPCR. Also clear the 2157 * occurs, with PECE1 and PECE0 set in LPCR.
1960 * runlatch bit before napping. 2158 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2159 * Also clear the runlatch bit before napping.
1961 */ 2160 */
1962kvm_do_nap: 2161kvm_do_nap:
1963 mfspr r2, SPRN_CTRLF 2162 mfspr r0, SPRN_CTRLF
1964 clrrdi r2, r2, 1 2163 clrrdi r0, r0, 1
1965 mtspr SPRN_CTRLT, r2 2164 mtspr SPRN_CTRLT, r0
1966 2165
1967 li r0,1 2166 li r0,1
1968 stb r0,HSTATE_HWTHREAD_REQ(r13) 2167 stb r0,HSTATE_HWTHREAD_REQ(r13)
1969 mfspr r5,SPRN_LPCR 2168 mfspr r5,SPRN_LPCR
1970 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2169 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
1971BEGIN_FTR_SECTION 2170BEGIN_FTR_SECTION
1972 oris r5,r5,LPCR_PECEDP@h 2171 ori r5, r5, LPCR_PECEDH
2172 rlwimi r5, r3, 0, LPCR_PECEDP
1973END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2173END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1974 mtspr SPRN_LPCR,r5 2174 mtspr SPRN_LPCR,r5
1975 isync 2175 isync
@@ -1994,9 +2194,23 @@ kvm_end_cede:
1994 /* Woken by external or decrementer interrupt */ 2194 /* Woken by external or decrementer interrupt */
1995 ld r1, HSTATE_HOST_R1(r13) 2195 ld r1, HSTATE_HOST_R1(r13)
1996 2196
2197#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2198 addi r3, r4, VCPU_TB_RMINTR
2199 bl kvmhv_accumulate_time
2200#endif
2201
1997 /* load up FP state */ 2202 /* load up FP state */
1998 bl kvmppc_load_fp 2203 bl kvmppc_load_fp
1999 2204
2205 /* Restore guest decrementer */
2206 ld r3, VCPU_DEC_EXPIRES(r4)
2207 ld r5, HSTATE_KVM_VCORE(r13)
2208 ld r6, VCORE_TB_OFFSET(r5)
2209 add r3, r3, r6 /* convert host TB to guest TB value */
2210 mftb r7
2211 subf r3, r7, r3
2212 mtspr SPRN_DEC, r3
2213
2000 /* Load NV GPRS */ 2214 /* Load NV GPRS */
2001 ld r14, VCPU_GPR(R14)(r4) 2215 ld r14, VCPU_GPR(R14)(r4)
2002 ld r15, VCPU_GPR(R15)(r4) 2216 ld r15, VCPU_GPR(R15)(r4)
@@ -2057,7 +2271,8 @@ kvm_cede_prodded:
2057 2271
2058 /* we've ceded but we want to give control to the host */ 2272 /* we've ceded but we want to give control to the host */
2059kvm_cede_exit: 2273kvm_cede_exit:
2060 b hcall_real_fallback 2274 ld r9, HSTATE_KVM_VCPU(r13)
2275 b guest_exit_cont
2061 2276
2062 /* Try to handle a machine check in real mode */ 2277 /* Try to handle a machine check in real mode */
2063machine_check_realmode: 2278machine_check_realmode:
@@ -2089,13 +2304,14 @@ machine_check_realmode:
2089 2304
2090/* 2305/*
2091 * Check the reason we woke from nap, and take appropriate action. 2306 * Check the reason we woke from nap, and take appropriate action.
2092 * Returns: 2307 * Returns (in r3):
2093 * 0 if nothing needs to be done 2308 * 0 if nothing needs to be done
2094 * 1 if something happened that needs to be handled by the host 2309 * 1 if something happened that needs to be handled by the host
2095 * -1 if there was a guest wakeup (IPI) 2310 * -1 if there was a guest wakeup (IPI or msgsnd)
2096 * 2311 *
2097 * Also sets r12 to the interrupt vector for any interrupt that needs 2312 * Also sets r12 to the interrupt vector for any interrupt that needs
2098 * to be handled now by the host (0x500 for external interrupt), or zero. 2313 * to be handled now by the host (0x500 for external interrupt), or zero.
2314 * Modifies r0, r6, r7, r8.
2099 */ 2315 */
2100kvmppc_check_wake_reason: 2316kvmppc_check_wake_reason:
2101 mfspr r6, SPRN_SRR1 2317 mfspr r6, SPRN_SRR1
@@ -2122,7 +2338,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2122 2338
2123 /* hypervisor doorbell */ 2339 /* hypervisor doorbell */
21243: li r12, BOOK3S_INTERRUPT_H_DOORBELL 23403: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2341 /* see if it's a host IPI */
2125 li r3, 1 2342 li r3, 1
2343 lbz r0, HSTATE_HOST_IPI(r13)
2344 cmpwi r0, 0
2345 bnelr
2346 /* if not, clear it and return -1 */
2347 lis r6, (PPC_DBELL_SERVER << (63-36))@h
2348 PPC_MSGCLR(6)
2349 li r3, -1
2126 blr 2350 blr
2127 2351
2128/* 2352/*
@@ -2131,6 +2355,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2131 * 0 if no interrupt is pending 2355 * 0 if no interrupt is pending
2132 * 1 if an interrupt is pending that needs to be handled by the host 2356 * 1 if an interrupt is pending that needs to be handled by the host
2133 * -1 if there was a guest wakeup IPI (which has now been cleared) 2357 * -1 if there was a guest wakeup IPI (which has now been cleared)
2358 * Modifies r0, r6, r7, r8, returns value in r3.
2134 */ 2359 */
2135kvmppc_read_intr: 2360kvmppc_read_intr:
2136 /* see if a host IPI is pending */ 2361 /* see if a host IPI is pending */
@@ -2185,6 +2410,7 @@ kvmppc_read_intr:
2185 bne- 43f 2410 bne- 43f
2186 2411
2187 /* OK, it's an IPI for us */ 2412 /* OK, it's an IPI for us */
2413 li r12, 0
2188 li r3, -1 2414 li r3, -1
21891: blr 24151: blr
2190 2416
@@ -2314,3 +2540,62 @@ kvmppc_fix_pmao:
2314 mtspr SPRN_PMC6, r3 2540 mtspr SPRN_PMC6, r3
2315 isync 2541 isync
2316 blr 2542 blr
2543
2544#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2545/*
2546 * Start timing an activity
2547 * r3 = pointer to time accumulation struct, r4 = vcpu
2548 */
2549kvmhv_start_timing:
2550 ld r5, HSTATE_KVM_VCORE(r13)
2551 lbz r6, VCORE_IN_GUEST(r5)
2552 cmpwi r6, 0
2553 beq 5f /* if in guest, need to */
2554 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
25555: mftb r5
2556 subf r5, r6, r5
2557 std r3, VCPU_CUR_ACTIVITY(r4)
2558 std r5, VCPU_ACTIVITY_START(r4)
2559 blr
2560
2561/*
2562 * Accumulate time to one activity and start another.
2563 * r3 = pointer to new time accumulation struct, r4 = vcpu
2564 */
2565kvmhv_accumulate_time:
2566 ld r5, HSTATE_KVM_VCORE(r13)
2567 lbz r8, VCORE_IN_GUEST(r5)
2568 cmpwi r8, 0
2569 beq 4f /* if in guest, need to */
2570 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
25714: ld r5, VCPU_CUR_ACTIVITY(r4)
2572 ld r6, VCPU_ACTIVITY_START(r4)
2573 std r3, VCPU_CUR_ACTIVITY(r4)
2574 mftb r7
2575 subf r7, r8, r7
2576 std r7, VCPU_ACTIVITY_START(r4)
2577 cmpdi r5, 0
2578 beqlr
2579 subf r3, r6, r7
2580 ld r8, TAS_SEQCOUNT(r5)
2581 cmpdi r8, 0
2582 addi r8, r8, 1
2583 std r8, TAS_SEQCOUNT(r5)
2584 lwsync
2585 ld r7, TAS_TOTAL(r5)
2586 add r7, r7, r3
2587 std r7, TAS_TOTAL(r5)
2588 ld r6, TAS_MIN(r5)
2589 ld r7, TAS_MAX(r5)
2590 beq 3f
2591 cmpd r3, r6
2592 bge 1f
25933: std r3, TAS_MIN(r5)
25941: cmpd r3, r7
2595 ble 2f
2596 std r3, TAS_MAX(r5)
25972: lwsync
2598 addi r8, r8, 1
2599 std r8, TAS_SEQCOUNT(r5)
2600 blr
2601#endif
diff --git a/arch/powerpc/kvm/book3s_pr_papr.c b/arch/powerpc/kvm/book3s_pr_papr.c
index ce3c893d509b..f2c75a1e0536 100644
--- a/arch/powerpc/kvm/book3s_pr_papr.c
+++ b/arch/powerpc/kvm/book3s_pr_papr.c
@@ -258,6 +258,28 @@ static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu)
258 return EMULATE_DONE; 258 return EMULATE_DONE;
259} 259}
260 260
261static int kvmppc_h_pr_logical_ci_load(struct kvm_vcpu *vcpu)
262{
263 long rc;
264
265 rc = kvmppc_h_logical_ci_load(vcpu);
266 if (rc == H_TOO_HARD)
267 return EMULATE_FAIL;
268 kvmppc_set_gpr(vcpu, 3, rc);
269 return EMULATE_DONE;
270}
271
272static int kvmppc_h_pr_logical_ci_store(struct kvm_vcpu *vcpu)
273{
274 long rc;
275
276 rc = kvmppc_h_logical_ci_store(vcpu);
277 if (rc == H_TOO_HARD)
278 return EMULATE_FAIL;
279 kvmppc_set_gpr(vcpu, 3, rc);
280 return EMULATE_DONE;
281}
282
261static int kvmppc_h_pr_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd) 283static int kvmppc_h_pr_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd)
262{ 284{
263 long rc = kvmppc_xics_hcall(vcpu, cmd); 285 long rc = kvmppc_xics_hcall(vcpu, cmd);
@@ -290,6 +312,10 @@ int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd)
290 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 312 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
291 vcpu->stat.halt_wakeup++; 313 vcpu->stat.halt_wakeup++;
292 return EMULATE_DONE; 314 return EMULATE_DONE;
315 case H_LOGICAL_CI_LOAD:
316 return kvmppc_h_pr_logical_ci_load(vcpu);
317 case H_LOGICAL_CI_STORE:
318 return kvmppc_h_pr_logical_ci_store(vcpu);
293 case H_XIRR: 319 case H_XIRR:
294 case H_CPPR: 320 case H_CPPR:
295 case H_EOI: 321 case H_EOI:
@@ -323,6 +349,8 @@ int kvmppc_hcall_impl_pr(unsigned long cmd)
323 case H_BULK_REMOVE: 349 case H_BULK_REMOVE:
324 case H_PUT_TCE: 350 case H_PUT_TCE:
325 case H_CEDE: 351 case H_CEDE:
352 case H_LOGICAL_CI_LOAD:
353 case H_LOGICAL_CI_STORE:
326#ifdef CONFIG_KVM_XICS 354#ifdef CONFIG_KVM_XICS
327 case H_XIRR: 355 case H_XIRR:
328 case H_CPPR: 356 case H_CPPR:
diff --git a/arch/powerpc/kvm/book3s_xics.c b/arch/powerpc/kvm/book3s_xics.c
index a4a8d9f0dcb7..8f3e6cc54d95 100644
--- a/arch/powerpc/kvm/book3s_xics.c
+++ b/arch/powerpc/kvm/book3s_xics.c
@@ -20,6 +20,7 @@
20#include <asm/xics.h> 20#include <asm/xics.h>
21#include <asm/debug.h> 21#include <asm/debug.h>
22#include <asm/time.h> 22#include <asm/time.h>
23#include <asm/spinlock.h>
23 24
24#include <linux/debugfs.h> 25#include <linux/debugfs.h>
25#include <linux/seq_file.h> 26#include <linux/seq_file.h>
@@ -39,7 +40,7 @@
39 * LOCKING 40 * LOCKING
40 * ======= 41 * =======
41 * 42 *
42 * Each ICS has a mutex protecting the information about the IRQ 43 * Each ICS has a spin lock protecting the information about the IRQ
43 * sources and avoiding simultaneous deliveries if the same interrupt. 44 * sources and avoiding simultaneous deliveries if the same interrupt.
44 * 45 *
45 * ICP operations are done via a single compare & swap transaction 46 * ICP operations are done via a single compare & swap transaction
@@ -109,7 +110,10 @@ static void ics_check_resend(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
109{ 110{
110 int i; 111 int i;
111 112
112 mutex_lock(&ics->lock); 113 unsigned long flags;
114
115 local_irq_save(flags);
116 arch_spin_lock(&ics->lock);
113 117
114 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 118 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
115 struct ics_irq_state *state = &ics->irq_state[i]; 119 struct ics_irq_state *state = &ics->irq_state[i];
@@ -120,12 +124,15 @@ static void ics_check_resend(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
120 XICS_DBG("resend %#x prio %#x\n", state->number, 124 XICS_DBG("resend %#x prio %#x\n", state->number,
121 state->priority); 125 state->priority);
122 126
123 mutex_unlock(&ics->lock); 127 arch_spin_unlock(&ics->lock);
128 local_irq_restore(flags);
124 icp_deliver_irq(xics, icp, state->number); 129 icp_deliver_irq(xics, icp, state->number);
125 mutex_lock(&ics->lock); 130 local_irq_save(flags);
131 arch_spin_lock(&ics->lock);
126 } 132 }
127 133
128 mutex_unlock(&ics->lock); 134 arch_spin_unlock(&ics->lock);
135 local_irq_restore(flags);
129} 136}
130 137
131static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics, 138static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
@@ -133,8 +140,10 @@ static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
133 u32 server, u32 priority, u32 saved_priority) 140 u32 server, u32 priority, u32 saved_priority)
134{ 141{
135 bool deliver; 142 bool deliver;
143 unsigned long flags;
136 144
137 mutex_lock(&ics->lock); 145 local_irq_save(flags);
146 arch_spin_lock(&ics->lock);
138 147
139 state->server = server; 148 state->server = server;
140 state->priority = priority; 149 state->priority = priority;
@@ -145,7 +154,8 @@ static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
145 deliver = true; 154 deliver = true;
146 } 155 }
147 156
148 mutex_unlock(&ics->lock); 157 arch_spin_unlock(&ics->lock);
158 local_irq_restore(flags);
149 159
150 return deliver; 160 return deliver;
151} 161}
@@ -186,6 +196,7 @@ int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority)
186 struct kvmppc_ics *ics; 196 struct kvmppc_ics *ics;
187 struct ics_irq_state *state; 197 struct ics_irq_state *state;
188 u16 src; 198 u16 src;
199 unsigned long flags;
189 200
190 if (!xics) 201 if (!xics)
191 return -ENODEV; 202 return -ENODEV;
@@ -195,10 +206,12 @@ int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority)
195 return -EINVAL; 206 return -EINVAL;
196 state = &ics->irq_state[src]; 207 state = &ics->irq_state[src];
197 208
198 mutex_lock(&ics->lock); 209 local_irq_save(flags);
210 arch_spin_lock(&ics->lock);
199 *server = state->server; 211 *server = state->server;
200 *priority = state->priority; 212 *priority = state->priority;
201 mutex_unlock(&ics->lock); 213 arch_spin_unlock(&ics->lock);
214 local_irq_restore(flags);
202 215
203 return 0; 216 return 0;
204} 217}
@@ -365,6 +378,7 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
365 struct kvmppc_ics *ics; 378 struct kvmppc_ics *ics;
366 u32 reject; 379 u32 reject;
367 u16 src; 380 u16 src;
381 unsigned long flags;
368 382
369 /* 383 /*
370 * This is used both for initial delivery of an interrupt and 384 * This is used both for initial delivery of an interrupt and
@@ -391,7 +405,8 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
391 state = &ics->irq_state[src]; 405 state = &ics->irq_state[src];
392 406
393 /* Get a lock on the ICS */ 407 /* Get a lock on the ICS */
394 mutex_lock(&ics->lock); 408 local_irq_save(flags);
409 arch_spin_lock(&ics->lock);
395 410
396 /* Get our server */ 411 /* Get our server */
397 if (!icp || state->server != icp->server_num) { 412 if (!icp || state->server != icp->server_num) {
@@ -434,7 +449,7 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
434 * 449 *
435 * Note that if successful, the new delivery might have itself 450 * Note that if successful, the new delivery might have itself
436 * rejected an interrupt that was "delivered" before we took the 451 * rejected an interrupt that was "delivered" before we took the
437 * icp mutex. 452 * ics spin lock.
438 * 453 *
439 * In this case we do the whole sequence all over again for the 454 * In this case we do the whole sequence all over again for the
440 * new guy. We cannot assume that the rejected interrupt is less 455 * new guy. We cannot assume that the rejected interrupt is less
@@ -448,7 +463,8 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
448 * Delivery was successful, did we reject somebody else ? 463 * Delivery was successful, did we reject somebody else ?
449 */ 464 */
450 if (reject && reject != XICS_IPI) { 465 if (reject && reject != XICS_IPI) {
451 mutex_unlock(&ics->lock); 466 arch_spin_unlock(&ics->lock);
467 local_irq_restore(flags);
452 new_irq = reject; 468 new_irq = reject;
453 goto again; 469 goto again;
454 } 470 }
@@ -468,12 +484,14 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
468 */ 484 */
469 smp_mb(); 485 smp_mb();
470 if (!icp->state.need_resend) { 486 if (!icp->state.need_resend) {
471 mutex_unlock(&ics->lock); 487 arch_spin_unlock(&ics->lock);
488 local_irq_restore(flags);
472 goto again; 489 goto again;
473 } 490 }
474 } 491 }
475 out: 492 out:
476 mutex_unlock(&ics->lock); 493 arch_spin_unlock(&ics->lock);
494 local_irq_restore(flags);
477} 495}
478 496
479static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 497static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
@@ -802,14 +820,22 @@ static noinline int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall)
802 XICS_DBG("XICS_RM: H_%x completing, act: %x state: %lx tgt: %p\n", 820 XICS_DBG("XICS_RM: H_%x completing, act: %x state: %lx tgt: %p\n",
803 hcall, icp->rm_action, icp->rm_dbgstate.raw, icp->rm_dbgtgt); 821 hcall, icp->rm_action, icp->rm_dbgstate.raw, icp->rm_dbgtgt);
804 822
805 if (icp->rm_action & XICS_RM_KICK_VCPU) 823 if (icp->rm_action & XICS_RM_KICK_VCPU) {
824 icp->n_rm_kick_vcpu++;
806 kvmppc_fast_vcpu_kick(icp->rm_kick_target); 825 kvmppc_fast_vcpu_kick(icp->rm_kick_target);
807 if (icp->rm_action & XICS_RM_CHECK_RESEND) 826 }
827 if (icp->rm_action & XICS_RM_CHECK_RESEND) {
828 icp->n_rm_check_resend++;
808 icp_check_resend(xics, icp->rm_resend_icp); 829 icp_check_resend(xics, icp->rm_resend_icp);
809 if (icp->rm_action & XICS_RM_REJECT) 830 }
831 if (icp->rm_action & XICS_RM_REJECT) {
832 icp->n_rm_reject++;
810 icp_deliver_irq(xics, icp, icp->rm_reject); 833 icp_deliver_irq(xics, icp, icp->rm_reject);
811 if (icp->rm_action & XICS_RM_NOTIFY_EOI) 834 }
835 if (icp->rm_action & XICS_RM_NOTIFY_EOI) {
836 icp->n_rm_notify_eoi++;
812 kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq); 837 kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq);
838 }
813 839
814 icp->rm_action = 0; 840 icp->rm_action = 0;
815 841
@@ -872,10 +898,21 @@ static int xics_debug_show(struct seq_file *m, void *private)
872 struct kvm *kvm = xics->kvm; 898 struct kvm *kvm = xics->kvm;
873 struct kvm_vcpu *vcpu; 899 struct kvm_vcpu *vcpu;
874 int icsid, i; 900 int icsid, i;
901 unsigned long flags;
902 unsigned long t_rm_kick_vcpu, t_rm_check_resend;
903 unsigned long t_rm_reject, t_rm_notify_eoi;
904 unsigned long t_reject, t_check_resend;
875 905
876 if (!kvm) 906 if (!kvm)
877 return 0; 907 return 0;
878 908
909 t_rm_kick_vcpu = 0;
910 t_rm_notify_eoi = 0;
911 t_rm_check_resend = 0;
912 t_rm_reject = 0;
913 t_check_resend = 0;
914 t_reject = 0;
915
879 seq_printf(m, "=========\nICP state\n=========\n"); 916 seq_printf(m, "=========\nICP state\n=========\n");
880 917
881 kvm_for_each_vcpu(i, vcpu, kvm) { 918 kvm_for_each_vcpu(i, vcpu, kvm) {
@@ -890,8 +927,19 @@ static int xics_debug_show(struct seq_file *m, void *private)
890 icp->server_num, state.xisr, 927 icp->server_num, state.xisr,
891 state.pending_pri, state.cppr, state.mfrr, 928 state.pending_pri, state.cppr, state.mfrr,
892 state.out_ee, state.need_resend); 929 state.out_ee, state.need_resend);
930 t_rm_kick_vcpu += icp->n_rm_kick_vcpu;
931 t_rm_notify_eoi += icp->n_rm_notify_eoi;
932 t_rm_check_resend += icp->n_rm_check_resend;
933 t_rm_reject += icp->n_rm_reject;
934 t_check_resend += icp->n_check_resend;
935 t_reject += icp->n_reject;
893 } 936 }
894 937
938 seq_printf(m, "ICP Guest->Host totals: kick_vcpu=%lu check_resend=%lu reject=%lu notify_eoi=%lu\n",
939 t_rm_kick_vcpu, t_rm_check_resend,
940 t_rm_reject, t_rm_notify_eoi);
941 seq_printf(m, "ICP Real Mode totals: check_resend=%lu resend=%lu\n",
942 t_check_resend, t_reject);
895 for (icsid = 0; icsid <= KVMPPC_XICS_MAX_ICS_ID; icsid++) { 943 for (icsid = 0; icsid <= KVMPPC_XICS_MAX_ICS_ID; icsid++) {
896 struct kvmppc_ics *ics = xics->ics[icsid]; 944 struct kvmppc_ics *ics = xics->ics[icsid];
897 945
@@ -901,7 +949,8 @@ static int xics_debug_show(struct seq_file *m, void *private)
901 seq_printf(m, "=========\nICS state for ICS 0x%x\n=========\n", 949 seq_printf(m, "=========\nICS state for ICS 0x%x\n=========\n",
902 icsid); 950 icsid);
903 951
904 mutex_lock(&ics->lock); 952 local_irq_save(flags);
953 arch_spin_lock(&ics->lock);
905 954
906 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 955 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
907 struct ics_irq_state *irq = &ics->irq_state[i]; 956 struct ics_irq_state *irq = &ics->irq_state[i];
@@ -912,7 +961,8 @@ static int xics_debug_show(struct seq_file *m, void *private)
912 irq->resend, irq->masked_pending); 961 irq->resend, irq->masked_pending);
913 962
914 } 963 }
915 mutex_unlock(&ics->lock); 964 arch_spin_unlock(&ics->lock);
965 local_irq_restore(flags);
916 } 966 }
917 return 0; 967 return 0;
918} 968}
@@ -965,7 +1015,6 @@ static struct kvmppc_ics *kvmppc_xics_create_ics(struct kvm *kvm,
965 if (!ics) 1015 if (!ics)
966 goto out; 1016 goto out;
967 1017
968 mutex_init(&ics->lock);
969 ics->icsid = icsid; 1018 ics->icsid = icsid;
970 1019
971 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 1020 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
@@ -1107,13 +1156,15 @@ static int xics_get_source(struct kvmppc_xics *xics, long irq, u64 addr)
1107 u64 __user *ubufp = (u64 __user *) addr; 1156 u64 __user *ubufp = (u64 __user *) addr;
1108 u16 idx; 1157 u16 idx;
1109 u64 val, prio; 1158 u64 val, prio;
1159 unsigned long flags;
1110 1160
1111 ics = kvmppc_xics_find_ics(xics, irq, &idx); 1161 ics = kvmppc_xics_find_ics(xics, irq, &idx);
1112 if (!ics) 1162 if (!ics)
1113 return -ENOENT; 1163 return -ENOENT;
1114 1164
1115 irqp = &ics->irq_state[idx]; 1165 irqp = &ics->irq_state[idx];
1116 mutex_lock(&ics->lock); 1166 local_irq_save(flags);
1167 arch_spin_lock(&ics->lock);
1117 ret = -ENOENT; 1168 ret = -ENOENT;
1118 if (irqp->exists) { 1169 if (irqp->exists) {
1119 val = irqp->server; 1170 val = irqp->server;
@@ -1129,7 +1180,8 @@ static int xics_get_source(struct kvmppc_xics *xics, long irq, u64 addr)
1129 val |= KVM_XICS_PENDING; 1180 val |= KVM_XICS_PENDING;
1130 ret = 0; 1181 ret = 0;
1131 } 1182 }
1132 mutex_unlock(&ics->lock); 1183 arch_spin_unlock(&ics->lock);
1184 local_irq_restore(flags);
1133 1185
1134 if (!ret && put_user(val, ubufp)) 1186 if (!ret && put_user(val, ubufp))
1135 ret = -EFAULT; 1187 ret = -EFAULT;
@@ -1146,6 +1198,7 @@ static int xics_set_source(struct kvmppc_xics *xics, long irq, u64 addr)
1146 u64 val; 1198 u64 val;
1147 u8 prio; 1199 u8 prio;
1148 u32 server; 1200 u32 server;
1201 unsigned long flags;
1149 1202
1150 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS) 1203 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
1151 return -ENOENT; 1204 return -ENOENT;
@@ -1166,7 +1219,8 @@ static int xics_set_source(struct kvmppc_xics *xics, long irq, u64 addr)
1166 kvmppc_xics_find_server(xics->kvm, server) == NULL) 1219 kvmppc_xics_find_server(xics->kvm, server) == NULL)
1167 return -EINVAL; 1220 return -EINVAL;
1168 1221
1169 mutex_lock(&ics->lock); 1222 local_irq_save(flags);
1223 arch_spin_lock(&ics->lock);
1170 irqp->server = server; 1224 irqp->server = server;
1171 irqp->saved_priority = prio; 1225 irqp->saved_priority = prio;
1172 if (val & KVM_XICS_MASKED) 1226 if (val & KVM_XICS_MASKED)
@@ -1178,7 +1232,8 @@ static int xics_set_source(struct kvmppc_xics *xics, long irq, u64 addr)
1178 if ((val & KVM_XICS_PENDING) && (val & KVM_XICS_LEVEL_SENSITIVE)) 1232 if ((val & KVM_XICS_PENDING) && (val & KVM_XICS_LEVEL_SENSITIVE))
1179 irqp->asserted = 1; 1233 irqp->asserted = 1;
1180 irqp->exists = 1; 1234 irqp->exists = 1;
1181 mutex_unlock(&ics->lock); 1235 arch_spin_unlock(&ics->lock);
1236 local_irq_restore(flags);
1182 1237
1183 if (val & KVM_XICS_PENDING) 1238 if (val & KVM_XICS_PENDING)
1184 icp_deliver_irq(xics, NULL, irqp->number); 1239 icp_deliver_irq(xics, NULL, irqp->number);
diff --git a/arch/powerpc/kvm/book3s_xics.h b/arch/powerpc/kvm/book3s_xics.h
index 73f0f2723c07..56ea44f9867f 100644
--- a/arch/powerpc/kvm/book3s_xics.h
+++ b/arch/powerpc/kvm/book3s_xics.h
@@ -78,13 +78,22 @@ struct kvmppc_icp {
78 u32 rm_reject; 78 u32 rm_reject;
79 u32 rm_eoied_irq; 79 u32 rm_eoied_irq;
80 80
81 /* Counters for each reason we exited real mode */
82 unsigned long n_rm_kick_vcpu;
83 unsigned long n_rm_check_resend;
84 unsigned long n_rm_reject;
85 unsigned long n_rm_notify_eoi;
86 /* Counters for handling ICP processing in real mode */
87 unsigned long n_check_resend;
88 unsigned long n_reject;
89
81 /* Debug stuff for real mode */ 90 /* Debug stuff for real mode */
82 union kvmppc_icp_state rm_dbgstate; 91 union kvmppc_icp_state rm_dbgstate;
83 struct kvm_vcpu *rm_dbgtgt; 92 struct kvm_vcpu *rm_dbgtgt;
84}; 93};
85 94
86struct kvmppc_ics { 95struct kvmppc_ics {
87 struct mutex lock; 96 arch_spinlock_t lock;
88 u16 icsid; 97 u16 icsid;
89 struct ics_irq_state irq_state[KVMPPC_XICS_IRQ_PER_ICS]; 98 struct ics_irq_state irq_state[KVMPPC_XICS_IRQ_PER_ICS];
90}; 99};
@@ -96,6 +105,8 @@ struct kvmppc_xics {
96 u32 max_icsid; 105 u32 max_icsid;
97 bool real_mode; 106 bool real_mode;
98 bool real_mode_dbg; 107 bool real_mode_dbg;
108 u32 err_noics;
109 u32 err_noicp;
99 struct kvmppc_ics *ics[KVMPPC_XICS_MAX_ICS_ID + 1]; 110 struct kvmppc_ics *ics[KVMPPC_XICS_MAX_ICS_ID + 1];
100}; 111};
101 112
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 91bbc845ac66..ac3ddf115f3d 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -529,6 +529,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
529 case KVM_CAP_PPC_RMA: 529 case KVM_CAP_PPC_RMA:
530 r = 0; 530 r = 0;
531 break; 531 break;
532 case KVM_CAP_PPC_HWRNG:
533 r = kvmppc_hwrng_present();
534 break;
532#endif 535#endif
533 case KVM_CAP_SYNC_MMU: 536 case KVM_CAP_SYNC_MMU:
534#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 537#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
diff --git a/arch/powerpc/platforms/powernv/rng.c b/arch/powerpc/platforms/powernv/rng.c
index 80db43944afe..6eb808ff637e 100644
--- a/arch/powerpc/platforms/powernv/rng.c
+++ b/arch/powerpc/platforms/powernv/rng.c
@@ -24,12 +24,22 @@
24 24
25struct powernv_rng { 25struct powernv_rng {
26 void __iomem *regs; 26 void __iomem *regs;
27 void __iomem *regs_real;
27 unsigned long mask; 28 unsigned long mask;
28}; 29};
29 30
30static DEFINE_PER_CPU(struct powernv_rng *, powernv_rng); 31static DEFINE_PER_CPU(struct powernv_rng *, powernv_rng);
31 32
32 33
34int powernv_hwrng_present(void)
35{
36 struct powernv_rng *rng;
37
38 rng = get_cpu_var(powernv_rng);
39 put_cpu_var(rng);
40 return rng != NULL;
41}
42
33static unsigned long rng_whiten(struct powernv_rng *rng, unsigned long val) 43static unsigned long rng_whiten(struct powernv_rng *rng, unsigned long val)
34{ 44{
35 unsigned long parity; 45 unsigned long parity;
@@ -46,6 +56,17 @@ static unsigned long rng_whiten(struct powernv_rng *rng, unsigned long val)
46 return val; 56 return val;
47} 57}
48 58
59int powernv_get_random_real_mode(unsigned long *v)
60{
61 struct powernv_rng *rng;
62
63 rng = raw_cpu_read(powernv_rng);
64
65 *v = rng_whiten(rng, in_rm64(rng->regs_real));
66
67 return 1;
68}
69
49int powernv_get_random_long(unsigned long *v) 70int powernv_get_random_long(unsigned long *v)
50{ 71{
51 struct powernv_rng *rng; 72 struct powernv_rng *rng;
@@ -80,12 +101,20 @@ static __init void rng_init_per_cpu(struct powernv_rng *rng,
80static __init int rng_create(struct device_node *dn) 101static __init int rng_create(struct device_node *dn)
81{ 102{
82 struct powernv_rng *rng; 103 struct powernv_rng *rng;
104 struct resource res;
83 unsigned long val; 105 unsigned long val;
84 106
85 rng = kzalloc(sizeof(*rng), GFP_KERNEL); 107 rng = kzalloc(sizeof(*rng), GFP_KERNEL);
86 if (!rng) 108 if (!rng)
87 return -ENOMEM; 109 return -ENOMEM;
88 110
111 if (of_address_to_resource(dn, 0, &res)) {
112 kfree(rng);
113 return -ENXIO;
114 }
115
116 rng->regs_real = (void __iomem *)res.start;
117
89 rng->regs = of_iomap(dn, 0); 118 rng->regs = of_iomap(dn, 0);
90 if (!rng->regs) { 119 if (!rng->regs) {
91 kfree(rng); 120 kfree(rng);
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index afa2bd750ffc..8cd8e7b288c5 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -110,7 +110,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
110/* upper facilities limit for kvm */ 110/* upper facilities limit for kvm */
111unsigned long kvm_s390_fac_list_mask[] = { 111unsigned long kvm_s390_fac_list_mask[] = {
112 0xffe6fffbfcfdfc40UL, 112 0xffe6fffbfcfdfc40UL,
113 0x205c800000000000UL, 113 0x005c800000000000UL,
114}; 114};
115 115
116unsigned long kvm_s390_fac_list_mask_size(void) 116unsigned long kvm_s390_fac_list_mask_size(void)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index d67206a7b99a..629af0f1c5c4 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -683,8 +683,7 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
683 unsigned long bitmap = 1; 683 unsigned long bitmap = 1;
684 struct kvm_lapic **dst; 684 struct kvm_lapic **dst;
685 int i; 685 int i;
686 bool ret = false; 686 bool ret, x2apic_ipi;
687 bool x2apic_ipi = src && apic_x2apic_mode(src);
688 687
689 *r = -1; 688 *r = -1;
690 689
@@ -696,16 +695,18 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
696 if (irq->shorthand) 695 if (irq->shorthand)
697 return false; 696 return false;
698 697
698 x2apic_ipi = src && apic_x2apic_mode(src);
699 if (irq->dest_id == (x2apic_ipi ? X2APIC_BROADCAST : APIC_BROADCAST)) 699 if (irq->dest_id == (x2apic_ipi ? X2APIC_BROADCAST : APIC_BROADCAST))
700 return false; 700 return false;
701 701
702 ret = true;
702 rcu_read_lock(); 703 rcu_read_lock();
703 map = rcu_dereference(kvm->arch.apic_map); 704 map = rcu_dereference(kvm->arch.apic_map);
704 705
705 if (!map) 706 if (!map) {
707 ret = false;
706 goto out; 708 goto out;
707 709 }
708 ret = true;
709 710
710 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 711 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
711 if (irq->dest_id >= ARRAY_SIZE(map->phys_map)) 712 if (irq->dest_id >= ARRAY_SIZE(map->phys_map))
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 146f295ee322..d43867c33bc4 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4481,9 +4481,11 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4481 pfn = spte_to_pfn(*sptep); 4481 pfn = spte_to_pfn(*sptep);
4482 4482
4483 /* 4483 /*
4484 * Only EPT supported for now; otherwise, one would need to 4484 * We cannot do huge page mapping for indirect shadow pages,
4485 * find out efficiently whether the guest page tables are 4485 * which are found on the last rmap (level = 1) when not using
4486 * also using huge pages. 4486 * tdp; such shadow pages are synced with the page table in
4487 * the guest, and the guest page table is using 4K page size
4488 * mapping if the indirect sp has level = 1.
4487 */ 4489 */
4488 if (sp->role.direct && 4490 if (sp->role.direct &&
4489 !kvm_is_reserved_pfn(pfn) && 4491 !kvm_is_reserved_pfn(pfn) &&
@@ -4504,19 +4506,12 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
4504 bool flush = false; 4506 bool flush = false;
4505 unsigned long *rmapp; 4507 unsigned long *rmapp;
4506 unsigned long last_index, index; 4508 unsigned long last_index, index;
4507 gfn_t gfn_start, gfn_end;
4508 4509
4509 spin_lock(&kvm->mmu_lock); 4510 spin_lock(&kvm->mmu_lock);
4510 4511
4511 gfn_start = memslot->base_gfn;
4512 gfn_end = memslot->base_gfn + memslot->npages - 1;
4513
4514 if (gfn_start >= gfn_end)
4515 goto out;
4516
4517 rmapp = memslot->arch.rmap[0]; 4512 rmapp = memslot->arch.rmap[0];
4518 last_index = gfn_to_index(gfn_end, memslot->base_gfn, 4513 last_index = gfn_to_index(memslot->base_gfn + memslot->npages - 1,
4519 PT_PAGE_TABLE_LEVEL); 4514 memslot->base_gfn, PT_PAGE_TABLE_LEVEL);
4520 4515
4521 for (index = 0; index <= last_index; ++index, ++rmapp) { 4516 for (index = 0; index <= last_index; ++index, ++rmapp) {
4522 if (*rmapp) 4517 if (*rmapp)
@@ -4534,7 +4529,6 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
4534 if (flush) 4529 if (flush)
4535 kvm_flush_remote_tlbs(kvm); 4530 kvm_flush_remote_tlbs(kvm);
4536 4531
4537out:
4538 spin_unlock(&kvm->mmu_lock); 4532 spin_unlock(&kvm->mmu_lock);
4539} 4533}
4540 4534
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index f5e8dce8046c..f7b61687bd79 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -3622,8 +3622,16 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
3622 3622
3623static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3623static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
3624{ 3624{
3625 unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ? 3625 /*
3626 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON); 3626 * Pass through host's Machine Check Enable value to hw_cr4, which
3627 * is in force while we are in guest mode. Do not let guests control
3628 * this bit, even if host CR4.MCE == 0.
3629 */
3630 unsigned long hw_cr4 =
3631 (cr4_read_shadow() & X86_CR4_MCE) |
3632 (cr4 & ~X86_CR4_MCE) |
3633 (to_vmx(vcpu)->rmode.vm86_active ?
3634 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
3627 3635
3628 if (cr4 & X86_CR4_VMXE) { 3636 if (cr4 & X86_CR4_VMXE) {
3629 /* 3637 /*
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index e1a81267f3f6..ed31c31b2485 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5799,7 +5799,6 @@ int kvm_arch_init(void *opaque)
5799 kvm_set_mmio_spte_mask(); 5799 kvm_set_mmio_spte_mask();
5800 5800
5801 kvm_x86_ops = ops; 5801 kvm_x86_ops = ops;
5802 kvm_init_msr_list();
5803 5802
5804 kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, 5803 kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
5805 PT_DIRTY_MASK, PT64_NX_MASK, 0); 5804 PT_DIRTY_MASK, PT64_NX_MASK, 0);
@@ -7253,7 +7252,14 @@ void kvm_arch_hardware_disable(void)
7253 7252
7254int kvm_arch_hardware_setup(void) 7253int kvm_arch_hardware_setup(void)
7255{ 7254{
7256 return kvm_x86_ops->hardware_setup(); 7255 int r;
7256
7257 r = kvm_x86_ops->hardware_setup();
7258 if (r != 0)
7259 return r;
7260
7261 kvm_init_msr_list();
7262 return 0;
7257} 7263}
7258 7264
7259void kvm_arch_hardware_unsetup(void) 7265void kvm_arch_hardware_unsetup(void)
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index f574d7be7631..4b60056776d1 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -813,6 +813,7 @@ struct kvm_ppc_smmu_info {
813#define KVM_CAP_MIPS_MSA 112 813#define KVM_CAP_MIPS_MSA 112
814#define KVM_CAP_S390_INJECT_IRQ 113 814#define KVM_CAP_S390_INJECT_IRQ 113
815#define KVM_CAP_S390_IRQ_STATE 114 815#define KVM_CAP_S390_IRQ_STATE 114
816#define KVM_CAP_PPC_HWRNG 115
816 817
817#ifdef KVM_CAP_IRQ_ROUTING 818#ifdef KVM_CAP_IRQ_ROUTING
818 819
diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c
index 8d550ff14700..78fb8201014f 100644
--- a/virt/kvm/arm/vgic.c
+++ b/virt/kvm/arm/vgic.c
@@ -1561,6 +1561,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1561 goto out; 1561 goto out;
1562 } 1562 }
1563 1563
1564 if (irq_num >= kvm->arch.vgic.nr_irqs)
1565 return -EINVAL;
1566
1564 vcpu_id = vgic_update_irq_pending(kvm, cpuid, irq_num, level); 1567 vcpu_id = vgic_update_irq_pending(kvm, cpuid, irq_num, level);
1565 if (vcpu_id >= 0) { 1568 if (vcpu_id >= 0) {
1566 /* kick the specified vcpu */ 1569 /* kick the specified vcpu */
@@ -2141,7 +2144,7 @@ int kvm_irq_map_gsi(struct kvm *kvm,
2141 struct kvm_kernel_irq_routing_entry *entries, 2144 struct kvm_kernel_irq_routing_entry *entries,
2142 int gsi) 2145 int gsi)
2143{ 2146{
2144 return gsi; 2147 return 0;
2145} 2148}
2146 2149
2147int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin) 2150int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index d3fc9399062a..90977418aeb6 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -89,6 +89,7 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
89static __read_mostly struct preempt_ops kvm_preempt_ops; 89static __read_mostly struct preempt_ops kvm_preempt_ops;
90 90
91struct dentry *kvm_debugfs_dir; 91struct dentry *kvm_debugfs_dir;
92EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
92 93
93static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, 94static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
94 unsigned long arg); 95 unsigned long arg);
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-01 05:19:57 -0500