1 files changed, 72 insertions, 27 deletions

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f702f82aa1e..aa75f21a9fb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -83,11 +83,14 @@
 #define KVM_NR_FIXED_MTRR_REGION 88
 #define KVM_NR_VAR_MTRR 8
 
+#define ASYNC_PF_PER_VCPU 64
+
 extern spinlock_t kvm_lock;
 extern struct list_head vm_list;
 
 struct kvm_vcpu;
 struct kvm;
+struct kvm_async_pf;
 
 enum kvm_reg {
         VCPU_REGS_RAX = 0,
@@ -114,6 +117,7 @@ enum kvm_reg {
 
 enum kvm_reg_ex {
         VCPU_EXREG_PDPTR = NR_VCPU_REGS,
+        VCPU_EXREG_CR3,
 };
 
 enum {
@@ -238,16 +242,18 @@ struct kvm_mmu {
         void (*new_cr3)(struct kvm_vcpu *vcpu);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
         unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
-        int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err);
-        void (*inject_page_fault)(struct kvm_vcpu *vcpu);
+        int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
+                          bool prefault);
+        void (*inject_page_fault)(struct kvm_vcpu *vcpu,
+                                  struct x86_exception *fault);
         void (*free)(struct kvm_vcpu *vcpu);
         gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
-                            u32 *error);
+                            struct x86_exception *exception);
         gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access);
         void (*prefetch_page)(struct kvm_vcpu *vcpu,
                               struct kvm_mmu_page *page);
         int (*sync_page)(struct kvm_vcpu *vcpu,
-                         struct kvm_mmu_page *sp, bool clear_unsync);
+                         struct kvm_mmu_page *sp);
         void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva);
         hpa_t root_hpa;
         int root_level;
@@ -315,16 +321,6 @@ struct kvm_vcpu_arch {
          */
         struct kvm_mmu *walk_mmu;
 
-        /*
-         * This struct is filled with the necessary information to propagate a
-         * page fault into the guest
-         */
-        struct {
-                u64      address;
-                unsigned error_code;
-                bool     nested;
-        } fault;
-
         /* only needed in kvm_pv_mmu_op() path, but it's hot so
          * put it here to avoid allocation */
         struct kvm_pv_mmu_op_buffer mmu_op_buffer;
@@ -412,6 +408,15 @@ struct kvm_vcpu_arch {
         u64 hv_vapic;
 
         cpumask_var_t wbinvd_dirty_mask;
+
+        struct {
+                bool halted;
+                gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)];
+                struct gfn_to_hva_cache data;
+                u64 msr_val;
+                u32 id;
+                bool send_user_only;
+        } apf;
 };
 
 struct kvm_arch {
@@ -456,6 +461,10 @@ struct kvm_arch {
         /* fields used by HYPER-V emulation */
         u64 hv_guest_os_id;
         u64 hv_hypercall;
+
+        #ifdef CONFIG_KVM_MMU_AUDIT
+        int audit_point;
+        #endif
 };
 
 struct kvm_vm_stat {
@@ -529,6 +538,7 @@ struct kvm_x86_ops {
                             struct kvm_segment *var, int seg);
         void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
         void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu);
+        void (*decache_cr3)(struct kvm_vcpu *vcpu);
         void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
         void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
@@ -582,9 +592,17 @@ struct kvm_x86_ops {
 
         void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
 
+        void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2);
         const struct trace_print_flags *exit_reasons_str;
 };
 
+struct kvm_arch_async_pf {
+        u32 token;
+        gfn_t gfn;
+        unsigned long cr3;
+        bool direct_map;
+};
+
 extern struct kvm_x86_ops *kvm_x86_ops;
 
 int kvm_mmu_module_init(void);
@@ -594,7 +612,6 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
 int kvm_mmu_create(struct kvm_vcpu *vcpu);
 int kvm_mmu_setup(struct kvm_vcpu *vcpu);
 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte);
-void kvm_mmu_set_base_ptes(u64 base_pte);
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask);
 
@@ -623,8 +640,15 @@ enum emulation_result {
 #define EMULTYPE_NO_DECODE          (1 << 0)
 #define EMULTYPE_TRAP_UD            (1 << 1)
 #define EMULTYPE_SKIP               (1 << 2)
-int emulate_instruction(struct kvm_vcpu *vcpu,
-                        unsigned long cr2, u16 error_code, int emulation_type);
+int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2,
+                            int emulation_type, void *insn, int insn_len);
+
+static inline int emulate_instruction(struct kvm_vcpu *vcpu,
+                        int emulation_type)
+{
+        return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
+}
+
 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
 void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
 
@@ -650,7 +674,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
-void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
@@ -668,11 +692,11 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu);
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
                             gfn_t gfn, void *data, int offset, int len,
                             u32 access);
-void kvm_propagate_fault(struct kvm_vcpu *vcpu);
+void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
 
 int kvm_pic_set_irq(void *opaque, int irq, int level);
@@ -690,16 +714,21 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
-gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
+gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
+                              struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception);
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
 int kvm_fix_hypercall(struct kvm_vcpu *vcpu);
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code);
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code,
+                       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 
 void kvm_enable_tdp(void);
@@ -766,20 +795,25 @@ enum {
 #define HF_VINTR_MASK           (1 << 2)
 #define HF_NMI_MASK             (1 << 3)
 #define HF_IRET_MASK            (1 << 4)
+#define HF_GUEST_MASK           (1 << 5) /* VCPU is in guest-mode */
 
 /*
  * Hardware virtualization extension instructions may fault if a
  * reboot turns off virtualization while processes are running.
  * Trap the fault and ignore the instruction if that happens.
  */
-asmlinkage void kvm_handle_fault_on_reboot(void);
+asmlinkage void kvm_spurious_fault(void);
+extern bool kvm_rebooting;
 
 #define __kvm_handle_fault_on_reboot(insn) \
         "666: " insn "\n\t" \
+        "668: \n\t"                           \
         ".pushsection .fixup, \"ax\" \n" \
         "667: \n\t" \
+        "cmpb $0, kvm_rebooting \n\t"         \
+        "jne 668b \n\t"                       \
         __ASM_SIZE(push) " $666b \n\t"        \
-        "jmp kvm_handle_fault_on_reboot \n\t" \
+        "call kvm_spurious_fault \n\t"        \
         ".popsection \n\t" \
         ".pushsection __ex_table, \"a\" \n\t" \
         _ASM_PTR " 666b, 667b \n\t" \
@@ -799,4 +833,15 @@ void kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
 
 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
 
+void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
+                                     struct kvm_async_pf *work);
+void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
+                                 struct kvm_async_pf *work);
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
+                               struct kvm_async_pf *work);
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
+extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
+
+void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
+
 #endif /* _ASM_X86_KVM_HOST_H */