8 files changed, 143 insertions, 41 deletions

diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index 0049211959c0..6040d115ef51 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -229,7 +229,26 @@ struct read_cache {
         unsigned long end;
 };
 
-struct decode_cache {
+struct x86_emulate_ctxt {
+        struct x86_emulate_ops *ops;
+
+        /* Register state before/after emulation. */
+        unsigned long eflags;
+        unsigned long eip; /* eip before instruction emulation */
+        /* Emulated execution mode, represented by an X86EMUL_MODE value. */
+        int mode;
+
+        /* interruptibility state, as a result of execution of STI or MOV SS */
+        int interruptibility;
+
+        bool guest_mode; /* guest running a nested guest */
+        bool perm_ok; /* do not check permissions if true */
+        bool only_vendor_specific_insn;
+
+        bool have_exception;
+        struct x86_exception exception;
+
+        /* decode cache */
         u8 twobyte;
         u8 b;
         u8 intercept;
@@ -246,8 +265,6 @@ struct decode_cache {
         unsigned int d;
         int (*execute)(struct x86_emulate_ctxt *ctxt);
         int (*check_perm)(struct x86_emulate_ctxt *ctxt);
-        unsigned long regs[NR_VCPU_REGS];
-        unsigned long eip;
         /* modrm */
         u8 modrm;
         u8 modrm_mod;
@@ -255,34 +272,14 @@ struct decode_cache {
         u8 modrm_rm;
         u8 modrm_seg;
         bool rip_relative;
+        unsigned long _eip;
+        /* Fields above regs are cleared together. */
+        unsigned long regs[NR_VCPU_REGS];
         struct fetch_cache fetch;
         struct read_cache io_read;
         struct read_cache mem_read;
 };
 
-struct x86_emulate_ctxt {
-        struct x86_emulate_ops *ops;
-
-        /* Register state before/after emulation. */
-        unsigned long eflags;
-        unsigned long eip; /* eip before instruction emulation */
-        /* Emulated execution mode, represented by an X86EMUL_MODE value. */
-        int mode;
-
-        /* interruptibility state, as a result of execution of STI or MOV SS */
-        int interruptibility;
-
-        bool guest_mode; /* guest running a nested guest */
-        bool perm_ok; /* do not check permissions if true */
-        bool only_vendor_specific_insn;
-
-        bool have_exception;
-        struct x86_exception exception;
-
-        /* decode cache */
-        struct decode_cache decode;
-};
-
 /* Repeat String Operation Prefix */
 #define REPE_PREFIX     0xf3
 #define REPNE_PREFIX    0xf2
@@ -373,6 +370,5 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt);
 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
                          u16 tss_selector, int reason,
                          bool has_error_code, u32 error_code);
-int emulate_int_real(struct x86_emulate_ctxt *ctxt,
-                     struct x86_emulate_ops *ops, int irq);
+int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq);
 #endif /* _ASM_X86_KVM_X86_EMULATE_H */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index d2ac8e2ee897..dd51c83aa5de 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -48,7 +48,7 @@
         (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
                           | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
                           | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR  \
-                          | X86_CR4_OSXSAVE \
+                          | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_RDWRGSFS \
                           | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
 
 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
@@ -205,6 +205,7 @@ union kvm_mmu_page_role {
                 unsigned invalid:1;
                 unsigned nxe:1;
                 unsigned cr0_wp:1;
+                unsigned smep_andnot_wp:1;
         };
 };
 
@@ -227,15 +228,17 @@ struct kvm_mmu_page {
          * in this shadow page.
          */
         DECLARE_BITMAP(slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
-        bool multimapped;         /* More than one parent_pte? */
         bool unsync;
         int root_count;          /* Currently serving as active root */
         unsigned int unsync_children;
-        union {
-                u64 *parent_pte;               /* !multimapped */
-                struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */
-        };
+        unsigned long parent_ptes;      /* Reverse mapping for parent_pte */
         DECLARE_BITMAP(unsync_child_bitmap, 512);
+
+#ifdef CONFIG_X86_32
+        int clear_spte_count;
+#endif
+
+        struct rcu_head rcu;
 };
 
 struct kvm_pv_mmu_op_buffer {
@@ -269,8 +272,6 @@ struct kvm_mmu {
         gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
                             struct x86_exception *exception);
         gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access);
-        void (*prefetch_page)(struct kvm_vcpu *vcpu,
-                              struct kvm_mmu_page *page);
         int (*sync_page)(struct kvm_vcpu *vcpu,
                          struct kvm_mmu_page *sp);
         void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva);
@@ -346,8 +347,7 @@ struct kvm_vcpu_arch {
          * put it here to avoid allocation */
         struct kvm_pv_mmu_op_buffer mmu_op_buffer;
 
-        struct kvm_mmu_memory_cache mmu_pte_chain_cache;
-        struct kvm_mmu_memory_cache mmu_rmap_desc_cache;
+        struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
         struct kvm_mmu_memory_cache mmu_page_cache;
         struct kvm_mmu_memory_cache mmu_page_header_cache;
 
@@ -393,6 +393,15 @@ struct kvm_vcpu_arch {
         unsigned int hw_tsc_khz;
         unsigned int time_offset;
         struct page *time_page;
+
+        struct {
+                u64 msr_val;
+                u64 last_steal;
+                u64 accum_steal;
+                struct gfn_to_hva_cache stime;
+                struct kvm_steal_time steal;
+        } st;
+
         u64 last_guest_tsc;
         u64 last_kernel_ns;
         u64 last_tsc_nsec;
@@ -419,6 +428,11 @@ struct kvm_vcpu_arch {
         u64 mcg_ctl;
         u64 *mce_banks;
 
+        /* Cache MMIO info */
+        u64 mmio_gva;
+        unsigned access;
+        gfn_t mmio_gfn;
+
         /* used for guest single stepping over the given code position */
         unsigned long singlestep_rip;
 
@@ -441,6 +455,7 @@ struct kvm_arch {
         unsigned int n_used_mmu_pages;
         unsigned int n_requested_mmu_pages;
         unsigned int n_max_mmu_pages;
+        unsigned int indirect_shadow_pages;
         atomic_t invlpg_counter;
         struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
         /*
@@ -477,6 +492,8 @@ struct kvm_arch {
         u64 hv_guest_os_id;
         u64 hv_hypercall;
 
+        atomic_t reader_counter;
+
         #ifdef CONFIG_KVM_MMU_AUDIT
         int audit_point;
         #endif
@@ -559,7 +576,7 @@ struct kvm_x86_ops {
         void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
         void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
-        void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
+        int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
         void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
         void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
         void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
@@ -636,7 +653,6 @@ void kvm_mmu_module_exit(void);
 void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
 int kvm_mmu_create(struct kvm_vcpu *vcpu);
 int kvm_mmu_setup(struct kvm_vcpu *vcpu);
-void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte);
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask);
 
@@ -830,11 +846,12 @@ enum {
 asmlinkage void kvm_spurious_fault(void);
 extern bool kvm_rebooting;
 
-#define __kvm_handle_fault_on_reboot(insn) \
+#define ____kvm_handle_fault_on_reboot(insn, cleanup_insn)      \
         "666: " insn "\n\t" \
         "668: \n\t"                           \
         ".pushsection .fixup, \"ax\" \n" \
         "667: \n\t" \
+        cleanup_insn "\n\t"                   \
         "cmpb $0, kvm_rebooting \n\t"         \
         "jne 668b \n\t"                       \
         __ASM_SIZE(push) " $666b \n\t"        \
@@ -844,6 +861,9 @@ extern bool kvm_rebooting;
         _ASM_PTR " 666b, 667b \n\t" \
         ".popsection"
 
+#define __kvm_handle_fault_on_reboot(insn)              \
+        ____kvm_handle_fault_on_reboot(insn, "")
+
 #define KVM_ARCH_WANT_MMU_NOTIFIER
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_age_hva(struct kvm *kvm, unsigned long hva);
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index a427bf77a93d..734c3767cfac 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -21,6 +21,7 @@
  */
 #define KVM_FEATURE_CLOCKSOURCE2        3
 #define KVM_FEATURE_ASYNC_PF            4
+#define KVM_FEATURE_STEAL_TIME          5
 
 /* The last 8 bits are used to indicate how to interpret the flags field
  * in pvclock structure. If no bits are set, all flags are ignored.
@@ -30,10 +31,23 @@
 #define MSR_KVM_WALL_CLOCK  0x11
 #define MSR_KVM_SYSTEM_TIME 0x12
 
+#define KVM_MSR_ENABLED 1
 /* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */
 #define MSR_KVM_WALL_CLOCK_NEW  0x4b564d00
 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01
 #define MSR_KVM_ASYNC_PF_EN 0x4b564d02
+#define MSR_KVM_STEAL_TIME  0x4b564d03
+
+struct kvm_steal_time {
+        __u64 steal;
+        __u32 version;
+        __u32 flags;
+        __u32 pad[12];
+};
+
+#define KVM_STEAL_ALIGNMENT_BITS 5
+#define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
+#define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)
 
 #define KVM_MAX_MMU_OP_BATCH           32
 
@@ -178,6 +192,7 @@ void __init kvm_guest_init(void);
 void kvm_async_pf_task_wait(u32 token);
 void kvm_async_pf_task_wake(u32 token);
 u32 kvm_read_and_reset_pf_reason(void);
+extern void kvm_disable_steal_time(void);
 #else
 #define kvm_guest_init() do { } while (0)
 #define kvm_async_pf_task_wait(T) do {} while(0)
@@ -186,6 +201,11 @@ static inline u32 kvm_read_and_reset_pf_reason(void)
 {
         return 0;
 }
+
+static inline void kvm_disable_steal_time(void)
+{
+        return;
+}
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index d96bdb25ca3d..d52609aeeab8 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -441,6 +441,18 @@
 #define MSR_IA32_VMX_VMCS_ENUM          0x0000048a
 #define MSR_IA32_VMX_PROCBASED_CTLS2    0x0000048b
 #define MSR_IA32_VMX_EPT_VPID_CAP       0x0000048c
+#define MSR_IA32_VMX_TRUE_PINBASED_CTLS  0x0000048d
+#define MSR_IA32_VMX_TRUE_PROCBASED_CTLS 0x0000048e
+#define MSR_IA32_VMX_TRUE_EXIT_CTLS      0x0000048f
+#define MSR_IA32_VMX_TRUE_ENTRY_CTLS     0x00000490
+
+/* VMX_BASIC bits and bitmasks */
+#define VMX_BASIC_VMCS_SIZE_SHIFT       32
+#define VMX_BASIC_64            0x0001000000000000LLU
+#define VMX_BASIC_MEM_TYPE_SHIFT        50
+#define VMX_BASIC_MEM_TYPE_MASK 0x003c000000000000LLU
+#define VMX_BASIC_MEM_TYPE_WB   6LLU
+#define VMX_BASIC_INOUT         0x0040000000000000LLU
 
 /* AMD-V MSRs */
 
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index ebbc4d8ab170..a7d2db9a74fb 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -230,6 +230,15 @@ static inline unsigned long long paravirt_sched_clock(void)
         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 }
 
+struct jump_label_key;
+extern struct jump_label_key paravirt_steal_enabled;
+extern struct jump_label_key paravirt_steal_rq_enabled;
+
+static inline u64 paravirt_steal_clock(int cpu)
+{
+        return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
+}
+
 static inline unsigned long long paravirt_read_pmc(int counter)
 {
         return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 82885099c869..2c7652163111 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -89,6 +89,7 @@ struct pv_lazy_ops {
 
 struct pv_time_ops {
         unsigned long long (*sched_clock)(void);
+        unsigned long long (*steal_clock)(int cpu);
         unsigned long (*get_tsc_khz)(void);
 };
 
diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/processor-flags.h
index 59ab4dffa377..2dddb317bb39 100644
--- a/arch/x86/include/asm/processor-flags.h
+++ b/arch/x86/include/asm/processor-flags.h
@@ -59,6 +59,7 @@
 #define X86_CR4_OSFXSR  0x00000200 /* enable fast FPU save and restore */
 #define X86_CR4_OSXMMEXCPT 0x00000400 /* enable unmasked SSE exceptions */
 #define X86_CR4_VMXE    0x00002000 /* enable VMX virtualization */
+#define X86_CR4_RDWRGSFS 0x00010000 /* enable RDWRGSFS support */
 #define X86_CR4_OSXSAVE 0x00040000 /* enable xsave and xrestore */
 #define X86_CR4_SMEP    0x00100000 /* enable SMEP support */
 
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 84471b810460..2caf290e9895 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -132,6 +132,8 @@ enum vmcs_field {
         GUEST_IA32_PAT_HIGH             = 0x00002805,
         GUEST_IA32_EFER                 = 0x00002806,
         GUEST_IA32_EFER_HIGH            = 0x00002807,
+        GUEST_IA32_PERF_GLOBAL_CTRL     = 0x00002808,
+        GUEST_IA32_PERF_GLOBAL_CTRL_HIGH= 0x00002809,
         GUEST_PDPTR0                    = 0x0000280a,
         GUEST_PDPTR0_HIGH               = 0x0000280b,
         GUEST_PDPTR1                    = 0x0000280c,
@@ -144,6 +146,8 @@ enum vmcs_field {
         HOST_IA32_PAT_HIGH              = 0x00002c01,
         HOST_IA32_EFER                  = 0x00002c02,
         HOST_IA32_EFER_HIGH             = 0x00002c03,
+        HOST_IA32_PERF_GLOBAL_CTRL      = 0x00002c04,
+        HOST_IA32_PERF_GLOBAL_CTRL_HIGH = 0x00002c05,
         PIN_BASED_VM_EXEC_CONTROL       = 0x00004000,
         CPU_BASED_VM_EXEC_CONTROL       = 0x00004002,
         EXCEPTION_BITMAP                = 0x00004004,
@@ -426,4 +430,43 @@ struct vmx_msr_entry {
         u64 value;
 } __aligned(16);
 
+/*
+ * Exit Qualifications for entry failure during or after loading guest state
+ */
+#define ENTRY_FAIL_DEFAULT              0
+#define ENTRY_FAIL_PDPTE                2
+#define ENTRY_FAIL_NMI                  3
+#define ENTRY_FAIL_VMCS_LINK_PTR        4
+
+/*
+ * VM-instruction error numbers
+ */
+enum vm_instruction_error_number {
+        VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1,
+        VMXERR_VMCLEAR_INVALID_ADDRESS = 2,
+        VMXERR_VMCLEAR_VMXON_POINTER = 3,
+        VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4,
+        VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5,
+        VMXERR_VMRESUME_AFTER_VMXOFF = 6,
+        VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7,
+        VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8,
+        VMXERR_VMPTRLD_INVALID_ADDRESS = 9,
+        VMXERR_VMPTRLD_VMXON_POINTER = 10,
+        VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11,
+        VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12,
+        VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13,
+        VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15,
+        VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16,
+        VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17,
+        VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18,
+        VMXERR_VMCALL_NONCLEAR_VMCS = 19,
+        VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20,
+        VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22,
+        VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23,
+        VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24,
+        VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25,
+        VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26,
+        VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28,
+};
+
 #endif