diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2009-12-08 11:02:38 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2009-12-08 11:02:38 -0500 |
commit | ed9216c1717a3f3738a77908aff78995ea69e7ff (patch) | |
tree | c6b5ace7c333dabbf1d94074a13a98244bcdfb26 /arch/x86/kvm | |
parent | d7fc02c7bae7b1cf69269992cf880a43a350cdaa (diff) | |
parent | d5696725b2a4c59503f5e0bc33adeee7f30cd45b (diff) |
Merge branch 'kvm-updates/2.6.33' of git://git.kernel.org/pub/scm/virt/kvm/kvm
* 'kvm-updates/2.6.33' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (84 commits)
KVM: VMX: Fix comparison of guest efer with stale host value
KVM: s390: Fix prefix register checking in arch/s390/kvm/sigp.c
KVM: Drop user return notifier when disabling virtualization on a cpu
KVM: VMX: Disable unrestricted guest when EPT disabled
KVM: x86 emulator: limit instructions to 15 bytes
KVM: s390: Make psw available on all exits, not just a subset
KVM: x86: Add KVM_GET/SET_VCPU_EVENTS
KVM: VMX: Report unexpected simultaneous exceptions as internal errors
KVM: Allow internal errors reported to userspace to carry extra data
KVM: Reorder IOCTLs in main kvm.h
KVM: x86: Polish exception injection via KVM_SET_GUEST_DEBUG
KVM: only clear irq_source_id if irqchip is present
KVM: x86: disallow KVM_{SET,GET}_LAPIC without allocated in-kernel lapic
KVM: x86: disallow multiple KVM_CREATE_IRQCHIP
KVM: VMX: Remove vmx->msr_offset_efer
KVM: MMU: update invlpg handler comment
KVM: VMX: move CR3/PDPTR update to vmx_set_cr3
KVM: remove duplicated task_switch check
KVM: powerpc: Fix BUILD_BUG_ON condition
KVM: VMX: Use shared msr infrastructure
...
Trivial conflicts due to new Kconfig options in arch/Kconfig and kernel/Makefile
Diffstat (limited to 'arch/x86/kvm')
-rw-r--r-- | arch/x86/kvm/Kconfig | 1 | ||||
-rw-r--r-- | arch/x86/kvm/Makefile | 3 | ||||
-rw-r--r-- | arch/x86/kvm/emulate.c | 159 | ||||
-rw-r--r-- | arch/x86/kvm/i8254.c | 2 | ||||
-rw-r--r-- | arch/x86/kvm/i8259.c | 44 | ||||
-rw-r--r-- | arch/x86/kvm/irq.h | 7 | ||||
-rw-r--r-- | arch/x86/kvm/lapic.c | 8 | ||||
-rw-r--r-- | arch/x86/kvm/mmu.c | 3 | ||||
-rw-r--r-- | arch/x86/kvm/paging_tmpl.h | 1 | ||||
-rw-r--r-- | arch/x86/kvm/svm.c | 331 | ||||
-rw-r--r-- | arch/x86/kvm/trace.h | 165 | ||||
-rw-r--r-- | arch/x86/kvm/vmx.c | 448 | ||||
-rw-r--r-- | arch/x86/kvm/x86.c | 550 |
13 files changed, 1239 insertions, 483 deletions
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index b84e571f4175..4cd498332466 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig | |||
@@ -28,6 +28,7 @@ config KVM | |||
28 | select HAVE_KVM_IRQCHIP | 28 | select HAVE_KVM_IRQCHIP |
29 | select HAVE_KVM_EVENTFD | 29 | select HAVE_KVM_EVENTFD |
30 | select KVM_APIC_ARCHITECTURE | 30 | select KVM_APIC_ARCHITECTURE |
31 | select USER_RETURN_NOTIFIER | ||
31 | ---help--- | 32 | ---help--- |
32 | Support hosting fully virtualized guest machines using hardware | 33 | Support hosting fully virtualized guest machines using hardware |
33 | virtualization extensions. You will need a fairly recent | 34 | virtualization extensions. You will need a fairly recent |
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 0e7fe78d0f74..31a7035c4bd9 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile | |||
@@ -6,7 +6,8 @@ CFLAGS_svm.o := -I. | |||
6 | CFLAGS_vmx.o := -I. | 6 | CFLAGS_vmx.o := -I. |
7 | 7 | ||
8 | kvm-y += $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \ | 8 | kvm-y += $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \ |
9 | coalesced_mmio.o irq_comm.o eventfd.o) | 9 | coalesced_mmio.o irq_comm.o eventfd.o \ |
10 | assigned-dev.o) | ||
10 | kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o) | 11 | kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o) |
11 | 12 | ||
12 | kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ | 13 | kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ |
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 1be5cd640e93..7e8faea4651e 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c | |||
@@ -75,6 +75,8 @@ | |||
75 | #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */ | 75 | #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */ |
76 | #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */ | 76 | #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */ |
77 | #define GroupMask 0xff /* Group number stored in bits 0:7 */ | 77 | #define GroupMask 0xff /* Group number stored in bits 0:7 */ |
78 | /* Misc flags */ | ||
79 | #define No64 (1<<28) | ||
78 | /* Source 2 operand type */ | 80 | /* Source 2 operand type */ |
79 | #define Src2None (0<<29) | 81 | #define Src2None (0<<29) |
80 | #define Src2CL (1<<29) | 82 | #define Src2CL (1<<29) |
@@ -92,19 +94,23 @@ static u32 opcode_table[256] = { | |||
92 | /* 0x00 - 0x07 */ | 94 | /* 0x00 - 0x07 */ |
93 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, | 95 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, |
94 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, | 96 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, |
95 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, 0, 0, | 97 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, |
98 | ImplicitOps | Stack | No64, ImplicitOps | Stack | No64, | ||
96 | /* 0x08 - 0x0F */ | 99 | /* 0x08 - 0x0F */ |
97 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, | 100 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, |
98 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, | 101 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, |
99 | 0, 0, 0, 0, | 102 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, |
103 | ImplicitOps | Stack | No64, 0, | ||
100 | /* 0x10 - 0x17 */ | 104 | /* 0x10 - 0x17 */ |
101 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, | 105 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, |
102 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, | 106 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, |
103 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, 0, 0, | 107 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, |
108 | ImplicitOps | Stack | No64, ImplicitOps | Stack | No64, | ||
104 | /* 0x18 - 0x1F */ | 109 | /* 0x18 - 0x1F */ |
105 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, | 110 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, |
106 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, | 111 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, |
107 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, 0, 0, | 112 | ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, |
113 | ImplicitOps | Stack | No64, ImplicitOps | Stack | No64, | ||
108 | /* 0x20 - 0x27 */ | 114 | /* 0x20 - 0x27 */ |
109 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, | 115 | ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, |
110 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, | 116 | ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, |
@@ -133,7 +139,8 @@ static u32 opcode_table[256] = { | |||
133 | DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, | 139 | DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, |
134 | DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, | 140 | DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, |
135 | /* 0x60 - 0x67 */ | 141 | /* 0x60 - 0x67 */ |
136 | 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , | 142 | ImplicitOps | Stack | No64, ImplicitOps | Stack | No64, |
143 | 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , | ||
137 | 0, 0, 0, 0, | 144 | 0, 0, 0, 0, |
138 | /* 0x68 - 0x6F */ | 145 | /* 0x68 - 0x6F */ |
139 | SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0, | 146 | SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0, |
@@ -158,7 +165,7 @@ static u32 opcode_table[256] = { | |||
158 | /* 0x90 - 0x97 */ | 165 | /* 0x90 - 0x97 */ |
159 | DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, | 166 | DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, |
160 | /* 0x98 - 0x9F */ | 167 | /* 0x98 - 0x9F */ |
161 | 0, 0, SrcImm | Src2Imm16, 0, | 168 | 0, 0, SrcImm | Src2Imm16 | No64, 0, |
162 | ImplicitOps | Stack, ImplicitOps | Stack, 0, 0, | 169 | ImplicitOps | Stack, ImplicitOps | Stack, 0, 0, |
163 | /* 0xA0 - 0xA7 */ | 170 | /* 0xA0 - 0xA7 */ |
164 | ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, | 171 | ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, |
@@ -185,7 +192,7 @@ static u32 opcode_table[256] = { | |||
185 | ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, | 192 | ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, |
186 | /* 0xC8 - 0xCF */ | 193 | /* 0xC8 - 0xCF */ |
187 | 0, 0, 0, ImplicitOps | Stack, | 194 | 0, 0, 0, ImplicitOps | Stack, |
188 | ImplicitOps, SrcImmByte, ImplicitOps, ImplicitOps, | 195 | ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps, |
189 | /* 0xD0 - 0xD7 */ | 196 | /* 0xD0 - 0xD7 */ |
190 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, | 197 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, |
191 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, | 198 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, |
@@ -198,7 +205,7 @@ static u32 opcode_table[256] = { | |||
198 | ByteOp | SrcImmUByte, SrcImmUByte, | 205 | ByteOp | SrcImmUByte, SrcImmUByte, |
199 | /* 0xE8 - 0xEF */ | 206 | /* 0xE8 - 0xEF */ |
200 | SrcImm | Stack, SrcImm | ImplicitOps, | 207 | SrcImm | Stack, SrcImm | ImplicitOps, |
201 | SrcImmU | Src2Imm16, SrcImmByte | ImplicitOps, | 208 | SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps, |
202 | SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, | 209 | SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, |
203 | SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, | 210 | SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, |
204 | /* 0xF0 - 0xF7 */ | 211 | /* 0xF0 - 0xF7 */ |
@@ -244,11 +251,13 @@ static u32 twobyte_table[256] = { | |||
244 | /* 0x90 - 0x9F */ | 251 | /* 0x90 - 0x9F */ |
245 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | 252 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
246 | /* 0xA0 - 0xA7 */ | 253 | /* 0xA0 - 0xA7 */ |
247 | 0, 0, 0, DstMem | SrcReg | ModRM | BitOp, | 254 | ImplicitOps | Stack, ImplicitOps | Stack, |
255 | 0, DstMem | SrcReg | ModRM | BitOp, | ||
248 | DstMem | SrcReg | Src2ImmByte | ModRM, | 256 | DstMem | SrcReg | Src2ImmByte | ModRM, |
249 | DstMem | SrcReg | Src2CL | ModRM, 0, 0, | 257 | DstMem | SrcReg | Src2CL | ModRM, 0, 0, |
250 | /* 0xA8 - 0xAF */ | 258 | /* 0xA8 - 0xAF */ |
251 | 0, 0, 0, DstMem | SrcReg | ModRM | BitOp, | 259 | ImplicitOps | Stack, ImplicitOps | Stack, |
260 | 0, DstMem | SrcReg | ModRM | BitOp, | ||
252 | DstMem | SrcReg | Src2ImmByte | ModRM, | 261 | DstMem | SrcReg | Src2ImmByte | ModRM, |
253 | DstMem | SrcReg | Src2CL | ModRM, | 262 | DstMem | SrcReg | Src2CL | ModRM, |
254 | ModRM, 0, | 263 | ModRM, 0, |
@@ -613,6 +622,9 @@ static int do_insn_fetch(struct x86_emulate_ctxt *ctxt, | |||
613 | { | 622 | { |
614 | int rc = 0; | 623 | int rc = 0; |
615 | 624 | ||
625 | /* x86 instructions are limited to 15 bytes. */ | ||
626 | if (eip + size - ctxt->decode.eip_orig > 15) | ||
627 | return X86EMUL_UNHANDLEABLE; | ||
616 | eip += ctxt->cs_base; | 628 | eip += ctxt->cs_base; |
617 | while (size--) { | 629 | while (size--) { |
618 | rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++); | 630 | rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++); |
@@ -871,7 +883,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) | |||
871 | /* Shadow copy of register state. Committed on successful emulation. */ | 883 | /* Shadow copy of register state. Committed on successful emulation. */ |
872 | 884 | ||
873 | memset(c, 0, sizeof(struct decode_cache)); | 885 | memset(c, 0, sizeof(struct decode_cache)); |
874 | c->eip = kvm_rip_read(ctxt->vcpu); | 886 | c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu); |
875 | ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS); | 887 | ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS); |
876 | memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); | 888 | memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); |
877 | 889 | ||
@@ -962,6 +974,11 @@ done_prefixes: | |||
962 | } | 974 | } |
963 | } | 975 | } |
964 | 976 | ||
977 | if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) { | ||
978 | kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");; | ||
979 | return -1; | ||
980 | } | ||
981 | |||
965 | if (c->d & Group) { | 982 | if (c->d & Group) { |
966 | group = c->d & GroupMask; | 983 | group = c->d & GroupMask; |
967 | c->modrm = insn_fetch(u8, 1, c->eip); | 984 | c->modrm = insn_fetch(u8, 1, c->eip); |
@@ -1186,6 +1203,69 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt, | |||
1186 | return rc; | 1203 | return rc; |
1187 | } | 1204 | } |
1188 | 1205 | ||
1206 | static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg) | ||
1207 | { | ||
1208 | struct decode_cache *c = &ctxt->decode; | ||
1209 | struct kvm_segment segment; | ||
1210 | |||
1211 | kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg); | ||
1212 | |||
1213 | c->src.val = segment.selector; | ||
1214 | emulate_push(ctxt); | ||
1215 | } | ||
1216 | |||
1217 | static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, | ||
1218 | struct x86_emulate_ops *ops, int seg) | ||
1219 | { | ||
1220 | struct decode_cache *c = &ctxt->decode; | ||
1221 | unsigned long selector; | ||
1222 | int rc; | ||
1223 | |||
1224 | rc = emulate_pop(ctxt, ops, &selector, c->op_bytes); | ||
1225 | if (rc != 0) | ||
1226 | return rc; | ||
1227 | |||
1228 | rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg); | ||
1229 | return rc; | ||
1230 | } | ||
1231 | |||
1232 | static void emulate_pusha(struct x86_emulate_ctxt *ctxt) | ||
1233 | { | ||
1234 | struct decode_cache *c = &ctxt->decode; | ||
1235 | unsigned long old_esp = c->regs[VCPU_REGS_RSP]; | ||
1236 | int reg = VCPU_REGS_RAX; | ||
1237 | |||
1238 | while (reg <= VCPU_REGS_RDI) { | ||
1239 | (reg == VCPU_REGS_RSP) ? | ||
1240 | (c->src.val = old_esp) : (c->src.val = c->regs[reg]); | ||
1241 | |||
1242 | emulate_push(ctxt); | ||
1243 | ++reg; | ||
1244 | } | ||
1245 | } | ||
1246 | |||
1247 | static int emulate_popa(struct x86_emulate_ctxt *ctxt, | ||
1248 | struct x86_emulate_ops *ops) | ||
1249 | { | ||
1250 | struct decode_cache *c = &ctxt->decode; | ||
1251 | int rc = 0; | ||
1252 | int reg = VCPU_REGS_RDI; | ||
1253 | |||
1254 | while (reg >= VCPU_REGS_RAX) { | ||
1255 | if (reg == VCPU_REGS_RSP) { | ||
1256 | register_address_increment(c, &c->regs[VCPU_REGS_RSP], | ||
1257 | c->op_bytes); | ||
1258 | --reg; | ||
1259 | } | ||
1260 | |||
1261 | rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes); | ||
1262 | if (rc != 0) | ||
1263 | break; | ||
1264 | --reg; | ||
1265 | } | ||
1266 | return rc; | ||
1267 | } | ||
1268 | |||
1189 | static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, | 1269 | static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, |
1190 | struct x86_emulate_ops *ops) | 1270 | struct x86_emulate_ops *ops) |
1191 | { | 1271 | { |
@@ -1707,18 +1787,45 @@ special_insn: | |||
1707 | add: /* add */ | 1787 | add: /* add */ |
1708 | emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags); | 1788 | emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags); |
1709 | break; | 1789 | break; |
1790 | case 0x06: /* push es */ | ||
1791 | emulate_push_sreg(ctxt, VCPU_SREG_ES); | ||
1792 | break; | ||
1793 | case 0x07: /* pop es */ | ||
1794 | rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES); | ||
1795 | if (rc != 0) | ||
1796 | goto done; | ||
1797 | break; | ||
1710 | case 0x08 ... 0x0d: | 1798 | case 0x08 ... 0x0d: |
1711 | or: /* or */ | 1799 | or: /* or */ |
1712 | emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags); | 1800 | emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags); |
1713 | break; | 1801 | break; |
1802 | case 0x0e: /* push cs */ | ||
1803 | emulate_push_sreg(ctxt, VCPU_SREG_CS); | ||
1804 | break; | ||
1714 | case 0x10 ... 0x15: | 1805 | case 0x10 ... 0x15: |
1715 | adc: /* adc */ | 1806 | adc: /* adc */ |
1716 | emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags); | 1807 | emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags); |
1717 | break; | 1808 | break; |
1809 | case 0x16: /* push ss */ | ||
1810 | emulate_push_sreg(ctxt, VCPU_SREG_SS); | ||
1811 | break; | ||
1812 | case 0x17: /* pop ss */ | ||
1813 | rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS); | ||
1814 | if (rc != 0) | ||
1815 | goto done; | ||
1816 | break; | ||
1718 | case 0x18 ... 0x1d: | 1817 | case 0x18 ... 0x1d: |
1719 | sbb: /* sbb */ | 1818 | sbb: /* sbb */ |
1720 | emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags); | 1819 | emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags); |
1721 | break; | 1820 | break; |
1821 | case 0x1e: /* push ds */ | ||
1822 | emulate_push_sreg(ctxt, VCPU_SREG_DS); | ||
1823 | break; | ||
1824 | case 0x1f: /* pop ds */ | ||
1825 | rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS); | ||
1826 | if (rc != 0) | ||
1827 | goto done; | ||
1828 | break; | ||
1722 | case 0x20 ... 0x25: | 1829 | case 0x20 ... 0x25: |
1723 | and: /* and */ | 1830 | and: /* and */ |
1724 | emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags); | 1831 | emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags); |
@@ -1750,6 +1857,14 @@ special_insn: | |||
1750 | if (rc != 0) | 1857 | if (rc != 0) |
1751 | goto done; | 1858 | goto done; |
1752 | break; | 1859 | break; |
1860 | case 0x60: /* pusha */ | ||
1861 | emulate_pusha(ctxt); | ||
1862 | break; | ||
1863 | case 0x61: /* popa */ | ||
1864 | rc = emulate_popa(ctxt, ops); | ||
1865 | if (rc != 0) | ||
1866 | goto done; | ||
1867 | break; | ||
1753 | case 0x63: /* movsxd */ | 1868 | case 0x63: /* movsxd */ |
1754 | if (ctxt->mode != X86EMUL_MODE_PROT64) | 1869 | if (ctxt->mode != X86EMUL_MODE_PROT64) |
1755 | goto cannot_emulate; | 1870 | goto cannot_emulate; |
@@ -1761,7 +1876,7 @@ special_insn: | |||
1761 | break; | 1876 | break; |
1762 | case 0x6c: /* insb */ | 1877 | case 0x6c: /* insb */ |
1763 | case 0x6d: /* insw/insd */ | 1878 | case 0x6d: /* insw/insd */ |
1764 | if (kvm_emulate_pio_string(ctxt->vcpu, NULL, | 1879 | if (kvm_emulate_pio_string(ctxt->vcpu, |
1765 | 1, | 1880 | 1, |
1766 | (c->d & ByteOp) ? 1 : c->op_bytes, | 1881 | (c->d & ByteOp) ? 1 : c->op_bytes, |
1767 | c->rep_prefix ? | 1882 | c->rep_prefix ? |
@@ -1777,7 +1892,7 @@ special_insn: | |||
1777 | return 0; | 1892 | return 0; |
1778 | case 0x6e: /* outsb */ | 1893 | case 0x6e: /* outsb */ |
1779 | case 0x6f: /* outsw/outsd */ | 1894 | case 0x6f: /* outsw/outsd */ |
1780 | if (kvm_emulate_pio_string(ctxt->vcpu, NULL, | 1895 | if (kvm_emulate_pio_string(ctxt->vcpu, |
1781 | 0, | 1896 | 0, |
1782 | (c->d & ByteOp) ? 1 : c->op_bytes, | 1897 | (c->d & ByteOp) ? 1 : c->op_bytes, |
1783 | c->rep_prefix ? | 1898 | c->rep_prefix ? |
@@ -2070,7 +2185,7 @@ special_insn: | |||
2070 | case 0xef: /* out (e/r)ax,dx */ | 2185 | case 0xef: /* out (e/r)ax,dx */ |
2071 | port = c->regs[VCPU_REGS_RDX]; | 2186 | port = c->regs[VCPU_REGS_RDX]; |
2072 | io_dir_in = 0; | 2187 | io_dir_in = 0; |
2073 | do_io: if (kvm_emulate_pio(ctxt->vcpu, NULL, io_dir_in, | 2188 | do_io: if (kvm_emulate_pio(ctxt->vcpu, io_dir_in, |
2074 | (c->d & ByteOp) ? 1 : c->op_bytes, | 2189 | (c->d & ByteOp) ? 1 : c->op_bytes, |
2075 | port) != 0) { | 2190 | port) != 0) { |
2076 | c->eip = saved_eip; | 2191 | c->eip = saved_eip; |
@@ -2297,6 +2412,14 @@ twobyte_insn: | |||
2297 | jmp_rel(c, c->src.val); | 2412 | jmp_rel(c, c->src.val); |
2298 | c->dst.type = OP_NONE; | 2413 | c->dst.type = OP_NONE; |
2299 | break; | 2414 | break; |
2415 | case 0xa0: /* push fs */ | ||
2416 | emulate_push_sreg(ctxt, VCPU_SREG_FS); | ||
2417 | break; | ||
2418 | case 0xa1: /* pop fs */ | ||
2419 | rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS); | ||
2420 | if (rc != 0) | ||
2421 | goto done; | ||
2422 | break; | ||
2300 | case 0xa3: | 2423 | case 0xa3: |
2301 | bt: /* bt */ | 2424 | bt: /* bt */ |
2302 | c->dst.type = OP_NONE; | 2425 | c->dst.type = OP_NONE; |
@@ -2308,6 +2431,14 @@ twobyte_insn: | |||
2308 | case 0xa5: /* shld cl, r, r/m */ | 2431 | case 0xa5: /* shld cl, r, r/m */ |
2309 | emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags); | 2432 | emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags); |
2310 | break; | 2433 | break; |
2434 | case 0xa8: /* push gs */ | ||
2435 | emulate_push_sreg(ctxt, VCPU_SREG_GS); | ||
2436 | break; | ||
2437 | case 0xa9: /* pop gs */ | ||
2438 | rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS); | ||
2439 | if (rc != 0) | ||
2440 | goto done; | ||
2441 | break; | ||
2311 | case 0xab: | 2442 | case 0xab: |
2312 | bts: /* bts */ | 2443 | bts: /* bts */ |
2313 | /* only subword offset */ | 2444 | /* only subword offset */ |
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index 144e7f60b5e2..fab7440c9bb2 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c | |||
@@ -688,10 +688,8 @@ static void __inject_pit_timer_intr(struct kvm *kvm) | |||
688 | struct kvm_vcpu *vcpu; | 688 | struct kvm_vcpu *vcpu; |
689 | int i; | 689 | int i; |
690 | 690 | ||
691 | mutex_lock(&kvm->irq_lock); | ||
692 | kvm_set_irq(kvm, kvm->arch.vpit->irq_source_id, 0, 1); | 691 | kvm_set_irq(kvm, kvm->arch.vpit->irq_source_id, 0, 1); |
693 | kvm_set_irq(kvm, kvm->arch.vpit->irq_source_id, 0, 0); | 692 | kvm_set_irq(kvm, kvm->arch.vpit->irq_source_id, 0, 0); |
694 | mutex_unlock(&kvm->irq_lock); | ||
695 | 693 | ||
696 | /* | 694 | /* |
697 | * Provides NMI watchdog support via Virtual Wire mode. | 695 | * Provides NMI watchdog support via Virtual Wire mode. |
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index 01f151682802..d057c0cbd245 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c | |||
@@ -38,7 +38,15 @@ static void pic_clear_isr(struct kvm_kpic_state *s, int irq) | |||
38 | s->isr_ack |= (1 << irq); | 38 | s->isr_ack |= (1 << irq); |
39 | if (s != &s->pics_state->pics[0]) | 39 | if (s != &s->pics_state->pics[0]) |
40 | irq += 8; | 40 | irq += 8; |
41 | /* | ||
42 | * We are dropping lock while calling ack notifiers since ack | ||
43 | * notifier callbacks for assigned devices call into PIC recursively. | ||
44 | * Other interrupt may be delivered to PIC while lock is dropped but | ||
45 | * it should be safe since PIC state is already updated at this stage. | ||
46 | */ | ||
47 | spin_unlock(&s->pics_state->lock); | ||
41 | kvm_notify_acked_irq(s->pics_state->kvm, SELECT_PIC(irq), irq); | 48 | kvm_notify_acked_irq(s->pics_state->kvm, SELECT_PIC(irq), irq); |
49 | spin_lock(&s->pics_state->lock); | ||
42 | } | 50 | } |
43 | 51 | ||
44 | void kvm_pic_clear_isr_ack(struct kvm *kvm) | 52 | void kvm_pic_clear_isr_ack(struct kvm *kvm) |
@@ -176,16 +184,18 @@ int kvm_pic_set_irq(void *opaque, int irq, int level) | |||
176 | static inline void pic_intack(struct kvm_kpic_state *s, int irq) | 184 | static inline void pic_intack(struct kvm_kpic_state *s, int irq) |
177 | { | 185 | { |
178 | s->isr |= 1 << irq; | 186 | s->isr |= 1 << irq; |
179 | if (s->auto_eoi) { | ||
180 | if (s->rotate_on_auto_eoi) | ||
181 | s->priority_add = (irq + 1) & 7; | ||
182 | pic_clear_isr(s, irq); | ||
183 | } | ||
184 | /* | 187 | /* |
185 | * We don't clear a level sensitive interrupt here | 188 | * We don't clear a level sensitive interrupt here |
186 | */ | 189 | */ |
187 | if (!(s->elcr & (1 << irq))) | 190 | if (!(s->elcr & (1 << irq))) |
188 | s->irr &= ~(1 << irq); | 191 | s->irr &= ~(1 << irq); |
192 | |||
193 | if (s->auto_eoi) { | ||
194 | if (s->rotate_on_auto_eoi) | ||
195 | s->priority_add = (irq + 1) & 7; | ||
196 | pic_clear_isr(s, irq); | ||
197 | } | ||
198 | |||
189 | } | 199 | } |
190 | 200 | ||
191 | int kvm_pic_read_irq(struct kvm *kvm) | 201 | int kvm_pic_read_irq(struct kvm *kvm) |
@@ -225,22 +235,11 @@ int kvm_pic_read_irq(struct kvm *kvm) | |||
225 | 235 | ||
226 | void kvm_pic_reset(struct kvm_kpic_state *s) | 236 | void kvm_pic_reset(struct kvm_kpic_state *s) |
227 | { | 237 | { |
228 | int irq, irqbase, n; | 238 | int irq; |
229 | struct kvm *kvm = s->pics_state->irq_request_opaque; | 239 | struct kvm *kvm = s->pics_state->irq_request_opaque; |
230 | struct kvm_vcpu *vcpu0 = kvm->bsp_vcpu; | 240 | struct kvm_vcpu *vcpu0 = kvm->bsp_vcpu; |
241 | u8 irr = s->irr, isr = s->imr; | ||
231 | 242 | ||
232 | if (s == &s->pics_state->pics[0]) | ||
233 | irqbase = 0; | ||
234 | else | ||
235 | irqbase = 8; | ||
236 | |||
237 | for (irq = 0; irq < PIC_NUM_PINS/2; irq++) { | ||
238 | if (vcpu0 && kvm_apic_accept_pic_intr(vcpu0)) | ||
239 | if (s->irr & (1 << irq) || s->isr & (1 << irq)) { | ||
240 | n = irq + irqbase; | ||
241 | kvm_notify_acked_irq(kvm, SELECT_PIC(n), n); | ||
242 | } | ||
243 | } | ||
244 | s->last_irr = 0; | 243 | s->last_irr = 0; |
245 | s->irr = 0; | 244 | s->irr = 0; |
246 | s->imr = 0; | 245 | s->imr = 0; |
@@ -256,6 +255,13 @@ void kvm_pic_reset(struct kvm_kpic_state *s) | |||
256 | s->rotate_on_auto_eoi = 0; | 255 | s->rotate_on_auto_eoi = 0; |
257 | s->special_fully_nested_mode = 0; | 256 | s->special_fully_nested_mode = 0; |
258 | s->init4 = 0; | 257 | s->init4 = 0; |
258 | |||
259 | for (irq = 0; irq < PIC_NUM_PINS/2; irq++) { | ||
260 | if (vcpu0 && kvm_apic_accept_pic_intr(vcpu0)) | ||
261 | if (irr & (1 << irq) || isr & (1 << irq)) { | ||
262 | pic_clear_isr(s, irq); | ||
263 | } | ||
264 | } | ||
259 | } | 265 | } |
260 | 266 | ||
261 | static void pic_ioport_write(void *opaque, u32 addr, u32 val) | 267 | static void pic_ioport_write(void *opaque, u32 addr, u32 val) |
@@ -298,9 +304,9 @@ static void pic_ioport_write(void *opaque, u32 addr, u32 val) | |||
298 | priority = get_priority(s, s->isr); | 304 | priority = get_priority(s, s->isr); |
299 | if (priority != 8) { | 305 | if (priority != 8) { |
300 | irq = (priority + s->priority_add) & 7; | 306 | irq = (priority + s->priority_add) & 7; |
301 | pic_clear_isr(s, irq); | ||
302 | if (cmd == 5) | 307 | if (cmd == 5) |
303 | s->priority_add = (irq + 1) & 7; | 308 | s->priority_add = (irq + 1) & 7; |
309 | pic_clear_isr(s, irq); | ||
304 | pic_update_irq(s->pics_state); | 310 | pic_update_irq(s->pics_state); |
305 | } | 311 | } |
306 | break; | 312 | break; |
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 7d6058a2fd38..be399e207d57 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h | |||
@@ -71,6 +71,7 @@ struct kvm_pic { | |||
71 | int output; /* intr from master PIC */ | 71 | int output; /* intr from master PIC */ |
72 | struct kvm_io_device dev; | 72 | struct kvm_io_device dev; |
73 | void (*ack_notifier)(void *opaque, int irq); | 73 | void (*ack_notifier)(void *opaque, int irq); |
74 | unsigned long irq_states[16]; | ||
74 | }; | 75 | }; |
75 | 76 | ||
76 | struct kvm_pic *kvm_create_pic(struct kvm *kvm); | 77 | struct kvm_pic *kvm_create_pic(struct kvm *kvm); |
@@ -85,7 +86,11 @@ static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) | |||
85 | 86 | ||
86 | static inline int irqchip_in_kernel(struct kvm *kvm) | 87 | static inline int irqchip_in_kernel(struct kvm *kvm) |
87 | { | 88 | { |
88 | return pic_irqchip(kvm) != NULL; | 89 | int ret; |
90 | |||
91 | ret = (pic_irqchip(kvm) != NULL); | ||
92 | smp_rmb(); | ||
93 | return ret; | ||
89 | } | 94 | } |
90 | 95 | ||
91 | void kvm_pic_reset(struct kvm_kpic_state *s); | 96 | void kvm_pic_reset(struct kvm_kpic_state *s); |
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 23c217692ea9..cd60c0bd1b32 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c | |||
@@ -32,7 +32,6 @@ | |||
32 | #include <asm/current.h> | 32 | #include <asm/current.h> |
33 | #include <asm/apicdef.h> | 33 | #include <asm/apicdef.h> |
34 | #include <asm/atomic.h> | 34 | #include <asm/atomic.h> |
35 | #include <asm/apicdef.h> | ||
36 | #include "kvm_cache_regs.h" | 35 | #include "kvm_cache_regs.h" |
37 | #include "irq.h" | 36 | #include "irq.h" |
38 | #include "trace.h" | 37 | #include "trace.h" |
@@ -471,11 +470,8 @@ static void apic_set_eoi(struct kvm_lapic *apic) | |||
471 | trigger_mode = IOAPIC_LEVEL_TRIG; | 470 | trigger_mode = IOAPIC_LEVEL_TRIG; |
472 | else | 471 | else |
473 | trigger_mode = IOAPIC_EDGE_TRIG; | 472 | trigger_mode = IOAPIC_EDGE_TRIG; |
474 | if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)) { | 473 | if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)) |
475 | mutex_lock(&apic->vcpu->kvm->irq_lock); | ||
476 | kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode); | 474 | kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode); |
477 | mutex_unlock(&apic->vcpu->kvm->irq_lock); | ||
478 | } | ||
479 | } | 475 | } |
480 | 476 | ||
481 | static void apic_send_ipi(struct kvm_lapic *apic) | 477 | static void apic_send_ipi(struct kvm_lapic *apic) |
@@ -504,9 +500,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) | |||
504 | irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, | 500 | irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, |
505 | irq.vector); | 501 | irq.vector); |
506 | 502 | ||
507 | mutex_lock(&apic->vcpu->kvm->irq_lock); | ||
508 | kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq); | 503 | kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq); |
509 | mutex_unlock(&apic->vcpu->kvm->irq_lock); | ||
510 | } | 504 | } |
511 | 505 | ||
512 | static u32 apic_get_tmcct(struct kvm_lapic *apic) | 506 | static u32 apic_get_tmcct(struct kvm_lapic *apic) |
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 818b92ad82cf..4c3e5b2314cb 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c | |||
@@ -2789,7 +2789,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) | |||
2789 | if (r) | 2789 | if (r) |
2790 | goto out; | 2790 | goto out; |
2791 | 2791 | ||
2792 | er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); | 2792 | er = emulate_instruction(vcpu, cr2, error_code, 0); |
2793 | 2793 | ||
2794 | switch (er) { | 2794 | switch (er) { |
2795 | case EMULATE_DONE: | 2795 | case EMULATE_DONE: |
@@ -2800,6 +2800,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) | |||
2800 | case EMULATE_FAIL: | 2800 | case EMULATE_FAIL: |
2801 | vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; | 2801 | vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
2802 | vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; | 2802 | vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; |
2803 | vcpu->run->internal.ndata = 0; | ||
2803 | return 0; | 2804 | return 0; |
2804 | default: | 2805 | default: |
2805 | BUG(); | 2806 | BUG(); |
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 72558f8ff3f5..a6017132fba8 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h | |||
@@ -467,7 +467,6 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva) | |||
467 | level = iterator.level; | 467 | level = iterator.level; |
468 | sptep = iterator.sptep; | 468 | sptep = iterator.sptep; |
469 | 469 | ||
470 | /* FIXME: properly handle invlpg on large guest pages */ | ||
471 | if (level == PT_PAGE_TABLE_LEVEL || | 470 | if (level == PT_PAGE_TABLE_LEVEL || |
472 | ((level == PT_DIRECTORY_LEVEL && is_large_pte(*sptep))) || | 471 | ((level == PT_DIRECTORY_LEVEL && is_large_pte(*sptep))) || |
473 | ((level == PT_PDPE_LEVEL && is_large_pte(*sptep)))) { | 472 | ((level == PT_PDPE_LEVEL && is_large_pte(*sptep)))) { |
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index c17404add91f..3de0b37ec038 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c | |||
@@ -46,6 +46,7 @@ MODULE_LICENSE("GPL"); | |||
46 | #define SVM_FEATURE_NPT (1 << 0) | 46 | #define SVM_FEATURE_NPT (1 << 0) |
47 | #define SVM_FEATURE_LBRV (1 << 1) | 47 | #define SVM_FEATURE_LBRV (1 << 1) |
48 | #define SVM_FEATURE_SVML (1 << 2) | 48 | #define SVM_FEATURE_SVML (1 << 2) |
49 | #define SVM_FEATURE_PAUSE_FILTER (1 << 10) | ||
49 | 50 | ||
50 | #define NESTED_EXIT_HOST 0 /* Exit handled on host level */ | 51 | #define NESTED_EXIT_HOST 0 /* Exit handled on host level */ |
51 | #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ | 52 | #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ |
@@ -53,15 +54,6 @@ MODULE_LICENSE("GPL"); | |||
53 | 54 | ||
54 | #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) | 55 | #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) |
55 | 56 | ||
56 | /* Turn on to get debugging output*/ | ||
57 | /* #define NESTED_DEBUG */ | ||
58 | |||
59 | #ifdef NESTED_DEBUG | ||
60 | #define nsvm_printk(fmt, args...) printk(KERN_INFO fmt, ## args) | ||
61 | #else | ||
62 | #define nsvm_printk(fmt, args...) do {} while(0) | ||
63 | #endif | ||
64 | |||
65 | static const u32 host_save_user_msrs[] = { | 57 | static const u32 host_save_user_msrs[] = { |
66 | #ifdef CONFIG_X86_64 | 58 | #ifdef CONFIG_X86_64 |
67 | MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, | 59 | MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, |
@@ -85,6 +77,9 @@ struct nested_state { | |||
85 | /* gpa pointers to the real vectors */ | 77 | /* gpa pointers to the real vectors */ |
86 | u64 vmcb_msrpm; | 78 | u64 vmcb_msrpm; |
87 | 79 | ||
80 | /* A VMEXIT is required but not yet emulated */ | ||
81 | bool exit_required; | ||
82 | |||
88 | /* cache for intercepts of the guest */ | 83 | /* cache for intercepts of the guest */ |
89 | u16 intercept_cr_read; | 84 | u16 intercept_cr_read; |
90 | u16 intercept_cr_write; | 85 | u16 intercept_cr_write; |
@@ -112,6 +107,8 @@ struct vcpu_svm { | |||
112 | u32 *msrpm; | 107 | u32 *msrpm; |
113 | 108 | ||
114 | struct nested_state nested; | 109 | struct nested_state nested; |
110 | |||
111 | bool nmi_singlestep; | ||
115 | }; | 112 | }; |
116 | 113 | ||
117 | /* enable NPT for AMD64 and X86 with PAE */ | 114 | /* enable NPT for AMD64 and X86 with PAE */ |
@@ -286,7 +283,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) | |||
286 | struct vcpu_svm *svm = to_svm(vcpu); | 283 | struct vcpu_svm *svm = to_svm(vcpu); |
287 | 284 | ||
288 | if (!svm->next_rip) { | 285 | if (!svm->next_rip) { |
289 | if (emulate_instruction(vcpu, vcpu->run, 0, 0, EMULTYPE_SKIP) != | 286 | if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) != |
290 | EMULATE_DONE) | 287 | EMULATE_DONE) |
291 | printk(KERN_DEBUG "%s: NOP\n", __func__); | 288 | printk(KERN_DEBUG "%s: NOP\n", __func__); |
292 | return; | 289 | return; |
@@ -316,7 +313,7 @@ static void svm_hardware_disable(void *garbage) | |||
316 | cpu_svm_disable(); | 313 | cpu_svm_disable(); |
317 | } | 314 | } |
318 | 315 | ||
319 | static void svm_hardware_enable(void *garbage) | 316 | static int svm_hardware_enable(void *garbage) |
320 | { | 317 | { |
321 | 318 | ||
322 | struct svm_cpu_data *svm_data; | 319 | struct svm_cpu_data *svm_data; |
@@ -325,16 +322,21 @@ static void svm_hardware_enable(void *garbage) | |||
325 | struct desc_struct *gdt; | 322 | struct desc_struct *gdt; |
326 | int me = raw_smp_processor_id(); | 323 | int me = raw_smp_processor_id(); |
327 | 324 | ||
325 | rdmsrl(MSR_EFER, efer); | ||
326 | if (efer & EFER_SVME) | ||
327 | return -EBUSY; | ||
328 | |||
328 | if (!has_svm()) { | 329 | if (!has_svm()) { |
329 | printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me); | 330 | printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n", |
330 | return; | 331 | me); |
332 | return -EINVAL; | ||
331 | } | 333 | } |
332 | svm_data = per_cpu(svm_data, me); | 334 | svm_data = per_cpu(svm_data, me); |
333 | 335 | ||
334 | if (!svm_data) { | 336 | if (!svm_data) { |
335 | printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n", | 337 | printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n", |
336 | me); | 338 | me); |
337 | return; | 339 | return -EINVAL; |
338 | } | 340 | } |
339 | 341 | ||
340 | svm_data->asid_generation = 1; | 342 | svm_data->asid_generation = 1; |
@@ -345,11 +347,12 @@ static void svm_hardware_enable(void *garbage) | |||
345 | gdt = (struct desc_struct *)gdt_descr.base; | 347 | gdt = (struct desc_struct *)gdt_descr.base; |
346 | svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); | 348 | svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); |
347 | 349 | ||
348 | rdmsrl(MSR_EFER, efer); | ||
349 | wrmsrl(MSR_EFER, efer | EFER_SVME); | 350 | wrmsrl(MSR_EFER, efer | EFER_SVME); |
350 | 351 | ||
351 | wrmsrl(MSR_VM_HSAVE_PA, | 352 | wrmsrl(MSR_VM_HSAVE_PA, |
352 | page_to_pfn(svm_data->save_area) << PAGE_SHIFT); | 353 | page_to_pfn(svm_data->save_area) << PAGE_SHIFT); |
354 | |||
355 | return 0; | ||
353 | } | 356 | } |
354 | 357 | ||
355 | static void svm_cpu_uninit(int cpu) | 358 | static void svm_cpu_uninit(int cpu) |
@@ -476,7 +479,7 @@ static __init int svm_hardware_setup(void) | |||
476 | kvm_enable_efer_bits(EFER_SVME); | 479 | kvm_enable_efer_bits(EFER_SVME); |
477 | } | 480 | } |
478 | 481 | ||
479 | for_each_online_cpu(cpu) { | 482 | for_each_possible_cpu(cpu) { |
480 | r = svm_cpu_init(cpu); | 483 | r = svm_cpu_init(cpu); |
481 | if (r) | 484 | if (r) |
482 | goto err; | 485 | goto err; |
@@ -510,7 +513,7 @@ static __exit void svm_hardware_unsetup(void) | |||
510 | { | 513 | { |
511 | int cpu; | 514 | int cpu; |
512 | 515 | ||
513 | for_each_online_cpu(cpu) | 516 | for_each_possible_cpu(cpu) |
514 | svm_cpu_uninit(cpu); | 517 | svm_cpu_uninit(cpu); |
515 | 518 | ||
516 | __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER); | 519 | __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER); |
@@ -625,11 +628,12 @@ static void init_vmcb(struct vcpu_svm *svm) | |||
625 | save->rip = 0x0000fff0; | 628 | save->rip = 0x0000fff0; |
626 | svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; | 629 | svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; |
627 | 630 | ||
628 | /* | 631 | /* This is the guest-visible cr0 value. |
629 | * cr0 val on cpu init should be 0x60000010, we enable cpu | 632 | * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0. |
630 | * cache by default. the orderly way is to enable cache in bios. | ||
631 | */ | 633 | */ |
632 | save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP; | 634 | svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET; |
635 | kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0); | ||
636 | |||
633 | save->cr4 = X86_CR4_PAE; | 637 | save->cr4 = X86_CR4_PAE; |
634 | /* rdx = ?? */ | 638 | /* rdx = ?? */ |
635 | 639 | ||
@@ -644,8 +648,6 @@ static void init_vmcb(struct vcpu_svm *svm) | |||
644 | control->intercept_cr_write &= ~(INTERCEPT_CR0_MASK| | 648 | control->intercept_cr_write &= ~(INTERCEPT_CR0_MASK| |
645 | INTERCEPT_CR3_MASK); | 649 | INTERCEPT_CR3_MASK); |
646 | save->g_pat = 0x0007040600070406ULL; | 650 | save->g_pat = 0x0007040600070406ULL; |
647 | /* enable caching because the QEMU Bios doesn't enable it */ | ||
648 | save->cr0 = X86_CR0_ET; | ||
649 | save->cr3 = 0; | 651 | save->cr3 = 0; |
650 | save->cr4 = 0; | 652 | save->cr4 = 0; |
651 | } | 653 | } |
@@ -654,6 +656,11 @@ static void init_vmcb(struct vcpu_svm *svm) | |||
654 | svm->nested.vmcb = 0; | 656 | svm->nested.vmcb = 0; |
655 | svm->vcpu.arch.hflags = 0; | 657 | svm->vcpu.arch.hflags = 0; |
656 | 658 | ||
659 | if (svm_has(SVM_FEATURE_PAUSE_FILTER)) { | ||
660 | control->pause_filter_count = 3000; | ||
661 | control->intercept |= (1ULL << INTERCEPT_PAUSE); | ||
662 | } | ||
663 | |||
657 | enable_gif(svm); | 664 | enable_gif(svm); |
658 | } | 665 | } |
659 | 666 | ||
@@ -758,14 +765,13 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) | |||
758 | int i; | 765 | int i; |
759 | 766 | ||
760 | if (unlikely(cpu != vcpu->cpu)) { | 767 | if (unlikely(cpu != vcpu->cpu)) { |
761 | u64 tsc_this, delta; | 768 | u64 delta; |
762 | 769 | ||
763 | /* | 770 | /* |
764 | * Make sure that the guest sees a monotonically | 771 | * Make sure that the guest sees a monotonically |
765 | * increasing TSC. | 772 | * increasing TSC. |
766 | */ | 773 | */ |
767 | rdtscll(tsc_this); | 774 | delta = vcpu->arch.host_tsc - native_read_tsc(); |
768 | delta = vcpu->arch.host_tsc - tsc_this; | ||
769 | svm->vmcb->control.tsc_offset += delta; | 775 | svm->vmcb->control.tsc_offset += delta; |
770 | if (is_nested(svm)) | 776 | if (is_nested(svm)) |
771 | svm->nested.hsave->control.tsc_offset += delta; | 777 | svm->nested.hsave->control.tsc_offset += delta; |
@@ -787,7 +793,7 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) | |||
787 | for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) | 793 | for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) |
788 | wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); | 794 | wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); |
789 | 795 | ||
790 | rdtscll(vcpu->arch.host_tsc); | 796 | vcpu->arch.host_tsc = native_read_tsc(); |
791 | } | 797 | } |
792 | 798 | ||
793 | static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) | 799 | static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) |
@@ -1045,7 +1051,7 @@ static void update_db_intercept(struct kvm_vcpu *vcpu) | |||
1045 | svm->vmcb->control.intercept_exceptions &= | 1051 | svm->vmcb->control.intercept_exceptions &= |
1046 | ~((1 << DB_VECTOR) | (1 << BP_VECTOR)); | 1052 | ~((1 << DB_VECTOR) | (1 << BP_VECTOR)); |
1047 | 1053 | ||
1048 | if (vcpu->arch.singlestep) | 1054 | if (svm->nmi_singlestep) |
1049 | svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR); | 1055 | svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR); |
1050 | 1056 | ||
1051 | if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { | 1057 | if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { |
@@ -1060,26 +1066,16 @@ static void update_db_intercept(struct kvm_vcpu *vcpu) | |||
1060 | vcpu->guest_debug = 0; | 1066 | vcpu->guest_debug = 0; |
1061 | } | 1067 | } |
1062 | 1068 | ||
1063 | static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) | 1069 | static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) |
1064 | { | 1070 | { |
1065 | int old_debug = vcpu->guest_debug; | ||
1066 | struct vcpu_svm *svm = to_svm(vcpu); | 1071 | struct vcpu_svm *svm = to_svm(vcpu); |
1067 | 1072 | ||
1068 | vcpu->guest_debug = dbg->control; | ||
1069 | |||
1070 | update_db_intercept(vcpu); | ||
1071 | |||
1072 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | 1073 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) |
1073 | svm->vmcb->save.dr7 = dbg->arch.debugreg[7]; | 1074 | svm->vmcb->save.dr7 = dbg->arch.debugreg[7]; |
1074 | else | 1075 | else |
1075 | svm->vmcb->save.dr7 = vcpu->arch.dr7; | 1076 | svm->vmcb->save.dr7 = vcpu->arch.dr7; |
1076 | 1077 | ||
1077 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | 1078 | update_db_intercept(vcpu); |
1078 | svm->vmcb->save.rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF; | ||
1079 | else if (old_debug & KVM_GUESTDBG_SINGLESTEP) | ||
1080 | svm->vmcb->save.rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | ||
1081 | |||
1082 | return 0; | ||
1083 | } | 1079 | } |
1084 | 1080 | ||
1085 | static void load_host_msrs(struct kvm_vcpu *vcpu) | 1081 | static void load_host_msrs(struct kvm_vcpu *vcpu) |
@@ -1180,7 +1176,7 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, | |||
1180 | } | 1176 | } |
1181 | } | 1177 | } |
1182 | 1178 | ||
1183 | static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1179 | static int pf_interception(struct vcpu_svm *svm) |
1184 | { | 1180 | { |
1185 | u64 fault_address; | 1181 | u64 fault_address; |
1186 | u32 error_code; | 1182 | u32 error_code; |
@@ -1194,17 +1190,19 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1194 | return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); | 1190 | return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); |
1195 | } | 1191 | } |
1196 | 1192 | ||
1197 | static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1193 | static int db_interception(struct vcpu_svm *svm) |
1198 | { | 1194 | { |
1195 | struct kvm_run *kvm_run = svm->vcpu.run; | ||
1196 | |||
1199 | if (!(svm->vcpu.guest_debug & | 1197 | if (!(svm->vcpu.guest_debug & |
1200 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && | 1198 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && |
1201 | !svm->vcpu.arch.singlestep) { | 1199 | !svm->nmi_singlestep) { |
1202 | kvm_queue_exception(&svm->vcpu, DB_VECTOR); | 1200 | kvm_queue_exception(&svm->vcpu, DB_VECTOR); |
1203 | return 1; | 1201 | return 1; |
1204 | } | 1202 | } |
1205 | 1203 | ||
1206 | if (svm->vcpu.arch.singlestep) { | 1204 | if (svm->nmi_singlestep) { |
1207 | svm->vcpu.arch.singlestep = false; | 1205 | svm->nmi_singlestep = false; |
1208 | if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) | 1206 | if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) |
1209 | svm->vmcb->save.rflags &= | 1207 | svm->vmcb->save.rflags &= |
1210 | ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | 1208 | ~(X86_EFLAGS_TF | X86_EFLAGS_RF); |
@@ -1223,25 +1221,27 @@ static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1223 | return 1; | 1221 | return 1; |
1224 | } | 1222 | } |
1225 | 1223 | ||
1226 | static int bp_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1224 | static int bp_interception(struct vcpu_svm *svm) |
1227 | { | 1225 | { |
1226 | struct kvm_run *kvm_run = svm->vcpu.run; | ||
1227 | |||
1228 | kvm_run->exit_reason = KVM_EXIT_DEBUG; | 1228 | kvm_run->exit_reason = KVM_EXIT_DEBUG; |
1229 | kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; | 1229 | kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; |
1230 | kvm_run->debug.arch.exception = BP_VECTOR; | 1230 | kvm_run->debug.arch.exception = BP_VECTOR; |
1231 | return 0; | 1231 | return 0; |
1232 | } | 1232 | } |
1233 | 1233 | ||
1234 | static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1234 | static int ud_interception(struct vcpu_svm *svm) |
1235 | { | 1235 | { |
1236 | int er; | 1236 | int er; |
1237 | 1237 | ||
1238 | er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD); | 1238 | er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD); |
1239 | if (er != EMULATE_DONE) | 1239 | if (er != EMULATE_DONE) |
1240 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); | 1240 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); |
1241 | return 1; | 1241 | return 1; |
1242 | } | 1242 | } |
1243 | 1243 | ||
1244 | static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1244 | static int nm_interception(struct vcpu_svm *svm) |
1245 | { | 1245 | { |
1246 | svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); | 1246 | svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); |
1247 | if (!(svm->vcpu.arch.cr0 & X86_CR0_TS)) | 1247 | if (!(svm->vcpu.arch.cr0 & X86_CR0_TS)) |
@@ -1251,7 +1251,7 @@ static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1251 | return 1; | 1251 | return 1; |
1252 | } | 1252 | } |
1253 | 1253 | ||
1254 | static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1254 | static int mc_interception(struct vcpu_svm *svm) |
1255 | { | 1255 | { |
1256 | /* | 1256 | /* |
1257 | * On an #MC intercept the MCE handler is not called automatically in | 1257 | * On an #MC intercept the MCE handler is not called automatically in |
@@ -1264,8 +1264,10 @@ static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1264 | return 1; | 1264 | return 1; |
1265 | } | 1265 | } |
1266 | 1266 | ||
1267 | static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1267 | static int shutdown_interception(struct vcpu_svm *svm) |
1268 | { | 1268 | { |
1269 | struct kvm_run *kvm_run = svm->vcpu.run; | ||
1270 | |||
1269 | /* | 1271 | /* |
1270 | * VMCB is undefined after a SHUTDOWN intercept | 1272 | * VMCB is undefined after a SHUTDOWN intercept |
1271 | * so reinitialize it. | 1273 | * so reinitialize it. |
@@ -1277,7 +1279,7 @@ static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1277 | return 0; | 1279 | return 0; |
1278 | } | 1280 | } |
1279 | 1281 | ||
1280 | static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1282 | static int io_interception(struct vcpu_svm *svm) |
1281 | { | 1283 | { |
1282 | u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ | 1284 | u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ |
1283 | int size, in, string; | 1285 | int size, in, string; |
@@ -1291,7 +1293,7 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1291 | 1293 | ||
1292 | if (string) { | 1294 | if (string) { |
1293 | if (emulate_instruction(&svm->vcpu, | 1295 | if (emulate_instruction(&svm->vcpu, |
1294 | kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) | 1296 | 0, 0, 0) == EMULATE_DO_MMIO) |
1295 | return 0; | 1297 | return 0; |
1296 | return 1; | 1298 | return 1; |
1297 | } | 1299 | } |
@@ -1301,33 +1303,33 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1301 | size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; | 1303 | size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; |
1302 | 1304 | ||
1303 | skip_emulated_instruction(&svm->vcpu); | 1305 | skip_emulated_instruction(&svm->vcpu); |
1304 | return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port); | 1306 | return kvm_emulate_pio(&svm->vcpu, in, size, port); |
1305 | } | 1307 | } |
1306 | 1308 | ||
1307 | static int nmi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1309 | static int nmi_interception(struct vcpu_svm *svm) |
1308 | { | 1310 | { |
1309 | return 1; | 1311 | return 1; |
1310 | } | 1312 | } |
1311 | 1313 | ||
1312 | static int intr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1314 | static int intr_interception(struct vcpu_svm *svm) |
1313 | { | 1315 | { |
1314 | ++svm->vcpu.stat.irq_exits; | 1316 | ++svm->vcpu.stat.irq_exits; |
1315 | return 1; | 1317 | return 1; |
1316 | } | 1318 | } |
1317 | 1319 | ||
1318 | static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1320 | static int nop_on_interception(struct vcpu_svm *svm) |
1319 | { | 1321 | { |
1320 | return 1; | 1322 | return 1; |
1321 | } | 1323 | } |
1322 | 1324 | ||
1323 | static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1325 | static int halt_interception(struct vcpu_svm *svm) |
1324 | { | 1326 | { |
1325 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 1; | 1327 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 1; |
1326 | skip_emulated_instruction(&svm->vcpu); | 1328 | skip_emulated_instruction(&svm->vcpu); |
1327 | return kvm_emulate_halt(&svm->vcpu); | 1329 | return kvm_emulate_halt(&svm->vcpu); |
1328 | } | 1330 | } |
1329 | 1331 | ||
1330 | static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1332 | static int vmmcall_interception(struct vcpu_svm *svm) |
1331 | { | 1333 | { |
1332 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; | 1334 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; |
1333 | skip_emulated_instruction(&svm->vcpu); | 1335 | skip_emulated_instruction(&svm->vcpu); |
@@ -1378,8 +1380,15 @@ static inline int nested_svm_intr(struct vcpu_svm *svm) | |||
1378 | 1380 | ||
1379 | svm->vmcb->control.exit_code = SVM_EXIT_INTR; | 1381 | svm->vmcb->control.exit_code = SVM_EXIT_INTR; |
1380 | 1382 | ||
1381 | if (nested_svm_exit_handled(svm)) { | 1383 | if (svm->nested.intercept & 1ULL) { |
1382 | nsvm_printk("VMexit -> INTR\n"); | 1384 | /* |
1385 | * The #vmexit can't be emulated here directly because this | ||
1386 | * code path runs with irqs and preemtion disabled. A | ||
1387 | * #vmexit emulation might sleep. Only signal request for | ||
1388 | * the #vmexit here. | ||
1389 | */ | ||
1390 | svm->nested.exit_required = true; | ||
1391 | trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); | ||
1383 | return 1; | 1392 | return 1; |
1384 | } | 1393 | } |
1385 | 1394 | ||
@@ -1390,10 +1399,7 @@ static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, enum km_type idx) | |||
1390 | { | 1399 | { |
1391 | struct page *page; | 1400 | struct page *page; |
1392 | 1401 | ||
1393 | down_read(¤t->mm->mmap_sem); | ||
1394 | page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT); | 1402 | page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT); |
1395 | up_read(¤t->mm->mmap_sem); | ||
1396 | |||
1397 | if (is_error_page(page)) | 1403 | if (is_error_page(page)) |
1398 | goto error; | 1404 | goto error; |
1399 | 1405 | ||
@@ -1532,14 +1538,12 @@ static int nested_svm_exit_handled(struct vcpu_svm *svm) | |||
1532 | } | 1538 | } |
1533 | default: { | 1539 | default: { |
1534 | u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); | 1540 | u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); |
1535 | nsvm_printk("exit code: 0x%x\n", exit_code); | ||
1536 | if (svm->nested.intercept & exit_bits) | 1541 | if (svm->nested.intercept & exit_bits) |
1537 | vmexit = NESTED_EXIT_DONE; | 1542 | vmexit = NESTED_EXIT_DONE; |
1538 | } | 1543 | } |
1539 | } | 1544 | } |
1540 | 1545 | ||
1541 | if (vmexit == NESTED_EXIT_DONE) { | 1546 | if (vmexit == NESTED_EXIT_DONE) { |
1542 | nsvm_printk("#VMEXIT reason=%04x\n", exit_code); | ||
1543 | nested_svm_vmexit(svm); | 1547 | nested_svm_vmexit(svm); |
1544 | } | 1548 | } |
1545 | 1549 | ||
@@ -1584,6 +1588,12 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) | |||
1584 | struct vmcb *hsave = svm->nested.hsave; | 1588 | struct vmcb *hsave = svm->nested.hsave; |
1585 | struct vmcb *vmcb = svm->vmcb; | 1589 | struct vmcb *vmcb = svm->vmcb; |
1586 | 1590 | ||
1591 | trace_kvm_nested_vmexit_inject(vmcb->control.exit_code, | ||
1592 | vmcb->control.exit_info_1, | ||
1593 | vmcb->control.exit_info_2, | ||
1594 | vmcb->control.exit_int_info, | ||
1595 | vmcb->control.exit_int_info_err); | ||
1596 | |||
1587 | nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, KM_USER0); | 1597 | nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, KM_USER0); |
1588 | if (!nested_vmcb) | 1598 | if (!nested_vmcb) |
1589 | return 1; | 1599 | return 1; |
@@ -1617,6 +1627,22 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) | |||
1617 | nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2; | 1627 | nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2; |
1618 | nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info; | 1628 | nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info; |
1619 | nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err; | 1629 | nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err; |
1630 | |||
1631 | /* | ||
1632 | * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have | ||
1633 | * to make sure that we do not lose injected events. So check event_inj | ||
1634 | * here and copy it to exit_int_info if it is valid. | ||
1635 | * Exit_int_info and event_inj can't be both valid because the case | ||
1636 | * below only happens on a VMRUN instruction intercept which has | ||
1637 | * no valid exit_int_info set. | ||
1638 | */ | ||
1639 | if (vmcb->control.event_inj & SVM_EVTINJ_VALID) { | ||
1640 | struct vmcb_control_area *nc = &nested_vmcb->control; | ||
1641 | |||
1642 | nc->exit_int_info = vmcb->control.event_inj; | ||
1643 | nc->exit_int_info_err = vmcb->control.event_inj_err; | ||
1644 | } | ||
1645 | |||
1620 | nested_vmcb->control.tlb_ctl = 0; | 1646 | nested_vmcb->control.tlb_ctl = 0; |
1621 | nested_vmcb->control.event_inj = 0; | 1647 | nested_vmcb->control.event_inj = 0; |
1622 | nested_vmcb->control.event_inj_err = 0; | 1648 | nested_vmcb->control.event_inj_err = 0; |
@@ -1628,10 +1654,6 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) | |||
1628 | /* Restore the original control entries */ | 1654 | /* Restore the original control entries */ |
1629 | copy_vmcb_control_area(vmcb, hsave); | 1655 | copy_vmcb_control_area(vmcb, hsave); |
1630 | 1656 | ||
1631 | /* Kill any pending exceptions */ | ||
1632 | if (svm->vcpu.arch.exception.pending == true) | ||
1633 | nsvm_printk("WARNING: Pending Exception\n"); | ||
1634 | |||
1635 | kvm_clear_exception_queue(&svm->vcpu); | 1657 | kvm_clear_exception_queue(&svm->vcpu); |
1636 | kvm_clear_interrupt_queue(&svm->vcpu); | 1658 | kvm_clear_interrupt_queue(&svm->vcpu); |
1637 | 1659 | ||
@@ -1702,6 +1724,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) | |||
1702 | /* nested_vmcb is our indicator if nested SVM is activated */ | 1724 | /* nested_vmcb is our indicator if nested SVM is activated */ |
1703 | svm->nested.vmcb = svm->vmcb->save.rax; | 1725 | svm->nested.vmcb = svm->vmcb->save.rax; |
1704 | 1726 | ||
1727 | trace_kvm_nested_vmrun(svm->vmcb->save.rip - 3, svm->nested.vmcb, | ||
1728 | nested_vmcb->save.rip, | ||
1729 | nested_vmcb->control.int_ctl, | ||
1730 | nested_vmcb->control.event_inj, | ||
1731 | nested_vmcb->control.nested_ctl); | ||
1732 | |||
1705 | /* Clear internal status */ | 1733 | /* Clear internal status */ |
1706 | kvm_clear_exception_queue(&svm->vcpu); | 1734 | kvm_clear_exception_queue(&svm->vcpu); |
1707 | kvm_clear_interrupt_queue(&svm->vcpu); | 1735 | kvm_clear_interrupt_queue(&svm->vcpu); |
@@ -1789,28 +1817,15 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) | |||
1789 | svm->nested.intercept = nested_vmcb->control.intercept; | 1817 | svm->nested.intercept = nested_vmcb->control.intercept; |
1790 | 1818 | ||
1791 | force_new_asid(&svm->vcpu); | 1819 | force_new_asid(&svm->vcpu); |
1792 | svm->vmcb->control.exit_int_info = nested_vmcb->control.exit_int_info; | ||
1793 | svm->vmcb->control.exit_int_info_err = nested_vmcb->control.exit_int_info_err; | ||
1794 | svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; | 1820 | svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; |
1795 | if (nested_vmcb->control.int_ctl & V_IRQ_MASK) { | ||
1796 | nsvm_printk("nSVM Injecting Interrupt: 0x%x\n", | ||
1797 | nested_vmcb->control.int_ctl); | ||
1798 | } | ||
1799 | if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) | 1821 | if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) |
1800 | svm->vcpu.arch.hflags |= HF_VINTR_MASK; | 1822 | svm->vcpu.arch.hflags |= HF_VINTR_MASK; |
1801 | else | 1823 | else |
1802 | svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; | 1824 | svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; |
1803 | 1825 | ||
1804 | nsvm_printk("nSVM exit_int_info: 0x%x | int_state: 0x%x\n", | ||
1805 | nested_vmcb->control.exit_int_info, | ||
1806 | nested_vmcb->control.int_state); | ||
1807 | |||
1808 | svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; | 1826 | svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; |
1809 | svm->vmcb->control.int_state = nested_vmcb->control.int_state; | 1827 | svm->vmcb->control.int_state = nested_vmcb->control.int_state; |
1810 | svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset; | 1828 | svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset; |
1811 | if (nested_vmcb->control.event_inj & SVM_EVTINJ_VALID) | ||
1812 | nsvm_printk("Injecting Event: 0x%x\n", | ||
1813 | nested_vmcb->control.event_inj); | ||
1814 | svm->vmcb->control.event_inj = nested_vmcb->control.event_inj; | 1829 | svm->vmcb->control.event_inj = nested_vmcb->control.event_inj; |
1815 | svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err; | 1830 | svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err; |
1816 | 1831 | ||
@@ -1837,7 +1852,7 @@ static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb) | |||
1837 | to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; | 1852 | to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; |
1838 | } | 1853 | } |
1839 | 1854 | ||
1840 | static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1855 | static int vmload_interception(struct vcpu_svm *svm) |
1841 | { | 1856 | { |
1842 | struct vmcb *nested_vmcb; | 1857 | struct vmcb *nested_vmcb; |
1843 | 1858 | ||
@@ -1857,7 +1872,7 @@ static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1857 | return 1; | 1872 | return 1; |
1858 | } | 1873 | } |
1859 | 1874 | ||
1860 | static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1875 | static int vmsave_interception(struct vcpu_svm *svm) |
1861 | { | 1876 | { |
1862 | struct vmcb *nested_vmcb; | 1877 | struct vmcb *nested_vmcb; |
1863 | 1878 | ||
@@ -1877,10 +1892,8 @@ static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1877 | return 1; | 1892 | return 1; |
1878 | } | 1893 | } |
1879 | 1894 | ||
1880 | static int vmrun_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1895 | static int vmrun_interception(struct vcpu_svm *svm) |
1881 | { | 1896 | { |
1882 | nsvm_printk("VMrun\n"); | ||
1883 | |||
1884 | if (nested_svm_check_permissions(svm)) | 1897 | if (nested_svm_check_permissions(svm)) |
1885 | return 1; | 1898 | return 1; |
1886 | 1899 | ||
@@ -1907,7 +1920,7 @@ failed: | |||
1907 | return 1; | 1920 | return 1; |
1908 | } | 1921 | } |
1909 | 1922 | ||
1910 | static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1923 | static int stgi_interception(struct vcpu_svm *svm) |
1911 | { | 1924 | { |
1912 | if (nested_svm_check_permissions(svm)) | 1925 | if (nested_svm_check_permissions(svm)) |
1913 | return 1; | 1926 | return 1; |
@@ -1920,7 +1933,7 @@ static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1920 | return 1; | 1933 | return 1; |
1921 | } | 1934 | } |
1922 | 1935 | ||
1923 | static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1936 | static int clgi_interception(struct vcpu_svm *svm) |
1924 | { | 1937 | { |
1925 | if (nested_svm_check_permissions(svm)) | 1938 | if (nested_svm_check_permissions(svm)) |
1926 | return 1; | 1939 | return 1; |
@@ -1937,10 +1950,12 @@ static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1937 | return 1; | 1950 | return 1; |
1938 | } | 1951 | } |
1939 | 1952 | ||
1940 | static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1953 | static int invlpga_interception(struct vcpu_svm *svm) |
1941 | { | 1954 | { |
1942 | struct kvm_vcpu *vcpu = &svm->vcpu; | 1955 | struct kvm_vcpu *vcpu = &svm->vcpu; |
1943 | nsvm_printk("INVLPGA\n"); | 1956 | |
1957 | trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX], | ||
1958 | vcpu->arch.regs[VCPU_REGS_RAX]); | ||
1944 | 1959 | ||
1945 | /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ | 1960 | /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ |
1946 | kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]); | 1961 | kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]); |
@@ -1950,15 +1965,21 @@ static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1950 | return 1; | 1965 | return 1; |
1951 | } | 1966 | } |
1952 | 1967 | ||
1953 | static int invalid_op_interception(struct vcpu_svm *svm, | 1968 | static int skinit_interception(struct vcpu_svm *svm) |
1954 | struct kvm_run *kvm_run) | ||
1955 | { | 1969 | { |
1970 | trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]); | ||
1971 | |||
1956 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); | 1972 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); |
1957 | return 1; | 1973 | return 1; |
1958 | } | 1974 | } |
1959 | 1975 | ||
1960 | static int task_switch_interception(struct vcpu_svm *svm, | 1976 | static int invalid_op_interception(struct vcpu_svm *svm) |
1961 | struct kvm_run *kvm_run) | 1977 | { |
1978 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); | ||
1979 | return 1; | ||
1980 | } | ||
1981 | |||
1982 | static int task_switch_interception(struct vcpu_svm *svm) | ||
1962 | { | 1983 | { |
1963 | u16 tss_selector; | 1984 | u16 tss_selector; |
1964 | int reason; | 1985 | int reason; |
@@ -2008,14 +2029,14 @@ static int task_switch_interception(struct vcpu_svm *svm, | |||
2008 | return kvm_task_switch(&svm->vcpu, tss_selector, reason); | 2029 | return kvm_task_switch(&svm->vcpu, tss_selector, reason); |
2009 | } | 2030 | } |
2010 | 2031 | ||
2011 | static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2032 | static int cpuid_interception(struct vcpu_svm *svm) |
2012 | { | 2033 | { |
2013 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 2; | 2034 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 2; |
2014 | kvm_emulate_cpuid(&svm->vcpu); | 2035 | kvm_emulate_cpuid(&svm->vcpu); |
2015 | return 1; | 2036 | return 1; |
2016 | } | 2037 | } |
2017 | 2038 | ||
2018 | static int iret_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2039 | static int iret_interception(struct vcpu_svm *svm) |
2019 | { | 2040 | { |
2020 | ++svm->vcpu.stat.nmi_window_exits; | 2041 | ++svm->vcpu.stat.nmi_window_exits; |
2021 | svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET); | 2042 | svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET); |
@@ -2023,26 +2044,27 @@ static int iret_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
2023 | return 1; | 2044 | return 1; |
2024 | } | 2045 | } |
2025 | 2046 | ||
2026 | static int invlpg_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2047 | static int invlpg_interception(struct vcpu_svm *svm) |
2027 | { | 2048 | { |
2028 | if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0) != EMULATE_DONE) | 2049 | if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE) |
2029 | pr_unimpl(&svm->vcpu, "%s: failed\n", __func__); | 2050 | pr_unimpl(&svm->vcpu, "%s: failed\n", __func__); |
2030 | return 1; | 2051 | return 1; |
2031 | } | 2052 | } |
2032 | 2053 | ||
2033 | static int emulate_on_interception(struct vcpu_svm *svm, | 2054 | static int emulate_on_interception(struct vcpu_svm *svm) |
2034 | struct kvm_run *kvm_run) | ||
2035 | { | 2055 | { |
2036 | if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE) | 2056 | if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE) |
2037 | pr_unimpl(&svm->vcpu, "%s: failed\n", __func__); | 2057 | pr_unimpl(&svm->vcpu, "%s: failed\n", __func__); |
2038 | return 1; | 2058 | return 1; |
2039 | } | 2059 | } |
2040 | 2060 | ||
2041 | static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2061 | static int cr8_write_interception(struct vcpu_svm *svm) |
2042 | { | 2062 | { |
2063 | struct kvm_run *kvm_run = svm->vcpu.run; | ||
2064 | |||
2043 | u8 cr8_prev = kvm_get_cr8(&svm->vcpu); | 2065 | u8 cr8_prev = kvm_get_cr8(&svm->vcpu); |
2044 | /* instruction emulation calls kvm_set_cr8() */ | 2066 | /* instruction emulation calls kvm_set_cr8() */ |
2045 | emulate_instruction(&svm->vcpu, NULL, 0, 0, 0); | 2067 | emulate_instruction(&svm->vcpu, 0, 0, 0); |
2046 | if (irqchip_in_kernel(svm->vcpu.kvm)) { | 2068 | if (irqchip_in_kernel(svm->vcpu.kvm)) { |
2047 | svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK; | 2069 | svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK; |
2048 | return 1; | 2070 | return 1; |
@@ -2128,7 +2150,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) | |||
2128 | return 0; | 2150 | return 0; |
2129 | } | 2151 | } |
2130 | 2152 | ||
2131 | static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2153 | static int rdmsr_interception(struct vcpu_svm *svm) |
2132 | { | 2154 | { |
2133 | u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; | 2155 | u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; |
2134 | u64 data; | 2156 | u64 data; |
@@ -2221,7 +2243,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) | |||
2221 | return 0; | 2243 | return 0; |
2222 | } | 2244 | } |
2223 | 2245 | ||
2224 | static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2246 | static int wrmsr_interception(struct vcpu_svm *svm) |
2225 | { | 2247 | { |
2226 | u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; | 2248 | u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; |
2227 | u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u) | 2249 | u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u) |
@@ -2237,17 +2259,18 @@ static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
2237 | return 1; | 2259 | return 1; |
2238 | } | 2260 | } |
2239 | 2261 | ||
2240 | static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 2262 | static int msr_interception(struct vcpu_svm *svm) |
2241 | { | 2263 | { |
2242 | if (svm->vmcb->control.exit_info_1) | 2264 | if (svm->vmcb->control.exit_info_1) |
2243 | return wrmsr_interception(svm, kvm_run); | 2265 | return wrmsr_interception(svm); |
2244 | else | 2266 | else |
2245 | return rdmsr_interception(svm, kvm_run); | 2267 | return rdmsr_interception(svm); |
2246 | } | 2268 | } |
2247 | 2269 | ||
2248 | static int interrupt_window_interception(struct vcpu_svm *svm, | 2270 | static int interrupt_window_interception(struct vcpu_svm *svm) |
2249 | struct kvm_run *kvm_run) | ||
2250 | { | 2271 | { |
2272 | struct kvm_run *kvm_run = svm->vcpu.run; | ||
2273 | |||
2251 | svm_clear_vintr(svm); | 2274 | svm_clear_vintr(svm); |
2252 | svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; | 2275 | svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; |
2253 | /* | 2276 | /* |
@@ -2265,8 +2288,13 @@ static int interrupt_window_interception(struct vcpu_svm *svm, | |||
2265 | return 1; | 2288 | return 1; |
2266 | } | 2289 | } |
2267 | 2290 | ||
2268 | static int (*svm_exit_handlers[])(struct vcpu_svm *svm, | 2291 | static int pause_interception(struct vcpu_svm *svm) |
2269 | struct kvm_run *kvm_run) = { | 2292 | { |
2293 | kvm_vcpu_on_spin(&(svm->vcpu)); | ||
2294 | return 1; | ||
2295 | } | ||
2296 | |||
2297 | static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { | ||
2270 | [SVM_EXIT_READ_CR0] = emulate_on_interception, | 2298 | [SVM_EXIT_READ_CR0] = emulate_on_interception, |
2271 | [SVM_EXIT_READ_CR3] = emulate_on_interception, | 2299 | [SVM_EXIT_READ_CR3] = emulate_on_interception, |
2272 | [SVM_EXIT_READ_CR4] = emulate_on_interception, | 2300 | [SVM_EXIT_READ_CR4] = emulate_on_interception, |
@@ -2301,6 +2329,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, | |||
2301 | [SVM_EXIT_CPUID] = cpuid_interception, | 2329 | [SVM_EXIT_CPUID] = cpuid_interception, |
2302 | [SVM_EXIT_IRET] = iret_interception, | 2330 | [SVM_EXIT_IRET] = iret_interception, |
2303 | [SVM_EXIT_INVD] = emulate_on_interception, | 2331 | [SVM_EXIT_INVD] = emulate_on_interception, |
2332 | [SVM_EXIT_PAUSE] = pause_interception, | ||
2304 | [SVM_EXIT_HLT] = halt_interception, | 2333 | [SVM_EXIT_HLT] = halt_interception, |
2305 | [SVM_EXIT_INVLPG] = invlpg_interception, | 2334 | [SVM_EXIT_INVLPG] = invlpg_interception, |
2306 | [SVM_EXIT_INVLPGA] = invlpga_interception, | 2335 | [SVM_EXIT_INVLPGA] = invlpga_interception, |
@@ -2314,26 +2343,36 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, | |||
2314 | [SVM_EXIT_VMSAVE] = vmsave_interception, | 2343 | [SVM_EXIT_VMSAVE] = vmsave_interception, |
2315 | [SVM_EXIT_STGI] = stgi_interception, | 2344 | [SVM_EXIT_STGI] = stgi_interception, |
2316 | [SVM_EXIT_CLGI] = clgi_interception, | 2345 | [SVM_EXIT_CLGI] = clgi_interception, |
2317 | [SVM_EXIT_SKINIT] = invalid_op_interception, | 2346 | [SVM_EXIT_SKINIT] = skinit_interception, |
2318 | [SVM_EXIT_WBINVD] = emulate_on_interception, | 2347 | [SVM_EXIT_WBINVD] = emulate_on_interception, |
2319 | [SVM_EXIT_MONITOR] = invalid_op_interception, | 2348 | [SVM_EXIT_MONITOR] = invalid_op_interception, |
2320 | [SVM_EXIT_MWAIT] = invalid_op_interception, | 2349 | [SVM_EXIT_MWAIT] = invalid_op_interception, |
2321 | [SVM_EXIT_NPF] = pf_interception, | 2350 | [SVM_EXIT_NPF] = pf_interception, |
2322 | }; | 2351 | }; |
2323 | 2352 | ||
2324 | static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | 2353 | static int handle_exit(struct kvm_vcpu *vcpu) |
2325 | { | 2354 | { |
2326 | struct vcpu_svm *svm = to_svm(vcpu); | 2355 | struct vcpu_svm *svm = to_svm(vcpu); |
2356 | struct kvm_run *kvm_run = vcpu->run; | ||
2327 | u32 exit_code = svm->vmcb->control.exit_code; | 2357 | u32 exit_code = svm->vmcb->control.exit_code; |
2328 | 2358 | ||
2329 | trace_kvm_exit(exit_code, svm->vmcb->save.rip); | 2359 | trace_kvm_exit(exit_code, svm->vmcb->save.rip); |
2330 | 2360 | ||
2361 | if (unlikely(svm->nested.exit_required)) { | ||
2362 | nested_svm_vmexit(svm); | ||
2363 | svm->nested.exit_required = false; | ||
2364 | |||
2365 | return 1; | ||
2366 | } | ||
2367 | |||
2331 | if (is_nested(svm)) { | 2368 | if (is_nested(svm)) { |
2332 | int vmexit; | 2369 | int vmexit; |
2333 | 2370 | ||
2334 | nsvm_printk("nested handle_exit: 0x%x | 0x%lx | 0x%lx | 0x%lx\n", | 2371 | trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code, |
2335 | exit_code, svm->vmcb->control.exit_info_1, | 2372 | svm->vmcb->control.exit_info_1, |
2336 | svm->vmcb->control.exit_info_2, svm->vmcb->save.rip); | 2373 | svm->vmcb->control.exit_info_2, |
2374 | svm->vmcb->control.exit_int_info, | ||
2375 | svm->vmcb->control.exit_int_info_err); | ||
2337 | 2376 | ||
2338 | vmexit = nested_svm_exit_special(svm); | 2377 | vmexit = nested_svm_exit_special(svm); |
2339 | 2378 | ||
@@ -2383,7 +2422,7 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | |||
2383 | return 0; | 2422 | return 0; |
2384 | } | 2423 | } |
2385 | 2424 | ||
2386 | return svm_exit_handlers[exit_code](svm, kvm_run); | 2425 | return svm_exit_handlers[exit_code](svm); |
2387 | } | 2426 | } |
2388 | 2427 | ||
2389 | static void reload_tss(struct kvm_vcpu *vcpu) | 2428 | static void reload_tss(struct kvm_vcpu *vcpu) |
@@ -2460,20 +2499,47 @@ static int svm_nmi_allowed(struct kvm_vcpu *vcpu) | |||
2460 | !(svm->vcpu.arch.hflags & HF_NMI_MASK); | 2499 | !(svm->vcpu.arch.hflags & HF_NMI_MASK); |
2461 | } | 2500 | } |
2462 | 2501 | ||
2502 | static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) | ||
2503 | { | ||
2504 | struct vcpu_svm *svm = to_svm(vcpu); | ||
2505 | |||
2506 | return !!(svm->vcpu.arch.hflags & HF_NMI_MASK); | ||
2507 | } | ||
2508 | |||
2509 | static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) | ||
2510 | { | ||
2511 | struct vcpu_svm *svm = to_svm(vcpu); | ||
2512 | |||
2513 | if (masked) { | ||
2514 | svm->vcpu.arch.hflags |= HF_NMI_MASK; | ||
2515 | svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET); | ||
2516 | } else { | ||
2517 | svm->vcpu.arch.hflags &= ~HF_NMI_MASK; | ||
2518 | svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET); | ||
2519 | } | ||
2520 | } | ||
2521 | |||
2463 | static int svm_interrupt_allowed(struct kvm_vcpu *vcpu) | 2522 | static int svm_interrupt_allowed(struct kvm_vcpu *vcpu) |
2464 | { | 2523 | { |
2465 | struct vcpu_svm *svm = to_svm(vcpu); | 2524 | struct vcpu_svm *svm = to_svm(vcpu); |
2466 | struct vmcb *vmcb = svm->vmcb; | 2525 | struct vmcb *vmcb = svm->vmcb; |
2467 | return (vmcb->save.rflags & X86_EFLAGS_IF) && | 2526 | int ret; |
2468 | !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) && | 2527 | |
2469 | gif_set(svm) && | 2528 | if (!gif_set(svm) || |
2470 | !(is_nested(svm) && (svm->vcpu.arch.hflags & HF_VINTR_MASK)); | 2529 | (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)) |
2530 | return 0; | ||
2531 | |||
2532 | ret = !!(vmcb->save.rflags & X86_EFLAGS_IF); | ||
2533 | |||
2534 | if (is_nested(svm)) | ||
2535 | return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK); | ||
2536 | |||
2537 | return ret; | ||
2471 | } | 2538 | } |
2472 | 2539 | ||
2473 | static void enable_irq_window(struct kvm_vcpu *vcpu) | 2540 | static void enable_irq_window(struct kvm_vcpu *vcpu) |
2474 | { | 2541 | { |
2475 | struct vcpu_svm *svm = to_svm(vcpu); | 2542 | struct vcpu_svm *svm = to_svm(vcpu); |
2476 | nsvm_printk("Trying to open IRQ window\n"); | ||
2477 | 2543 | ||
2478 | nested_svm_intr(svm); | 2544 | nested_svm_intr(svm); |
2479 | 2545 | ||
@@ -2498,7 +2564,7 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu) | |||
2498 | /* Something prevents NMI from been injected. Single step over | 2564 | /* Something prevents NMI from been injected. Single step over |
2499 | possible problem (IRET or exception injection or interrupt | 2565 | possible problem (IRET or exception injection or interrupt |
2500 | shadow) */ | 2566 | shadow) */ |
2501 | vcpu->arch.singlestep = true; | 2567 | svm->nmi_singlestep = true; |
2502 | svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); | 2568 | svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); |
2503 | update_db_intercept(vcpu); | 2569 | update_db_intercept(vcpu); |
2504 | } | 2570 | } |
@@ -2588,13 +2654,20 @@ static void svm_complete_interrupts(struct vcpu_svm *svm) | |||
2588 | #define R "e" | 2654 | #define R "e" |
2589 | #endif | 2655 | #endif |
2590 | 2656 | ||
2591 | static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2657 | static void svm_vcpu_run(struct kvm_vcpu *vcpu) |
2592 | { | 2658 | { |
2593 | struct vcpu_svm *svm = to_svm(vcpu); | 2659 | struct vcpu_svm *svm = to_svm(vcpu); |
2594 | u16 fs_selector; | 2660 | u16 fs_selector; |
2595 | u16 gs_selector; | 2661 | u16 gs_selector; |
2596 | u16 ldt_selector; | 2662 | u16 ldt_selector; |
2597 | 2663 | ||
2664 | /* | ||
2665 | * A vmexit emulation is required before the vcpu can be executed | ||
2666 | * again. | ||
2667 | */ | ||
2668 | if (unlikely(svm->nested.exit_required)) | ||
2669 | return; | ||
2670 | |||
2598 | svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; | 2671 | svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; |
2599 | svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; | 2672 | svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; |
2600 | svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; | 2673 | svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; |
@@ -2893,6 +2966,8 @@ static struct kvm_x86_ops svm_x86_ops = { | |||
2893 | .queue_exception = svm_queue_exception, | 2966 | .queue_exception = svm_queue_exception, |
2894 | .interrupt_allowed = svm_interrupt_allowed, | 2967 | .interrupt_allowed = svm_interrupt_allowed, |
2895 | .nmi_allowed = svm_nmi_allowed, | 2968 | .nmi_allowed = svm_nmi_allowed, |
2969 | .get_nmi_mask = svm_get_nmi_mask, | ||
2970 | .set_nmi_mask = svm_set_nmi_mask, | ||
2896 | .enable_nmi_window = enable_nmi_window, | 2971 | .enable_nmi_window = enable_nmi_window, |
2897 | .enable_irq_window = enable_irq_window, | 2972 | .enable_irq_window = enable_irq_window, |
2898 | .update_cr8_intercept = update_cr8_intercept, | 2973 | .update_cr8_intercept = update_cr8_intercept, |
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 0d480e77eacf..816e0449db0b 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h | |||
@@ -349,6 +349,171 @@ TRACE_EVENT(kvm_apic_accept_irq, | |||
349 | __entry->coalesced ? " (coalesced)" : "") | 349 | __entry->coalesced ? " (coalesced)" : "") |
350 | ); | 350 | ); |
351 | 351 | ||
352 | /* | ||
353 | * Tracepoint for nested VMRUN | ||
354 | */ | ||
355 | TRACE_EVENT(kvm_nested_vmrun, | ||
356 | TP_PROTO(__u64 rip, __u64 vmcb, __u64 nested_rip, __u32 int_ctl, | ||
357 | __u32 event_inj, bool npt), | ||
358 | TP_ARGS(rip, vmcb, nested_rip, int_ctl, event_inj, npt), | ||
359 | |||
360 | TP_STRUCT__entry( | ||
361 | __field( __u64, rip ) | ||
362 | __field( __u64, vmcb ) | ||
363 | __field( __u64, nested_rip ) | ||
364 | __field( __u32, int_ctl ) | ||
365 | __field( __u32, event_inj ) | ||
366 | __field( bool, npt ) | ||
367 | ), | ||
368 | |||
369 | TP_fast_assign( | ||
370 | __entry->rip = rip; | ||
371 | __entry->vmcb = vmcb; | ||
372 | __entry->nested_rip = nested_rip; | ||
373 | __entry->int_ctl = int_ctl; | ||
374 | __entry->event_inj = event_inj; | ||
375 | __entry->npt = npt; | ||
376 | ), | ||
377 | |||
378 | TP_printk("rip: 0x%016llx vmcb: 0x%016llx nrip: 0x%016llx int_ctl: 0x%08x " | ||
379 | "event_inj: 0x%08x npt: %s\n", | ||
380 | __entry->rip, __entry->vmcb, __entry->nested_rip, | ||
381 | __entry->int_ctl, __entry->event_inj, | ||
382 | __entry->npt ? "on" : "off") | ||
383 | ); | ||
384 | |||
385 | /* | ||
386 | * Tracepoint for #VMEXIT while nested | ||
387 | */ | ||
388 | TRACE_EVENT(kvm_nested_vmexit, | ||
389 | TP_PROTO(__u64 rip, __u32 exit_code, | ||
390 | __u64 exit_info1, __u64 exit_info2, | ||
391 | __u32 exit_int_info, __u32 exit_int_info_err), | ||
392 | TP_ARGS(rip, exit_code, exit_info1, exit_info2, | ||
393 | exit_int_info, exit_int_info_err), | ||
394 | |||
395 | TP_STRUCT__entry( | ||
396 | __field( __u64, rip ) | ||
397 | __field( __u32, exit_code ) | ||
398 | __field( __u64, exit_info1 ) | ||
399 | __field( __u64, exit_info2 ) | ||
400 | __field( __u32, exit_int_info ) | ||
401 | __field( __u32, exit_int_info_err ) | ||
402 | ), | ||
403 | |||
404 | TP_fast_assign( | ||
405 | __entry->rip = rip; | ||
406 | __entry->exit_code = exit_code; | ||
407 | __entry->exit_info1 = exit_info1; | ||
408 | __entry->exit_info2 = exit_info2; | ||
409 | __entry->exit_int_info = exit_int_info; | ||
410 | __entry->exit_int_info_err = exit_int_info_err; | ||
411 | ), | ||
412 | TP_printk("rip: 0x%016llx reason: %s ext_inf1: 0x%016llx " | ||
413 | "ext_inf2: 0x%016llx ext_int: 0x%08x ext_int_err: 0x%08x\n", | ||
414 | __entry->rip, | ||
415 | ftrace_print_symbols_seq(p, __entry->exit_code, | ||
416 | kvm_x86_ops->exit_reasons_str), | ||
417 | __entry->exit_info1, __entry->exit_info2, | ||
418 | __entry->exit_int_info, __entry->exit_int_info_err) | ||
419 | ); | ||
420 | |||
421 | /* | ||
422 | * Tracepoint for #VMEXIT reinjected to the guest | ||
423 | */ | ||
424 | TRACE_EVENT(kvm_nested_vmexit_inject, | ||
425 | TP_PROTO(__u32 exit_code, | ||
426 | __u64 exit_info1, __u64 exit_info2, | ||
427 | __u32 exit_int_info, __u32 exit_int_info_err), | ||
428 | TP_ARGS(exit_code, exit_info1, exit_info2, | ||
429 | exit_int_info, exit_int_info_err), | ||
430 | |||
431 | TP_STRUCT__entry( | ||
432 | __field( __u32, exit_code ) | ||
433 | __field( __u64, exit_info1 ) | ||
434 | __field( __u64, exit_info2 ) | ||
435 | __field( __u32, exit_int_info ) | ||
436 | __field( __u32, exit_int_info_err ) | ||
437 | ), | ||
438 | |||
439 | TP_fast_assign( | ||
440 | __entry->exit_code = exit_code; | ||
441 | __entry->exit_info1 = exit_info1; | ||
442 | __entry->exit_info2 = exit_info2; | ||
443 | __entry->exit_int_info = exit_int_info; | ||
444 | __entry->exit_int_info_err = exit_int_info_err; | ||
445 | ), | ||
446 | |||
447 | TP_printk("reason: %s ext_inf1: 0x%016llx " | ||
448 | "ext_inf2: 0x%016llx ext_int: 0x%08x ext_int_err: 0x%08x\n", | ||
449 | ftrace_print_symbols_seq(p, __entry->exit_code, | ||
450 | kvm_x86_ops->exit_reasons_str), | ||
451 | __entry->exit_info1, __entry->exit_info2, | ||
452 | __entry->exit_int_info, __entry->exit_int_info_err) | ||
453 | ); | ||
454 | |||
455 | /* | ||
456 | * Tracepoint for nested #vmexit because of interrupt pending | ||
457 | */ | ||
458 | TRACE_EVENT(kvm_nested_intr_vmexit, | ||
459 | TP_PROTO(__u64 rip), | ||
460 | TP_ARGS(rip), | ||
461 | |||
462 | TP_STRUCT__entry( | ||
463 | __field( __u64, rip ) | ||
464 | ), | ||
465 | |||
466 | TP_fast_assign( | ||
467 | __entry->rip = rip | ||
468 | ), | ||
469 | |||
470 | TP_printk("rip: 0x%016llx\n", __entry->rip) | ||
471 | ); | ||
472 | |||
473 | /* | ||
474 | * Tracepoint for nested #vmexit because of interrupt pending | ||
475 | */ | ||
476 | TRACE_EVENT(kvm_invlpga, | ||
477 | TP_PROTO(__u64 rip, int asid, u64 address), | ||
478 | TP_ARGS(rip, asid, address), | ||
479 | |||
480 | TP_STRUCT__entry( | ||
481 | __field( __u64, rip ) | ||
482 | __field( int, asid ) | ||
483 | __field( __u64, address ) | ||
484 | ), | ||
485 | |||
486 | TP_fast_assign( | ||
487 | __entry->rip = rip; | ||
488 | __entry->asid = asid; | ||
489 | __entry->address = address; | ||
490 | ), | ||
491 | |||
492 | TP_printk("rip: 0x%016llx asid: %d address: 0x%016llx\n", | ||
493 | __entry->rip, __entry->asid, __entry->address) | ||
494 | ); | ||
495 | |||
496 | /* | ||
497 | * Tracepoint for nested #vmexit because of interrupt pending | ||
498 | */ | ||
499 | TRACE_EVENT(kvm_skinit, | ||
500 | TP_PROTO(__u64 rip, __u32 slb), | ||
501 | TP_ARGS(rip, slb), | ||
502 | |||
503 | TP_STRUCT__entry( | ||
504 | __field( __u64, rip ) | ||
505 | __field( __u32, slb ) | ||
506 | ), | ||
507 | |||
508 | TP_fast_assign( | ||
509 | __entry->rip = rip; | ||
510 | __entry->slb = slb; | ||
511 | ), | ||
512 | |||
513 | TP_printk("rip: 0x%016llx slb: 0x%08x\n", | ||
514 | __entry->rip, __entry->slb) | ||
515 | ); | ||
516 | |||
352 | #endif /* _TRACE_KVM_H */ | 517 | #endif /* _TRACE_KVM_H */ |
353 | 518 | ||
354 | /* This part must be outside protection */ | 519 | /* This part must be outside protection */ |
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index ed53b42caba1..d4918d6fc924 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c | |||
@@ -61,12 +61,37 @@ module_param_named(unrestricted_guest, | |||
61 | static int __read_mostly emulate_invalid_guest_state = 0; | 61 | static int __read_mostly emulate_invalid_guest_state = 0; |
62 | module_param(emulate_invalid_guest_state, bool, S_IRUGO); | 62 | module_param(emulate_invalid_guest_state, bool, S_IRUGO); |
63 | 63 | ||
64 | /* | ||
65 | * These 2 parameters are used to config the controls for Pause-Loop Exiting: | ||
66 | * ple_gap: upper bound on the amount of time between two successive | ||
67 | * executions of PAUSE in a loop. Also indicate if ple enabled. | ||
68 | * According to test, this time is usually small than 41 cycles. | ||
69 | * ple_window: upper bound on the amount of time a guest is allowed to execute | ||
70 | * in a PAUSE loop. Tests indicate that most spinlocks are held for | ||
71 | * less than 2^12 cycles | ||
72 | * Time is measured based on a counter that runs at the same rate as the TSC, | ||
73 | * refer SDM volume 3b section 21.6.13 & 22.1.3. | ||
74 | */ | ||
75 | #define KVM_VMX_DEFAULT_PLE_GAP 41 | ||
76 | #define KVM_VMX_DEFAULT_PLE_WINDOW 4096 | ||
77 | static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP; | ||
78 | module_param(ple_gap, int, S_IRUGO); | ||
79 | |||
80 | static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; | ||
81 | module_param(ple_window, int, S_IRUGO); | ||
82 | |||
64 | struct vmcs { | 83 | struct vmcs { |
65 | u32 revision_id; | 84 | u32 revision_id; |
66 | u32 abort; | 85 | u32 abort; |
67 | char data[0]; | 86 | char data[0]; |
68 | }; | 87 | }; |
69 | 88 | ||
89 | struct shared_msr_entry { | ||
90 | unsigned index; | ||
91 | u64 data; | ||
92 | u64 mask; | ||
93 | }; | ||
94 | |||
70 | struct vcpu_vmx { | 95 | struct vcpu_vmx { |
71 | struct kvm_vcpu vcpu; | 96 | struct kvm_vcpu vcpu; |
72 | struct list_head local_vcpus_link; | 97 | struct list_head local_vcpus_link; |
@@ -74,13 +99,12 @@ struct vcpu_vmx { | |||
74 | int launched; | 99 | int launched; |
75 | u8 fail; | 100 | u8 fail; |
76 | u32 idt_vectoring_info; | 101 | u32 idt_vectoring_info; |
77 | struct kvm_msr_entry *guest_msrs; | 102 | struct shared_msr_entry *guest_msrs; |
78 | struct kvm_msr_entry *host_msrs; | ||
79 | int nmsrs; | 103 | int nmsrs; |
80 | int save_nmsrs; | 104 | int save_nmsrs; |
81 | int msr_offset_efer; | ||
82 | #ifdef CONFIG_X86_64 | 105 | #ifdef CONFIG_X86_64 |
83 | int msr_offset_kernel_gs_base; | 106 | u64 msr_host_kernel_gs_base; |
107 | u64 msr_guest_kernel_gs_base; | ||
84 | #endif | 108 | #endif |
85 | struct vmcs *vmcs; | 109 | struct vmcs *vmcs; |
86 | struct { | 110 | struct { |
@@ -88,7 +112,6 @@ struct vcpu_vmx { | |||
88 | u16 fs_sel, gs_sel, ldt_sel; | 112 | u16 fs_sel, gs_sel, ldt_sel; |
89 | int gs_ldt_reload_needed; | 113 | int gs_ldt_reload_needed; |
90 | int fs_reload_needed; | 114 | int fs_reload_needed; |
91 | int guest_efer_loaded; | ||
92 | } host_state; | 115 | } host_state; |
93 | struct { | 116 | struct { |
94 | int vm86_active; | 117 | int vm86_active; |
@@ -107,7 +130,6 @@ struct vcpu_vmx { | |||
107 | } rmode; | 130 | } rmode; |
108 | int vpid; | 131 | int vpid; |
109 | bool emulation_required; | 132 | bool emulation_required; |
110 | enum emulation_result invalid_state_emulation_result; | ||
111 | 133 | ||
112 | /* Support for vnmi-less CPUs */ | 134 | /* Support for vnmi-less CPUs */ |
113 | int soft_vnmi_blocked; | 135 | int soft_vnmi_blocked; |
@@ -176,6 +198,8 @@ static struct kvm_vmx_segment_field { | |||
176 | VMX_SEGMENT_FIELD(LDTR), | 198 | VMX_SEGMENT_FIELD(LDTR), |
177 | }; | 199 | }; |
178 | 200 | ||
201 | static u64 host_efer; | ||
202 | |||
179 | static void ept_save_pdptrs(struct kvm_vcpu *vcpu); | 203 | static void ept_save_pdptrs(struct kvm_vcpu *vcpu); |
180 | 204 | ||
181 | /* | 205 | /* |
@@ -184,28 +208,12 @@ static void ept_save_pdptrs(struct kvm_vcpu *vcpu); | |||
184 | */ | 208 | */ |
185 | static const u32 vmx_msr_index[] = { | 209 | static const u32 vmx_msr_index[] = { |
186 | #ifdef CONFIG_X86_64 | 210 | #ifdef CONFIG_X86_64 |
187 | MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE, | 211 | MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, |
188 | #endif | 212 | #endif |
189 | MSR_EFER, MSR_K6_STAR, | 213 | MSR_EFER, MSR_K6_STAR, |
190 | }; | 214 | }; |
191 | #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index) | 215 | #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index) |
192 | 216 | ||
193 | static void load_msrs(struct kvm_msr_entry *e, int n) | ||
194 | { | ||
195 | int i; | ||
196 | |||
197 | for (i = 0; i < n; ++i) | ||
198 | wrmsrl(e[i].index, e[i].data); | ||
199 | } | ||
200 | |||
201 | static void save_msrs(struct kvm_msr_entry *e, int n) | ||
202 | { | ||
203 | int i; | ||
204 | |||
205 | for (i = 0; i < n; ++i) | ||
206 | rdmsrl(e[i].index, e[i].data); | ||
207 | } | ||
208 | |||
209 | static inline int is_page_fault(u32 intr_info) | 217 | static inline int is_page_fault(u32 intr_info) |
210 | { | 218 | { |
211 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | | 219 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | |
@@ -320,6 +328,12 @@ static inline int cpu_has_vmx_unrestricted_guest(void) | |||
320 | SECONDARY_EXEC_UNRESTRICTED_GUEST; | 328 | SECONDARY_EXEC_UNRESTRICTED_GUEST; |
321 | } | 329 | } |
322 | 330 | ||
331 | static inline int cpu_has_vmx_ple(void) | ||
332 | { | ||
333 | return vmcs_config.cpu_based_2nd_exec_ctrl & | ||
334 | SECONDARY_EXEC_PAUSE_LOOP_EXITING; | ||
335 | } | ||
336 | |||
323 | static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm) | 337 | static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm) |
324 | { | 338 | { |
325 | return flexpriority_enabled && | 339 | return flexpriority_enabled && |
@@ -348,7 +362,7 @@ static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr) | |||
348 | int i; | 362 | int i; |
349 | 363 | ||
350 | for (i = 0; i < vmx->nmsrs; ++i) | 364 | for (i = 0; i < vmx->nmsrs; ++i) |
351 | if (vmx->guest_msrs[i].index == msr) | 365 | if (vmx_msr_index[vmx->guest_msrs[i].index] == msr) |
352 | return i; | 366 | return i; |
353 | return -1; | 367 | return -1; |
354 | } | 368 | } |
@@ -379,7 +393,7 @@ static inline void __invept(int ext, u64 eptp, gpa_t gpa) | |||
379 | : : "a" (&operand), "c" (ext) : "cc", "memory"); | 393 | : : "a" (&operand), "c" (ext) : "cc", "memory"); |
380 | } | 394 | } |
381 | 395 | ||
382 | static struct kvm_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr) | 396 | static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr) |
383 | { | 397 | { |
384 | int i; | 398 | int i; |
385 | 399 | ||
@@ -570,17 +584,12 @@ static void reload_tss(void) | |||
570 | load_TR_desc(); | 584 | load_TR_desc(); |
571 | } | 585 | } |
572 | 586 | ||
573 | static void load_transition_efer(struct vcpu_vmx *vmx) | 587 | static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset) |
574 | { | 588 | { |
575 | int efer_offset = vmx->msr_offset_efer; | ||
576 | u64 host_efer; | ||
577 | u64 guest_efer; | 589 | u64 guest_efer; |
578 | u64 ignore_bits; | 590 | u64 ignore_bits; |
579 | 591 | ||
580 | if (efer_offset < 0) | 592 | guest_efer = vmx->vcpu.arch.shadow_efer; |
581 | return; | ||
582 | host_efer = vmx->host_msrs[efer_offset].data; | ||
583 | guest_efer = vmx->guest_msrs[efer_offset].data; | ||
584 | 593 | ||
585 | /* | 594 | /* |
586 | * NX is emulated; LMA and LME handled by hardware; SCE meaninless | 595 | * NX is emulated; LMA and LME handled by hardware; SCE meaninless |
@@ -593,27 +602,17 @@ static void load_transition_efer(struct vcpu_vmx *vmx) | |||
593 | if (guest_efer & EFER_LMA) | 602 | if (guest_efer & EFER_LMA) |
594 | ignore_bits &= ~(u64)EFER_SCE; | 603 | ignore_bits &= ~(u64)EFER_SCE; |
595 | #endif | 604 | #endif |
596 | if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits)) | ||
597 | return; | ||
598 | |||
599 | vmx->host_state.guest_efer_loaded = 1; | ||
600 | guest_efer &= ~ignore_bits; | 605 | guest_efer &= ~ignore_bits; |
601 | guest_efer |= host_efer & ignore_bits; | 606 | guest_efer |= host_efer & ignore_bits; |
602 | wrmsrl(MSR_EFER, guest_efer); | 607 | vmx->guest_msrs[efer_offset].data = guest_efer; |
603 | vmx->vcpu.stat.efer_reload++; | 608 | vmx->guest_msrs[efer_offset].mask = ~ignore_bits; |
604 | } | 609 | return true; |
605 | |||
606 | static void reload_host_efer(struct vcpu_vmx *vmx) | ||
607 | { | ||
608 | if (vmx->host_state.guest_efer_loaded) { | ||
609 | vmx->host_state.guest_efer_loaded = 0; | ||
610 | load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); | ||
611 | } | ||
612 | } | 610 | } |
613 | 611 | ||
614 | static void vmx_save_host_state(struct kvm_vcpu *vcpu) | 612 | static void vmx_save_host_state(struct kvm_vcpu *vcpu) |
615 | { | 613 | { |
616 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 614 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
615 | int i; | ||
617 | 616 | ||
618 | if (vmx->host_state.loaded) | 617 | if (vmx->host_state.loaded) |
619 | return; | 618 | return; |
@@ -650,13 +649,15 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) | |||
650 | #endif | 649 | #endif |
651 | 650 | ||
652 | #ifdef CONFIG_X86_64 | 651 | #ifdef CONFIG_X86_64 |
653 | if (is_long_mode(&vmx->vcpu)) | 652 | if (is_long_mode(&vmx->vcpu)) { |
654 | save_msrs(vmx->host_msrs + | 653 | rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); |
655 | vmx->msr_offset_kernel_gs_base, 1); | 654 | wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); |
656 | 655 | } | |
657 | #endif | 656 | #endif |
658 | load_msrs(vmx->guest_msrs, vmx->save_nmsrs); | 657 | for (i = 0; i < vmx->save_nmsrs; ++i) |
659 | load_transition_efer(vmx); | 658 | kvm_set_shared_msr(vmx->guest_msrs[i].index, |
659 | vmx->guest_msrs[i].data, | ||
660 | vmx->guest_msrs[i].mask); | ||
660 | } | 661 | } |
661 | 662 | ||
662 | static void __vmx_load_host_state(struct vcpu_vmx *vmx) | 663 | static void __vmx_load_host_state(struct vcpu_vmx *vmx) |
@@ -684,9 +685,12 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx) | |||
684 | local_irq_restore(flags); | 685 | local_irq_restore(flags); |
685 | } | 686 | } |
686 | reload_tss(); | 687 | reload_tss(); |
687 | save_msrs(vmx->guest_msrs, vmx->save_nmsrs); | 688 | #ifdef CONFIG_X86_64 |
688 | load_msrs(vmx->host_msrs, vmx->save_nmsrs); | 689 | if (is_long_mode(&vmx->vcpu)) { |
689 | reload_host_efer(vmx); | 690 | rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); |
691 | wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); | ||
692 | } | ||
693 | #endif | ||
690 | } | 694 | } |
691 | 695 | ||
692 | static void vmx_load_host_state(struct vcpu_vmx *vmx) | 696 | static void vmx_load_host_state(struct vcpu_vmx *vmx) |
@@ -877,19 +881,14 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, | |||
877 | /* | 881 | /* |
878 | * Swap MSR entry in host/guest MSR entry array. | 882 | * Swap MSR entry in host/guest MSR entry array. |
879 | */ | 883 | */ |
880 | #ifdef CONFIG_X86_64 | ||
881 | static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) | 884 | static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) |
882 | { | 885 | { |
883 | struct kvm_msr_entry tmp; | 886 | struct shared_msr_entry tmp; |
884 | 887 | ||
885 | tmp = vmx->guest_msrs[to]; | 888 | tmp = vmx->guest_msrs[to]; |
886 | vmx->guest_msrs[to] = vmx->guest_msrs[from]; | 889 | vmx->guest_msrs[to] = vmx->guest_msrs[from]; |
887 | vmx->guest_msrs[from] = tmp; | 890 | vmx->guest_msrs[from] = tmp; |
888 | tmp = vmx->host_msrs[to]; | ||
889 | vmx->host_msrs[to] = vmx->host_msrs[from]; | ||
890 | vmx->host_msrs[from] = tmp; | ||
891 | } | 891 | } |
892 | #endif | ||
893 | 892 | ||
894 | /* | 893 | /* |
895 | * Set up the vmcs to automatically save and restore system | 894 | * Set up the vmcs to automatically save and restore system |
@@ -898,15 +897,13 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) | |||
898 | */ | 897 | */ |
899 | static void setup_msrs(struct vcpu_vmx *vmx) | 898 | static void setup_msrs(struct vcpu_vmx *vmx) |
900 | { | 899 | { |
901 | int save_nmsrs; | 900 | int save_nmsrs, index; |
902 | unsigned long *msr_bitmap; | 901 | unsigned long *msr_bitmap; |
903 | 902 | ||
904 | vmx_load_host_state(vmx); | 903 | vmx_load_host_state(vmx); |
905 | save_nmsrs = 0; | 904 | save_nmsrs = 0; |
906 | #ifdef CONFIG_X86_64 | 905 | #ifdef CONFIG_X86_64 |
907 | if (is_long_mode(&vmx->vcpu)) { | 906 | if (is_long_mode(&vmx->vcpu)) { |
908 | int index; | ||
909 | |||
910 | index = __find_msr_index(vmx, MSR_SYSCALL_MASK); | 907 | index = __find_msr_index(vmx, MSR_SYSCALL_MASK); |
911 | if (index >= 0) | 908 | if (index >= 0) |
912 | move_msr_up(vmx, index, save_nmsrs++); | 909 | move_msr_up(vmx, index, save_nmsrs++); |
@@ -916,9 +913,6 @@ static void setup_msrs(struct vcpu_vmx *vmx) | |||
916 | index = __find_msr_index(vmx, MSR_CSTAR); | 913 | index = __find_msr_index(vmx, MSR_CSTAR); |
917 | if (index >= 0) | 914 | if (index >= 0) |
918 | move_msr_up(vmx, index, save_nmsrs++); | 915 | move_msr_up(vmx, index, save_nmsrs++); |
919 | index = __find_msr_index(vmx, MSR_KERNEL_GS_BASE); | ||
920 | if (index >= 0) | ||
921 | move_msr_up(vmx, index, save_nmsrs++); | ||
922 | /* | 916 | /* |
923 | * MSR_K6_STAR is only needed on long mode guests, and only | 917 | * MSR_K6_STAR is only needed on long mode guests, and only |
924 | * if efer.sce is enabled. | 918 | * if efer.sce is enabled. |
@@ -928,13 +922,11 @@ static void setup_msrs(struct vcpu_vmx *vmx) | |||
928 | move_msr_up(vmx, index, save_nmsrs++); | 922 | move_msr_up(vmx, index, save_nmsrs++); |
929 | } | 923 | } |
930 | #endif | 924 | #endif |
931 | vmx->save_nmsrs = save_nmsrs; | 925 | index = __find_msr_index(vmx, MSR_EFER); |
926 | if (index >= 0 && update_transition_efer(vmx, index)) | ||
927 | move_msr_up(vmx, index, save_nmsrs++); | ||
932 | 928 | ||
933 | #ifdef CONFIG_X86_64 | 929 | vmx->save_nmsrs = save_nmsrs; |
934 | vmx->msr_offset_kernel_gs_base = | ||
935 | __find_msr_index(vmx, MSR_KERNEL_GS_BASE); | ||
936 | #endif | ||
937 | vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER); | ||
938 | 930 | ||
939 | if (cpu_has_vmx_msr_bitmap()) { | 931 | if (cpu_has_vmx_msr_bitmap()) { |
940 | if (is_long_mode(&vmx->vcpu)) | 932 | if (is_long_mode(&vmx->vcpu)) |
@@ -976,7 +968,7 @@ static void guest_write_tsc(u64 guest_tsc, u64 host_tsc) | |||
976 | static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) | 968 | static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) |
977 | { | 969 | { |
978 | u64 data; | 970 | u64 data; |
979 | struct kvm_msr_entry *msr; | 971 | struct shared_msr_entry *msr; |
980 | 972 | ||
981 | if (!pdata) { | 973 | if (!pdata) { |
982 | printk(KERN_ERR "BUG: get_msr called with NULL pdata\n"); | 974 | printk(KERN_ERR "BUG: get_msr called with NULL pdata\n"); |
@@ -991,9 +983,13 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) | |||
991 | case MSR_GS_BASE: | 983 | case MSR_GS_BASE: |
992 | data = vmcs_readl(GUEST_GS_BASE); | 984 | data = vmcs_readl(GUEST_GS_BASE); |
993 | break; | 985 | break; |
986 | case MSR_KERNEL_GS_BASE: | ||
987 | vmx_load_host_state(to_vmx(vcpu)); | ||
988 | data = to_vmx(vcpu)->msr_guest_kernel_gs_base; | ||
989 | break; | ||
990 | #endif | ||
994 | case MSR_EFER: | 991 | case MSR_EFER: |
995 | return kvm_get_msr_common(vcpu, msr_index, pdata); | 992 | return kvm_get_msr_common(vcpu, msr_index, pdata); |
996 | #endif | ||
997 | case MSR_IA32_TSC: | 993 | case MSR_IA32_TSC: |
998 | data = guest_read_tsc(); | 994 | data = guest_read_tsc(); |
999 | break; | 995 | break; |
@@ -1007,6 +1003,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) | |||
1007 | data = vmcs_readl(GUEST_SYSENTER_ESP); | 1003 | data = vmcs_readl(GUEST_SYSENTER_ESP); |
1008 | break; | 1004 | break; |
1009 | default: | 1005 | default: |
1006 | vmx_load_host_state(to_vmx(vcpu)); | ||
1010 | msr = find_msr_entry(to_vmx(vcpu), msr_index); | 1007 | msr = find_msr_entry(to_vmx(vcpu), msr_index); |
1011 | if (msr) { | 1008 | if (msr) { |
1012 | vmx_load_host_state(to_vmx(vcpu)); | 1009 | vmx_load_host_state(to_vmx(vcpu)); |
@@ -1028,7 +1025,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) | |||
1028 | static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) | 1025 | static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) |
1029 | { | 1026 | { |
1030 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 1027 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
1031 | struct kvm_msr_entry *msr; | 1028 | struct shared_msr_entry *msr; |
1032 | u64 host_tsc; | 1029 | u64 host_tsc; |
1033 | int ret = 0; | 1030 | int ret = 0; |
1034 | 1031 | ||
@@ -1044,6 +1041,10 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) | |||
1044 | case MSR_GS_BASE: | 1041 | case MSR_GS_BASE: |
1045 | vmcs_writel(GUEST_GS_BASE, data); | 1042 | vmcs_writel(GUEST_GS_BASE, data); |
1046 | break; | 1043 | break; |
1044 | case MSR_KERNEL_GS_BASE: | ||
1045 | vmx_load_host_state(vmx); | ||
1046 | vmx->msr_guest_kernel_gs_base = data; | ||
1047 | break; | ||
1047 | #endif | 1048 | #endif |
1048 | case MSR_IA32_SYSENTER_CS: | 1049 | case MSR_IA32_SYSENTER_CS: |
1049 | vmcs_write32(GUEST_SYSENTER_CS, data); | 1050 | vmcs_write32(GUEST_SYSENTER_CS, data); |
@@ -1097,30 +1098,14 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) | |||
1097 | } | 1098 | } |
1098 | } | 1099 | } |
1099 | 1100 | ||
1100 | static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) | 1101 | static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) |
1101 | { | 1102 | { |
1102 | int old_debug = vcpu->guest_debug; | ||
1103 | unsigned long flags; | ||
1104 | |||
1105 | vcpu->guest_debug = dbg->control; | ||
1106 | if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) | ||
1107 | vcpu->guest_debug = 0; | ||
1108 | |||
1109 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | 1103 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) |
1110 | vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]); | 1104 | vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]); |
1111 | else | 1105 | else |
1112 | vmcs_writel(GUEST_DR7, vcpu->arch.dr7); | 1106 | vmcs_writel(GUEST_DR7, vcpu->arch.dr7); |
1113 | 1107 | ||
1114 | flags = vmcs_readl(GUEST_RFLAGS); | ||
1115 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
1116 | flags |= X86_EFLAGS_TF | X86_EFLAGS_RF; | ||
1117 | else if (old_debug & KVM_GUESTDBG_SINGLESTEP) | ||
1118 | flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | ||
1119 | vmcs_writel(GUEST_RFLAGS, flags); | ||
1120 | |||
1121 | update_exception_bitmap(vcpu); | 1108 | update_exception_bitmap(vcpu); |
1122 | |||
1123 | return 0; | ||
1124 | } | 1109 | } |
1125 | 1110 | ||
1126 | static __init int cpu_has_kvm_support(void) | 1111 | static __init int cpu_has_kvm_support(void) |
@@ -1139,12 +1124,15 @@ static __init int vmx_disabled_by_bios(void) | |||
1139 | /* locked but not enabled */ | 1124 | /* locked but not enabled */ |
1140 | } | 1125 | } |
1141 | 1126 | ||
1142 | static void hardware_enable(void *garbage) | 1127 | static int hardware_enable(void *garbage) |
1143 | { | 1128 | { |
1144 | int cpu = raw_smp_processor_id(); | 1129 | int cpu = raw_smp_processor_id(); |
1145 | u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); | 1130 | u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); |
1146 | u64 old; | 1131 | u64 old; |
1147 | 1132 | ||
1133 | if (read_cr4() & X86_CR4_VMXE) | ||
1134 | return -EBUSY; | ||
1135 | |||
1148 | INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu)); | 1136 | INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu)); |
1149 | rdmsrl(MSR_IA32_FEATURE_CONTROL, old); | 1137 | rdmsrl(MSR_IA32_FEATURE_CONTROL, old); |
1150 | if ((old & (FEATURE_CONTROL_LOCKED | | 1138 | if ((old & (FEATURE_CONTROL_LOCKED | |
@@ -1159,6 +1147,10 @@ static void hardware_enable(void *garbage) | |||
1159 | asm volatile (ASM_VMX_VMXON_RAX | 1147 | asm volatile (ASM_VMX_VMXON_RAX |
1160 | : : "a"(&phys_addr), "m"(phys_addr) | 1148 | : : "a"(&phys_addr), "m"(phys_addr) |
1161 | : "memory", "cc"); | 1149 | : "memory", "cc"); |
1150 | |||
1151 | ept_sync_global(); | ||
1152 | |||
1153 | return 0; | ||
1162 | } | 1154 | } |
1163 | 1155 | ||
1164 | static void vmclear_local_vcpus(void) | 1156 | static void vmclear_local_vcpus(void) |
@@ -1250,7 +1242,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) | |||
1250 | SECONDARY_EXEC_WBINVD_EXITING | | 1242 | SECONDARY_EXEC_WBINVD_EXITING | |
1251 | SECONDARY_EXEC_ENABLE_VPID | | 1243 | SECONDARY_EXEC_ENABLE_VPID | |
1252 | SECONDARY_EXEC_ENABLE_EPT | | 1244 | SECONDARY_EXEC_ENABLE_EPT | |
1253 | SECONDARY_EXEC_UNRESTRICTED_GUEST; | 1245 | SECONDARY_EXEC_UNRESTRICTED_GUEST | |
1246 | SECONDARY_EXEC_PAUSE_LOOP_EXITING; | ||
1254 | if (adjust_vmx_controls(min2, opt2, | 1247 | if (adjust_vmx_controls(min2, opt2, |
1255 | MSR_IA32_VMX_PROCBASED_CTLS2, | 1248 | MSR_IA32_VMX_PROCBASED_CTLS2, |
1256 | &_cpu_based_2nd_exec_control) < 0) | 1249 | &_cpu_based_2nd_exec_control) < 0) |
@@ -1344,15 +1337,17 @@ static void free_kvm_area(void) | |||
1344 | { | 1337 | { |
1345 | int cpu; | 1338 | int cpu; |
1346 | 1339 | ||
1347 | for_each_online_cpu(cpu) | 1340 | for_each_possible_cpu(cpu) { |
1348 | free_vmcs(per_cpu(vmxarea, cpu)); | 1341 | free_vmcs(per_cpu(vmxarea, cpu)); |
1342 | per_cpu(vmxarea, cpu) = NULL; | ||
1343 | } | ||
1349 | } | 1344 | } |
1350 | 1345 | ||
1351 | static __init int alloc_kvm_area(void) | 1346 | static __init int alloc_kvm_area(void) |
1352 | { | 1347 | { |
1353 | int cpu; | 1348 | int cpu; |
1354 | 1349 | ||
1355 | for_each_online_cpu(cpu) { | 1350 | for_each_possible_cpu(cpu) { |
1356 | struct vmcs *vmcs; | 1351 | struct vmcs *vmcs; |
1357 | 1352 | ||
1358 | vmcs = alloc_vmcs_cpu(cpu); | 1353 | vmcs = alloc_vmcs_cpu(cpu); |
@@ -1394,6 +1389,9 @@ static __init int hardware_setup(void) | |||
1394 | if (enable_ept && !cpu_has_vmx_ept_2m_page()) | 1389 | if (enable_ept && !cpu_has_vmx_ept_2m_page()) |
1395 | kvm_disable_largepages(); | 1390 | kvm_disable_largepages(); |
1396 | 1391 | ||
1392 | if (!cpu_has_vmx_ple()) | ||
1393 | ple_gap = 0; | ||
1394 | |||
1397 | return alloc_kvm_area(); | 1395 | return alloc_kvm_area(); |
1398 | } | 1396 | } |
1399 | 1397 | ||
@@ -1536,8 +1534,16 @@ continue_rmode: | |||
1536 | static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) | 1534 | static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) |
1537 | { | 1535 | { |
1538 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 1536 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
1539 | struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); | 1537 | struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); |
1538 | |||
1539 | if (!msr) | ||
1540 | return; | ||
1540 | 1541 | ||
1542 | /* | ||
1543 | * Force kernel_gs_base reloading before EFER changes, as control | ||
1544 | * of this msr depends on is_long_mode(). | ||
1545 | */ | ||
1546 | vmx_load_host_state(to_vmx(vcpu)); | ||
1541 | vcpu->arch.shadow_efer = efer; | 1547 | vcpu->arch.shadow_efer = efer; |
1542 | if (!msr) | 1548 | if (!msr) |
1543 | return; | 1549 | return; |
@@ -1727,6 +1733,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) | |||
1727 | vmcs_write64(EPT_POINTER, eptp); | 1733 | vmcs_write64(EPT_POINTER, eptp); |
1728 | guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 : | 1734 | guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 : |
1729 | vcpu->kvm->arch.ept_identity_map_addr; | 1735 | vcpu->kvm->arch.ept_identity_map_addr; |
1736 | ept_load_pdptrs(vcpu); | ||
1730 | } | 1737 | } |
1731 | 1738 | ||
1732 | vmx_flush_tlb(vcpu); | 1739 | vmx_flush_tlb(vcpu); |
@@ -2302,13 +2309,22 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) | |||
2302 | ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; | 2309 | ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; |
2303 | if (vmx->vpid == 0) | 2310 | if (vmx->vpid == 0) |
2304 | exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; | 2311 | exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; |
2305 | if (!enable_ept) | 2312 | if (!enable_ept) { |
2306 | exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; | 2313 | exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; |
2314 | enable_unrestricted_guest = 0; | ||
2315 | } | ||
2307 | if (!enable_unrestricted_guest) | 2316 | if (!enable_unrestricted_guest) |
2308 | exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; | 2317 | exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; |
2318 | if (!ple_gap) | ||
2319 | exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; | ||
2309 | vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); | 2320 | vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); |
2310 | } | 2321 | } |
2311 | 2322 | ||
2323 | if (ple_gap) { | ||
2324 | vmcs_write32(PLE_GAP, ple_gap); | ||
2325 | vmcs_write32(PLE_WINDOW, ple_window); | ||
2326 | } | ||
2327 | |||
2312 | vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); | 2328 | vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); |
2313 | vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); | 2329 | vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); |
2314 | vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ | 2330 | vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ |
@@ -2376,10 +2392,9 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) | |||
2376 | if (wrmsr_safe(index, data_low, data_high) < 0) | 2392 | if (wrmsr_safe(index, data_low, data_high) < 0) |
2377 | continue; | 2393 | continue; |
2378 | data = data_low | ((u64)data_high << 32); | 2394 | data = data_low | ((u64)data_high << 32); |
2379 | vmx->host_msrs[j].index = index; | 2395 | vmx->guest_msrs[j].index = i; |
2380 | vmx->host_msrs[j].reserved = 0; | 2396 | vmx->guest_msrs[j].data = 0; |
2381 | vmx->host_msrs[j].data = data; | 2397 | vmx->guest_msrs[j].mask = -1ull; |
2382 | vmx->guest_msrs[j] = vmx->host_msrs[j]; | ||
2383 | ++vmx->nmsrs; | 2398 | ++vmx->nmsrs; |
2384 | } | 2399 | } |
2385 | 2400 | ||
@@ -2510,7 +2525,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) | |||
2510 | if (vmx->vpid != 0) | 2525 | if (vmx->vpid != 0) |
2511 | vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); | 2526 | vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); |
2512 | 2527 | ||
2513 | vmx->vcpu.arch.cr0 = 0x60000010; | 2528 | vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET; |
2514 | vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ | 2529 | vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ |
2515 | vmx_set_cr4(&vmx->vcpu, 0); | 2530 | vmx_set_cr4(&vmx->vcpu, 0); |
2516 | vmx_set_efer(&vmx->vcpu, 0); | 2531 | vmx_set_efer(&vmx->vcpu, 0); |
@@ -2627,6 +2642,34 @@ static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) | |||
2627 | GUEST_INTR_STATE_NMI)); | 2642 | GUEST_INTR_STATE_NMI)); |
2628 | } | 2643 | } |
2629 | 2644 | ||
2645 | static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) | ||
2646 | { | ||
2647 | if (!cpu_has_virtual_nmis()) | ||
2648 | return to_vmx(vcpu)->soft_vnmi_blocked; | ||
2649 | else | ||
2650 | return !!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & | ||
2651 | GUEST_INTR_STATE_NMI); | ||
2652 | } | ||
2653 | |||
2654 | static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) | ||
2655 | { | ||
2656 | struct vcpu_vmx *vmx = to_vmx(vcpu); | ||
2657 | |||
2658 | if (!cpu_has_virtual_nmis()) { | ||
2659 | if (vmx->soft_vnmi_blocked != masked) { | ||
2660 | vmx->soft_vnmi_blocked = masked; | ||
2661 | vmx->vnmi_blocked_time = 0; | ||
2662 | } | ||
2663 | } else { | ||
2664 | if (masked) | ||
2665 | vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, | ||
2666 | GUEST_INTR_STATE_NMI); | ||
2667 | else | ||
2668 | vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, | ||
2669 | GUEST_INTR_STATE_NMI); | ||
2670 | } | ||
2671 | } | ||
2672 | |||
2630 | static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu) | 2673 | static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu) |
2631 | { | 2674 | { |
2632 | return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && | 2675 | return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && |
@@ -2659,7 +2702,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, | |||
2659 | * Cause the #SS fault with 0 error code in VM86 mode. | 2702 | * Cause the #SS fault with 0 error code in VM86 mode. |
2660 | */ | 2703 | */ |
2661 | if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) | 2704 | if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) |
2662 | if (emulate_instruction(vcpu, NULL, 0, 0, 0) == EMULATE_DONE) | 2705 | if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE) |
2663 | return 1; | 2706 | return 1; |
2664 | /* | 2707 | /* |
2665 | * Forward all other exceptions that are valid in real mode. | 2708 | * Forward all other exceptions that are valid in real mode. |
@@ -2710,15 +2753,16 @@ static void kvm_machine_check(void) | |||
2710 | #endif | 2753 | #endif |
2711 | } | 2754 | } |
2712 | 2755 | ||
2713 | static int handle_machine_check(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2756 | static int handle_machine_check(struct kvm_vcpu *vcpu) |
2714 | { | 2757 | { |
2715 | /* already handled by vcpu_run */ | 2758 | /* already handled by vcpu_run */ |
2716 | return 1; | 2759 | return 1; |
2717 | } | 2760 | } |
2718 | 2761 | ||
2719 | static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2762 | static int handle_exception(struct kvm_vcpu *vcpu) |
2720 | { | 2763 | { |
2721 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 2764 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
2765 | struct kvm_run *kvm_run = vcpu->run; | ||
2722 | u32 intr_info, ex_no, error_code; | 2766 | u32 intr_info, ex_no, error_code; |
2723 | unsigned long cr2, rip, dr6; | 2767 | unsigned long cr2, rip, dr6; |
2724 | u32 vect_info; | 2768 | u32 vect_info; |
@@ -2728,12 +2772,17 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2728 | intr_info = vmcs_read32(VM_EXIT_INTR_INFO); | 2772 | intr_info = vmcs_read32(VM_EXIT_INTR_INFO); |
2729 | 2773 | ||
2730 | if (is_machine_check(intr_info)) | 2774 | if (is_machine_check(intr_info)) |
2731 | return handle_machine_check(vcpu, kvm_run); | 2775 | return handle_machine_check(vcpu); |
2732 | 2776 | ||
2733 | if ((vect_info & VECTORING_INFO_VALID_MASK) && | 2777 | if ((vect_info & VECTORING_INFO_VALID_MASK) && |
2734 | !is_page_fault(intr_info)) | 2778 | !is_page_fault(intr_info)) { |
2735 | printk(KERN_ERR "%s: unexpected, vectoring info 0x%x " | 2779 | vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
2736 | "intr info 0x%x\n", __func__, vect_info, intr_info); | 2780 | vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; |
2781 | vcpu->run->internal.ndata = 2; | ||
2782 | vcpu->run->internal.data[0] = vect_info; | ||
2783 | vcpu->run->internal.data[1] = intr_info; | ||
2784 | return 0; | ||
2785 | } | ||
2737 | 2786 | ||
2738 | if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR) | 2787 | if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR) |
2739 | return 1; /* already handled by vmx_vcpu_run() */ | 2788 | return 1; /* already handled by vmx_vcpu_run() */ |
@@ -2744,7 +2793,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2744 | } | 2793 | } |
2745 | 2794 | ||
2746 | if (is_invalid_opcode(intr_info)) { | 2795 | if (is_invalid_opcode(intr_info)) { |
2747 | er = emulate_instruction(vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD); | 2796 | er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD); |
2748 | if (er != EMULATE_DONE) | 2797 | if (er != EMULATE_DONE) |
2749 | kvm_queue_exception(vcpu, UD_VECTOR); | 2798 | kvm_queue_exception(vcpu, UD_VECTOR); |
2750 | return 1; | 2799 | return 1; |
@@ -2803,20 +2852,19 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2803 | return 0; | 2852 | return 0; |
2804 | } | 2853 | } |
2805 | 2854 | ||
2806 | static int handle_external_interrupt(struct kvm_vcpu *vcpu, | 2855 | static int handle_external_interrupt(struct kvm_vcpu *vcpu) |
2807 | struct kvm_run *kvm_run) | ||
2808 | { | 2856 | { |
2809 | ++vcpu->stat.irq_exits; | 2857 | ++vcpu->stat.irq_exits; |
2810 | return 1; | 2858 | return 1; |
2811 | } | 2859 | } |
2812 | 2860 | ||
2813 | static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2861 | static int handle_triple_fault(struct kvm_vcpu *vcpu) |
2814 | { | 2862 | { |
2815 | kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; | 2863 | vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; |
2816 | return 0; | 2864 | return 0; |
2817 | } | 2865 | } |
2818 | 2866 | ||
2819 | static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2867 | static int handle_io(struct kvm_vcpu *vcpu) |
2820 | { | 2868 | { |
2821 | unsigned long exit_qualification; | 2869 | unsigned long exit_qualification; |
2822 | int size, in, string; | 2870 | int size, in, string; |
@@ -2827,8 +2875,7 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2827 | string = (exit_qualification & 16) != 0; | 2875 | string = (exit_qualification & 16) != 0; |
2828 | 2876 | ||
2829 | if (string) { | 2877 | if (string) { |
2830 | if (emulate_instruction(vcpu, | 2878 | if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DO_MMIO) |
2831 | kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) | ||
2832 | return 0; | 2879 | return 0; |
2833 | return 1; | 2880 | return 1; |
2834 | } | 2881 | } |
@@ -2838,7 +2885,7 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2838 | port = exit_qualification >> 16; | 2885 | port = exit_qualification >> 16; |
2839 | 2886 | ||
2840 | skip_emulated_instruction(vcpu); | 2887 | skip_emulated_instruction(vcpu); |
2841 | return kvm_emulate_pio(vcpu, kvm_run, in, size, port); | 2888 | return kvm_emulate_pio(vcpu, in, size, port); |
2842 | } | 2889 | } |
2843 | 2890 | ||
2844 | static void | 2891 | static void |
@@ -2852,7 +2899,7 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) | |||
2852 | hypercall[2] = 0xc1; | 2899 | hypercall[2] = 0xc1; |
2853 | } | 2900 | } |
2854 | 2901 | ||
2855 | static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2902 | static int handle_cr(struct kvm_vcpu *vcpu) |
2856 | { | 2903 | { |
2857 | unsigned long exit_qualification, val; | 2904 | unsigned long exit_qualification, val; |
2858 | int cr; | 2905 | int cr; |
@@ -2887,7 +2934,7 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2887 | return 1; | 2934 | return 1; |
2888 | if (cr8_prev <= cr8) | 2935 | if (cr8_prev <= cr8) |
2889 | return 1; | 2936 | return 1; |
2890 | kvm_run->exit_reason = KVM_EXIT_SET_TPR; | 2937 | vcpu->run->exit_reason = KVM_EXIT_SET_TPR; |
2891 | return 0; | 2938 | return 0; |
2892 | } | 2939 | } |
2893 | }; | 2940 | }; |
@@ -2922,13 +2969,13 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2922 | default: | 2969 | default: |
2923 | break; | 2970 | break; |
2924 | } | 2971 | } |
2925 | kvm_run->exit_reason = 0; | 2972 | vcpu->run->exit_reason = 0; |
2926 | pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n", | 2973 | pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n", |
2927 | (int)(exit_qualification >> 4) & 3, cr); | 2974 | (int)(exit_qualification >> 4) & 3, cr); |
2928 | return 0; | 2975 | return 0; |
2929 | } | 2976 | } |
2930 | 2977 | ||
2931 | static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2978 | static int handle_dr(struct kvm_vcpu *vcpu) |
2932 | { | 2979 | { |
2933 | unsigned long exit_qualification; | 2980 | unsigned long exit_qualification; |
2934 | unsigned long val; | 2981 | unsigned long val; |
@@ -2944,13 +2991,13 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2944 | * guest debugging itself. | 2991 | * guest debugging itself. |
2945 | */ | 2992 | */ |
2946 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { | 2993 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { |
2947 | kvm_run->debug.arch.dr6 = vcpu->arch.dr6; | 2994 | vcpu->run->debug.arch.dr6 = vcpu->arch.dr6; |
2948 | kvm_run->debug.arch.dr7 = dr; | 2995 | vcpu->run->debug.arch.dr7 = dr; |
2949 | kvm_run->debug.arch.pc = | 2996 | vcpu->run->debug.arch.pc = |
2950 | vmcs_readl(GUEST_CS_BASE) + | 2997 | vmcs_readl(GUEST_CS_BASE) + |
2951 | vmcs_readl(GUEST_RIP); | 2998 | vmcs_readl(GUEST_RIP); |
2952 | kvm_run->debug.arch.exception = DB_VECTOR; | 2999 | vcpu->run->debug.arch.exception = DB_VECTOR; |
2953 | kvm_run->exit_reason = KVM_EXIT_DEBUG; | 3000 | vcpu->run->exit_reason = KVM_EXIT_DEBUG; |
2954 | return 0; | 3001 | return 0; |
2955 | } else { | 3002 | } else { |
2956 | vcpu->arch.dr7 &= ~DR7_GD; | 3003 | vcpu->arch.dr7 &= ~DR7_GD; |
@@ -3016,13 +3063,13 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3016 | return 1; | 3063 | return 1; |
3017 | } | 3064 | } |
3018 | 3065 | ||
3019 | static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3066 | static int handle_cpuid(struct kvm_vcpu *vcpu) |
3020 | { | 3067 | { |
3021 | kvm_emulate_cpuid(vcpu); | 3068 | kvm_emulate_cpuid(vcpu); |
3022 | return 1; | 3069 | return 1; |
3023 | } | 3070 | } |
3024 | 3071 | ||
3025 | static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3072 | static int handle_rdmsr(struct kvm_vcpu *vcpu) |
3026 | { | 3073 | { |
3027 | u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; | 3074 | u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; |
3028 | u64 data; | 3075 | u64 data; |
@@ -3041,7 +3088,7 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3041 | return 1; | 3088 | return 1; |
3042 | } | 3089 | } |
3043 | 3090 | ||
3044 | static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3091 | static int handle_wrmsr(struct kvm_vcpu *vcpu) |
3045 | { | 3092 | { |
3046 | u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; | 3093 | u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; |
3047 | u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u) | 3094 | u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u) |
@@ -3058,14 +3105,12 @@ static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3058 | return 1; | 3105 | return 1; |
3059 | } | 3106 | } |
3060 | 3107 | ||
3061 | static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu, | 3108 | static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) |
3062 | struct kvm_run *kvm_run) | ||
3063 | { | 3109 | { |
3064 | return 1; | 3110 | return 1; |
3065 | } | 3111 | } |
3066 | 3112 | ||
3067 | static int handle_interrupt_window(struct kvm_vcpu *vcpu, | 3113 | static int handle_interrupt_window(struct kvm_vcpu *vcpu) |
3068 | struct kvm_run *kvm_run) | ||
3069 | { | 3114 | { |
3070 | u32 cpu_based_vm_exec_control; | 3115 | u32 cpu_based_vm_exec_control; |
3071 | 3116 | ||
@@ -3081,34 +3126,34 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu, | |||
3081 | * possible | 3126 | * possible |
3082 | */ | 3127 | */ |
3083 | if (!irqchip_in_kernel(vcpu->kvm) && | 3128 | if (!irqchip_in_kernel(vcpu->kvm) && |
3084 | kvm_run->request_interrupt_window && | 3129 | vcpu->run->request_interrupt_window && |
3085 | !kvm_cpu_has_interrupt(vcpu)) { | 3130 | !kvm_cpu_has_interrupt(vcpu)) { |
3086 | kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; | 3131 | vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; |
3087 | return 0; | 3132 | return 0; |
3088 | } | 3133 | } |
3089 | return 1; | 3134 | return 1; |
3090 | } | 3135 | } |
3091 | 3136 | ||
3092 | static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3137 | static int handle_halt(struct kvm_vcpu *vcpu) |
3093 | { | 3138 | { |
3094 | skip_emulated_instruction(vcpu); | 3139 | skip_emulated_instruction(vcpu); |
3095 | return kvm_emulate_halt(vcpu); | 3140 | return kvm_emulate_halt(vcpu); |
3096 | } | 3141 | } |
3097 | 3142 | ||
3098 | static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3143 | static int handle_vmcall(struct kvm_vcpu *vcpu) |
3099 | { | 3144 | { |
3100 | skip_emulated_instruction(vcpu); | 3145 | skip_emulated_instruction(vcpu); |
3101 | kvm_emulate_hypercall(vcpu); | 3146 | kvm_emulate_hypercall(vcpu); |
3102 | return 1; | 3147 | return 1; |
3103 | } | 3148 | } |
3104 | 3149 | ||
3105 | static int handle_vmx_insn(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3150 | static int handle_vmx_insn(struct kvm_vcpu *vcpu) |
3106 | { | 3151 | { |
3107 | kvm_queue_exception(vcpu, UD_VECTOR); | 3152 | kvm_queue_exception(vcpu, UD_VECTOR); |
3108 | return 1; | 3153 | return 1; |
3109 | } | 3154 | } |
3110 | 3155 | ||
3111 | static int handle_invlpg(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3156 | static int handle_invlpg(struct kvm_vcpu *vcpu) |
3112 | { | 3157 | { |
3113 | unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); | 3158 | unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); |
3114 | 3159 | ||
@@ -3117,14 +3162,14 @@ static int handle_invlpg(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3117 | return 1; | 3162 | return 1; |
3118 | } | 3163 | } |
3119 | 3164 | ||
3120 | static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3165 | static int handle_wbinvd(struct kvm_vcpu *vcpu) |
3121 | { | 3166 | { |
3122 | skip_emulated_instruction(vcpu); | 3167 | skip_emulated_instruction(vcpu); |
3123 | /* TODO: Add support for VT-d/pass-through device */ | 3168 | /* TODO: Add support for VT-d/pass-through device */ |
3124 | return 1; | 3169 | return 1; |
3125 | } | 3170 | } |
3126 | 3171 | ||
3127 | static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3172 | static int handle_apic_access(struct kvm_vcpu *vcpu) |
3128 | { | 3173 | { |
3129 | unsigned long exit_qualification; | 3174 | unsigned long exit_qualification; |
3130 | enum emulation_result er; | 3175 | enum emulation_result er; |
@@ -3133,7 +3178,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3133 | exit_qualification = vmcs_readl(EXIT_QUALIFICATION); | 3178 | exit_qualification = vmcs_readl(EXIT_QUALIFICATION); |
3134 | offset = exit_qualification & 0xffful; | 3179 | offset = exit_qualification & 0xffful; |
3135 | 3180 | ||
3136 | er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); | 3181 | er = emulate_instruction(vcpu, 0, 0, 0); |
3137 | 3182 | ||
3138 | if (er != EMULATE_DONE) { | 3183 | if (er != EMULATE_DONE) { |
3139 | printk(KERN_ERR | 3184 | printk(KERN_ERR |
@@ -3144,7 +3189,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3144 | return 1; | 3189 | return 1; |
3145 | } | 3190 | } |
3146 | 3191 | ||
3147 | static int handle_task_switch(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3192 | static int handle_task_switch(struct kvm_vcpu *vcpu) |
3148 | { | 3193 | { |
3149 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3194 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
3150 | unsigned long exit_qualification; | 3195 | unsigned long exit_qualification; |
@@ -3198,7 +3243,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3198 | return 1; | 3243 | return 1; |
3199 | } | 3244 | } |
3200 | 3245 | ||
3201 | static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3246 | static int handle_ept_violation(struct kvm_vcpu *vcpu) |
3202 | { | 3247 | { |
3203 | unsigned long exit_qualification; | 3248 | unsigned long exit_qualification; |
3204 | gpa_t gpa; | 3249 | gpa_t gpa; |
@@ -3219,8 +3264,8 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3219 | vmcs_readl(GUEST_LINEAR_ADDRESS)); | 3264 | vmcs_readl(GUEST_LINEAR_ADDRESS)); |
3220 | printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n", | 3265 | printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n", |
3221 | (long unsigned int)exit_qualification); | 3266 | (long unsigned int)exit_qualification); |
3222 | kvm_run->exit_reason = KVM_EXIT_UNKNOWN; | 3267 | vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; |
3223 | kvm_run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION; | 3268 | vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION; |
3224 | return 0; | 3269 | return 0; |
3225 | } | 3270 | } |
3226 | 3271 | ||
@@ -3290,7 +3335,7 @@ static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte, | |||
3290 | } | 3335 | } |
3291 | } | 3336 | } |
3292 | 3337 | ||
3293 | static int handle_ept_misconfig(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3338 | static int handle_ept_misconfig(struct kvm_vcpu *vcpu) |
3294 | { | 3339 | { |
3295 | u64 sptes[4]; | 3340 | u64 sptes[4]; |
3296 | int nr_sptes, i; | 3341 | int nr_sptes, i; |
@@ -3306,13 +3351,13 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3306 | for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i) | 3351 | for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i) |
3307 | ept_misconfig_inspect_spte(vcpu, sptes[i-1], i); | 3352 | ept_misconfig_inspect_spte(vcpu, sptes[i-1], i); |
3308 | 3353 | ||
3309 | kvm_run->exit_reason = KVM_EXIT_UNKNOWN; | 3354 | vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; |
3310 | kvm_run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG; | 3355 | vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG; |
3311 | 3356 | ||
3312 | return 0; | 3357 | return 0; |
3313 | } | 3358 | } |
3314 | 3359 | ||
3315 | static int handle_nmi_window(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3360 | static int handle_nmi_window(struct kvm_vcpu *vcpu) |
3316 | { | 3361 | { |
3317 | u32 cpu_based_vm_exec_control; | 3362 | u32 cpu_based_vm_exec_control; |
3318 | 3363 | ||
@@ -3325,36 +3370,50 @@ static int handle_nmi_window(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3325 | return 1; | 3370 | return 1; |
3326 | } | 3371 | } |
3327 | 3372 | ||
3328 | static void handle_invalid_guest_state(struct kvm_vcpu *vcpu, | 3373 | static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) |
3329 | struct kvm_run *kvm_run) | ||
3330 | { | 3374 | { |
3331 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3375 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
3332 | enum emulation_result err = EMULATE_DONE; | 3376 | enum emulation_result err = EMULATE_DONE; |
3333 | 3377 | int ret = 1; | |
3334 | local_irq_enable(); | ||
3335 | preempt_enable(); | ||
3336 | 3378 | ||
3337 | while (!guest_state_valid(vcpu)) { | 3379 | while (!guest_state_valid(vcpu)) { |
3338 | err = emulate_instruction(vcpu, kvm_run, 0, 0, 0); | 3380 | err = emulate_instruction(vcpu, 0, 0, 0); |
3339 | 3381 | ||
3340 | if (err == EMULATE_DO_MMIO) | 3382 | if (err == EMULATE_DO_MMIO) { |
3341 | break; | 3383 | ret = 0; |
3384 | goto out; | ||
3385 | } | ||
3342 | 3386 | ||
3343 | if (err != EMULATE_DONE) { | 3387 | if (err != EMULATE_DONE) { |
3344 | kvm_report_emulation_failure(vcpu, "emulation failure"); | 3388 | kvm_report_emulation_failure(vcpu, "emulation failure"); |
3345 | break; | 3389 | vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
3390 | vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; | ||
3391 | vcpu->run->internal.ndata = 0; | ||
3392 | ret = 0; | ||
3393 | goto out; | ||
3346 | } | 3394 | } |
3347 | 3395 | ||
3348 | if (signal_pending(current)) | 3396 | if (signal_pending(current)) |
3349 | break; | 3397 | goto out; |
3350 | if (need_resched()) | 3398 | if (need_resched()) |
3351 | schedule(); | 3399 | schedule(); |
3352 | } | 3400 | } |
3353 | 3401 | ||
3354 | preempt_disable(); | 3402 | vmx->emulation_required = 0; |
3355 | local_irq_disable(); | 3403 | out: |
3404 | return ret; | ||
3405 | } | ||
3356 | 3406 | ||
3357 | vmx->invalid_state_emulation_result = err; | 3407 | /* |
3408 | * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE | ||
3409 | * exiting, so only get here on cpu with PAUSE-Loop-Exiting. | ||
3410 | */ | ||
3411 | static int handle_pause(struct kvm_vcpu *vcpu) | ||
3412 | { | ||
3413 | skip_emulated_instruction(vcpu); | ||
3414 | kvm_vcpu_on_spin(vcpu); | ||
3415 | |||
3416 | return 1; | ||
3358 | } | 3417 | } |
3359 | 3418 | ||
3360 | /* | 3419 | /* |
@@ -3362,8 +3421,7 @@ static void handle_invalid_guest_state(struct kvm_vcpu *vcpu, | |||
3362 | * may resume. Otherwise they set the kvm_run parameter to indicate what needs | 3421 | * may resume. Otherwise they set the kvm_run parameter to indicate what needs |
3363 | * to be done to userspace and return 0. | 3422 | * to be done to userspace and return 0. |
3364 | */ | 3423 | */ |
3365 | static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, | 3424 | static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { |
3366 | struct kvm_run *kvm_run) = { | ||
3367 | [EXIT_REASON_EXCEPTION_NMI] = handle_exception, | 3425 | [EXIT_REASON_EXCEPTION_NMI] = handle_exception, |
3368 | [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, | 3426 | [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, |
3369 | [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, | 3427 | [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, |
@@ -3394,6 +3452,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, | |||
3394 | [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, | 3452 | [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, |
3395 | [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, | 3453 | [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, |
3396 | [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, | 3454 | [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, |
3455 | [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, | ||
3397 | }; | 3456 | }; |
3398 | 3457 | ||
3399 | static const int kvm_vmx_max_exit_handlers = | 3458 | static const int kvm_vmx_max_exit_handlers = |
@@ -3403,7 +3462,7 @@ static const int kvm_vmx_max_exit_handlers = | |||
3403 | * The guest has exited. See if we can fix it or if we need userspace | 3462 | * The guest has exited. See if we can fix it or if we need userspace |
3404 | * assistance. | 3463 | * assistance. |
3405 | */ | 3464 | */ |
3406 | static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | 3465 | static int vmx_handle_exit(struct kvm_vcpu *vcpu) |
3407 | { | 3466 | { |
3408 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3467 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
3409 | u32 exit_reason = vmx->exit_reason; | 3468 | u32 exit_reason = vmx->exit_reason; |
@@ -3411,13 +3470,9 @@ static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | |||
3411 | 3470 | ||
3412 | trace_kvm_exit(exit_reason, kvm_rip_read(vcpu)); | 3471 | trace_kvm_exit(exit_reason, kvm_rip_read(vcpu)); |
3413 | 3472 | ||
3414 | /* If we need to emulate an MMIO from handle_invalid_guest_state | 3473 | /* If guest state is invalid, start emulating */ |
3415 | * we just return 0 */ | 3474 | if (vmx->emulation_required && emulate_invalid_guest_state) |
3416 | if (vmx->emulation_required && emulate_invalid_guest_state) { | 3475 | return handle_invalid_guest_state(vcpu); |
3417 | if (guest_state_valid(vcpu)) | ||
3418 | vmx->emulation_required = 0; | ||
3419 | return vmx->invalid_state_emulation_result != EMULATE_DO_MMIO; | ||
3420 | } | ||
3421 | 3476 | ||
3422 | /* Access CR3 don't cause VMExit in paging mode, so we need | 3477 | /* Access CR3 don't cause VMExit in paging mode, so we need |
3423 | * to sync with guest real CR3. */ | 3478 | * to sync with guest real CR3. */ |
@@ -3425,8 +3480,8 @@ static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | |||
3425 | vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); | 3480 | vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); |
3426 | 3481 | ||
3427 | if (unlikely(vmx->fail)) { | 3482 | if (unlikely(vmx->fail)) { |
3428 | kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; | 3483 | vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; |
3429 | kvm_run->fail_entry.hardware_entry_failure_reason | 3484 | vcpu->run->fail_entry.hardware_entry_failure_reason |
3430 | = vmcs_read32(VM_INSTRUCTION_ERROR); | 3485 | = vmcs_read32(VM_INSTRUCTION_ERROR); |
3431 | return 0; | 3486 | return 0; |
3432 | } | 3487 | } |
@@ -3459,10 +3514,10 @@ static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | |||
3459 | 3514 | ||
3460 | if (exit_reason < kvm_vmx_max_exit_handlers | 3515 | if (exit_reason < kvm_vmx_max_exit_handlers |
3461 | && kvm_vmx_exit_handlers[exit_reason]) | 3516 | && kvm_vmx_exit_handlers[exit_reason]) |
3462 | return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run); | 3517 | return kvm_vmx_exit_handlers[exit_reason](vcpu); |
3463 | else { | 3518 | else { |
3464 | kvm_run->exit_reason = KVM_EXIT_UNKNOWN; | 3519 | vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; |
3465 | kvm_run->hw.hardware_exit_reason = exit_reason; | 3520 | vcpu->run->hw.hardware_exit_reason = exit_reason; |
3466 | } | 3521 | } |
3467 | return 0; | 3522 | return 0; |
3468 | } | 3523 | } |
@@ -3600,23 +3655,18 @@ static void fixup_rmode_irq(struct vcpu_vmx *vmx) | |||
3600 | #define Q "l" | 3655 | #define Q "l" |
3601 | #endif | 3656 | #endif |
3602 | 3657 | ||
3603 | static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3658 | static void vmx_vcpu_run(struct kvm_vcpu *vcpu) |
3604 | { | 3659 | { |
3605 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3660 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
3606 | 3661 | ||
3607 | if (enable_ept && is_paging(vcpu)) { | ||
3608 | vmcs_writel(GUEST_CR3, vcpu->arch.cr3); | ||
3609 | ept_load_pdptrs(vcpu); | ||
3610 | } | ||
3611 | /* Record the guest's net vcpu time for enforced NMI injections. */ | 3662 | /* Record the guest's net vcpu time for enforced NMI injections. */ |
3612 | if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) | 3663 | if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) |
3613 | vmx->entry_time = ktime_get(); | 3664 | vmx->entry_time = ktime_get(); |
3614 | 3665 | ||
3615 | /* Handle invalid guest state instead of entering VMX */ | 3666 | /* Don't enter VMX if guest state is invalid, let the exit handler |
3616 | if (vmx->emulation_required && emulate_invalid_guest_state) { | 3667 | start emulation until we arrive back to a valid state */ |
3617 | handle_invalid_guest_state(vcpu, kvm_run); | 3668 | if (vmx->emulation_required && emulate_invalid_guest_state) |
3618 | return; | 3669 | return; |
3619 | } | ||
3620 | 3670 | ||
3621 | if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty)) | 3671 | if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty)) |
3622 | vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); | 3672 | vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); |
@@ -3775,7 +3825,6 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu) | |||
3775 | __clear_bit(vmx->vpid, vmx_vpid_bitmap); | 3825 | __clear_bit(vmx->vpid, vmx_vpid_bitmap); |
3776 | spin_unlock(&vmx_vpid_lock); | 3826 | spin_unlock(&vmx_vpid_lock); |
3777 | vmx_free_vmcs(vcpu); | 3827 | vmx_free_vmcs(vcpu); |
3778 | kfree(vmx->host_msrs); | ||
3779 | kfree(vmx->guest_msrs); | 3828 | kfree(vmx->guest_msrs); |
3780 | kvm_vcpu_uninit(vcpu); | 3829 | kvm_vcpu_uninit(vcpu); |
3781 | kmem_cache_free(kvm_vcpu_cache, vmx); | 3830 | kmem_cache_free(kvm_vcpu_cache, vmx); |
@@ -3802,10 +3851,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) | |||
3802 | goto uninit_vcpu; | 3851 | goto uninit_vcpu; |
3803 | } | 3852 | } |
3804 | 3853 | ||
3805 | vmx->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); | ||
3806 | if (!vmx->host_msrs) | ||
3807 | goto free_guest_msrs; | ||
3808 | |||
3809 | vmx->vmcs = alloc_vmcs(); | 3854 | vmx->vmcs = alloc_vmcs(); |
3810 | if (!vmx->vmcs) | 3855 | if (!vmx->vmcs) |
3811 | goto free_msrs; | 3856 | goto free_msrs; |
@@ -3836,8 +3881,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) | |||
3836 | free_vmcs: | 3881 | free_vmcs: |
3837 | free_vmcs(vmx->vmcs); | 3882 | free_vmcs(vmx->vmcs); |
3838 | free_msrs: | 3883 | free_msrs: |
3839 | kfree(vmx->host_msrs); | ||
3840 | free_guest_msrs: | ||
3841 | kfree(vmx->guest_msrs); | 3884 | kfree(vmx->guest_msrs); |
3842 | uninit_vcpu: | 3885 | uninit_vcpu: |
3843 | kvm_vcpu_uninit(&vmx->vcpu); | 3886 | kvm_vcpu_uninit(&vmx->vcpu); |
@@ -3973,6 +4016,8 @@ static struct kvm_x86_ops vmx_x86_ops = { | |||
3973 | .queue_exception = vmx_queue_exception, | 4016 | .queue_exception = vmx_queue_exception, |
3974 | .interrupt_allowed = vmx_interrupt_allowed, | 4017 | .interrupt_allowed = vmx_interrupt_allowed, |
3975 | .nmi_allowed = vmx_nmi_allowed, | 4018 | .nmi_allowed = vmx_nmi_allowed, |
4019 | .get_nmi_mask = vmx_get_nmi_mask, | ||
4020 | .set_nmi_mask = vmx_set_nmi_mask, | ||
3976 | .enable_nmi_window = enable_nmi_window, | 4021 | .enable_nmi_window = enable_nmi_window, |
3977 | .enable_irq_window = enable_irq_window, | 4022 | .enable_irq_window = enable_irq_window, |
3978 | .update_cr8_intercept = update_cr8_intercept, | 4023 | .update_cr8_intercept = update_cr8_intercept, |
@@ -3987,7 +4032,12 @@ static struct kvm_x86_ops vmx_x86_ops = { | |||
3987 | 4032 | ||
3988 | static int __init vmx_init(void) | 4033 | static int __init vmx_init(void) |
3989 | { | 4034 | { |
3990 | int r; | 4035 | int r, i; |
4036 | |||
4037 | rdmsrl_safe(MSR_EFER, &host_efer); | ||
4038 | |||
4039 | for (i = 0; i < NR_VMX_MSR; ++i) | ||
4040 | kvm_define_shared_msr(i, vmx_msr_index[i]); | ||
3991 | 4041 | ||
3992 | vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL); | 4042 | vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL); |
3993 | if (!vmx_io_bitmap_a) | 4043 | if (!vmx_io_bitmap_a) |
@@ -4049,8 +4099,6 @@ static int __init vmx_init(void) | |||
4049 | if (bypass_guest_pf) | 4099 | if (bypass_guest_pf) |
4050 | kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull); | 4100 | kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull); |
4051 | 4101 | ||
4052 | ept_sync_global(); | ||
4053 | |||
4054 | return 0; | 4102 | return 0; |
4055 | 4103 | ||
4056 | out3: | 4104 | out3: |
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4fc80174191c..9d068966fb2a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c | |||
@@ -37,6 +37,7 @@ | |||
37 | #include <linux/iommu.h> | 37 | #include <linux/iommu.h> |
38 | #include <linux/intel-iommu.h> | 38 | #include <linux/intel-iommu.h> |
39 | #include <linux/cpufreq.h> | 39 | #include <linux/cpufreq.h> |
40 | #include <linux/user-return-notifier.h> | ||
40 | #include <trace/events/kvm.h> | 41 | #include <trace/events/kvm.h> |
41 | #undef TRACE_INCLUDE_FILE | 42 | #undef TRACE_INCLUDE_FILE |
42 | #define CREATE_TRACE_POINTS | 43 | #define CREATE_TRACE_POINTS |
@@ -88,6 +89,25 @@ EXPORT_SYMBOL_GPL(kvm_x86_ops); | |||
88 | int ignore_msrs = 0; | 89 | int ignore_msrs = 0; |
89 | module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR); | 90 | module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR); |
90 | 91 | ||
92 | #define KVM_NR_SHARED_MSRS 16 | ||
93 | |||
94 | struct kvm_shared_msrs_global { | ||
95 | int nr; | ||
96 | struct kvm_shared_msr { | ||
97 | u32 msr; | ||
98 | u64 value; | ||
99 | } msrs[KVM_NR_SHARED_MSRS]; | ||
100 | }; | ||
101 | |||
102 | struct kvm_shared_msrs { | ||
103 | struct user_return_notifier urn; | ||
104 | bool registered; | ||
105 | u64 current_value[KVM_NR_SHARED_MSRS]; | ||
106 | }; | ||
107 | |||
108 | static struct kvm_shared_msrs_global __read_mostly shared_msrs_global; | ||
109 | static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs); | ||
110 | |||
91 | struct kvm_stats_debugfs_item debugfs_entries[] = { | 111 | struct kvm_stats_debugfs_item debugfs_entries[] = { |
92 | { "pf_fixed", VCPU_STAT(pf_fixed) }, | 112 | { "pf_fixed", VCPU_STAT(pf_fixed) }, |
93 | { "pf_guest", VCPU_STAT(pf_guest) }, | 113 | { "pf_guest", VCPU_STAT(pf_guest) }, |
@@ -124,6 +144,72 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { | |||
124 | { NULL } | 144 | { NULL } |
125 | }; | 145 | }; |
126 | 146 | ||
147 | static void kvm_on_user_return(struct user_return_notifier *urn) | ||
148 | { | ||
149 | unsigned slot; | ||
150 | struct kvm_shared_msr *global; | ||
151 | struct kvm_shared_msrs *locals | ||
152 | = container_of(urn, struct kvm_shared_msrs, urn); | ||
153 | |||
154 | for (slot = 0; slot < shared_msrs_global.nr; ++slot) { | ||
155 | global = &shared_msrs_global.msrs[slot]; | ||
156 | if (global->value != locals->current_value[slot]) { | ||
157 | wrmsrl(global->msr, global->value); | ||
158 | locals->current_value[slot] = global->value; | ||
159 | } | ||
160 | } | ||
161 | locals->registered = false; | ||
162 | user_return_notifier_unregister(urn); | ||
163 | } | ||
164 | |||
165 | void kvm_define_shared_msr(unsigned slot, u32 msr) | ||
166 | { | ||
167 | int cpu; | ||
168 | u64 value; | ||
169 | |||
170 | if (slot >= shared_msrs_global.nr) | ||
171 | shared_msrs_global.nr = slot + 1; | ||
172 | shared_msrs_global.msrs[slot].msr = msr; | ||
173 | rdmsrl_safe(msr, &value); | ||
174 | shared_msrs_global.msrs[slot].value = value; | ||
175 | for_each_online_cpu(cpu) | ||
176 | per_cpu(shared_msrs, cpu).current_value[slot] = value; | ||
177 | } | ||
178 | EXPORT_SYMBOL_GPL(kvm_define_shared_msr); | ||
179 | |||
180 | static void kvm_shared_msr_cpu_online(void) | ||
181 | { | ||
182 | unsigned i; | ||
183 | struct kvm_shared_msrs *locals = &__get_cpu_var(shared_msrs); | ||
184 | |||
185 | for (i = 0; i < shared_msrs_global.nr; ++i) | ||
186 | locals->current_value[i] = shared_msrs_global.msrs[i].value; | ||
187 | } | ||
188 | |||
189 | void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask) | ||
190 | { | ||
191 | struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs); | ||
192 | |||
193 | if (((value ^ smsr->current_value[slot]) & mask) == 0) | ||
194 | return; | ||
195 | smsr->current_value[slot] = value; | ||
196 | wrmsrl(shared_msrs_global.msrs[slot].msr, value); | ||
197 | if (!smsr->registered) { | ||
198 | smsr->urn.on_user_return = kvm_on_user_return; | ||
199 | user_return_notifier_register(&smsr->urn); | ||
200 | smsr->registered = true; | ||
201 | } | ||
202 | } | ||
203 | EXPORT_SYMBOL_GPL(kvm_set_shared_msr); | ||
204 | |||
205 | static void drop_user_return_notifiers(void *ignore) | ||
206 | { | ||
207 | struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs); | ||
208 | |||
209 | if (smsr->registered) | ||
210 | kvm_on_user_return(&smsr->urn); | ||
211 | } | ||
212 | |||
127 | unsigned long segment_base(u16 selector) | 213 | unsigned long segment_base(u16 selector) |
128 | { | 214 | { |
129 | struct descriptor_table gdt; | 215 | struct descriptor_table gdt; |
@@ -485,16 +571,19 @@ static inline u32 bit(int bitno) | |||
485 | * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. | 571 | * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. |
486 | * | 572 | * |
487 | * This list is modified at module load time to reflect the | 573 | * This list is modified at module load time to reflect the |
488 | * capabilities of the host cpu. | 574 | * capabilities of the host cpu. This capabilities test skips MSRs that are |
575 | * kvm-specific. Those are put in the beginning of the list. | ||
489 | */ | 576 | */ |
577 | |||
578 | #define KVM_SAVE_MSRS_BEGIN 2 | ||
490 | static u32 msrs_to_save[] = { | 579 | static u32 msrs_to_save[] = { |
580 | MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, | ||
491 | MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, | 581 | MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, |
492 | MSR_K6_STAR, | 582 | MSR_K6_STAR, |
493 | #ifdef CONFIG_X86_64 | 583 | #ifdef CONFIG_X86_64 |
494 | MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, | 584 | MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, |
495 | #endif | 585 | #endif |
496 | MSR_IA32_TSC, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, | 586 | MSR_IA32_TSC, MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA |
497 | MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA | ||
498 | }; | 587 | }; |
499 | 588 | ||
500 | static unsigned num_msrs_to_save; | 589 | static unsigned num_msrs_to_save; |
@@ -678,7 +767,8 @@ static void kvm_write_guest_time(struct kvm_vcpu *v) | |||
678 | /* With all the info we got, fill in the values */ | 767 | /* With all the info we got, fill in the values */ |
679 | 768 | ||
680 | vcpu->hv_clock.system_time = ts.tv_nsec + | 769 | vcpu->hv_clock.system_time = ts.tv_nsec + |
681 | (NSEC_PER_SEC * (u64)ts.tv_sec); | 770 | (NSEC_PER_SEC * (u64)ts.tv_sec) + v->kvm->arch.kvmclock_offset; |
771 | |||
682 | /* | 772 | /* |
683 | * The interface expects us to write an even number signaling that the | 773 | * The interface expects us to write an even number signaling that the |
684 | * update is finished. Since the guest won't see the intermediate | 774 | * update is finished. Since the guest won't see the intermediate |
@@ -836,6 +926,38 @@ static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
836 | return 0; | 926 | return 0; |
837 | } | 927 | } |
838 | 928 | ||
929 | static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data) | ||
930 | { | ||
931 | struct kvm *kvm = vcpu->kvm; | ||
932 | int lm = is_long_mode(vcpu); | ||
933 | u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64 | ||
934 | : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32; | ||
935 | u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64 | ||
936 | : kvm->arch.xen_hvm_config.blob_size_32; | ||
937 | u32 page_num = data & ~PAGE_MASK; | ||
938 | u64 page_addr = data & PAGE_MASK; | ||
939 | u8 *page; | ||
940 | int r; | ||
941 | |||
942 | r = -E2BIG; | ||
943 | if (page_num >= blob_size) | ||
944 | goto out; | ||
945 | r = -ENOMEM; | ||
946 | page = kzalloc(PAGE_SIZE, GFP_KERNEL); | ||
947 | if (!page) | ||
948 | goto out; | ||
949 | r = -EFAULT; | ||
950 | if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE)) | ||
951 | goto out_free; | ||
952 | if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE)) | ||
953 | goto out_free; | ||
954 | r = 0; | ||
955 | out_free: | ||
956 | kfree(page); | ||
957 | out: | ||
958 | return r; | ||
959 | } | ||
960 | |||
839 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | 961 | int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) |
840 | { | 962 | { |
841 | switch (msr) { | 963 | switch (msr) { |
@@ -951,6 +1073,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
951 | "0x%x data 0x%llx\n", msr, data); | 1073 | "0x%x data 0x%llx\n", msr, data); |
952 | break; | 1074 | break; |
953 | default: | 1075 | default: |
1076 | if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr)) | ||
1077 | return xen_hvm_config(vcpu, data); | ||
954 | if (!ignore_msrs) { | 1078 | if (!ignore_msrs) { |
955 | pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", | 1079 | pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", |
956 | msr, data); | 1080 | msr, data); |
@@ -1225,6 +1349,9 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
1225 | case KVM_CAP_PIT2: | 1349 | case KVM_CAP_PIT2: |
1226 | case KVM_CAP_PIT_STATE2: | 1350 | case KVM_CAP_PIT_STATE2: |
1227 | case KVM_CAP_SET_IDENTITY_MAP_ADDR: | 1351 | case KVM_CAP_SET_IDENTITY_MAP_ADDR: |
1352 | case KVM_CAP_XEN_HVM: | ||
1353 | case KVM_CAP_ADJUST_CLOCK: | ||
1354 | case KVM_CAP_VCPU_EVENTS: | ||
1228 | r = 1; | 1355 | r = 1; |
1229 | break; | 1356 | break; |
1230 | case KVM_CAP_COALESCED_MMIO: | 1357 | case KVM_CAP_COALESCED_MMIO: |
@@ -1239,8 +1366,8 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
1239 | case KVM_CAP_NR_MEMSLOTS: | 1366 | case KVM_CAP_NR_MEMSLOTS: |
1240 | r = KVM_MEMORY_SLOTS; | 1367 | r = KVM_MEMORY_SLOTS; |
1241 | break; | 1368 | break; |
1242 | case KVM_CAP_PV_MMU: | 1369 | case KVM_CAP_PV_MMU: /* obsolete */ |
1243 | r = !tdp_enabled; | 1370 | r = 0; |
1244 | break; | 1371 | break; |
1245 | case KVM_CAP_IOMMU: | 1372 | case KVM_CAP_IOMMU: |
1246 | r = iommu_found(); | 1373 | r = iommu_found(); |
@@ -1327,6 +1454,12 @@ out: | |||
1327 | void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) | 1454 | void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) |
1328 | { | 1455 | { |
1329 | kvm_x86_ops->vcpu_load(vcpu, cpu); | 1456 | kvm_x86_ops->vcpu_load(vcpu, cpu); |
1457 | if (unlikely(per_cpu(cpu_tsc_khz, cpu) == 0)) { | ||
1458 | unsigned long khz = cpufreq_quick_get(cpu); | ||
1459 | if (!khz) | ||
1460 | khz = tsc_khz; | ||
1461 | per_cpu(cpu_tsc_khz, cpu) = khz; | ||
1462 | } | ||
1330 | kvm_request_guest_time_update(vcpu); | 1463 | kvm_request_guest_time_update(vcpu); |
1331 | } | 1464 | } |
1332 | 1465 | ||
@@ -1760,6 +1893,61 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, | |||
1760 | return 0; | 1893 | return 0; |
1761 | } | 1894 | } |
1762 | 1895 | ||
1896 | static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, | ||
1897 | struct kvm_vcpu_events *events) | ||
1898 | { | ||
1899 | vcpu_load(vcpu); | ||
1900 | |||
1901 | events->exception.injected = vcpu->arch.exception.pending; | ||
1902 | events->exception.nr = vcpu->arch.exception.nr; | ||
1903 | events->exception.has_error_code = vcpu->arch.exception.has_error_code; | ||
1904 | events->exception.error_code = vcpu->arch.exception.error_code; | ||
1905 | |||
1906 | events->interrupt.injected = vcpu->arch.interrupt.pending; | ||
1907 | events->interrupt.nr = vcpu->arch.interrupt.nr; | ||
1908 | events->interrupt.soft = vcpu->arch.interrupt.soft; | ||
1909 | |||
1910 | events->nmi.injected = vcpu->arch.nmi_injected; | ||
1911 | events->nmi.pending = vcpu->arch.nmi_pending; | ||
1912 | events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu); | ||
1913 | |||
1914 | events->sipi_vector = vcpu->arch.sipi_vector; | ||
1915 | |||
1916 | events->flags = 0; | ||
1917 | |||
1918 | vcpu_put(vcpu); | ||
1919 | } | ||
1920 | |||
1921 | static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, | ||
1922 | struct kvm_vcpu_events *events) | ||
1923 | { | ||
1924 | if (events->flags) | ||
1925 | return -EINVAL; | ||
1926 | |||
1927 | vcpu_load(vcpu); | ||
1928 | |||
1929 | vcpu->arch.exception.pending = events->exception.injected; | ||
1930 | vcpu->arch.exception.nr = events->exception.nr; | ||
1931 | vcpu->arch.exception.has_error_code = events->exception.has_error_code; | ||
1932 | vcpu->arch.exception.error_code = events->exception.error_code; | ||
1933 | |||
1934 | vcpu->arch.interrupt.pending = events->interrupt.injected; | ||
1935 | vcpu->arch.interrupt.nr = events->interrupt.nr; | ||
1936 | vcpu->arch.interrupt.soft = events->interrupt.soft; | ||
1937 | if (vcpu->arch.interrupt.pending && irqchip_in_kernel(vcpu->kvm)) | ||
1938 | kvm_pic_clear_isr_ack(vcpu->kvm); | ||
1939 | |||
1940 | vcpu->arch.nmi_injected = events->nmi.injected; | ||
1941 | vcpu->arch.nmi_pending = events->nmi.pending; | ||
1942 | kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked); | ||
1943 | |||
1944 | vcpu->arch.sipi_vector = events->sipi_vector; | ||
1945 | |||
1946 | vcpu_put(vcpu); | ||
1947 | |||
1948 | return 0; | ||
1949 | } | ||
1950 | |||
1763 | long kvm_arch_vcpu_ioctl(struct file *filp, | 1951 | long kvm_arch_vcpu_ioctl(struct file *filp, |
1764 | unsigned int ioctl, unsigned long arg) | 1952 | unsigned int ioctl, unsigned long arg) |
1765 | { | 1953 | { |
@@ -1770,6 +1958,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1770 | 1958 | ||
1771 | switch (ioctl) { | 1959 | switch (ioctl) { |
1772 | case KVM_GET_LAPIC: { | 1960 | case KVM_GET_LAPIC: { |
1961 | r = -EINVAL; | ||
1962 | if (!vcpu->arch.apic) | ||
1963 | goto out; | ||
1773 | lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); | 1964 | lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); |
1774 | 1965 | ||
1775 | r = -ENOMEM; | 1966 | r = -ENOMEM; |
@@ -1785,6 +1976,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1785 | break; | 1976 | break; |
1786 | } | 1977 | } |
1787 | case KVM_SET_LAPIC: { | 1978 | case KVM_SET_LAPIC: { |
1979 | r = -EINVAL; | ||
1980 | if (!vcpu->arch.apic) | ||
1981 | goto out; | ||
1788 | lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); | 1982 | lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); |
1789 | r = -ENOMEM; | 1983 | r = -ENOMEM; |
1790 | if (!lapic) | 1984 | if (!lapic) |
@@ -1911,6 +2105,27 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1911 | r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); | 2105 | r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); |
1912 | break; | 2106 | break; |
1913 | } | 2107 | } |
2108 | case KVM_GET_VCPU_EVENTS: { | ||
2109 | struct kvm_vcpu_events events; | ||
2110 | |||
2111 | kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); | ||
2112 | |||
2113 | r = -EFAULT; | ||
2114 | if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events))) | ||
2115 | break; | ||
2116 | r = 0; | ||
2117 | break; | ||
2118 | } | ||
2119 | case KVM_SET_VCPU_EVENTS: { | ||
2120 | struct kvm_vcpu_events events; | ||
2121 | |||
2122 | r = -EFAULT; | ||
2123 | if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events))) | ||
2124 | break; | ||
2125 | |||
2126 | r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); | ||
2127 | break; | ||
2128 | } | ||
1914 | default: | 2129 | default: |
1915 | r = -EINVAL; | 2130 | r = -EINVAL; |
1916 | } | 2131 | } |
@@ -2039,9 +2254,7 @@ static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) | |||
2039 | sizeof(struct kvm_pic_state)); | 2254 | sizeof(struct kvm_pic_state)); |
2040 | break; | 2255 | break; |
2041 | case KVM_IRQCHIP_IOAPIC: | 2256 | case KVM_IRQCHIP_IOAPIC: |
2042 | memcpy(&chip->chip.ioapic, | 2257 | r = kvm_get_ioapic(kvm, &chip->chip.ioapic); |
2043 | ioapic_irqchip(kvm), | ||
2044 | sizeof(struct kvm_ioapic_state)); | ||
2045 | break; | 2258 | break; |
2046 | default: | 2259 | default: |
2047 | r = -EINVAL; | 2260 | r = -EINVAL; |
@@ -2071,11 +2284,7 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) | |||
2071 | spin_unlock(&pic_irqchip(kvm)->lock); | 2284 | spin_unlock(&pic_irqchip(kvm)->lock); |
2072 | break; | 2285 | break; |
2073 | case KVM_IRQCHIP_IOAPIC: | 2286 | case KVM_IRQCHIP_IOAPIC: |
2074 | mutex_lock(&kvm->irq_lock); | 2287 | r = kvm_set_ioapic(kvm, &chip->chip.ioapic); |
2075 | memcpy(ioapic_irqchip(kvm), | ||
2076 | &chip->chip.ioapic, | ||
2077 | sizeof(struct kvm_ioapic_state)); | ||
2078 | mutex_unlock(&kvm->irq_lock); | ||
2079 | break; | 2288 | break; |
2080 | default: | 2289 | default: |
2081 | r = -EINVAL; | 2290 | r = -EINVAL; |
@@ -2183,7 +2392,7 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
2183 | { | 2392 | { |
2184 | struct kvm *kvm = filp->private_data; | 2393 | struct kvm *kvm = filp->private_data; |
2185 | void __user *argp = (void __user *)arg; | 2394 | void __user *argp = (void __user *)arg; |
2186 | int r = -EINVAL; | 2395 | int r = -ENOTTY; |
2187 | /* | 2396 | /* |
2188 | * This union makes it completely explicit to gcc-3.x | 2397 | * This union makes it completely explicit to gcc-3.x |
2189 | * that these two variables' stack usage should be | 2398 | * that these two variables' stack usage should be |
@@ -2245,25 +2454,39 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
2245 | if (r) | 2454 | if (r) |
2246 | goto out; | 2455 | goto out; |
2247 | break; | 2456 | break; |
2248 | case KVM_CREATE_IRQCHIP: | 2457 | case KVM_CREATE_IRQCHIP: { |
2458 | struct kvm_pic *vpic; | ||
2459 | |||
2460 | mutex_lock(&kvm->lock); | ||
2461 | r = -EEXIST; | ||
2462 | if (kvm->arch.vpic) | ||
2463 | goto create_irqchip_unlock; | ||
2249 | r = -ENOMEM; | 2464 | r = -ENOMEM; |
2250 | kvm->arch.vpic = kvm_create_pic(kvm); | 2465 | vpic = kvm_create_pic(kvm); |
2251 | if (kvm->arch.vpic) { | 2466 | if (vpic) { |
2252 | r = kvm_ioapic_init(kvm); | 2467 | r = kvm_ioapic_init(kvm); |
2253 | if (r) { | 2468 | if (r) { |
2254 | kfree(kvm->arch.vpic); | 2469 | kfree(vpic); |
2255 | kvm->arch.vpic = NULL; | 2470 | goto create_irqchip_unlock; |
2256 | goto out; | ||
2257 | } | 2471 | } |
2258 | } else | 2472 | } else |
2259 | goto out; | 2473 | goto create_irqchip_unlock; |
2474 | smp_wmb(); | ||
2475 | kvm->arch.vpic = vpic; | ||
2476 | smp_wmb(); | ||
2260 | r = kvm_setup_default_irq_routing(kvm); | 2477 | r = kvm_setup_default_irq_routing(kvm); |
2261 | if (r) { | 2478 | if (r) { |
2479 | mutex_lock(&kvm->irq_lock); | ||
2262 | kfree(kvm->arch.vpic); | 2480 | kfree(kvm->arch.vpic); |
2263 | kfree(kvm->arch.vioapic); | 2481 | kfree(kvm->arch.vioapic); |
2264 | goto out; | 2482 | kvm->arch.vpic = NULL; |
2483 | kvm->arch.vioapic = NULL; | ||
2484 | mutex_unlock(&kvm->irq_lock); | ||
2265 | } | 2485 | } |
2486 | create_irqchip_unlock: | ||
2487 | mutex_unlock(&kvm->lock); | ||
2266 | break; | 2488 | break; |
2489 | } | ||
2267 | case KVM_CREATE_PIT: | 2490 | case KVM_CREATE_PIT: |
2268 | u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; | 2491 | u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; |
2269 | goto create_pit; | 2492 | goto create_pit; |
@@ -2293,10 +2516,8 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
2293 | goto out; | 2516 | goto out; |
2294 | if (irqchip_in_kernel(kvm)) { | 2517 | if (irqchip_in_kernel(kvm)) { |
2295 | __s32 status; | 2518 | __s32 status; |
2296 | mutex_lock(&kvm->irq_lock); | ||
2297 | status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, | 2519 | status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, |
2298 | irq_event.irq, irq_event.level); | 2520 | irq_event.irq, irq_event.level); |
2299 | mutex_unlock(&kvm->irq_lock); | ||
2300 | if (ioctl == KVM_IRQ_LINE_STATUS) { | 2521 | if (ioctl == KVM_IRQ_LINE_STATUS) { |
2301 | irq_event.status = status; | 2522 | irq_event.status = status; |
2302 | if (copy_to_user(argp, &irq_event, | 2523 | if (copy_to_user(argp, &irq_event, |
@@ -2422,6 +2643,55 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
2422 | r = 0; | 2643 | r = 0; |
2423 | break; | 2644 | break; |
2424 | } | 2645 | } |
2646 | case KVM_XEN_HVM_CONFIG: { | ||
2647 | r = -EFAULT; | ||
2648 | if (copy_from_user(&kvm->arch.xen_hvm_config, argp, | ||
2649 | sizeof(struct kvm_xen_hvm_config))) | ||
2650 | goto out; | ||
2651 | r = -EINVAL; | ||
2652 | if (kvm->arch.xen_hvm_config.flags) | ||
2653 | goto out; | ||
2654 | r = 0; | ||
2655 | break; | ||
2656 | } | ||
2657 | case KVM_SET_CLOCK: { | ||
2658 | struct timespec now; | ||
2659 | struct kvm_clock_data user_ns; | ||
2660 | u64 now_ns; | ||
2661 | s64 delta; | ||
2662 | |||
2663 | r = -EFAULT; | ||
2664 | if (copy_from_user(&user_ns, argp, sizeof(user_ns))) | ||
2665 | goto out; | ||
2666 | |||
2667 | r = -EINVAL; | ||
2668 | if (user_ns.flags) | ||
2669 | goto out; | ||
2670 | |||
2671 | r = 0; | ||
2672 | ktime_get_ts(&now); | ||
2673 | now_ns = timespec_to_ns(&now); | ||
2674 | delta = user_ns.clock - now_ns; | ||
2675 | kvm->arch.kvmclock_offset = delta; | ||
2676 | break; | ||
2677 | } | ||
2678 | case KVM_GET_CLOCK: { | ||
2679 | struct timespec now; | ||
2680 | struct kvm_clock_data user_ns; | ||
2681 | u64 now_ns; | ||
2682 | |||
2683 | ktime_get_ts(&now); | ||
2684 | now_ns = timespec_to_ns(&now); | ||
2685 | user_ns.clock = kvm->arch.kvmclock_offset + now_ns; | ||
2686 | user_ns.flags = 0; | ||
2687 | |||
2688 | r = -EFAULT; | ||
2689 | if (copy_to_user(argp, &user_ns, sizeof(user_ns))) | ||
2690 | goto out; | ||
2691 | r = 0; | ||
2692 | break; | ||
2693 | } | ||
2694 | |||
2425 | default: | 2695 | default: |
2426 | ; | 2696 | ; |
2427 | } | 2697 | } |
@@ -2434,7 +2704,8 @@ static void kvm_init_msr_list(void) | |||
2434 | u32 dummy[2]; | 2704 | u32 dummy[2]; |
2435 | unsigned i, j; | 2705 | unsigned i, j; |
2436 | 2706 | ||
2437 | for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { | 2707 | /* skip the first msrs in the list. KVM-specific */ |
2708 | for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) { | ||
2438 | if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) | 2709 | if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) |
2439 | continue; | 2710 | continue; |
2440 | if (j < i) | 2711 | if (j < i) |
@@ -2758,13 +3029,13 @@ static void cache_all_regs(struct kvm_vcpu *vcpu) | |||
2758 | } | 3029 | } |
2759 | 3030 | ||
2760 | int emulate_instruction(struct kvm_vcpu *vcpu, | 3031 | int emulate_instruction(struct kvm_vcpu *vcpu, |
2761 | struct kvm_run *run, | ||
2762 | unsigned long cr2, | 3032 | unsigned long cr2, |
2763 | u16 error_code, | 3033 | u16 error_code, |
2764 | int emulation_type) | 3034 | int emulation_type) |
2765 | { | 3035 | { |
2766 | int r, shadow_mask; | 3036 | int r, shadow_mask; |
2767 | struct decode_cache *c; | 3037 | struct decode_cache *c; |
3038 | struct kvm_run *run = vcpu->run; | ||
2768 | 3039 | ||
2769 | kvm_clear_exception_queue(vcpu); | 3040 | kvm_clear_exception_queue(vcpu); |
2770 | vcpu->arch.mmio_fault_cr2 = cr2; | 3041 | vcpu->arch.mmio_fault_cr2 = cr2; |
@@ -2784,7 +3055,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, | |||
2784 | kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); | 3055 | kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); |
2785 | 3056 | ||
2786 | vcpu->arch.emulate_ctxt.vcpu = vcpu; | 3057 | vcpu->arch.emulate_ctxt.vcpu = vcpu; |
2787 | vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); | 3058 | vcpu->arch.emulate_ctxt.eflags = kvm_get_rflags(vcpu); |
2788 | vcpu->arch.emulate_ctxt.mode = | 3059 | vcpu->arch.emulate_ctxt.mode = |
2789 | (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM) | 3060 | (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM) |
2790 | ? X86EMUL_MODE_REAL : cs_l | 3061 | ? X86EMUL_MODE_REAL : cs_l |
@@ -2862,7 +3133,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, | |||
2862 | return EMULATE_DO_MMIO; | 3133 | return EMULATE_DO_MMIO; |
2863 | } | 3134 | } |
2864 | 3135 | ||
2865 | kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); | 3136 | kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); |
2866 | 3137 | ||
2867 | if (vcpu->mmio_is_write) { | 3138 | if (vcpu->mmio_is_write) { |
2868 | vcpu->mmio_needed = 0; | 3139 | vcpu->mmio_needed = 0; |
@@ -2970,8 +3241,7 @@ static int pio_string_write(struct kvm_vcpu *vcpu) | |||
2970 | return r; | 3241 | return r; |
2971 | } | 3242 | } |
2972 | 3243 | ||
2973 | int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | 3244 | int kvm_emulate_pio(struct kvm_vcpu *vcpu, int in, int size, unsigned port) |
2974 | int size, unsigned port) | ||
2975 | { | 3245 | { |
2976 | unsigned long val; | 3246 | unsigned long val; |
2977 | 3247 | ||
@@ -3000,7 +3270,7 @@ int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
3000 | } | 3270 | } |
3001 | EXPORT_SYMBOL_GPL(kvm_emulate_pio); | 3271 | EXPORT_SYMBOL_GPL(kvm_emulate_pio); |
3002 | 3272 | ||
3003 | int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | 3273 | int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, int in, |
3004 | int size, unsigned long count, int down, | 3274 | int size, unsigned long count, int down, |
3005 | gva_t address, int rep, unsigned port) | 3275 | gva_t address, int rep, unsigned port) |
3006 | { | 3276 | { |
@@ -3073,9 +3343,6 @@ static void bounce_off(void *info) | |||
3073 | /* nothing */ | 3343 | /* nothing */ |
3074 | } | 3344 | } |
3075 | 3345 | ||
3076 | static unsigned int ref_freq; | ||
3077 | static unsigned long tsc_khz_ref; | ||
3078 | |||
3079 | static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, | 3346 | static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, |
3080 | void *data) | 3347 | void *data) |
3081 | { | 3348 | { |
@@ -3084,14 +3351,11 @@ static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long va | |||
3084 | struct kvm_vcpu *vcpu; | 3351 | struct kvm_vcpu *vcpu; |
3085 | int i, send_ipi = 0; | 3352 | int i, send_ipi = 0; |
3086 | 3353 | ||
3087 | if (!ref_freq) | ||
3088 | ref_freq = freq->old; | ||
3089 | |||
3090 | if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) | 3354 | if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) |
3091 | return 0; | 3355 | return 0; |
3092 | if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) | 3356 | if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) |
3093 | return 0; | 3357 | return 0; |
3094 | per_cpu(cpu_tsc_khz, freq->cpu) = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); | 3358 | per_cpu(cpu_tsc_khz, freq->cpu) = freq->new; |
3095 | 3359 | ||
3096 | spin_lock(&kvm_lock); | 3360 | spin_lock(&kvm_lock); |
3097 | list_for_each_entry(kvm, &vm_list, vm_list) { | 3361 | list_for_each_entry(kvm, &vm_list, vm_list) { |
@@ -3128,9 +3392,28 @@ static struct notifier_block kvmclock_cpufreq_notifier_block = { | |||
3128 | .notifier_call = kvmclock_cpufreq_notifier | 3392 | .notifier_call = kvmclock_cpufreq_notifier |
3129 | }; | 3393 | }; |
3130 | 3394 | ||
3395 | static void kvm_timer_init(void) | ||
3396 | { | ||
3397 | int cpu; | ||
3398 | |||
3399 | if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { | ||
3400 | cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, | ||
3401 | CPUFREQ_TRANSITION_NOTIFIER); | ||
3402 | for_each_online_cpu(cpu) { | ||
3403 | unsigned long khz = cpufreq_get(cpu); | ||
3404 | if (!khz) | ||
3405 | khz = tsc_khz; | ||
3406 | per_cpu(cpu_tsc_khz, cpu) = khz; | ||
3407 | } | ||
3408 | } else { | ||
3409 | for_each_possible_cpu(cpu) | ||
3410 | per_cpu(cpu_tsc_khz, cpu) = tsc_khz; | ||
3411 | } | ||
3412 | } | ||
3413 | |||
3131 | int kvm_arch_init(void *opaque) | 3414 | int kvm_arch_init(void *opaque) |
3132 | { | 3415 | { |
3133 | int r, cpu; | 3416 | int r; |
3134 | struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; | 3417 | struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; |
3135 | 3418 | ||
3136 | if (kvm_x86_ops) { | 3419 | if (kvm_x86_ops) { |
@@ -3162,13 +3445,7 @@ int kvm_arch_init(void *opaque) | |||
3162 | kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, | 3445 | kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, |
3163 | PT_DIRTY_MASK, PT64_NX_MASK, 0); | 3446 | PT_DIRTY_MASK, PT64_NX_MASK, 0); |
3164 | 3447 | ||
3165 | for_each_possible_cpu(cpu) | 3448 | kvm_timer_init(); |
3166 | per_cpu(cpu_tsc_khz, cpu) = tsc_khz; | ||
3167 | if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { | ||
3168 | tsc_khz_ref = tsc_khz; | ||
3169 | cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, | ||
3170 | CPUFREQ_TRANSITION_NOTIFIER); | ||
3171 | } | ||
3172 | 3449 | ||
3173 | return 0; | 3450 | return 0; |
3174 | 3451 | ||
@@ -3296,7 +3573,7 @@ void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, | |||
3296 | unsigned long *rflags) | 3573 | unsigned long *rflags) |
3297 | { | 3574 | { |
3298 | kvm_lmsw(vcpu, msw); | 3575 | kvm_lmsw(vcpu, msw); |
3299 | *rflags = kvm_x86_ops->get_rflags(vcpu); | 3576 | *rflags = kvm_get_rflags(vcpu); |
3300 | } | 3577 | } |
3301 | 3578 | ||
3302 | unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) | 3579 | unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) |
@@ -3334,7 +3611,7 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, | |||
3334 | switch (cr) { | 3611 | switch (cr) { |
3335 | case 0: | 3612 | case 0: |
3336 | kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); | 3613 | kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); |
3337 | *rflags = kvm_x86_ops->get_rflags(vcpu); | 3614 | *rflags = kvm_get_rflags(vcpu); |
3338 | break; | 3615 | break; |
3339 | case 2: | 3616 | case 2: |
3340 | vcpu->arch.cr2 = val; | 3617 | vcpu->arch.cr2 = val; |
@@ -3454,18 +3731,18 @@ EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); | |||
3454 | * | 3731 | * |
3455 | * No need to exit to userspace if we already have an interrupt queued. | 3732 | * No need to exit to userspace if we already have an interrupt queued. |
3456 | */ | 3733 | */ |
3457 | static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, | 3734 | static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) |
3458 | struct kvm_run *kvm_run) | ||
3459 | { | 3735 | { |
3460 | return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) && | 3736 | return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) && |
3461 | kvm_run->request_interrupt_window && | 3737 | vcpu->run->request_interrupt_window && |
3462 | kvm_arch_interrupt_allowed(vcpu)); | 3738 | kvm_arch_interrupt_allowed(vcpu)); |
3463 | } | 3739 | } |
3464 | 3740 | ||
3465 | static void post_kvm_run_save(struct kvm_vcpu *vcpu, | 3741 | static void post_kvm_run_save(struct kvm_vcpu *vcpu) |
3466 | struct kvm_run *kvm_run) | ||
3467 | { | 3742 | { |
3468 | kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; | 3743 | struct kvm_run *kvm_run = vcpu->run; |
3744 | |||
3745 | kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0; | ||
3469 | kvm_run->cr8 = kvm_get_cr8(vcpu); | 3746 | kvm_run->cr8 = kvm_get_cr8(vcpu); |
3470 | kvm_run->apic_base = kvm_get_apic_base(vcpu); | 3747 | kvm_run->apic_base = kvm_get_apic_base(vcpu); |
3471 | if (irqchip_in_kernel(vcpu->kvm)) | 3748 | if (irqchip_in_kernel(vcpu->kvm)) |
@@ -3526,7 +3803,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) | |||
3526 | kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr); | 3803 | kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr); |
3527 | } | 3804 | } |
3528 | 3805 | ||
3529 | static void inject_pending_event(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3806 | static void inject_pending_event(struct kvm_vcpu *vcpu) |
3530 | { | 3807 | { |
3531 | /* try to reinject previous events if any */ | 3808 | /* try to reinject previous events if any */ |
3532 | if (vcpu->arch.exception.pending) { | 3809 | if (vcpu->arch.exception.pending) { |
@@ -3562,11 +3839,11 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3562 | } | 3839 | } |
3563 | } | 3840 | } |
3564 | 3841 | ||
3565 | static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3842 | static int vcpu_enter_guest(struct kvm_vcpu *vcpu) |
3566 | { | 3843 | { |
3567 | int r; | 3844 | int r; |
3568 | bool req_int_win = !irqchip_in_kernel(vcpu->kvm) && | 3845 | bool req_int_win = !irqchip_in_kernel(vcpu->kvm) && |
3569 | kvm_run->request_interrupt_window; | 3846 | vcpu->run->request_interrupt_window; |
3570 | 3847 | ||
3571 | if (vcpu->requests) | 3848 | if (vcpu->requests) |
3572 | if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) | 3849 | if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests)) |
@@ -3587,12 +3864,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3587 | kvm_x86_ops->tlb_flush(vcpu); | 3864 | kvm_x86_ops->tlb_flush(vcpu); |
3588 | if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS, | 3865 | if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS, |
3589 | &vcpu->requests)) { | 3866 | &vcpu->requests)) { |
3590 | kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS; | 3867 | vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; |
3591 | r = 0; | 3868 | r = 0; |
3592 | goto out; | 3869 | goto out; |
3593 | } | 3870 | } |
3594 | if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) { | 3871 | if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) { |
3595 | kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; | 3872 | vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; |
3596 | r = 0; | 3873 | r = 0; |
3597 | goto out; | 3874 | goto out; |
3598 | } | 3875 | } |
@@ -3616,7 +3893,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3616 | goto out; | 3893 | goto out; |
3617 | } | 3894 | } |
3618 | 3895 | ||
3619 | inject_pending_event(vcpu, kvm_run); | 3896 | inject_pending_event(vcpu); |
3620 | 3897 | ||
3621 | /* enable NMI/IRQ window open exits if needed */ | 3898 | /* enable NMI/IRQ window open exits if needed */ |
3622 | if (vcpu->arch.nmi_pending) | 3899 | if (vcpu->arch.nmi_pending) |
@@ -3642,7 +3919,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3642 | } | 3919 | } |
3643 | 3920 | ||
3644 | trace_kvm_entry(vcpu->vcpu_id); | 3921 | trace_kvm_entry(vcpu->vcpu_id); |
3645 | kvm_x86_ops->run(vcpu, kvm_run); | 3922 | kvm_x86_ops->run(vcpu); |
3646 | 3923 | ||
3647 | /* | 3924 | /* |
3648 | * If the guest has used debug registers, at least dr7 | 3925 | * If the guest has used debug registers, at least dr7 |
@@ -3684,13 +3961,13 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3684 | 3961 | ||
3685 | kvm_lapic_sync_from_vapic(vcpu); | 3962 | kvm_lapic_sync_from_vapic(vcpu); |
3686 | 3963 | ||
3687 | r = kvm_x86_ops->handle_exit(kvm_run, vcpu); | 3964 | r = kvm_x86_ops->handle_exit(vcpu); |
3688 | out: | 3965 | out: |
3689 | return r; | 3966 | return r; |
3690 | } | 3967 | } |
3691 | 3968 | ||
3692 | 3969 | ||
3693 | static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3970 | static int __vcpu_run(struct kvm_vcpu *vcpu) |
3694 | { | 3971 | { |
3695 | int r; | 3972 | int r; |
3696 | 3973 | ||
@@ -3710,7 +3987,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3710 | r = 1; | 3987 | r = 1; |
3711 | while (r > 0) { | 3988 | while (r > 0) { |
3712 | if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) | 3989 | if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) |
3713 | r = vcpu_enter_guest(vcpu, kvm_run); | 3990 | r = vcpu_enter_guest(vcpu); |
3714 | else { | 3991 | else { |
3715 | up_read(&vcpu->kvm->slots_lock); | 3992 | up_read(&vcpu->kvm->slots_lock); |
3716 | kvm_vcpu_block(vcpu); | 3993 | kvm_vcpu_block(vcpu); |
@@ -3738,14 +4015,14 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3738 | if (kvm_cpu_has_pending_timer(vcpu)) | 4015 | if (kvm_cpu_has_pending_timer(vcpu)) |
3739 | kvm_inject_pending_timer_irqs(vcpu); | 4016 | kvm_inject_pending_timer_irqs(vcpu); |
3740 | 4017 | ||
3741 | if (dm_request_for_irq_injection(vcpu, kvm_run)) { | 4018 | if (dm_request_for_irq_injection(vcpu)) { |
3742 | r = -EINTR; | 4019 | r = -EINTR; |
3743 | kvm_run->exit_reason = KVM_EXIT_INTR; | 4020 | vcpu->run->exit_reason = KVM_EXIT_INTR; |
3744 | ++vcpu->stat.request_irq_exits; | 4021 | ++vcpu->stat.request_irq_exits; |
3745 | } | 4022 | } |
3746 | if (signal_pending(current)) { | 4023 | if (signal_pending(current)) { |
3747 | r = -EINTR; | 4024 | r = -EINTR; |
3748 | kvm_run->exit_reason = KVM_EXIT_INTR; | 4025 | vcpu->run->exit_reason = KVM_EXIT_INTR; |
3749 | ++vcpu->stat.signal_exits; | 4026 | ++vcpu->stat.signal_exits; |
3750 | } | 4027 | } |
3751 | if (need_resched()) { | 4028 | if (need_resched()) { |
@@ -3756,7 +4033,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3756 | } | 4033 | } |
3757 | 4034 | ||
3758 | up_read(&vcpu->kvm->slots_lock); | 4035 | up_read(&vcpu->kvm->slots_lock); |
3759 | post_kvm_run_save(vcpu, kvm_run); | 4036 | post_kvm_run_save(vcpu); |
3760 | 4037 | ||
3761 | vapic_exit(vcpu); | 4038 | vapic_exit(vcpu); |
3762 | 4039 | ||
@@ -3789,15 +4066,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3789 | if (r) | 4066 | if (r) |
3790 | goto out; | 4067 | goto out; |
3791 | } | 4068 | } |
3792 | #if CONFIG_HAS_IOMEM | ||
3793 | if (vcpu->mmio_needed) { | 4069 | if (vcpu->mmio_needed) { |
3794 | memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); | 4070 | memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); |
3795 | vcpu->mmio_read_completed = 1; | 4071 | vcpu->mmio_read_completed = 1; |
3796 | vcpu->mmio_needed = 0; | 4072 | vcpu->mmio_needed = 0; |
3797 | 4073 | ||
3798 | down_read(&vcpu->kvm->slots_lock); | 4074 | down_read(&vcpu->kvm->slots_lock); |
3799 | r = emulate_instruction(vcpu, kvm_run, | 4075 | r = emulate_instruction(vcpu, vcpu->arch.mmio_fault_cr2, 0, |
3800 | vcpu->arch.mmio_fault_cr2, 0, | ||
3801 | EMULTYPE_NO_DECODE); | 4076 | EMULTYPE_NO_DECODE); |
3802 | up_read(&vcpu->kvm->slots_lock); | 4077 | up_read(&vcpu->kvm->slots_lock); |
3803 | if (r == EMULATE_DO_MMIO) { | 4078 | if (r == EMULATE_DO_MMIO) { |
@@ -3808,12 +4083,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3808 | goto out; | 4083 | goto out; |
3809 | } | 4084 | } |
3810 | } | 4085 | } |
3811 | #endif | ||
3812 | if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) | 4086 | if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) |
3813 | kvm_register_write(vcpu, VCPU_REGS_RAX, | 4087 | kvm_register_write(vcpu, VCPU_REGS_RAX, |
3814 | kvm_run->hypercall.ret); | 4088 | kvm_run->hypercall.ret); |
3815 | 4089 | ||
3816 | r = __vcpu_run(vcpu, kvm_run); | 4090 | r = __vcpu_run(vcpu); |
3817 | 4091 | ||
3818 | out: | 4092 | out: |
3819 | if (vcpu->sigset_active) | 4093 | if (vcpu->sigset_active) |
@@ -3847,13 +4121,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
3847 | #endif | 4121 | #endif |
3848 | 4122 | ||
3849 | regs->rip = kvm_rip_read(vcpu); | 4123 | regs->rip = kvm_rip_read(vcpu); |
3850 | regs->rflags = kvm_x86_ops->get_rflags(vcpu); | 4124 | regs->rflags = kvm_get_rflags(vcpu); |
3851 | |||
3852 | /* | ||
3853 | * Don't leak debug flags in case they were set for guest debugging | ||
3854 | */ | ||
3855 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
3856 | regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | ||
3857 | 4125 | ||
3858 | vcpu_put(vcpu); | 4126 | vcpu_put(vcpu); |
3859 | 4127 | ||
@@ -3881,12 +4149,10 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
3881 | kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13); | 4149 | kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13); |
3882 | kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14); | 4150 | kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14); |
3883 | kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15); | 4151 | kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15); |
3884 | |||
3885 | #endif | 4152 | #endif |
3886 | 4153 | ||
3887 | kvm_rip_write(vcpu, regs->rip); | 4154 | kvm_rip_write(vcpu, regs->rip); |
3888 | kvm_x86_ops->set_rflags(vcpu, regs->rflags); | 4155 | kvm_set_rflags(vcpu, regs->rflags); |
3889 | |||
3890 | 4156 | ||
3891 | vcpu->arch.exception.pending = false; | 4157 | vcpu->arch.exception.pending = false; |
3892 | 4158 | ||
@@ -4105,7 +4371,7 @@ static int is_vm86_segment(struct kvm_vcpu *vcpu, int seg) | |||
4105 | { | 4371 | { |
4106 | return (seg != VCPU_SREG_LDTR) && | 4372 | return (seg != VCPU_SREG_LDTR) && |
4107 | (seg != VCPU_SREG_TR) && | 4373 | (seg != VCPU_SREG_TR) && |
4108 | (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_VM); | 4374 | (kvm_get_rflags(vcpu) & X86_EFLAGS_VM); |
4109 | } | 4375 | } |
4110 | 4376 | ||
4111 | int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, | 4377 | int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, |
@@ -4133,7 +4399,7 @@ static void save_state_to_tss32(struct kvm_vcpu *vcpu, | |||
4133 | { | 4399 | { |
4134 | tss->cr3 = vcpu->arch.cr3; | 4400 | tss->cr3 = vcpu->arch.cr3; |
4135 | tss->eip = kvm_rip_read(vcpu); | 4401 | tss->eip = kvm_rip_read(vcpu); |
4136 | tss->eflags = kvm_x86_ops->get_rflags(vcpu); | 4402 | tss->eflags = kvm_get_rflags(vcpu); |
4137 | tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX); | 4403 | tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX); |
4138 | tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX); | 4404 | tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX); |
4139 | tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX); | 4405 | tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX); |
@@ -4157,7 +4423,7 @@ static int load_state_from_tss32(struct kvm_vcpu *vcpu, | |||
4157 | kvm_set_cr3(vcpu, tss->cr3); | 4423 | kvm_set_cr3(vcpu, tss->cr3); |
4158 | 4424 | ||
4159 | kvm_rip_write(vcpu, tss->eip); | 4425 | kvm_rip_write(vcpu, tss->eip); |
4160 | kvm_x86_ops->set_rflags(vcpu, tss->eflags | 2); | 4426 | kvm_set_rflags(vcpu, tss->eflags | 2); |
4161 | 4427 | ||
4162 | kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax); | 4428 | kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax); |
4163 | kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx); | 4429 | kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx); |
@@ -4195,7 +4461,7 @@ static void save_state_to_tss16(struct kvm_vcpu *vcpu, | |||
4195 | struct tss_segment_16 *tss) | 4461 | struct tss_segment_16 *tss) |
4196 | { | 4462 | { |
4197 | tss->ip = kvm_rip_read(vcpu); | 4463 | tss->ip = kvm_rip_read(vcpu); |
4198 | tss->flag = kvm_x86_ops->get_rflags(vcpu); | 4464 | tss->flag = kvm_get_rflags(vcpu); |
4199 | tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX); | 4465 | tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX); |
4200 | tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX); | 4466 | tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX); |
4201 | tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX); | 4467 | tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX); |
@@ -4210,14 +4476,13 @@ static void save_state_to_tss16(struct kvm_vcpu *vcpu, | |||
4210 | tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS); | 4476 | tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS); |
4211 | tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS); | 4477 | tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS); |
4212 | tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR); | 4478 | tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR); |
4213 | tss->prev_task_link = get_segment_selector(vcpu, VCPU_SREG_TR); | ||
4214 | } | 4479 | } |
4215 | 4480 | ||
4216 | static int load_state_from_tss16(struct kvm_vcpu *vcpu, | 4481 | static int load_state_from_tss16(struct kvm_vcpu *vcpu, |
4217 | struct tss_segment_16 *tss) | 4482 | struct tss_segment_16 *tss) |
4218 | { | 4483 | { |
4219 | kvm_rip_write(vcpu, tss->ip); | 4484 | kvm_rip_write(vcpu, tss->ip); |
4220 | kvm_x86_ops->set_rflags(vcpu, tss->flag | 2); | 4485 | kvm_set_rflags(vcpu, tss->flag | 2); |
4221 | kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax); | 4486 | kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax); |
4222 | kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx); | 4487 | kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx); |
4223 | kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx); | 4488 | kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx); |
@@ -4363,8 +4628,8 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
4363 | } | 4628 | } |
4364 | 4629 | ||
4365 | if (reason == TASK_SWITCH_IRET) { | 4630 | if (reason == TASK_SWITCH_IRET) { |
4366 | u32 eflags = kvm_x86_ops->get_rflags(vcpu); | 4631 | u32 eflags = kvm_get_rflags(vcpu); |
4367 | kvm_x86_ops->set_rflags(vcpu, eflags & ~X86_EFLAGS_NT); | 4632 | kvm_set_rflags(vcpu, eflags & ~X86_EFLAGS_NT); |
4368 | } | 4633 | } |
4369 | 4634 | ||
4370 | /* set back link to prev task only if NT bit is set in eflags | 4635 | /* set back link to prev task only if NT bit is set in eflags |
@@ -4372,11 +4637,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
4372 | if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE) | 4637 | if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE) |
4373 | old_tss_sel = 0xffff; | 4638 | old_tss_sel = 0xffff; |
4374 | 4639 | ||
4375 | /* set back link to prev task only if NT bit is set in eflags | ||
4376 | note that old_tss_sel is not used afetr this point */ | ||
4377 | if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE) | ||
4378 | old_tss_sel = 0xffff; | ||
4379 | |||
4380 | if (nseg_desc.type & 8) | 4640 | if (nseg_desc.type & 8) |
4381 | ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel, | 4641 | ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel, |
4382 | old_tss_base, &nseg_desc); | 4642 | old_tss_base, &nseg_desc); |
@@ -4385,8 +4645,8 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason) | |||
4385 | old_tss_base, &nseg_desc); | 4645 | old_tss_base, &nseg_desc); |
4386 | 4646 | ||
4387 | if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) { | 4647 | if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) { |
4388 | u32 eflags = kvm_x86_ops->get_rflags(vcpu); | 4648 | u32 eflags = kvm_get_rflags(vcpu); |
4389 | kvm_x86_ops->set_rflags(vcpu, eflags | X86_EFLAGS_NT); | 4649 | kvm_set_rflags(vcpu, eflags | X86_EFLAGS_NT); |
4390 | } | 4650 | } |
4391 | 4651 | ||
4392 | if (reason != TASK_SWITCH_IRET) { | 4652 | if (reason != TASK_SWITCH_IRET) { |
@@ -4438,8 +4698,10 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |||
4438 | 4698 | ||
4439 | mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4; | 4699 | mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4; |
4440 | kvm_x86_ops->set_cr4(vcpu, sregs->cr4); | 4700 | kvm_x86_ops->set_cr4(vcpu, sregs->cr4); |
4441 | if (!is_long_mode(vcpu) && is_pae(vcpu)) | 4701 | if (!is_long_mode(vcpu) && is_pae(vcpu)) { |
4442 | load_pdptrs(vcpu, vcpu->arch.cr3); | 4702 | load_pdptrs(vcpu, vcpu->arch.cr3); |
4703 | mmu_reset_needed = 1; | ||
4704 | } | ||
4443 | 4705 | ||
4444 | if (mmu_reset_needed) | 4706 | if (mmu_reset_needed) |
4445 | kvm_mmu_reset_context(vcpu); | 4707 | kvm_mmu_reset_context(vcpu); |
@@ -4480,12 +4742,32 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |||
4480 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, | 4742 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, |
4481 | struct kvm_guest_debug *dbg) | 4743 | struct kvm_guest_debug *dbg) |
4482 | { | 4744 | { |
4745 | unsigned long rflags; | ||
4483 | int i, r; | 4746 | int i, r; |
4484 | 4747 | ||
4485 | vcpu_load(vcpu); | 4748 | vcpu_load(vcpu); |
4486 | 4749 | ||
4487 | if ((dbg->control & (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP)) == | 4750 | if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { |
4488 | (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP)) { | 4751 | r = -EBUSY; |
4752 | if (vcpu->arch.exception.pending) | ||
4753 | goto unlock_out; | ||
4754 | if (dbg->control & KVM_GUESTDBG_INJECT_DB) | ||
4755 | kvm_queue_exception(vcpu, DB_VECTOR); | ||
4756 | else | ||
4757 | kvm_queue_exception(vcpu, BP_VECTOR); | ||
4758 | } | ||
4759 | |||
4760 | /* | ||
4761 | * Read rflags as long as potentially injected trace flags are still | ||
4762 | * filtered out. | ||
4763 | */ | ||
4764 | rflags = kvm_get_rflags(vcpu); | ||
4765 | |||
4766 | vcpu->guest_debug = dbg->control; | ||
4767 | if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) | ||
4768 | vcpu->guest_debug = 0; | ||
4769 | |||
4770 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { | ||
4489 | for (i = 0; i < KVM_NR_DB_REGS; ++i) | 4771 | for (i = 0; i < KVM_NR_DB_REGS; ++i) |
4490 | vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; | 4772 | vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; |
4491 | vcpu->arch.switch_db_regs = | 4773 | vcpu->arch.switch_db_regs = |
@@ -4496,13 +4778,23 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, | |||
4496 | vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK); | 4778 | vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK); |
4497 | } | 4779 | } |
4498 | 4780 | ||
4499 | r = kvm_x86_ops->set_guest_debug(vcpu, dbg); | 4781 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { |
4782 | vcpu->arch.singlestep_cs = | ||
4783 | get_segment_selector(vcpu, VCPU_SREG_CS); | ||
4784 | vcpu->arch.singlestep_rip = kvm_rip_read(vcpu); | ||
4785 | } | ||
4786 | |||
4787 | /* | ||
4788 | * Trigger an rflags update that will inject or remove the trace | ||
4789 | * flags. | ||
4790 | */ | ||
4791 | kvm_set_rflags(vcpu, rflags); | ||
4792 | |||
4793 | kvm_x86_ops->set_guest_debug(vcpu, dbg); | ||
4500 | 4794 | ||
4501 | if (dbg->control & KVM_GUESTDBG_INJECT_DB) | 4795 | r = 0; |
4502 | kvm_queue_exception(vcpu, DB_VECTOR); | ||
4503 | else if (dbg->control & KVM_GUESTDBG_INJECT_BP) | ||
4504 | kvm_queue_exception(vcpu, BP_VECTOR); | ||
4505 | 4796 | ||
4797 | unlock_out: | ||
4506 | vcpu_put(vcpu); | 4798 | vcpu_put(vcpu); |
4507 | 4799 | ||
4508 | return r; | 4800 | return r; |
@@ -4703,14 +4995,26 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) | |||
4703 | return kvm_x86_ops->vcpu_reset(vcpu); | 4995 | return kvm_x86_ops->vcpu_reset(vcpu); |
4704 | } | 4996 | } |
4705 | 4997 | ||
4706 | void kvm_arch_hardware_enable(void *garbage) | 4998 | int kvm_arch_hardware_enable(void *garbage) |
4707 | { | 4999 | { |
4708 | kvm_x86_ops->hardware_enable(garbage); | 5000 | /* |
5001 | * Since this may be called from a hotplug notifcation, | ||
5002 | * we can't get the CPU frequency directly. | ||
5003 | */ | ||
5004 | if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { | ||
5005 | int cpu = raw_smp_processor_id(); | ||
5006 | per_cpu(cpu_tsc_khz, cpu) = 0; | ||
5007 | } | ||
5008 | |||
5009 | kvm_shared_msr_cpu_online(); | ||
5010 | |||
5011 | return kvm_x86_ops->hardware_enable(garbage); | ||
4709 | } | 5012 | } |
4710 | 5013 | ||
4711 | void kvm_arch_hardware_disable(void *garbage) | 5014 | void kvm_arch_hardware_disable(void *garbage) |
4712 | { | 5015 | { |
4713 | kvm_x86_ops->hardware_disable(garbage); | 5016 | kvm_x86_ops->hardware_disable(garbage); |
5017 | drop_user_return_notifiers(garbage); | ||
4714 | } | 5018 | } |
4715 | 5019 | ||
4716 | int kvm_arch_hardware_setup(void) | 5020 | int kvm_arch_hardware_setup(void) |
@@ -4948,8 +5252,36 @@ int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) | |||
4948 | return kvm_x86_ops->interrupt_allowed(vcpu); | 5252 | return kvm_x86_ops->interrupt_allowed(vcpu); |
4949 | } | 5253 | } |
4950 | 5254 | ||
5255 | unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) | ||
5256 | { | ||
5257 | unsigned long rflags; | ||
5258 | |||
5259 | rflags = kvm_x86_ops->get_rflags(vcpu); | ||
5260 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
5261 | rflags &= ~(unsigned long)(X86_EFLAGS_TF | X86_EFLAGS_RF); | ||
5262 | return rflags; | ||
5263 | } | ||
5264 | EXPORT_SYMBOL_GPL(kvm_get_rflags); | ||
5265 | |||
5266 | void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) | ||
5267 | { | ||
5268 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && | ||
5269 | vcpu->arch.singlestep_cs == | ||
5270 | get_segment_selector(vcpu, VCPU_SREG_CS) && | ||
5271 | vcpu->arch.singlestep_rip == kvm_rip_read(vcpu)) | ||
5272 | rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF; | ||
5273 | kvm_x86_ops->set_rflags(vcpu, rflags); | ||
5274 | } | ||
5275 | EXPORT_SYMBOL_GPL(kvm_set_rflags); | ||
5276 | |||
4951 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); | 5277 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); |
4952 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); | 5278 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); |
4953 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); | 5279 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); |
4954 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); | 5280 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); |
4955 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); | 5281 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); |
5282 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun); | ||
5283 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); | ||
5284 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); | ||
5285 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); | ||
5286 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga); | ||
5287 | EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit); | ||