aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--drivers/kvm/kvm.h19
-rw-r--r--drivers/kvm/kvm_main.c113
-rw-r--r--drivers/kvm/x86.c157
-rw-r--r--drivers/kvm/x86.h3
4 files changed, 197 insertions, 95 deletions
diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index 64983240adca..bca07c6d0460 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -492,6 +492,8 @@ void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
492void vcpu_load(struct kvm_vcpu *vcpu); 492void vcpu_load(struct kvm_vcpu *vcpu);
493void vcpu_put(struct kvm_vcpu *vcpu); 493void vcpu_put(struct kvm_vcpu *vcpu);
494 494
495void decache_vcpus_on_cpu(int cpu);
496
495 497
496int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, 498int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
497 struct module *module); 499 struct module *module);
@@ -649,6 +651,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
649 651
650__init void kvm_arch_init(void); 652__init void kvm_arch_init(void);
651 653
654int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
655void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
656
657void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
658void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
659void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
660struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
661void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu);
662
663int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu);
664void kvm_arch_hardware_enable(void *garbage);
665void kvm_arch_hardware_disable(void *garbage);
666int kvm_arch_hardware_setup(void);
667void kvm_arch_hardware_unsetup(void);
668void kvm_arch_check_processor_compat(void *rtn);
669
670
652static inline void kvm_guest_enter(void) 671static inline void kvm_guest_enter(void)
653{ 672{
654 account_system_vtime(current); 673 account_system_vtime(current);
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 47a76c3a4c81..7fdfed52dbe7 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -50,8 +50,8 @@
50MODULE_AUTHOR("Qumranet"); 50MODULE_AUTHOR("Qumranet");
51MODULE_LICENSE("GPL"); 51MODULE_LICENSE("GPL");
52 52
53static DEFINE_SPINLOCK(kvm_lock); 53DEFINE_SPINLOCK(kvm_lock);
54static LIST_HEAD(vm_list); 54LIST_HEAD(vm_list);
55 55
56static cpumask_t cpus_hardware_enabled; 56static cpumask_t cpus_hardware_enabled;
57 57
@@ -124,13 +124,8 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
124 124
125 mutex_init(&vcpu->mutex); 125 mutex_init(&vcpu->mutex);
126 vcpu->cpu = -1; 126 vcpu->cpu = -1;
127 vcpu->mmu.root_hpa = INVALID_PAGE;
128 vcpu->kvm = kvm; 127 vcpu->kvm = kvm;
129 vcpu->vcpu_id = id; 128 vcpu->vcpu_id = id;
130 if (!irqchip_in_kernel(kvm) || id == 0)
131 vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
132 else
133 vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
134 init_waitqueue_head(&vcpu->wq); 129 init_waitqueue_head(&vcpu->wq);
135 130
136 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 131 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
@@ -140,29 +135,11 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
140 } 135 }
141 vcpu->run = page_address(page); 136 vcpu->run = page_address(page);
142 137
143 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 138 r = kvm_arch_vcpu_init(vcpu);
144 if (!page) {
145 r = -ENOMEM;
146 goto fail_free_run;
147 }
148 vcpu->pio_data = page_address(page);
149
150 r = kvm_mmu_create(vcpu);
151 if (r < 0) 139 if (r < 0)
152 goto fail_free_pio_data; 140 goto fail_free_run;
153
154 if (irqchip_in_kernel(kvm)) {
155 r = kvm_create_lapic(vcpu);
156 if (r < 0)
157 goto fail_mmu_destroy;
158 }
159
160 return 0; 141 return 0;
161 142
162fail_mmu_destroy:
163 kvm_mmu_destroy(vcpu);
164fail_free_pio_data:
165 free_page((unsigned long)vcpu->pio_data);
166fail_free_run: 143fail_free_run:
167 free_page((unsigned long)vcpu->run); 144 free_page((unsigned long)vcpu->run);
168fail: 145fail:
@@ -172,9 +149,7 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_init);
172 149
173void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) 150void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
174{ 151{
175 kvm_free_lapic(vcpu); 152 kvm_arch_vcpu_uninit(vcpu);
176 kvm_mmu_destroy(vcpu);
177 free_page((unsigned long)vcpu->pio_data);
178 free_page((unsigned long)vcpu->run); 153 free_page((unsigned long)vcpu->run);
179} 154}
180EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); 155EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
@@ -240,7 +215,7 @@ static void kvm_free_vcpus(struct kvm *kvm)
240 kvm_unload_vcpu_mmu(kvm->vcpus[i]); 215 kvm_unload_vcpu_mmu(kvm->vcpus[i]);
241 for (i = 0; i < KVM_MAX_VCPUS; ++i) { 216 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
242 if (kvm->vcpus[i]) { 217 if (kvm->vcpus[i]) {
243 kvm_x86_ops->vcpu_free(kvm->vcpus[i]); 218 kvm_arch_vcpu_free(kvm->vcpus[i]);
244 kvm->vcpus[i] = NULL; 219 kvm->vcpus[i] = NULL;
245 } 220 }
246 } 221 }
@@ -900,28 +875,17 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
900 if (!valid_vcpu(n)) 875 if (!valid_vcpu(n))
901 return -EINVAL; 876 return -EINVAL;
902 877
903 vcpu = kvm_x86_ops->vcpu_create(kvm, n); 878 vcpu = kvm_arch_vcpu_create(kvm, n);
904 if (IS_ERR(vcpu)) 879 if (IS_ERR(vcpu))
905 return PTR_ERR(vcpu); 880 return PTR_ERR(vcpu);
906 881
907 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); 882 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
908 883
909 /* We do fxsave: this must be aligned. */
910 BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);
911
912 vcpu_load(vcpu);
913 r = kvm_x86_ops->vcpu_reset(vcpu);
914 if (r == 0)
915 r = kvm_mmu_setup(vcpu);
916 vcpu_put(vcpu);
917 if (r < 0)
918 goto free_vcpu;
919
920 mutex_lock(&kvm->lock); 884 mutex_lock(&kvm->lock);
921 if (kvm->vcpus[n]) { 885 if (kvm->vcpus[n]) {
922 r = -EEXIST; 886 r = -EEXIST;
923 mutex_unlock(&kvm->lock); 887 mutex_unlock(&kvm->lock);
924 goto mmu_unload; 888 goto vcpu_destroy;
925 } 889 }
926 kvm->vcpus[n] = vcpu; 890 kvm->vcpus[n] = vcpu;
927 mutex_unlock(&kvm->lock); 891 mutex_unlock(&kvm->lock);
@@ -936,14 +900,8 @@ unlink:
936 mutex_lock(&kvm->lock); 900 mutex_lock(&kvm->lock);
937 kvm->vcpus[n] = NULL; 901 kvm->vcpus[n] = NULL;
938 mutex_unlock(&kvm->lock); 902 mutex_unlock(&kvm->lock);
939 903vcpu_destroy:
940mmu_unload: 904 kvm_arch_vcpu_destory(vcpu);
941 vcpu_load(vcpu);
942 kvm_mmu_unload(vcpu);
943 vcpu_put(vcpu);
944
945free_vcpu:
946 kvm_x86_ops->vcpu_free(vcpu);
947 return r; 905 return r;
948} 906}
949 907
@@ -1281,41 +1239,6 @@ static struct miscdevice kvm_dev = {
1281 &kvm_chardev_ops, 1239 &kvm_chardev_ops,
1282}; 1240};
1283 1241
1284/*
1285 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
1286 * cached on it.
1287 */
1288static void decache_vcpus_on_cpu(int cpu)
1289{
1290 struct kvm *vm;
1291 struct kvm_vcpu *vcpu;
1292 int i;
1293
1294 spin_lock(&kvm_lock);
1295 list_for_each_entry(vm, &vm_list, vm_list)
1296 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
1297 vcpu = vm->vcpus[i];
1298 if (!vcpu)
1299 continue;
1300 /*
1301 * If the vcpu is locked, then it is running on some
1302 * other cpu and therefore it is not cached on the
1303 * cpu in question.
1304 *
1305 * If it's not locked, check the last cpu it executed
1306 * on.
1307 */
1308 if (mutex_trylock(&vcpu->mutex)) {
1309 if (vcpu->cpu == cpu) {
1310 kvm_x86_ops->vcpu_decache(vcpu);
1311 vcpu->cpu = -1;
1312 }
1313 mutex_unlock(&vcpu->mutex);
1314 }
1315 }
1316 spin_unlock(&kvm_lock);
1317}
1318
1319static void hardware_enable(void *junk) 1242static void hardware_enable(void *junk)
1320{ 1243{
1321 int cpu = raw_smp_processor_id(); 1244 int cpu = raw_smp_processor_id();
@@ -1323,7 +1246,7 @@ static void hardware_enable(void *junk)
1323 if (cpu_isset(cpu, cpus_hardware_enabled)) 1246 if (cpu_isset(cpu, cpus_hardware_enabled))
1324 return; 1247 return;
1325 cpu_set(cpu, cpus_hardware_enabled); 1248 cpu_set(cpu, cpus_hardware_enabled);
1326 kvm_x86_ops->hardware_enable(NULL); 1249 kvm_arch_hardware_enable(NULL);
1327} 1250}
1328 1251
1329static void hardware_disable(void *junk) 1252static void hardware_disable(void *junk)
@@ -1334,7 +1257,7 @@ static void hardware_disable(void *junk)
1334 return; 1257 return;
1335 cpu_clear(cpu, cpus_hardware_enabled); 1258 cpu_clear(cpu, cpus_hardware_enabled);
1336 decache_vcpus_on_cpu(cpu); 1259 decache_vcpus_on_cpu(cpu);
1337 kvm_x86_ops->hardware_disable(NULL); 1260 kvm_arch_hardware_disable(NULL);
1338} 1261}
1339 1262
1340static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, 1263static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
@@ -1500,7 +1423,7 @@ static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
1500{ 1423{
1501 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 1424 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1502 1425
1503 kvm_x86_ops->vcpu_load(vcpu, cpu); 1426 kvm_arch_vcpu_load(vcpu, cpu);
1504} 1427}
1505 1428
1506static void kvm_sched_out(struct preempt_notifier *pn, 1429static void kvm_sched_out(struct preempt_notifier *pn,
@@ -1508,7 +1431,7 @@ static void kvm_sched_out(struct preempt_notifier *pn,
1508{ 1431{
1509 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); 1432 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1510 1433
1511 kvm_x86_ops->vcpu_put(vcpu); 1434 kvm_arch_vcpu_put(vcpu);
1512} 1435}
1513 1436
1514int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, 1437int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
@@ -1533,13 +1456,13 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
1533 1456
1534 kvm_x86_ops = ops; 1457 kvm_x86_ops = ops;
1535 1458
1536 r = kvm_x86_ops->hardware_setup(); 1459 r = kvm_arch_hardware_setup();
1537 if (r < 0) 1460 if (r < 0)
1538 goto out; 1461 goto out;
1539 1462
1540 for_each_online_cpu(cpu) { 1463 for_each_online_cpu(cpu) {
1541 smp_call_function_single(cpu, 1464 smp_call_function_single(cpu,
1542 kvm_x86_ops->check_processor_compatibility, 1465 kvm_arch_check_processor_compat,
1543 &r, 0, 1); 1466 &r, 0, 1);
1544 if (r < 0) 1467 if (r < 0)
1545 goto out_free_0; 1468 goto out_free_0;
@@ -1594,7 +1517,7 @@ out_free_2:
1594out_free_1: 1517out_free_1:
1595 on_each_cpu(hardware_disable, NULL, 0, 1); 1518 on_each_cpu(hardware_disable, NULL, 0, 1);
1596out_free_0: 1519out_free_0:
1597 kvm_x86_ops->hardware_unsetup(); 1520 kvm_arch_hardware_unsetup();
1598out: 1521out:
1599 kvm_x86_ops = NULL; 1522 kvm_x86_ops = NULL;
1600 return r; 1523 return r;
@@ -1610,7 +1533,7 @@ void kvm_exit_x86(void)
1610 unregister_reboot_notifier(&kvm_reboot_notifier); 1533 unregister_reboot_notifier(&kvm_reboot_notifier);
1611 unregister_cpu_notifier(&kvm_cpu_notifier); 1534 unregister_cpu_notifier(&kvm_cpu_notifier);
1612 on_each_cpu(hardware_disable, NULL, 0, 1); 1535 on_each_cpu(hardware_disable, NULL, 0, 1);
1613 kvm_x86_ops->hardware_unsetup(); 1536 kvm_arch_hardware_unsetup();
1614 kvm_x86_ops = NULL; 1537 kvm_x86_ops = NULL;
1615} 1538}
1616EXPORT_SYMBOL_GPL(kvm_exit_x86); 1539EXPORT_SYMBOL_GPL(kvm_exit_x86);
diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c
index 2edc53ec8b3b..4902b35060f5 100644
--- a/drivers/kvm/x86.c
+++ b/drivers/kvm/x86.c
@@ -564,6 +564,41 @@ out:
564 return r; 564 return r;
565} 565}
566 566
567/*
568 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
569 * cached on it.
570 */
571void decache_vcpus_on_cpu(int cpu)
572{
573 struct kvm *vm;
574 struct kvm_vcpu *vcpu;
575 int i;
576
577 spin_lock(&kvm_lock);
578 list_for_each_entry(vm, &vm_list, vm_list)
579 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
580 vcpu = vm->vcpus[i];
581 if (!vcpu)
582 continue;
583 /*
584 * If the vcpu is locked, then it is running on some
585 * other cpu and therefore it is not cached on the
586 * cpu in question.
587 *
588 * If it's not locked, check the last cpu it executed
589 * on.
590 */
591 if (mutex_trylock(&vcpu->mutex)) {
592 if (vcpu->cpu == cpu) {
593 kvm_x86_ops->vcpu_decache(vcpu);
594 vcpu->cpu = -1;
595 }
596 mutex_unlock(&vcpu->mutex);
597 }
598 }
599 spin_unlock(&kvm_lock);
600}
601
567long kvm_arch_dev_ioctl(struct file *filp, 602long kvm_arch_dev_ioctl(struct file *filp,
568 unsigned int ioctl, unsigned long arg) 603 unsigned int ioctl, unsigned long arg)
569{ 604{
@@ -2319,3 +2354,125 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
2319 fx_restore(&vcpu->host_fx_image); 2354 fx_restore(&vcpu->host_fx_image);
2320} 2355}
2321EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); 2356EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
2357
2358void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
2359{
2360 kvm_x86_ops->vcpu_free(vcpu);
2361}
2362
2363struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2364 unsigned int id)
2365{
2366 int r;
2367 struct kvm_vcpu *vcpu = kvm_x86_ops->vcpu_create(kvm, id);
2368
2369 if (IS_ERR(vcpu)) {
2370 r = -ENOMEM;
2371 goto fail;
2372 }
2373
2374 /* We do fxsave: this must be aligned. */
2375 BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);
2376
2377 vcpu_load(vcpu);
2378 r = kvm_arch_vcpu_reset(vcpu);
2379 if (r == 0)
2380 r = kvm_mmu_setup(vcpu);
2381 vcpu_put(vcpu);
2382 if (r < 0)
2383 goto free_vcpu;
2384
2385 return vcpu;
2386free_vcpu:
2387 kvm_x86_ops->vcpu_free(vcpu);
2388fail:
2389 return ERR_PTR(r);
2390}
2391
2392void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu)
2393{
2394 vcpu_load(vcpu);
2395 kvm_mmu_unload(vcpu);
2396 vcpu_put(vcpu);
2397
2398 kvm_x86_ops->vcpu_free(vcpu);
2399}
2400
2401int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
2402{
2403 return kvm_x86_ops->vcpu_reset(vcpu);
2404}
2405
2406void kvm_arch_hardware_enable(void *garbage)
2407{
2408 kvm_x86_ops->hardware_enable(garbage);
2409}
2410
2411void kvm_arch_hardware_disable(void *garbage)
2412{
2413 kvm_x86_ops->hardware_disable(garbage);
2414}
2415
2416int kvm_arch_hardware_setup(void)
2417{
2418 return kvm_x86_ops->hardware_setup();
2419}
2420
2421void kvm_arch_hardware_unsetup(void)
2422{
2423 kvm_x86_ops->hardware_unsetup();
2424}
2425
2426void kvm_arch_check_processor_compat(void *rtn)
2427{
2428 kvm_x86_ops->check_processor_compatibility(rtn);
2429}
2430
2431int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2432{
2433 struct page *page;
2434 struct kvm *kvm;
2435 int r;
2436
2437 BUG_ON(vcpu->kvm == NULL);
2438 kvm = vcpu->kvm;
2439
2440 vcpu->mmu.root_hpa = INVALID_PAGE;
2441 if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0)
2442 vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
2443 else
2444 vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
2445
2446 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2447 if (!page) {
2448 r = -ENOMEM;
2449 goto fail;
2450 }
2451 vcpu->pio_data = page_address(page);
2452
2453 r = kvm_mmu_create(vcpu);
2454 if (r < 0)
2455 goto fail_free_pio_data;
2456
2457 if (irqchip_in_kernel(kvm)) {
2458 r = kvm_create_lapic(vcpu);
2459 if (r < 0)
2460 goto fail_mmu_destroy;
2461 }
2462
2463 return 0;
2464
2465fail_mmu_destroy:
2466 kvm_mmu_destroy(vcpu);
2467fail_free_pio_data:
2468 free_page((unsigned long)vcpu->pio_data);
2469fail:
2470 return r;
2471}
2472
2473void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
2474{
2475 kvm_free_lapic(vcpu);
2476 kvm_mmu_destroy(vcpu);
2477 free_page((unsigned long)vcpu->pio_data);
2478}
diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h
index ec32c26a5118..4df064100226 100644
--- a/drivers/kvm/x86.h
+++ b/drivers/kvm/x86.h
@@ -19,6 +19,9 @@
19#include <linux/kvm.h> 19#include <linux/kvm.h>
20#include <linux/kvm_para.h> 20#include <linux/kvm_para.h>
21 21
22extern spinlock_t kvm_lock;
23extern struct list_head vm_list;
24
22struct kvm_vcpu { 25struct kvm_vcpu {
23 KVM_VCPU_COMM; 26 KVM_VCPU_COMM;
24 u64 host_tsc; 27 u64 host_tsc;