4 files changed, 35 insertions, 12 deletions

diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c
index 70d224d4264c..8213efe1998c 100644
--- a/arch/ia64/kvm/kvm-ia64.c
+++ b/arch/ia64/kvm/kvm-ia64.c
@@ -662,6 +662,7 @@ again:
                 goto vcpu_run_fail;
 
         srcu_read_unlock(&vcpu->kvm->srcu, idx);
+        vcpu->mode = IN_GUEST_MODE;
         kvm_guest_enter();
 
         /*
@@ -683,6 +684,7 @@ again:
          */
         barrier();
         kvm_guest_exit();
+        vcpu->mode = OUTSIDE_GUEST_MODE;
         preempt_enable();
 
         idx = srcu_read_lock(&vcpu->kvm->srcu);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 5eccdba08bd0..a7f65aa6eef6 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5210,14 +5210,18 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                 kvm_load_guest_fpu(vcpu);
         kvm_load_guest_xcr0(vcpu);
 
-        atomic_set(&vcpu->guest_mode, 1);
-        smp_wmb();
+        vcpu->mode = IN_GUEST_MODE;
+
+        /* We should set ->mode before check ->requests,
+         * see the comment in make_all_cpus_request.
+         */
+        smp_mb();
 
         local_irq_disable();
 
-        if (!atomic_read(&vcpu->guest_mode) || vcpu->requests
+        if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
             || need_resched() || signal_pending(current)) {
-                atomic_set(&vcpu->guest_mode, 0);
+                vcpu->mode = OUTSIDE_GUEST_MODE;
                 smp_wmb();
                 local_irq_enable();
                 preempt_enable();
@@ -5253,7 +5257,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 
         kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc);
 
-        atomic_set(&vcpu->guest_mode, 0);
+        vcpu->mode = OUTSIDE_GUEST_MODE;
         smp_wmb();
         local_irq_enable();
 
@@ -6157,7 +6161,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
 
         me = get_cpu();
         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
-                if (atomic_xchg(&vcpu->guest_mode, 0))
+                if (kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE)
                         smp_send_reschedule(cpu);
         put_cpu();
 }
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index b5021db21858..b99eacd988ab 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -98,19 +98,26 @@ int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
 #endif
 
+enum {
+        OUTSIDE_GUEST_MODE,
+        IN_GUEST_MODE,
+        EXITING_GUEST_MODE
+};
+
 struct kvm_vcpu {
         struct kvm *kvm;
 #ifdef CONFIG_PREEMPT_NOTIFIERS
         struct preempt_notifier preempt_notifier;
 #endif
+        int cpu;
         int vcpu_id;
-        struct mutex mutex;
-        int   cpu;
-        atomic_t guest_mode;
-        struct kvm_run *run;
+        int srcu_idx;
+        int mode;
         unsigned long requests;
         unsigned long guest_debug;
-        int srcu_idx;
+
+        struct mutex mutex;
+        struct kvm_run *run;
 
         int fpu_active;
         int guest_fpu_loaded, guest_xcr0_loaded;
@@ -140,6 +147,11 @@ struct kvm_vcpu {
         struct kvm_vcpu_arch arch;
 };
 
+static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
+{
+        return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
+}
+
 /*
  * Some of the bitops functions do not support too long bitmaps.
  * This number must be determined not to exceed such limits.
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index fd67bcde9980..19209f849cf7 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -171,7 +171,12 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
                 if (kvm_make_check_request(req, vcpu))
                         continue;
                 cpu = vcpu->cpu;
-                if (cpus != NULL && cpu != -1 && cpu != me)
+
+                /* Set ->requests bit before we read ->mode */
+                smp_mb();
+
+                if (cpus != NULL && cpu != -1 && cpu != me &&
+                      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
                         cpumask_set_cpu(cpu, cpus);
         }
         if (unlikely(cpus == NULL))