1 files changed, 188 insertions, 6 deletions

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 93c15743f1ee..ce878788a39f 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -107,6 +107,16 @@ static inline void apic_clear_vector(int vec, void *bitmap)
         clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
 }
 
+static inline int __apic_test_and_set_vector(int vec, void *bitmap)
+{
+        return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
+}
+
+static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
+{
+        return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
+}
+
 static inline int apic_hw_enabled(struct kvm_lapic *apic)
 {
         return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
@@ -210,6 +220,16 @@ static int find_highest_vector(void *bitmap)
                 return fls(word[word_offset << 2]) - 1 + (word_offset << 5);
 }
 
+static u8 count_vectors(void *bitmap)
+{
+        u32 *word = bitmap;
+        int word_offset;
+        u8 count = 0;
+        for (word_offset = 0; word_offset < MAX_APIC_VECTOR >> 5; ++word_offset)
+                count += hweight32(word[word_offset << 2]);
+        return count;
+}
+
 static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic)
 {
         apic->irr_pending = true;
@@ -242,6 +262,27 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
                 apic->irr_pending = true;
 }
 
+static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
+{
+        if (!__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
+                ++apic->isr_count;
+        BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
+        /*
+         * ISR (in service register) bit is set when injecting an interrupt.
+         * The highest vector is injected. Thus the latest bit set matches
+         * the highest bit in ISR.
+         */
+        apic->highest_isr_cache = vec;
+}
+
+static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
+{
+        if (__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
+                --apic->isr_count;
+        BUG_ON(apic->isr_count < 0);
+        apic->highest_isr_cache = -1;
+}
+
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 {
         struct kvm_lapic *apic = vcpu->arch.apic;
@@ -270,9 +311,61 @@ int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq)
                         irq->level, irq->trig_mode);
 }
 
+static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
+{
+
+        return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
+                                      sizeof(val));
+}
+
+static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
+{
+
+        return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
+                                      sizeof(*val));
+}
+
+static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
+}
+
+static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
+{
+        u8 val;
+        if (pv_eoi_get_user(vcpu, &val) < 0)
+                apic_debug("Can't read EOI MSR value: 0x%llx\n",
+                           (unsigned long long)vcpi->arch.pv_eoi.msr_val);
+        return val & 0x1;
+}
+
+static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
+{
+        if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
+                apic_debug("Can't set EOI MSR value: 0x%llx\n",
+                           (unsigned long long)vcpi->arch.pv_eoi.msr_val);
+                return;
+        }
+        __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
+}
+
+static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
+{
+        if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
+                apic_debug("Can't clear EOI MSR value: 0x%llx\n",
+                           (unsigned long long)vcpi->arch.pv_eoi.msr_val);
+                return;
+        }
+        __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
+}
+
 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 {
         int result;
+        if (!apic->isr_count)
+                return -1;
+        if (likely(apic->highest_isr_cache != -1))
+                return apic->highest_isr_cache;
 
         result = find_highest_vector(apic->regs + APIC_ISR);
         ASSERT(result == -1 || result >= 16);
@@ -482,17 +575,20 @@ int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
 }
 
-static void apic_set_eoi(struct kvm_lapic *apic)
+static int apic_set_eoi(struct kvm_lapic *apic)
 {
         int vector = apic_find_highest_isr(apic);
+
+        trace_kvm_eoi(apic, vector);
+
         /*
          * Not every write EOI will has corresponding ISR,
          * one example is when Kernel check timer on setup_IO_APIC
          */
         if (vector == -1)
-                return;
+                return vector;
 
-        apic_clear_vector(vector, apic->regs + APIC_ISR);
+        apic_clear_isr(vector, apic);
         apic_update_ppr(apic);
 
         if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
@@ -505,6 +601,7 @@ static void apic_set_eoi(struct kvm_lapic *apic)
                 kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
         }
         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
+        return vector;
 }
 
 static void apic_send_ipi(struct kvm_lapic *apic)
@@ -1081,10 +1178,13 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
                 apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
         }
         apic->irr_pending = false;
+        apic->isr_count = 0;
+        apic->highest_isr_cache = -1;
         update_divide_count(apic);
         atomic_set(&apic->lapic_timer.pending, 0);
         if (kvm_vcpu_is_bsp(vcpu))
                 vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP;
+        vcpu->arch.pv_eoi.msr_val = 0;
         apic_update_ppr(apic);
 
         vcpu->arch.apic_arb_prio = 0;
@@ -1248,7 +1348,7 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
         if (vector == -1)
                 return -1;
 
-        apic_set_vector(vector, apic->regs + APIC_ISR);
+        apic_set_isr(vector, apic);
         apic_update_ppr(apic);
         apic_clear_irr(vector, apic);
         return vector;
@@ -1267,6 +1367,8 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
         update_divide_count(apic);
         start_apic_timer(apic);
         apic->irr_pending = true;
+        apic->isr_count = count_vectors(apic->regs + APIC_ISR);
+        apic->highest_isr_cache = -1;
         kvm_make_request(KVM_REQ_EVENT, vcpu);
 }
 
@@ -1283,11 +1385,51 @@ void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
 }
 
+/*
+ * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
+ *
+ * Detect whether guest triggered PV EOI since the
+ * last entry. If yes, set EOI on guests's behalf.
+ * Clear PV EOI in guest memory in any case.
+ */
+static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
+                                        struct kvm_lapic *apic)
+{
+        bool pending;
+        int vector;
+        /*
+         * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
+         * and KVM_PV_EOI_ENABLED in guest memory as follows:
+         *
+         * KVM_APIC_PV_EOI_PENDING is unset:
+         *      -> host disabled PV EOI.
+         * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
+         *      -> host enabled PV EOI, guest did not execute EOI yet.
+         * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
+         *      -> host enabled PV EOI, guest executed EOI.
+         */
+        BUG_ON(!pv_eoi_enabled(vcpu));
+        pending = pv_eoi_get_pending(vcpu);
+        /*
+         * Clear pending bit in any case: it will be set again on vmentry.
+         * While this might not be ideal from performance point of view,
+         * this makes sure pv eoi is only enabled when we know it's safe.
+         */
+        pv_eoi_clr_pending(vcpu);
+        if (pending)
+                return;
+        vector = apic_set_eoi(apic);
+        trace_kvm_pv_eoi(apic, vector);
+}
+
 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
 {
         u32 data;
         void *vapic;
 
+        if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
+                apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
+
         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
                 return;
 
@@ -1298,17 +1440,44 @@ void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
         apic_set_tpr(vcpu->arch.apic, data & 0xff);
 }
 
+/*
+ * apic_sync_pv_eoi_to_guest - called before vmentry
+ *
+ * Detect whether it's safe to enable PV EOI and
+ * if yes do so.
+ */
+static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
+                                        struct kvm_lapic *apic)
+{
+        if (!pv_eoi_enabled(vcpu) ||
+            /* IRR set or many bits in ISR: could be nested. */
+            apic->irr_pending ||
+            /* Cache not set: could be safe but we don't bother. */
+            apic->highest_isr_cache == -1 ||
+            /* Need EOI to update ioapic. */
+            kvm_ioapic_handles_vector(vcpu->kvm, apic->highest_isr_cache)) {
+                /*
+                 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
+                 * so we need not do anything here.
+                 */
+                return;
+        }
+
+        pv_eoi_set_pending(apic->vcpu);
+}
+
 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
 {
         u32 data, tpr;
         int max_irr, max_isr;
-        struct kvm_lapic *apic;
+        struct kvm_lapic *apic = vcpu->arch.apic;
         void *vapic;
 
+        apic_sync_pv_eoi_to_guest(vcpu, apic);
+
         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
                 return;
 
-        apic = vcpu->arch.apic;
         tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff;
         max_irr = apic_find_highest_irr(apic);
         if (max_irr < 0)
@@ -1394,3 +1563,16 @@ int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
 
         return 0;
 }
+
+int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
+{
+        u64 addr = data & ~KVM_MSR_ENABLED;
+        if (!IS_ALIGNED(addr, 4))
+                return 1;
+
+        vcpu->arch.pv_eoi.msr_val = data;
+        if (!pv_eoi_enabled(vcpu))
+                return 0;
+        return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
+                                         addr);
+}