Merge tag 'kvm-3.8-1' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM updates from Marcelo Tosatti:
 "Considerable KVM/PPC work, x86 kvmclock vsyscall support,
  IA32_TSC_ADJUST MSR emulation, amongst others."

Fix up trivial conflict in kernel/sched/core.c due to cross-cpu
migration notifier added next to rq migration call-back.

* tag 'kvm-3.8-1' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (156 commits)
  KVM: emulator: fix real mode segment checks in address linearization
  VMX: remove unneeded enable_unrestricted_guest check
  KVM: VMX: fix DPL during entry to protected mode
  x86/kexec: crash_vmclear_local_vmcss needs __rcu
  kvm: Fix irqfd resampler list walk
  KVM: VMX: provide the vmclear function and a bitmap to support VMCLEAR in kdump
  x86/kexec: VMCLEAR VMCSs loaded on all cpus if necessary
  KVM: MMU: optimize for set_spte
  KVM: PPC: booke: Get/set guest EPCR register using ONE_REG interface
  KVM: PPC: bookehv: Add EPCR support in mtspr/mfspr emulation
  KVM: PPC: bookehv: Add guest computation mode for irq delivery
  KVM: PPC: Make EPCR a valid field for booke64 and bookehv
  KVM: PPC: booke: Extend MAS2 EPN mask for 64-bit
  KVM: PPC: e500: Mask MAS2 EPN high 32-bits in 32/64 tlbwe emulation
  KVM: PPC: Mask ea's high 32-bits in 32/64 instr emulation
  KVM: PPC: e500: Add emulation helper for getting instruction ea
  KVM: PPC: bookehv64: Add support for interrupt handling
  KVM: PPC: bookehv: Remove GET_VCPU macro from exception handler
  KVM: PPC: booke: Fix get_tb() compile error on 64-bit
  KVM: PPC: e500: Silence bogus GCC warning in tlb code
  ...

4 files changed, 214 insertions, 69 deletions

diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index 13ad89971d4..74467feb4dc 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -16,6 +16,7 @@
 #include <linux/delay.h>
 #include <linux/elf.h>
 #include <linux/elfcore.h>
+#include <linux/module.h>
 
 #include <asm/processor.h>
 #include <asm/hardirq.h>
@@ -30,6 +31,27 @@
 
 int in_crash_kexec;
 
+/*
+ * This is used to VMCLEAR all VMCSs loaded on the
+ * processor. And when loading kvm_intel module, the
+ * callback function pointer will be assigned.
+ *
+ * protected by rcu.
+ */
+crash_vmclear_fn __rcu *crash_vmclear_loaded_vmcss = NULL;
+EXPORT_SYMBOL_GPL(crash_vmclear_loaded_vmcss);
+
+static inline void cpu_crash_vmclear_loaded_vmcss(void)
+{
+        crash_vmclear_fn *do_vmclear_operation = NULL;
+
+        rcu_read_lock();
+        do_vmclear_operation = rcu_dereference(crash_vmclear_loaded_vmcss);
+        if (do_vmclear_operation)
+                do_vmclear_operation();
+        rcu_read_unlock();
+}
+
 #if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC)
 
 static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
@@ -46,6 +68,11 @@ static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
 #endif
         crash_save_cpu(regs, cpu);
 
+        /*
+         * VMCLEAR VMCSs loaded on all cpus if needed.
+         */
+        cpu_crash_vmclear_loaded_vmcss();
+
         /* Disable VMX or SVM if needed.
          *
          * We need to disable virtualization on all CPUs.
@@ -88,6 +115,11 @@ void native_machine_crash_shutdown(struct pt_regs *regs)
 
         kdump_nmi_shootdown_cpus();
 
+        /*
+         * VMCLEAR VMCSs loaded on this cpu if needed.
+         */
+        cpu_crash_vmclear_loaded_vmcss();
+
         /* Booting kdump kernel with VMX or SVM enabled won't work,
          * because (among other limitations) we can't disable paging
          * with the virt flags.
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 4180a874c76..08b973f6403 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -42,6 +42,7 @@
 #include <asm/apic.h>
 #include <asm/apicdef.h>
 #include <asm/hypervisor.h>
+#include <asm/kvm_guest.h>
 
 static int kvmapf = 1;
 
@@ -62,6 +63,15 @@ static int parse_no_stealacc(char *arg)
 
 early_param("no-steal-acc", parse_no_stealacc);
 
+static int kvmclock_vsyscall = 1;
+static int parse_no_kvmclock_vsyscall(char *arg)
+{
+        kvmclock_vsyscall = 0;
+        return 0;
+}
+
+early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall);
+
 static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
 static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
 static int has_steal_clock = 0;
@@ -110,11 +120,6 @@ void kvm_async_pf_task_wait(u32 token)
         struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
         struct kvm_task_sleep_node n, *e;
         DEFINE_WAIT(wait);
-        int cpu, idle;
-
-        cpu = get_cpu();
-        idle = idle_cpu(cpu);
-        put_cpu();
 
         spin_lock(&b->lock);
         e = _find_apf_task(b, token);
@@ -128,7 +133,7 @@ void kvm_async_pf_task_wait(u32 token)
 
         n.token = token;
         n.cpu = smp_processor_id();
-        n.halted = idle || preempt_count() > 1;
+        n.halted = is_idle_task(current) || preempt_count() > 1;
         init_waitqueue_head(&n.wq);
         hlist_add_head(&n.link, &b->list);
         spin_unlock(&b->lock);
@@ -471,6 +476,9 @@ void __init kvm_guest_init(void)
         if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
                 apic_set_eoi_write(kvm_guest_apic_eoi_write);
 
+        if (kvmclock_vsyscall)
+                kvm_setup_vsyscall_timeinfo();
+
 #ifdef CONFIG_SMP
         smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
         register_cpu_notifier(&kvm_cpu_notifier);
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index f1b42b3a186..220a360010f 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -23,6 +23,7 @@
 #include <asm/apic.h>
 #include <linux/percpu.h>
 #include <linux/hardirq.h>
+#include <linux/memblock.h>
 
 #include <asm/x86_init.h>
 #include <asm/reboot.h>
@@ -39,7 +40,7 @@ static int parse_no_kvmclock(char *arg)
 early_param("no-kvmclock", parse_no_kvmclock);
 
 /* The hypervisor will put information about time periodically here */
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct pvclock_vcpu_time_info, hv_clock);
+static struct pvclock_vsyscall_time_info *hv_clock;
 static struct pvclock_wall_clock wall_clock;
 
 /*
@@ -52,15 +53,20 @@ static unsigned long kvm_get_wallclock(void)
         struct pvclock_vcpu_time_info *vcpu_time;
         struct timespec ts;
         int low, high;
+        int cpu;
 
         low = (int)__pa_symbol(&wall_clock);
         high = ((u64)__pa_symbol(&wall_clock) >> 32);
 
         native_write_msr(msr_kvm_wall_clock, low, high);
 
-        vcpu_time = &get_cpu_var(hv_clock);
+        preempt_disable();
+        cpu = smp_processor_id();
+
+        vcpu_time = &hv_clock[cpu].pvti;
         pvclock_read_wallclock(&wall_clock, vcpu_time, &ts);
-        put_cpu_var(hv_clock);
+
+        preempt_enable();
 
         return ts.tv_sec;
 }
@@ -74,9 +80,11 @@ static cycle_t kvm_clock_read(void)
 {
         struct pvclock_vcpu_time_info *src;
         cycle_t ret;
+        int cpu;
 
         preempt_disable_notrace();
-        src = &__get_cpu_var(hv_clock);
+        cpu = smp_processor_id();
+        src = &hv_clock[cpu].pvti;
         ret = pvclock_clocksource_read(src);
         preempt_enable_notrace();
         return ret;
@@ -99,8 +107,15 @@ static cycle_t kvm_clock_get_cycles(struct clocksource *cs)
 static unsigned long kvm_get_tsc_khz(void)
 {
         struct pvclock_vcpu_time_info *src;
-        src = &per_cpu(hv_clock, 0);
-        return pvclock_tsc_khz(src);
+        int cpu;
+        unsigned long tsc_khz;
+
+        preempt_disable();
+        cpu = smp_processor_id();
+        src = &hv_clock[cpu].pvti;
+        tsc_khz = pvclock_tsc_khz(src);
+        preempt_enable();
+        return tsc_khz;
 }
 
 static void kvm_get_preset_lpj(void)
@@ -119,10 +134,14 @@ bool kvm_check_and_clear_guest_paused(void)
 {
         bool ret = false;
         struct pvclock_vcpu_time_info *src;
+        int cpu = smp_processor_id();
 
-        src = &__get_cpu_var(hv_clock);
+        if (!hv_clock)
+                return ret;
+
+        src = &hv_clock[cpu].pvti;
         if ((src->flags & PVCLOCK_GUEST_STOPPED) != 0) {
-                __this_cpu_and(hv_clock.flags, ~PVCLOCK_GUEST_STOPPED);
+                src->flags &= ~PVCLOCK_GUEST_STOPPED;
                 ret = true;
         }
 
@@ -141,9 +160,10 @@ int kvm_register_clock(char *txt)
 {
         int cpu = smp_processor_id();
         int low, high, ret;
+        struct pvclock_vcpu_time_info *src = &hv_clock[cpu].pvti;
 
-        low = (int)__pa(&per_cpu(hv_clock, cpu)) | 1;
-        high = ((u64)__pa(&per_cpu(hv_clock, cpu)) >> 32);
+        low = (int)__pa(src) | 1;
+        high = ((u64)__pa(src) >> 32);
         ret = native_write_msr_safe(msr_kvm_system_time, low, high);
         printk(KERN_INFO "kvm-clock: cpu %d, msr %x:%x, %s\n",
                cpu, high, low, txt);
@@ -197,6 +217,8 @@ static void kvm_shutdown(void)
 
 void __init kvmclock_init(void)
 {
+        unsigned long mem;
+
         if (!kvm_para_available())
                 return;
 
@@ -209,8 +231,18 @@ void __init kvmclock_init(void)
         printk(KERN_INFO "kvm-clock: Using msrs %x and %x",
                 msr_kvm_system_time, msr_kvm_wall_clock);
 
-        if (kvm_register_clock("boot clock"))
+        mem = memblock_alloc(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS,
+                             PAGE_SIZE);
+        if (!mem)
+                return;
+        hv_clock = __va(mem);
+
+        if (kvm_register_clock("boot clock")) {
+                hv_clock = NULL;
+                memblock_free(mem,
+                        sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS);
                 return;
+        }
         pv_time_ops.sched_clock = kvm_clock_read;
         x86_platform.calibrate_tsc = kvm_get_tsc_khz;
         x86_platform.get_wallclock = kvm_get_wallclock;
@@ -233,3 +265,37 @@ void __init kvmclock_init(void)
         if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE_STABLE_BIT))
                 pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
 }
+
+int __init kvm_setup_vsyscall_timeinfo(void)
+{
+#ifdef CONFIG_X86_64
+        int cpu;
+        int ret;
+        u8 flags;
+        struct pvclock_vcpu_time_info *vcpu_time;
+        unsigned int size;
+
+        size = sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS;
+
+        preempt_disable();
+        cpu = smp_processor_id();
+
+        vcpu_time = &hv_clock[cpu].pvti;
+        flags = pvclock_read_flags(vcpu_time);
+
+        if (!(flags & PVCLOCK_TSC_STABLE_BIT)) {
+                preempt_enable();
+                return 1;
+        }
+
+        if ((ret = pvclock_init_vsyscall(hv_clock, size))) {
+                preempt_enable();
+                return ret;
+        }
+
+        preempt_enable();
+
+        kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK;
+#endif
+        return 0;
+}
diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 42eb3300dfc..85c39590c1a 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -17,23 +17,13 @@
 
 #include <linux/kernel.h>
 #include <linux/percpu.h>
+#include <linux/notifier.h>
+#include <linux/sched.h>
+#include <linux/gfp.h>
+#include <linux/bootmem.h>
+#include <asm/fixmap.h>
 #include <asm/pvclock.h>
 
-/*
- * These are perodically updated
- *    xen: magic shared_info page
- *    kvm: gpa registered via msr
- * and then copied here.
- */
-struct pvclock_shadow_time {
-        u64 tsc_timestamp;     /* TSC at last update of time vals.  */
-        u64 system_timestamp;  /* Time, in nanosecs, since boot.    */
-        u32 tsc_to_nsec_mul;
-        int tsc_shift;
-        u32 version;
-        u8  flags;
-};
-
 static u8 valid_flags __read_mostly = 0;
 
 void pvclock_set_flags(u8 flags)
@@ -41,34 +31,6 @@ void pvclock_set_flags(u8 flags)
         valid_flags = flags;
 }
 
-static u64 pvclock_get_nsec_offset(struct pvclock_shadow_time *shadow)
-{
-        u64 delta = native_read_tsc() - shadow->tsc_timestamp;
-        return pvclock_scale_delta(delta, shadow->tsc_to_nsec_mul,
-                                   shadow->tsc_shift);
-}
-
-/*
- * Reads a consistent set of time-base values from hypervisor,
- * into a shadow data area.
- */
-static unsigned pvclock_get_time_values(struct pvclock_shadow_time *dst,
-                                        struct pvclock_vcpu_time_info *src)
-{
-        do {
-                dst->version = src->version;
-                rmb();          /* fetch version before data */
-                dst->tsc_timestamp     = src->tsc_timestamp;
-                dst->system_timestamp  = src->system_time;
-                dst->tsc_to_nsec_mul   = src->tsc_to_system_mul;
-                dst->tsc_shift         = src->tsc_shift;
-                dst->flags             = src->flags;
-                rmb();          /* test version after fetching data */
-        } while ((src->version & 1) || (dst->version != src->version));
-
-        return dst->version;
-}
-
 unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src)
 {
         u64 pv_tsc_khz = 1000000ULL << 32;
@@ -88,23 +50,32 @@ void pvclock_resume(void)
         atomic64_set(&last_value, 0);
 }
 
+u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
+{
+        unsigned version;
+        cycle_t ret;
+        u8 flags;
+
+        do {
+                version = __pvclock_read_cycles(src, &ret, &flags);
+        } while ((src->version & 1) || version != src->version);
+
+        return flags & valid_flags;
+}
+
 cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
 {
-        struct pvclock_shadow_time shadow;
         unsigned version;
-        cycle_t ret, offset;
+        cycle_t ret;
         u64 last;
+        u8 flags;
 
         do {
-                version = pvclock_get_time_values(&shadow, src);
-                barrier();
-                offset = pvclock_get_nsec_offset(&shadow);
-                ret = shadow.system_timestamp + offset;
-                barrier();
-        } while (version != src->version);
+                version = __pvclock_read_cycles(src, &ret, &flags);
+        } while ((src->version & 1) || version != src->version);
 
         if ((valid_flags & PVCLOCK_TSC_STABLE_BIT) &&
-                (shadow.flags & PVCLOCK_TSC_STABLE_BIT))
+                (flags & PVCLOCK_TSC_STABLE_BIT))
                 return ret;
 
         /*
@@ -156,3 +127,71 @@ void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
 
         set_normalized_timespec(ts, now.tv_sec, now.tv_nsec);
 }
+
+static struct pvclock_vsyscall_time_info *pvclock_vdso_info;
+
+static struct pvclock_vsyscall_time_info *
+pvclock_get_vsyscall_user_time_info(int cpu)
+{
+        if (!pvclock_vdso_info) {
+                BUG();
+                return NULL;
+        }
+
+        return &pvclock_vdso_info[cpu];
+}
+
+struct pvclock_vcpu_time_info *pvclock_get_vsyscall_time_info(int cpu)
+{
+        return &pvclock_get_vsyscall_user_time_info(cpu)->pvti;
+}
+
+#ifdef CONFIG_X86_64
+static int pvclock_task_migrate(struct notifier_block *nb, unsigned long l,
+                                void *v)
+{
+        struct task_migration_notifier *mn = v;
+        struct pvclock_vsyscall_time_info *pvti;
+
+        pvti = pvclock_get_vsyscall_user_time_info(mn->from_cpu);
+
+        /* this is NULL when pvclock vsyscall is not initialized */
+        if (unlikely(pvti == NULL))
+                return NOTIFY_DONE;
+
+        pvti->migrate_count++;
+
+        return NOTIFY_DONE;
+}
+
+static struct notifier_block pvclock_migrate = {
+        .notifier_call = pvclock_task_migrate,
+};
+
+/*
+ * Initialize the generic pvclock vsyscall state.  This will allocate
+ * a/some page(s) for the per-vcpu pvclock information, set up a
+ * fixmap mapping for the page(s)
+ */
+
+int __init pvclock_init_vsyscall(struct pvclock_vsyscall_time_info *i,
+                                 int size)
+{
+        int idx;
+
+        WARN_ON (size != PVCLOCK_VSYSCALL_NR_PAGES*PAGE_SIZE);
+
+        pvclock_vdso_info = i;
+
+        for (idx = 0; idx <= (PVCLOCK_FIXMAP_END-PVCLOCK_FIXMAP_BEGIN); idx++) {
+                __set_fixmap(PVCLOCK_FIXMAP_BEGIN + idx,
+                             __pa_symbol(i) + (idx*PAGE_SIZE),
+                             PAGE_KERNEL_VVAR);
+        }
+
+
+        register_task_migration_notifier(&pvclock_migrate);
+
+        return 0;
+}
+#endif