19 files changed, 128 insertions, 44 deletions

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 2bb986f305ac..4f86928246e7 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -1443,8 +1443,12 @@ BUILD_INTERRUPT3(hv_stimer0_callback_vector, HYPERV_STIMER0_VECTOR,
 
 ENTRY(page_fault)
         ASM_CLAC
-        pushl   $0; /* %gs's slot on the stack */
+        pushl   $do_page_fault
+        jmp     common_exception_read_cr2
+END(page_fault)
 
+common_exception_read_cr2:
+        /* the function address is in %gs's slot on the stack */
         SAVE_ALL switch_stacks=1 skip_gs=1
 
         ENCODE_FRAME_POINTER
@@ -1452,6 +1456,7 @@ ENTRY(page_fault)
 
         /* fixup %gs */
         GS_TO_REG %ecx
+        movl    PT_GS(%esp), %edi
         REG_TO_PTGS %ecx
         SET_KERNEL_GS %ecx
 
@@ -1463,9 +1468,9 @@ ENTRY(page_fault)
 
         TRACE_IRQS_OFF
         movl    %esp, %eax                      # pt_regs pointer
-        call    do_page_fault
+        CALL_NOSPEC %edi
         jmp     ret_from_exception
-END(page_fault)
+END(common_exception_read_cr2)
 
 common_exception:
         /* the function address is in %gs's slot on the stack */
@@ -1595,7 +1600,7 @@ END(general_protection)
 ENTRY(async_page_fault)
         ASM_CLAC
         pushl   $do_async_page_fault
-        jmp     common_exception
+        jmp     common_exception_read_cr2
 END(async_page_fault)
 #endif
 
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 9e911a96972b..648260b5f367 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -20,7 +20,6 @@
 #include <asm/intel-family.h>
 #include <asm/apic.h>
 #include <asm/cpu_device_id.h>
-#include <asm/hypervisor.h>
 
 #include "../perf_event.h"
 
@@ -263,8 +262,8 @@ static struct event_constraint intel_icl_event_constraints[] = {
 };
 
 static struct extra_reg intel_icl_extra_regs[] __read_mostly = {
-        INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff9fffull, RSP_0),
-        INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff9fffull, RSP_1),
+        INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffbfffull, RSP_0),
+        INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffffbfffull, RSP_1),
         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
         INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
         EVENT_EXTRA_END
@@ -4053,7 +4052,7 @@ static bool check_msr(unsigned long msr, u64 mask)
          * Disable the check for real HW, so we don't
          * mess with potentionaly enabled registers:
          */
-        if (hypervisor_is_type(X86_HYPER_NATIVE))
+        if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
                 return true;
 
         /*
@@ -4955,6 +4954,7 @@ __init int intel_pmu_init(void)
 
         case INTEL_FAM6_SKYLAKE_X:
                 pmem = true;
+                /* fall through */
         case INTEL_FAM6_SKYLAKE_MOBILE:
         case INTEL_FAM6_SKYLAKE_DESKTOP:
         case INTEL_FAM6_KABYLAKE_MOBILE:
@@ -5004,6 +5004,7 @@ __init int intel_pmu_init(void)
         case INTEL_FAM6_ICELAKE_X:
         case INTEL_FAM6_ICELAKE_XEON_D:
                 pmem = true;
+                /* fall through */
         case INTEL_FAM6_ICELAKE_MOBILE:
         case INTEL_FAM6_ICELAKE_DESKTOP:
                 x86_pmu.late_ack = true;
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 2c8db2c19328..f1269e804e9b 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -851,7 +851,7 @@ struct event_constraint intel_skl_pebs_event_constraints[] = {
 
 struct event_constraint intel_icl_pebs_event_constraints[] = {
         INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x100000000ULL),   /* INST_RETIRED.PREC_DIST */
-        INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x400000000ULL),  /* SLOTS */
+        INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),  /* SLOTS */
 
         INTEL_PLD_CONSTRAINT(0x1cd, 0xff),                      /* MEM_TRANS_RETIRED.LOAD_LATENCY */
         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x1d0, 0xf),    /* MEM_INST_RETIRED.LOAD */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 8282b8d41209..7b0a4ee77313 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -607,15 +607,16 @@ struct kvm_vcpu_arch {
 
         /*
          * QEMU userspace and the guest each have their own FPU state.
-         * In vcpu_run, we switch between the user, maintained in the
-         * task_struct struct, and guest FPU contexts. While running a VCPU,
-         * the VCPU thread will have the guest FPU context.
+         * In vcpu_run, we switch between the user and guest FPU contexts.
+         * While running a VCPU, the VCPU thread will have the guest FPU
+         * context.
          *
          * Note that while the PKRU state lives inside the fpu registers,
          * it is switched out separately at VMENTER and VMEXIT time. The
          * "guest_fpu" state here contains the guest FPU context, with the
          * host PRKU bits.
          */
+        struct fpu *user_fpu;
         struct fpu *guest_fpu;
 
         u64 xcr0;
diff --git a/arch/x86/include/uapi/asm/byteorder.h b/arch/x86/include/uapi/asm/byteorder.h
index 484e3cfd7ef2..149143cab9ff 100644
--- a/arch/x86/include/uapi/asm/byteorder.h
+++ b/arch/x86/include/uapi/asm/byteorder.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 #ifndef _ASM_X86_BYTEORDER_H
 #define _ASM_X86_BYTEORDER_H
 
diff --git a/arch/x86/include/uapi/asm/hwcap2.h b/arch/x86/include/uapi/asm/hwcap2.h
index 6ebaae90e207..8b2effe6efb8 100644
--- a/arch/x86/include/uapi/asm/hwcap2.h
+++ b/arch/x86/include/uapi/asm/hwcap2.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 #ifndef _ASM_X86_HWCAP2_H
 #define _ASM_X86_HWCAP2_H
 
diff --git a/arch/x86/include/uapi/asm/sigcontext32.h b/arch/x86/include/uapi/asm/sigcontext32.h
index 6b18e88de8a6..7114801d0499 100644
--- a/arch/x86/include/uapi/asm/sigcontext32.h
+++ b/arch/x86/include/uapi/asm/sigcontext32.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 #ifndef _ASM_X86_SIGCONTEXT32_H
 #define _ASM_X86_SIGCONTEXT32_H
 
diff --git a/arch/x86/include/uapi/asm/types.h b/arch/x86/include/uapi/asm/types.h
index df55e1ddb0c9..9d5c11a24279 100644
--- a/arch/x86/include/uapi/asm/types.h
+++ b/arch/x86/include/uapi/asm/types.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 #ifndef _ASM_X86_TYPES_H
 #define _ASM_X86_TYPES_H
 
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 66ca906aa790..801ecd1c3fd5 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -1226,7 +1226,7 @@ static ssize_t l1tf_show_state(char *buf)
 
 static ssize_t mds_show_state(char *buf)
 {
-        if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
+        if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
                 return sprintf(buf, "%s; SMT Host state unknown\n",
                                mds_strings[mds_mitigation]);
         }
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index a6342c899be5..f3d3e9646a99 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -193,10 +193,10 @@ ENTRY(secondary_startup_64)
 
         /* Set up %gs.
          *
-         * The base of %gs always points to the bottom of the irqstack
-         * union.  If the stack protector canary is enabled, it is
-         * located at %gs:40.  Note that, on SMP, the boot cpu uses
-         * init data section till per cpu areas are set up.
+         * The base of %gs always points to fixed_percpu_data. If the
+         * stack protector canary is enabled, it is located at %gs:40.
+         * Note that, on SMP, the boot cpu uses init data section until
+         * the per cpu areas are set up.
          */
         movl    $MSR_GS_BASE,%ecx
         movl    initial_gs(%rip),%eax
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index c43e96a938d0..c6f791bc481e 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -827,10 +827,6 @@ int __init hpet_enable(void)
         if (!hpet_cfg_working())
                 goto out_nohpet;
 
-        /* Validate that the counter is counting */
-        if (!hpet_counting())
-                goto out_nohpet;
-
         /*
          * Read the period and check for a sane value:
          */
@@ -896,6 +892,14 @@ int __init hpet_enable(void)
         }
         hpet_print_config();
 
+        /*
+         * Validate that the counter is counting. This needs to be done
+         * after sanitizing the config registers to properly deal with
+         * force enabled HPETs.
+         */
+        if (!hpet_counting())
+                goto out_nohpet;
+
         clocksource_register_hz(&clocksource_hpet, (u32)hpet_freq);
 
         if (id & HPET_ID_LEGSUP) {
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index 4f36d3241faf..2d6898c2cb64 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -100,7 +100,7 @@ copy_stack_frame(const void __user *fp, struct stack_frame_user *frame)
 {
         int ret;
 
-        if (!access_ok(fp, sizeof(*frame)))
+        if (__range_not_ok(fp, sizeof(*frame), TASK_SIZE))
                 return 0;
 
         ret = 1;
diff --git a/arch/x86/kernel/sysfb_efi.c b/arch/x86/kernel/sysfb_efi.c
index 8eb67a670b10..653b7f617b61 100644
--- a/arch/x86/kernel/sysfb_efi.c
+++ b/arch/x86/kernel/sysfb_efi.c
@@ -230,9 +230,55 @@ static const struct dmi_system_id efifb_dmi_system_table[] __initconst = {
         {},
 };
 
+/*
+ * Some devices have a portrait LCD but advertise a landscape resolution (and
+ * pitch). We simply swap width and height for these devices so that we can
+ * correctly deal with some of them coming with multiple resolutions.
+ */
+static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
+        {
+                /*
+                 * Lenovo MIIX310-10ICR, only some batches have the troublesome
+                 * 800x1280 portrait screen. Luckily the portrait version has
+                 * its own BIOS version, so we match on that.
+                 */
+                .matches = {
+                        DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+                        DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "MIIX 310-10ICR"),
+                        DMI_EXACT_MATCH(DMI_BIOS_VERSION, "1HCN44WW"),
+                },
+        },
+        {
+                /* Lenovo MIIX 320-10ICR with 800x1280 portrait screen */
+                .matches = {
+                        DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+                        DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
+                                        "Lenovo MIIX 320-10ICR"),
+                },
+        },
+        {
+                /* Lenovo D330 with 800x1280 or 1200x1920 portrait screen */
+                .matches = {
+                        DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+                        DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
+                                        "Lenovo ideapad D330-10IGM"),
+                },
+        },
+        {},
+};
+
 __init void sysfb_apply_efi_quirks(void)
 {
         if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI ||
             !(screen_info.capabilities & VIDEO_CAPABILITY_SKIP_QUIRKS))
                 dmi_check_system(efifb_dmi_system_table);
+
+        if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI &&
+            dmi_check_system(efifb_dmi_swap_width_height)) {
+                u16 temp = screen_info.lfb_width;
+
+                screen_info.lfb_width = screen_info.lfb_height;
+                screen_info.lfb_height = temp;
+                screen_info.lfb_linelength = 4 * screen_info.lfb_width;
+        }
 }
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 8f72526e2f68..24843cf49579 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3466,7 +3466,7 @@ static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
                 /*
                  * Currently, fast page fault only works for direct mapping
                  * since the gfn is not stable for indirect shadow page. See
-                 * Documentation/virtual/kvm/locking.txt to get more detail.
+                 * Documentation/virt/kvm/locking.txt to get more detail.
                  */
                 fault_handled = fast_pf_fix_direct_spte(vcpu, sp,
                                                         iterator.sptep, spte,
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 19f69df96758..7eafc6907861 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -2143,12 +2143,20 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
                 goto out;
         }
 
+        svm->vcpu.arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
+                                                     GFP_KERNEL_ACCOUNT);
+        if (!svm->vcpu.arch.user_fpu) {
+                printk(KERN_ERR "kvm: failed to allocate kvm userspace's fpu\n");
+                err = -ENOMEM;
+                goto free_partial_svm;
+        }
+
         svm->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
                                                      GFP_KERNEL_ACCOUNT);
         if (!svm->vcpu.arch.guest_fpu) {
                 printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n");
                 err = -ENOMEM;
-                goto free_partial_svm;
+                goto free_user_fpu;
         }
 
         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
@@ -2211,6 +2219,8 @@ uninit:
         kvm_vcpu_uninit(&svm->vcpu);
 free_svm:
         kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.guest_fpu);
+free_user_fpu:
+        kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.user_fpu);
 free_partial_svm:
         kmem_cache_free(kvm_vcpu_cache, svm);
 out:
@@ -2241,6 +2251,7 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu)
         __free_page(virt_to_page(svm->nested.hsave));
         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
         kvm_vcpu_uninit(vcpu);
+        kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.user_fpu);
         kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.guest_fpu);
         kmem_cache_free(kvm_vcpu_cache, svm);
 }
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 0f1378789bd0..ced9fba32598 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -220,6 +220,8 @@ static void free_nested(struct kvm_vcpu *vcpu)
         if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
                 return;
 
+        kvm_clear_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
+
         vmx->nested.vmxon = false;
         vmx->nested.smm.vmxon = false;
         free_vpid(vmx->nested.vpid02);
@@ -232,7 +234,9 @@ static void free_nested(struct kvm_vcpu *vcpu)
                 vmx->vmcs01.shadow_vmcs = NULL;
         }
         kfree(vmx->nested.cached_vmcs12);
+        vmx->nested.cached_vmcs12 = NULL;
         kfree(vmx->nested.cached_shadow_vmcs12);
+        vmx->nested.cached_shadow_vmcs12 = NULL;
         /* Unpin physical memory we referred to in the vmcs02 */
         if (vmx->nested.apic_access_page) {
                 kvm_release_page_dirty(vmx->nested.apic_access_page);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index a279447eb75b..074385c86c09 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -6598,6 +6598,7 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
         free_loaded_vmcs(vmx->loaded_vmcs);
         kfree(vmx->guest_msrs);
         kvm_vcpu_uninit(vcpu);
+        kmem_cache_free(x86_fpu_cache, vmx->vcpu.arch.user_fpu);
         kmem_cache_free(x86_fpu_cache, vmx->vcpu.arch.guest_fpu);
         kmem_cache_free(kvm_vcpu_cache, vmx);
 }
@@ -6613,12 +6614,20 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
         if (!vmx)
                 return ERR_PTR(-ENOMEM);
 
+        vmx->vcpu.arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
+                        GFP_KERNEL_ACCOUNT);
+        if (!vmx->vcpu.arch.user_fpu) {
+                printk(KERN_ERR "kvm: failed to allocate kvm userspace's fpu\n");
+                err = -ENOMEM;
+                goto free_partial_vcpu;
+        }
+
         vmx->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
                         GFP_KERNEL_ACCOUNT);
         if (!vmx->vcpu.arch.guest_fpu) {
                 printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n");
                 err = -ENOMEM;
-                goto free_partial_vcpu;
+                goto free_user_fpu;
         }
 
         vmx->vpid = allocate_vpid();
@@ -6721,6 +6730,8 @@ uninit_vcpu:
 free_vcpu:
         free_vpid(vmx->vpid);
         kmem_cache_free(x86_fpu_cache, vmx->vcpu.arch.guest_fpu);
+free_user_fpu:
+        kmem_cache_free(x86_fpu_cache, vmx->vcpu.arch.user_fpu);
 free_partial_vcpu:
         kmem_cache_free(kvm_vcpu_cache, vmx);
         return ERR_PTR(err);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 58305cf81182..c6d951cbd76c 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3306,6 +3306,10 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
         kvm_x86_ops->vcpu_load(vcpu, cpu);
 
+        fpregs_assert_state_consistent();
+        if (test_thread_flag(TIF_NEED_FPU_LOAD))
+                switch_fpu_return();
+
         /* Apply any externally detected TSC adjustments (due to suspend) */
         if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
                 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
@@ -7202,7 +7206,7 @@ static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
 
         rcu_read_unlock();
 
-        if (target)
+        if (target && READ_ONCE(target->ready))
                 kvm_vcpu_yield_to(target);
 }
 
@@ -7242,6 +7246,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
                 break;
         case KVM_HC_KICK_CPU:
                 kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
+                kvm_sched_yield(vcpu->kvm, a1);
                 ret = 0;
                 break;
 #ifdef CONFIG_X86_64
@@ -7990,9 +7995,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
         trace_kvm_entry(vcpu->vcpu_id);
         guest_enter_irqoff();
 
-        fpregs_assert_state_consistent();
-        if (test_thread_flag(TIF_NEED_FPU_LOAD))
-                switch_fpu_return();
+        /* The preempt notifier should have taken care of the FPU already.  */
+        WARN_ON_ONCE(test_thread_flag(TIF_NEED_FPU_LOAD));
 
         if (unlikely(vcpu->arch.switch_db_regs)) {
                 set_debugreg(0, 7);
@@ -8270,7 +8274,7 @@ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 {
         fpregs_lock();
 
-        copy_fpregs_to_fpstate(&current->thread.fpu);
+        copy_fpregs_to_fpstate(vcpu->arch.user_fpu);
         /* PKRU is separately restored in kvm_x86_ops->run.  */
         __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
                                 ~XFEATURE_MASK_PKRU);
@@ -8287,7 +8291,7 @@ static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
         fpregs_lock();
 
         copy_fpregs_to_fpstate(vcpu->arch.guest_fpu);
-        copy_kernel_to_fpregs(&current->thread.fpu.state);
+        copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
 
         fpregs_mark_activate();
         fpregs_unlock();
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 6c46095cd0d9..9ceacd1156db 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -177,13 +177,14 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
 
         pmd = pmd_offset(pud, address);
         pmd_k = pmd_offset(pud_k, address);
-        if (!pmd_present(*pmd_k))
-                return NULL;
 
-        if (!pmd_present(*pmd))
+        if (pmd_present(*pmd) != pmd_present(*pmd_k))
                 set_pmd(pmd, *pmd_k);
+
+        if (!pmd_present(*pmd_k))
+                return NULL;
         else
-                BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
+                BUG_ON(pmd_pfn(*pmd) != pmd_pfn(*pmd_k));
 
         return pmd_k;
 }
@@ -203,17 +204,13 @@ void vmalloc_sync_all(void)
                 spin_lock(&pgd_lock);
                 list_for_each_entry(page, &pgd_list, lru) {
                         spinlock_t *pgt_lock;
-                        pmd_t *ret;
 
                         /* the pgt_lock only for Xen */
                         pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
 
                         spin_lock(pgt_lock);
-                        ret = vmalloc_sync_one(page_address(page), address);
+                        vmalloc_sync_one(page_address(page), address);
                         spin_unlock(pgt_lock);
-
-                        if (!ret)
-                                break;
                 }
                 spin_unlock(&pgd_lock);
         }