Merge branch 'sched/urgent' into sched/core, to pick up fixes before applying new patches

Signed-off-by: Ingo Molnar <mingo@kernel.org>

32 files changed, 452 insertions, 303 deletions

diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index 92003f3c8a42..efc3b22d896e 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -213,7 +213,15 @@ void register_lapic_address(unsigned long address);
 extern void setup_boot_APIC_clock(void);
 extern void setup_secondary_APIC_clock(void);
 extern int APIC_init_uniprocessor(void);
+
+#ifdef CONFIG_X86_64
+static inline int apic_force_enable(unsigned long addr)
+{
+        return -1;
+}
+#else
 extern int apic_force_enable(unsigned long addr);
+#endif
 
 extern int apic_bsp_setup(bool upmode);
 extern void apic_ap_setup(void);
diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h
index 0dbc08282291..72ba21a8b5fc 100644
--- a/arch/x86/include/asm/fpu-internal.h
+++ b/arch/x86/include/asm/fpu-internal.h
@@ -370,7 +370,7 @@ static inline void drop_fpu(struct task_struct *tsk)
         preempt_disable();
         tsk->thread.fpu_counter = 0;
         __drop_fpu(tsk);
-        clear_used_math();
+        clear_stopped_child_used_math(tsk);
         preempt_enable();
 }
 
diff --git a/arch/x86/include/asm/imr.h b/arch/x86/include/asm/imr.h
new file mode 100644
index 000000000000..cd2ce4068441
--- /dev/null
+++ b/arch/x86/include/asm/imr.h
@@ -0,0 +1,60 @@
+/*
+ * imr.h: Isolated Memory Region API
+ *
+ * Copyright(c) 2013 Intel Corporation.
+ * Copyright(c) 2015 Bryan O'Donoghue <pure.logic@nexus-software.ie>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+#ifndef _IMR_H
+#define _IMR_H
+
+#include <linux/types.h>
+
+/*
+ * IMR agent access mask bits
+ * See section 12.7.4.7 from quark-x1000-datasheet.pdf for register
+ * definitions.
+ */
+#define IMR_ESRAM_FLUSH         BIT(31)
+#define IMR_CPU_SNOOP           BIT(30)         /* Applicable only to write */
+#define IMR_RMU                 BIT(29)
+#define IMR_VC1_SAI_ID3         BIT(15)
+#define IMR_VC1_SAI_ID2         BIT(14)
+#define IMR_VC1_SAI_ID1         BIT(13)
+#define IMR_VC1_SAI_ID0         BIT(12)
+#define IMR_VC0_SAI_ID3         BIT(11)
+#define IMR_VC0_SAI_ID2         BIT(10)
+#define IMR_VC0_SAI_ID1         BIT(9)
+#define IMR_VC0_SAI_ID0         BIT(8)
+#define IMR_CPU_0               BIT(1)          /* SMM mode */
+#define IMR_CPU                 BIT(0)          /* Non SMM mode */
+#define IMR_ACCESS_NONE         0
+
+/*
+ * Read/Write access-all bits here include some reserved bits
+ * These are the values firmware uses and are accepted by hardware.
+ * The kernel defines read/write access-all in the same way as firmware
+ * in order to have a consistent and crisp definition across firmware,
+ * bootloader and kernel.
+ */
+#define IMR_READ_ACCESS_ALL     0xBFFFFFFF
+#define IMR_WRITE_ACCESS_ALL    0xFFFFFFFF
+
+/* Number of IMRs provided by Quark X1000 SoC */
+#define QUARK_X1000_IMR_MAX     0x08
+#define QUARK_X1000_IMR_REGBASE 0x40
+
+/* IMR alignment bits - only bits 31:10 are checked for IMR validity */
+#define IMR_ALIGN               0x400
+#define IMR_MASK                (IMR_ALIGN - 1)
+
+int imr_add_range(phys_addr_t base, size_t size,
+                  unsigned int rmask, unsigned int wmask, bool lock);
+
+int imr_remove_range(phys_addr_t base, size_t size);
+
+#endif /* _IMR_H */
diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h
index e34e097b6f9d..705d35708a50 100644
--- a/arch/x86/include/asm/intel-mid.h
+++ b/arch/x86/include/asm/intel-mid.h
@@ -136,9 +136,6 @@ extern enum intel_mid_timer_options intel_mid_timer_options;
 #define SFI_MTMR_MAX_NUM 8
 #define SFI_MRTC_MAX    8
 
-extern struct console early_mrst_console;
-extern void mrst_early_console_init(void);
-
 extern struct console early_hsu_console;
 extern void hsu_early_console_init(const char *);
 
diff --git a/arch/x86/include/asm/kasan.h b/arch/x86/include/asm/kasan.h
new file mode 100644
index 000000000000..8b22422fbad8
--- /dev/null
+++ b/arch/x86/include/asm/kasan.h
@@ -0,0 +1,31 @@
+#ifndef _ASM_X86_KASAN_H
+#define _ASM_X86_KASAN_H
+
+/*
+ * Compiler uses shadow offset assuming that addresses start
+ * from 0. Kernel addresses don't start from 0, so shadow
+ * for kernel really starts from compiler's shadow offset +
+ * 'kernel address space start' >> KASAN_SHADOW_SCALE_SHIFT
+ */
+#define KASAN_SHADOW_START      (KASAN_SHADOW_OFFSET + \
+                                        (0xffff800000000000ULL >> 3))
+/* 47 bits for kernel address -> (47 - 3) bits for shadow */
+#define KASAN_SHADOW_END        (KASAN_SHADOW_START + (1ULL << (47 - 3)))
+
+#ifndef __ASSEMBLY__
+
+extern pte_t kasan_zero_pte[];
+extern pte_t kasan_zero_pmd[];
+extern pte_t kasan_zero_pud[];
+
+#ifdef CONFIG_KASAN
+void __init kasan_map_early_shadow(pgd_t *pgd);
+void __init kasan_init(void);
+#else
+static inline void kasan_map_early_shadow(pgd_t *pgd) { }
+static inline void kasan_init(void) { }
+#endif
+
+#endif
+
+#endif
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index eb181178fe0b..57a9d94fe160 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -208,6 +208,7 @@ struct x86_emulate_ops {
 
         void (*get_cpuid)(struct x86_emulate_ctxt *ctxt,
                           u32 *eax, u32 *ebx, u32 *ecx, u32 *edx);
+        void (*set_nmi_mask)(struct x86_emulate_ctxt *ctxt, bool masked);
 };
 
 typedef u32 __attribute__((vector_size(16))) sse128_t;
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index d89c6b828c96..a236e39cc385 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -38,8 +38,6 @@
 #define KVM_PRIVATE_MEM_SLOTS 3
 #define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 
-#define KVM_MMIO_SIZE 16
-
 #define KVM_PIO_PAGE_OFFSET 1
 #define KVM_COALESCED_MMIO_PAGE_OFFSET 2
 
@@ -51,7 +49,7 @@
                           | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
 
 #define CR3_L_MODE_RESERVED_BITS 0xFFFFFF0000000000ULL
-#define CR3_PCID_INVD            (1UL << 63)
+#define CR3_PCID_INVD            BIT_64(63)
 #define CR4_RESERVED_BITS                                               \
         (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
                           | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
@@ -160,6 +158,18 @@ enum {
 #define DR7_FIXED_1     0x00000400
 #define DR7_VOLATILE    0xffff2bff
 
+#define PFERR_PRESENT_BIT 0
+#define PFERR_WRITE_BIT 1
+#define PFERR_USER_BIT 2
+#define PFERR_RSVD_BIT 3
+#define PFERR_FETCH_BIT 4
+
+#define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
+#define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
+#define PFERR_USER_MASK (1U << PFERR_USER_BIT)
+#define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
+#define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
+
 /* apic attention bits */
 #define KVM_APIC_CHECK_VAPIC    0
 /*
@@ -615,6 +625,8 @@ struct kvm_arch {
         #ifdef CONFIG_KVM_MMU_AUDIT
         int audit_point;
         #endif
+
+        bool boot_vcpu_runs_old_kvmclock;
 };
 
 struct kvm_vm_stat {
@@ -643,6 +655,7 @@ struct kvm_vcpu_stat {
         u32 irq_window_exits;
         u32 nmi_window_exits;
         u32 halt_exits;
+        u32 halt_successful_poll;
         u32 halt_wakeup;
         u32 request_irq_exits;
         u32 irq_exits;
@@ -787,6 +800,31 @@ struct kvm_x86_ops {
         int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
 
         void (*sched_in)(struct kvm_vcpu *kvm, int cpu);
+
+        /*
+         * Arch-specific dirty logging hooks. These hooks are only supposed to
+         * be valid if the specific arch has hardware-accelerated dirty logging
+         * mechanism. Currently only for PML on VMX.
+         *
+         *  - slot_enable_log_dirty:
+         *      called when enabling log dirty mode for the slot.
+         *  - slot_disable_log_dirty:
+         *      called when disabling log dirty mode for the slot.
+         *      also called when slot is created with log dirty disabled.
+         *  - flush_log_dirty:
+         *      called before reporting dirty_bitmap to userspace.
+         *  - enable_log_dirty_pt_masked:
+         *      called when reenabling log dirty for the GFNs in the mask after
+         *      corresponding bits are cleared in slot->dirty_bitmap.
+         */
+        void (*slot_enable_log_dirty)(struct kvm *kvm,
+                                      struct kvm_memory_slot *slot);
+        void (*slot_disable_log_dirty)(struct kvm *kvm,
+                                       struct kvm_memory_slot *slot);
+        void (*flush_log_dirty)(struct kvm *kvm);
+        void (*enable_log_dirty_pt_masked)(struct kvm *kvm,
+                                           struct kvm_memory_slot *slot,
+                                           gfn_t offset, unsigned long mask);
 };
 
 struct kvm_arch_async_pf {
@@ -819,10 +857,17 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask);
 
 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
-void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot);
-void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
-                                     struct kvm_memory_slot *slot,
-                                     gfn_t gfn_offset, unsigned long mask);
+void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
+                                      struct kvm_memory_slot *memslot);
+void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
+                                   struct kvm_memory_slot *memslot);
+void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
+                                        struct kvm_memory_slot *memslot);
+void kvm_mmu_slot_set_dirty(struct kvm *kvm,
+                            struct kvm_memory_slot *memslot);
+void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
+                                   struct kvm_memory_slot *slot,
+                                   gfn_t gfn_offset, unsigned long mask);
 void kvm_mmu_zap_all(struct kvm *kvm);
 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm);
 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm);
diff --git a/arch/x86/include/asm/lguest_hcall.h b/arch/x86/include/asm/lguest_hcall.h
index 879fd7d33877..ef01fef3eebc 100644
--- a/arch/x86/include/asm/lguest_hcall.h
+++ b/arch/x86/include/asm/lguest_hcall.h
@@ -16,7 +16,6 @@
 #define LHCALL_SET_PTE          14
 #define LHCALL_SET_PGD          15
 #define LHCALL_LOAD_TLS         16
-#define LHCALL_NOTIFY           17
 #define LHCALL_LOAD_GDT_ENTRY   18
 #define LHCALL_SEND_INTERRUPTS  19
 
diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h
new file mode 100644
index 000000000000..a455a53d789a
--- /dev/null
+++ b/arch/x86/include/asm/livepatch.h
@@ -0,0 +1,46 @@
+/*
+ * livepatch.h - x86-specific Kernel Live Patching Core
+ *
+ * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
+ * Copyright (C) 2014 SUSE
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _ASM_X86_LIVEPATCH_H
+#define _ASM_X86_LIVEPATCH_H
+
+#include <linux/module.h>
+#include <linux/ftrace.h>
+
+#ifdef CONFIG_LIVEPATCH
+static inline int klp_check_compiler_support(void)
+{
+#ifndef CC_USING_FENTRY
+        return 1;
+#endif
+        return 0;
+}
+extern int klp_write_module_reloc(struct module *mod, unsigned long type,
+                                  unsigned long loc, unsigned long value);
+
+static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip)
+{
+        regs->ip = ip;
+}
+#else
+#error Live patching support is disabled; check CONFIG_LIVEPATCH
+#endif
+
+#endif /* _ASM_X86_LIVEPATCH_H */
diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h
index 876e74e8eec7..09b9620a73b4 100644
--- a/arch/x86/include/asm/mmu.h
+++ b/arch/x86/include/asm/mmu.h
@@ -19,6 +19,8 @@ typedef struct {
 
         struct mutex lock;
         void __user *vdso;
+
+        atomic_t perf_rdpmc_allowed;    /* nonzero if rdpmc is allowed */
 } mm_context_t;
 
 #ifdef CONFIG_SMP
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 4b75d591eb5e..883f6b933fa4 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -18,6 +18,21 @@ static inline void paravirt_activate_mm(struct mm_struct *prev,
 }
 #endif  /* !CONFIG_PARAVIRT */
 
+#ifdef CONFIG_PERF_EVENTS
+extern struct static_key rdpmc_always_available;
+
+static inline void load_mm_cr4(struct mm_struct *mm)
+{
+        if (static_key_true(&rdpmc_always_available) ||
+            atomic_read(&mm->context.perf_rdpmc_allowed))
+                cr4_set_bits(X86_CR4_PCE);
+        else
+                cr4_clear_bits(X86_CR4_PCE);
+}
+#else
+static inline void load_mm_cr4(struct mm_struct *mm) {}
+#endif
+
 /*
  * Used for LDT copy/destruction.
  */
@@ -52,15 +67,20 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                 /* Stop flush ipis for the previous mm */
                 cpumask_clear_cpu(cpu, mm_cpumask(prev));
 
+                /* Load per-mm CR4 state */
+                load_mm_cr4(next);
+
                 /*
                  * Load the LDT, if the LDT is different.
                  *
-                 * It's possible leave_mm(prev) has been called.  If so,
-                 * then prev->context.ldt could be out of sync with the
-                 * LDT descriptor or the LDT register.  This can only happen
-                 * if prev->context.ldt is non-null, since we never free
-                 * an LDT.  But LDTs can't be shared across mms, so
-                 * prev->context.ldt won't be equal to next->context.ldt.
+                 * It's possible that prev->context.ldt doesn't match
+                 * the LDT register.  This can happen if leave_mm(prev)
+                 * was called and then modify_ldt changed
+                 * prev->context.ldt but suppressed an IPI to this CPU.
+                 * In this case, prev->context.ldt != NULL, because we
+                 * never free an LDT while the mm still exists.  That
+                 * means that next->context.ldt != prev->context.ldt,
+                 * because mms never share an LDT.
                  */
                 if (unlikely(prev->context.ldt != next->context.ldt))
                         load_LDT_nolock(&next->context);
@@ -85,6 +105,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                          */
                         load_cr3(next->pgd);
                         trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
+                        load_mm_cr4(next);
                         load_LDT_nolock(&next->context);
                 }
         }
diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h
index 75450b2c7be4..4edd53b79a81 100644
--- a/arch/x86/include/asm/page_64_types.h
+++ b/arch/x86/include/asm/page_64_types.h
@@ -1,17 +1,23 @@
 #ifndef _ASM_X86_PAGE_64_DEFS_H
 #define _ASM_X86_PAGE_64_DEFS_H
 
-#define THREAD_SIZE_ORDER       2
+#ifdef CONFIG_KASAN
+#define KASAN_STACK_ORDER 1
+#else
+#define KASAN_STACK_ORDER 0
+#endif
+
+#define THREAD_SIZE_ORDER       (2 + KASAN_STACK_ORDER)
 #define THREAD_SIZE  (PAGE_SIZE << THREAD_SIZE_ORDER)
 #define CURRENT_MASK (~(THREAD_SIZE - 1))
 
-#define EXCEPTION_STACK_ORDER 0
+#define EXCEPTION_STACK_ORDER (0 + KASAN_STACK_ORDER)
 #define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER)
 
 #define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1)
 #define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER)
 
-#define IRQ_STACK_ORDER 2
+#define IRQ_STACK_ORDER (2 + KASAN_STACK_ORDER)
 #define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER)
 
 #define DOUBLEFAULT_STACK 1
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 32444ae939ca..965c47d254aa 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -80,16 +80,16 @@ static inline void write_cr3(unsigned long x)
         PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
 }
 
-static inline unsigned long read_cr4(void)
+static inline unsigned long __read_cr4(void)
 {
         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
 }
-static inline unsigned long read_cr4_safe(void)
+static inline unsigned long __read_cr4_safe(void)
 {
         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
 }
 
-static inline void write_cr4(unsigned long x)
+static inline void __write_cr4(unsigned long x)
 {
         PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
 }
diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h
index 206a87fdd22d..fd74a11959de 100644
--- a/arch/x86/include/asm/pgtable-2level.h
+++ b/arch/x86/include/asm/pgtable-2level.h
@@ -62,44 +62,8 @@ static inline unsigned long pte_bitop(unsigned long value, unsigned int rightshi
         return ((value >> rightshift) & mask) << leftshift;
 }
 
-/*
- * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE and _PAGE_BIT_PROTNONE are taken,
- * split up the 29 bits of offset into this range.
- */
-#define PTE_FILE_MAX_BITS       29
-#define PTE_FILE_SHIFT1         (_PAGE_BIT_PRESENT + 1)
-#define PTE_FILE_SHIFT2         (_PAGE_BIT_FILE + 1)
-#define PTE_FILE_SHIFT3         (_PAGE_BIT_PROTNONE + 1)
-#define PTE_FILE_BITS1          (PTE_FILE_SHIFT2 - PTE_FILE_SHIFT1 - 1)
-#define PTE_FILE_BITS2          (PTE_FILE_SHIFT3 - PTE_FILE_SHIFT2 - 1)
-
-#define PTE_FILE_MASK1          ((1U << PTE_FILE_BITS1) - 1)
-#define PTE_FILE_MASK2          ((1U << PTE_FILE_BITS2) - 1)
-
-#define PTE_FILE_LSHIFT2        (PTE_FILE_BITS1)
-#define PTE_FILE_LSHIFT3        (PTE_FILE_BITS1 + PTE_FILE_BITS2)
-
-static __always_inline pgoff_t pte_to_pgoff(pte_t pte)
-{
-        return (pgoff_t)
-                (pte_bitop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1,  0)                +
-                 pte_bitop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2,  PTE_FILE_LSHIFT2) +
-                 pte_bitop(pte.pte_low, PTE_FILE_SHIFT3,           -1UL,  PTE_FILE_LSHIFT3));
-}
-
-static __always_inline pte_t pgoff_to_pte(pgoff_t off)
-{
-        return (pte_t){
-                .pte_low =
-                        pte_bitop(off,                0, PTE_FILE_MASK1,  PTE_FILE_SHIFT1) +
-                        pte_bitop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2,  PTE_FILE_SHIFT2) +
-                        pte_bitop(off, PTE_FILE_LSHIFT3,           -1UL,  PTE_FILE_SHIFT3) +
-                        _PAGE_FILE,
-        };
-}
-
 /* Encode and de-code a swap entry */
-#define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1)
+#define SWP_TYPE_BITS 5
 #define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1)
 
 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)
diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index 81bb91b49a88..cdaa58c9b39e 100644
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -176,18 +176,6 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
 #endif
 
-/*
- * Bits 0, 6 and 7 are taken in the low part of the pte,
- * put the 32 bits of offset into the high part.
- *
- * For soft-dirty tracking 11 bit is taken from
- * the low part of pte as well.
- */
-#define pte_to_pgoff(pte) ((pte).pte_high)
-#define pgoff_to_pte(off)                                               \
-        ((pte_t) { { .pte_low = _PAGE_FILE, .pte_high = (off) } })
-#define PTE_FILE_MAX_BITS       32
-
 /* Encode and de-code a swap entry */
 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5)
 #define __swp_type(x)                   (((x).val) & 0x1f)
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index e8a5454acc99..a0c35bf6cb92 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -115,11 +115,6 @@ static inline int pte_write(pte_t pte)
         return pte_flags(pte) & _PAGE_RW;
 }
 
-static inline int pte_file(pte_t pte)
-{
-        return pte_flags(pte) & _PAGE_FILE;
-}
-
 static inline int pte_huge(pte_t pte)
 {
         return pte_flags(pte) & _PAGE_PSE;
@@ -137,13 +132,7 @@ static inline int pte_exec(pte_t pte)
 
 static inline int pte_special(pte_t pte)
 {
-        /*
-         * See CONFIG_NUMA_BALANCING pte_numa in include/asm-generic/pgtable.h.
-         * On x86 we have _PAGE_BIT_NUMA == _PAGE_BIT_GLOBAL+1 ==
-         * __PAGE_BIT_SOFTW1 == _PAGE_BIT_SPECIAL.
-         */
-        return (pte_flags(pte) & _PAGE_SPECIAL) &&
-                (pte_flags(pte) & (_PAGE_PRESENT|_PAGE_PROTNONE));
+        return pte_flags(pte) & _PAGE_SPECIAL;
 }
 
 static inline unsigned long pte_pfn(pte_t pte)
@@ -305,7 +294,7 @@ static inline pmd_t pmd_mkwrite(pmd_t pmd)
 
 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
 {
-        return pmd_clear_flags(pmd, _PAGE_PRESENT);
+        return pmd_clear_flags(pmd, _PAGE_PRESENT | _PAGE_PROTNONE);
 }
 
 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
@@ -329,21 +318,6 @@ static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
         return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
 }
 
-static inline pte_t pte_file_clear_soft_dirty(pte_t pte)
-{
-        return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
-}
-
-static inline pte_t pte_file_mksoft_dirty(pte_t pte)
-{
-        return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
-}
-
-static inline int pte_file_soft_dirty(pte_t pte)
-{
-        return pte_flags(pte) & _PAGE_SOFT_DIRTY;
-}
-
 #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
 
 /*
@@ -463,13 +437,6 @@ static inline int pte_same(pte_t a, pte_t b)
 
 static inline int pte_present(pte_t a)
 {
-        return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE |
-                               _PAGE_NUMA);
-}
-
-#define pte_present_nonuma pte_present_nonuma
-static inline int pte_present_nonuma(pte_t a)
-{
         return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
 }
 
@@ -479,7 +446,7 @@ static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
         if (pte_flags(a) & _PAGE_PRESENT)
                 return true;
 
-        if ((pte_flags(a) & (_PAGE_PROTNONE | _PAGE_NUMA)) &&
+        if ((pte_flags(a) & _PAGE_PROTNONE) &&
                         mm_tlb_flush_pending(mm))
                 return true;
 
@@ -499,10 +466,27 @@ static inline int pmd_present(pmd_t pmd)
          * the _PAGE_PSE flag will remain set at all times while the
          * _PAGE_PRESENT bit is clear).
          */
-        return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE |
-                                 _PAGE_NUMA);
+        return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+/*
+ * These work without NUMA balancing but the kernel does not care. See the
+ * comment in include/asm-generic/pgtable.h
+ */
+static inline int pte_protnone(pte_t pte)
+{
+        return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
+                == _PAGE_PROTNONE;
+}
+
+static inline int pmd_protnone(pmd_t pmd)
+{
+        return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
+                == _PAGE_PROTNONE;
+}
+#endif /* CONFIG_NUMA_BALANCING */
+
 static inline int pmd_none(pmd_t pmd)
 {
         /* Only check low word on 32-bit platforms, since it might be
@@ -559,11 +543,6 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
 
 static inline int pmd_bad(pmd_t pmd)
 {
-#ifdef CONFIG_NUMA_BALANCING
-        /* pmd_numa check */
-        if ((pmd_flags(pmd) & (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA)
-                return 0;
-#endif
         return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
 }
 
@@ -882,19 +861,16 @@ static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
 static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
 {
-        VM_BUG_ON(pte_present_nonuma(pte));
         return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
 }
 
 static inline int pte_swp_soft_dirty(pte_t pte)
 {
-        VM_BUG_ON(pte_present_nonuma(pte));
         return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
 }
 
 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
 {
-        VM_BUG_ON(pte_present_nonuma(pte));
         return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
 }
 #endif
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 4572b2f30237..2ee781114d34 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -133,10 +133,6 @@ static inline int pgd_large(pgd_t pgd) { return 0; }
 /* PUD - Level3 access */
 
 /* PMD  - Level 2 access */
-#define pte_to_pgoff(pte) ((pte_val((pte)) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
-#define pgoff_to_pte(off) ((pte_t) { .pte = ((off) << PAGE_SHIFT) |     \
-                                            _PAGE_FILE })
-#define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT
 
 /* PTE - Level 1 access. */
 
@@ -145,13 +141,8 @@ static inline int pgd_large(pgd_t pgd) { return 0; }
 #define pte_unmap(pte) ((void)(pte))/* NOP */
 
 /* Encode and de-code a swap entry */
-#define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1)
-#ifdef CONFIG_NUMA_BALANCING
-/* Automatic NUMA balancing needs to be distinguishable from swap entries */
-#define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 2)
-#else
+#define SWP_TYPE_BITS 5
 #define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1)
-#endif
 
 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)
 
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 25bcd4a89517..8c7c10802e9c 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -4,7 +4,7 @@
 #include <linux/const.h>
 #include <asm/page_types.h>
 
-#define FIRST_USER_ADDRESS      0
+#define FIRST_USER_ADDRESS      0UL
 
 #define _PAGE_BIT_PRESENT       0       /* is present */
 #define _PAGE_BIT_RW            1       /* writeable */
@@ -27,19 +27,9 @@
 #define _PAGE_BIT_SOFT_DIRTY    _PAGE_BIT_SOFTW3 /* software dirty tracking */
 #define _PAGE_BIT_NX           63       /* No execute: only valid after cpuid check */
 
-/*
- * Swap offsets on configurations that allow automatic NUMA balancing use the
- * bits after _PAGE_BIT_GLOBAL. To uniquely distinguish NUMA hinting PTEs from
- * swap entries, we use the first bit after _PAGE_BIT_GLOBAL and shrink the
- * maximum possible swap space from 16TB to 8TB.
- */
-#define _PAGE_BIT_NUMA          (_PAGE_BIT_GLOBAL+1)
-
 /* If _PAGE_BIT_PRESENT is clear, we use these: */
 /* - if the user mapped it with PROT_NONE; pte_present gives true */
 #define _PAGE_BIT_PROTNONE      _PAGE_BIT_GLOBAL
-/* - set: nonlinear file mapping, saved PTE; unset:swap */
-#define _PAGE_BIT_FILE          _PAGE_BIT_DIRTY
 
 #define _PAGE_PRESENT   (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT)
 #define _PAGE_RW        (_AT(pteval_t, 1) << _PAGE_BIT_RW)
@@ -78,21 +68,6 @@
 #endif
 
 /*
- * _PAGE_NUMA distinguishes between a numa hinting minor fault and a page
- * that is not present. The hinting fault gathers numa placement statistics
- * (see pte_numa()). The bit is always zero when the PTE is not present.
- *
- * The bit picked must be always zero when the pmd is present and not
- * present, so that we don't lose information when we set it while
- * atomically clearing the present bit.
- */
-#ifdef CONFIG_NUMA_BALANCING
-#define _PAGE_NUMA      (_AT(pteval_t, 1) << _PAGE_BIT_NUMA)
-#else
-#define _PAGE_NUMA      (_AT(pteval_t, 0))
-#endif
-
-/*
  * Tracking soft dirty bit when a page goes to a swap is tricky.
  * We need a bit which can be stored in pte _and_ not conflict
  * with swap entry format. On x86 bits 6 and 7 are *not* involved
@@ -114,7 +89,6 @@
 #define _PAGE_NX        (_AT(pteval_t, 0))
 #endif
 
-#define _PAGE_FILE      (_AT(pteval_t, 1) << _PAGE_BIT_FILE)
 #define _PAGE_PROTNONE  (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
 
 #define _PAGE_TABLE     (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |        \
@@ -125,8 +99,8 @@
 /* Set of bits not changed in pte_modify */
 #define _PAGE_CHG_MASK  (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |         \
                          _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \
-                         _PAGE_SOFT_DIRTY | _PAGE_NUMA)
-#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_NUMA)
+                         _PAGE_SOFT_DIRTY)
+#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)
 
 /*
  * The cache modes defined here are used to translate between pure SW usage
@@ -327,20 +301,6 @@ static inline pteval_t pte_flags(pte_t pte)
         return native_pte_val(pte) & PTE_FLAGS_MASK;
 }
 
-#ifdef CONFIG_NUMA_BALANCING
-/* Set of bits that distinguishes present, prot_none and numa ptes */
-#define _PAGE_NUMA_MASK (_PAGE_NUMA|_PAGE_PROTNONE|_PAGE_PRESENT)
-static inline pteval_t ptenuma_flags(pte_t pte)
-{
-        return pte_flags(pte) & _PAGE_NUMA_MASK;
-}
-
-static inline pmdval_t pmdnuma_flags(pmd_t pmd)
-{
-        return pmd_flags(pmd) & _PAGE_NUMA_MASK;
-}
-#endif /* CONFIG_NUMA_BALANCING */
-
 #define pgprot_val(x)   ((x).pgprot)
 #define __pgprot(x)     ((pgprot_t) { (x) } )
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index a092a0cce0b7..ec1c93588cef 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -579,39 +579,6 @@ static inline void load_sp0(struct tss_struct *tss,
 #define set_iopl_mask native_set_iopl_mask
 #endif /* CONFIG_PARAVIRT */
 
-/*
- * Save the cr4 feature set we're using (ie
- * Pentium 4MB enable and PPro Global page
- * enable), so that any CPU's that boot up
- * after us can get the correct flags.
- */
-extern unsigned long mmu_cr4_features;
-extern u32 *trampoline_cr4_features;
-
-static inline void set_in_cr4(unsigned long mask)
-{
-        unsigned long cr4;
-
-        mmu_cr4_features |= mask;
-        if (trampoline_cr4_features)
-                *trampoline_cr4_features = mmu_cr4_features;
-        cr4 = read_cr4();
-        cr4 |= mask;
-        write_cr4(cr4);
-}
-
-static inline void clear_in_cr4(unsigned long mask)
-{
-        unsigned long cr4;
-
-        mmu_cr4_features &= ~mask;
-        if (trampoline_cr4_features)
-                *trampoline_cr4_features = mmu_cr4_features;
-        cr4 = read_cr4();
-        cr4 &= ~mask;
-        write_cr4(cr4);
-}
-
 typedef struct {
         unsigned long           seg;
 } mm_segment_t;
diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h
index e820c080a4e9..6a4b00fafb00 100644
--- a/arch/x86/include/asm/special_insns.h
+++ b/arch/x86/include/asm/special_insns.h
@@ -137,17 +137,17 @@ static inline void write_cr3(unsigned long x)
         native_write_cr3(x);
 }
 
-static inline unsigned long read_cr4(void)
+static inline unsigned long __read_cr4(void)
 {
         return native_read_cr4();
 }
 
-static inline unsigned long read_cr4_safe(void)
+static inline unsigned long __read_cr4_safe(void)
 {
         return native_read_cr4_safe();
 }
 
-static inline void write_cr4(unsigned long x)
+static inline void __write_cr4(unsigned long x)
 {
         native_write_cr4(x);
 }
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 625660f8a2fc..cf87de3fc390 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -46,7 +46,7 @@ static __always_inline bool static_key_false(struct static_key *key);
 
 static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)
 {
-        set_bit(0, (volatile unsigned long *)&lock->tickets.tail);
+        set_bit(0, (volatile unsigned long *)&lock->tickets.head);
 }
 
 #else  /* !CONFIG_PARAVIRT_SPINLOCKS */
@@ -60,10 +60,30 @@ static inline void __ticket_unlock_kick(arch_spinlock_t *lock,
 }
 
 #endif /* CONFIG_PARAVIRT_SPINLOCKS */
+static inline int  __tickets_equal(__ticket_t one, __ticket_t two)
+{
+        return !((one ^ two) & ~TICKET_SLOWPATH_FLAG);
+}
+
+static inline void __ticket_check_and_clear_slowpath(arch_spinlock_t *lock,
+                                                        __ticket_t head)
+{
+        if (head & TICKET_SLOWPATH_FLAG) {
+                arch_spinlock_t old, new;
+
+                old.tickets.head = head;
+                new.tickets.head = head & ~TICKET_SLOWPATH_FLAG;
+                old.tickets.tail = new.tickets.head + TICKET_LOCK_INC;
+                new.tickets.tail = old.tickets.tail;
+
+                /* try to clear slowpath flag when there are no contenders */
+                cmpxchg(&lock->head_tail, old.head_tail, new.head_tail);
+        }
+}
 
 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
 {
-        return lock.tickets.head == lock.tickets.tail;
+        return __tickets_equal(lock.tickets.head, lock.tickets.tail);
 }
 
 /*
@@ -87,18 +107,21 @@ static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
         if (likely(inc.head == inc.tail))
                 goto out;
 
-        inc.tail &= ~TICKET_SLOWPATH_FLAG;
         for (;;) {
                 unsigned count = SPIN_THRESHOLD;
 
                 do {
-                        if (READ_ONCE(lock->tickets.head) == inc.tail)
-                                goto out;
+                        inc.head = READ_ONCE(lock->tickets.head);
+                        if (__tickets_equal(inc.head, inc.tail))
+                                goto clear_slowpath;
                         cpu_relax();
                 } while (--count);
                 __ticket_lock_spinning(lock, inc.tail);
         }
-out:    barrier();      /* make sure nothing creeps before the lock is taken */
+clear_slowpath:
+        __ticket_check_and_clear_slowpath(lock, inc.head);
+out:
+        barrier();      /* make sure nothing creeps before the lock is taken */
 }
 
 static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
@@ -106,56 +129,30 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
         arch_spinlock_t old, new;
 
         old.tickets = READ_ONCE(lock->tickets);
-        if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG))
+        if (!__tickets_equal(old.tickets.head, old.tickets.tail))
                 return 0;
 
         new.head_tail = old.head_tail + (TICKET_LOCK_INC << TICKET_SHIFT);
+        new.head_tail &= ~TICKET_SLOWPATH_FLAG;
 
         /* cmpxchg is a full barrier, so nothing can move before it */
         return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
 }
 
-static inline void __ticket_unlock_slowpath(arch_spinlock_t *lock,
-                                            arch_spinlock_t old)
-{
-        arch_spinlock_t new;
-
-        BUILD_BUG_ON(((__ticket_t)NR_CPUS) != NR_CPUS);
-
-        /* Perform the unlock on the "before" copy */
-        old.tickets.head += TICKET_LOCK_INC;
-
-        /* Clear the slowpath flag */
-        new.head_tail = old.head_tail & ~(TICKET_SLOWPATH_FLAG << TICKET_SHIFT);
-
-        /*
-         * If the lock is uncontended, clear the flag - use cmpxchg in
-         * case it changes behind our back though.
-         */
-        if (new.tickets.head != new.tickets.tail ||
-            cmpxchg(&lock->head_tail, old.head_tail,
-                                        new.head_tail) != old.head_tail) {
-                /*
-                 * Lock still has someone queued for it, so wake up an
-                 * appropriate waiter.
-                 */
-                __ticket_unlock_kick(lock, old.tickets.head);
-        }
-}
-
 static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
 {
         if (TICKET_SLOWPATH_FLAG &&
-            static_key_false(&paravirt_ticketlocks_enabled)) {
-                arch_spinlock_t prev;
+                static_key_false(&paravirt_ticketlocks_enabled)) {
+                __ticket_t head;
 
-                prev = *lock;
-                add_smp(&lock->tickets.head, TICKET_LOCK_INC);
+                BUILD_BUG_ON(((__ticket_t)NR_CPUS) != NR_CPUS);
 
-                /* add_smp() is a full mb() */
+                head = xadd(&lock->tickets.head, TICKET_LOCK_INC);
 
-                if (unlikely(lock->tickets.tail & TICKET_SLOWPATH_FLAG))
-                        __ticket_unlock_slowpath(lock, prev);
+                if (unlikely(head & TICKET_SLOWPATH_FLAG)) {
+                        head &= ~TICKET_SLOWPATH_FLAG;
+                        __ticket_unlock_kick(lock, (head + TICKET_LOCK_INC));
+                }
         } else
                 __add(&lock->tickets.head, TICKET_LOCK_INC, UNLOCK_LOCK_PREFIX);
 }
@@ -164,14 +161,15 @@ static inline int arch_spin_is_locked(arch_spinlock_t *lock)
 {
         struct __raw_tickets tmp = READ_ONCE(lock->tickets);
 
-        return tmp.tail != tmp.head;
+        return !__tickets_equal(tmp.tail, tmp.head);
 }
 
 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
 {
         struct __raw_tickets tmp = READ_ONCE(lock->tickets);
 
-        return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC;
+        tmp.head &= ~TICKET_SLOWPATH_FLAG;
+        return (tmp.tail - tmp.head) > TICKET_LOCK_INC;
 }
 #define arch_spin_is_contended  arch_spin_is_contended
 
@@ -183,16 +181,16 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
 
 static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
 {
-        __ticket_t head = ACCESS_ONCE(lock->tickets.head);
+        __ticket_t head = READ_ONCE(lock->tickets.head);
 
         for (;;) {
-                struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);
+                struct __raw_tickets tmp = READ_ONCE(lock->tickets);
                 /*
                  * We need to check "unlocked" in a loop, tmp.head == head
                  * can be false positive because of overflow.
                  */
-                if (tmp.head == (tmp.tail & ~TICKET_SLOWPATH_FLAG) ||
-                    tmp.head != head)
+                if (__tickets_equal(tmp.head, tmp.tail) ||
+                                !__tickets_equal(tmp.head, head))
                         break;
 
                 cpu_relax();
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 19e2c468fc2c..e4661196994e 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -27,11 +27,12 @@ static __always_inline void *__inline_memcpy(void *to, const void *from, size_t
    function. */
 
 #define __HAVE_ARCH_MEMCPY 1
+extern void *__memcpy(void *to, const void *from, size_t len);
+
 #ifndef CONFIG_KMEMCHECK
 #if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4
 extern void *memcpy(void *to, const void *from, size_t len);
 #else
-extern void *__memcpy(void *to, const void *from, size_t len);
 #define memcpy(dst, src, len)                                   \
 ({                                                              \
         size_t __len = (len);                                   \
@@ -53,9 +54,11 @@ extern void *__memcpy(void *to, const void *from, size_t len);
 
 #define __HAVE_ARCH_MEMSET
 void *memset(void *s, int c, size_t n);
+void *__memset(void *s, int c, size_t n);
 
 #define __HAVE_ARCH_MEMMOVE
 void *memmove(void *dest, const void *src, size_t count);
+void *__memmove(void *dest, const void *src, size_t count);
 
 int memcmp(const void *cs, const void *ct, size_t count);
 size_t strlen(const char *s);
@@ -63,6 +66,19 @@ char *strcpy(char *dest, const char *src);
 char *strcat(char *dest, const char *src);
 int strcmp(const char *cs, const char *ct);
 
+#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
+
+/*
+ * For files that not instrumented (e.g. mm/slub.c) we
+ * should use not instrumented version of mem* functions.
+ */
+
+#undef memcpy
+#define memcpy(dst, src, len) __memcpy(dst, src, len)
+#define memmove(dst, src, len) __memmove(dst, src, len)
+#define memset(s, c, n) __memset(s, c, n)
+#endif
+
 #endif /* __KERNEL__ */
 
 #endif /* _ASM_X86_STRING_64_H */
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index e82e95abc92b..1d4e4f279a32 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -31,7 +31,6 @@ struct thread_info {
         __u32                   cpu;            /* current CPU */
         int                     saved_preempt_count;
         mm_segment_t            addr_limit;
-        struct restart_block    restart_block;
         void __user             *sysenter_return;
         unsigned int            sig_on_uaccess_error:1;
         unsigned int            uaccess_err:1;  /* uaccess failed */
@@ -45,9 +44,6 @@ struct thread_info {
         .cpu            = 0,                    \
         .saved_preempt_count = INIT_PREEMPT_COUNT,      \
         .addr_limit     = KERNEL_DS,            \
-        .restart_block = {                      \
-                .fn = do_no_restart_syscall,    \
-        },                                      \
 }
 
 #define init_thread_info        (init_thread_union.thread_info)
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 04905bfc508b..cd791948b286 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -15,6 +15,75 @@
 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
 #endif
 
+struct tlb_state {
+#ifdef CONFIG_SMP
+        struct mm_struct *active_mm;
+        int state;
+#endif
+
+        /*
+         * Access to this CR4 shadow and to H/W CR4 is protected by
+         * disabling interrupts when modifying either one.
+         */
+        unsigned long cr4;
+};
+DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
+
+/* Initialize cr4 shadow for this CPU. */
+static inline void cr4_init_shadow(void)
+{
+        this_cpu_write(cpu_tlbstate.cr4, __read_cr4());
+}
+
+/* Set in this cpu's CR4. */
+static inline void cr4_set_bits(unsigned long mask)
+{
+        unsigned long cr4;
+
+        cr4 = this_cpu_read(cpu_tlbstate.cr4);
+        if ((cr4 | mask) != cr4) {
+                cr4 |= mask;
+                this_cpu_write(cpu_tlbstate.cr4, cr4);
+                __write_cr4(cr4);
+        }
+}
+
+/* Clear in this cpu's CR4. */
+static inline void cr4_clear_bits(unsigned long mask)
+{
+        unsigned long cr4;
+
+        cr4 = this_cpu_read(cpu_tlbstate.cr4);
+        if ((cr4 & ~mask) != cr4) {
+                cr4 &= ~mask;
+                this_cpu_write(cpu_tlbstate.cr4, cr4);
+                __write_cr4(cr4);
+        }
+}
+
+/* Read the CR4 shadow. */
+static inline unsigned long cr4_read_shadow(void)
+{
+        return this_cpu_read(cpu_tlbstate.cr4);
+}
+
+/*
+ * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
+ * enable and PPro Global page enable), so that any CPU's that boot
+ * up after us can get the correct flags.  This should only be used
+ * during boot on the boot cpu.
+ */
+extern unsigned long mmu_cr4_features;
+extern u32 *trampoline_cr4_features;
+
+static inline void cr4_set_bits_and_update_boot(unsigned long mask)
+{
+        mmu_cr4_features |= mask;
+        if (trampoline_cr4_features)
+                *trampoline_cr4_features = mmu_cr4_features;
+        cr4_set_bits(mask);
+}
+
 static inline void __native_flush_tlb(void)
 {
         native_write_cr3(native_read_cr3());
@@ -24,7 +93,7 @@ static inline void __native_flush_tlb_global_irq_disabled(void)
 {
         unsigned long cr4;
 
-        cr4 = native_read_cr4();
+        cr4 = this_cpu_read(cpu_tlbstate.cr4);
         /* clear PGE */
         native_write_cr4(cr4 & ~X86_CR4_PGE);
         /* write old PGE again and flush TLBs */
@@ -184,12 +253,6 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
 #define TLBSTATE_OK     1
 #define TLBSTATE_LAZY   2
 
-struct tlb_state {
-        struct mm_struct *active_mm;
-        int state;
-};
-DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
-
 static inline void reset_lazy_tlbstate(void)
 {
         this_cpu_write(cpu_tlbstate.state, 0);
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 0d592e0a5b84..ace9dec050b1 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -179,7 +179,7 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
         asm volatile("call __get_user_%P3"                              \
                      : "=a" (__ret_gu), "=r" (__val_gu)                 \
                      : "0" (ptr), "i" (sizeof(*(ptr))));                \
-        (x) = (__typeof__(*(ptr))) __val_gu;                            \
+        (x) = (__force __typeof__(*(ptr))) __val_gu;                    \
         __ret_gu;                                                       \
 })
 
diff --git a/arch/x86/include/asm/virtext.h b/arch/x86/include/asm/virtext.h
index 5da71c27cc59..cce9ee68e335 100644
--- a/arch/x86/include/asm/virtext.h
+++ b/arch/x86/include/asm/virtext.h
@@ -19,6 +19,7 @@
 
 #include <asm/vmx.h>
 #include <asm/svm.h>
+#include <asm/tlbflush.h>
 
 /*
  * VMX functions:
@@ -40,12 +41,12 @@ static inline int cpu_has_vmx(void)
 static inline void cpu_vmxoff(void)
 {
         asm volatile (ASM_VMX_VMXOFF : : : "cc");
-        write_cr4(read_cr4() & ~X86_CR4_VMXE);
+        cr4_clear_bits(X86_CR4_VMXE);
 }
 
 static inline int cpu_vmx_enabled(void)
 {
-        return read_cr4() & X86_CR4_VMXE;
+        return __read_cr4() & X86_CR4_VMXE;
 }
 
 /** Disable VMX if it is enabled on the current CPU
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 45afaee9555c..da772edd19ab 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -69,6 +69,7 @@
 #define SECONDARY_EXEC_PAUSE_LOOP_EXITING       0x00000400
 #define SECONDARY_EXEC_ENABLE_INVPCID           0x00001000
 #define SECONDARY_EXEC_SHADOW_VMCS              0x00004000
+#define SECONDARY_EXEC_ENABLE_PML               0x00020000
 #define SECONDARY_EXEC_XSAVES                   0x00100000
 
 
@@ -121,6 +122,7 @@ enum vmcs_field {
         GUEST_LDTR_SELECTOR             = 0x0000080c,
         GUEST_TR_SELECTOR               = 0x0000080e,
         GUEST_INTR_STATUS               = 0x00000810,
+        GUEST_PML_INDEX                 = 0x00000812,
         HOST_ES_SELECTOR                = 0x00000c00,
         HOST_CS_SELECTOR                = 0x00000c02,
         HOST_SS_SELECTOR                = 0x00000c04,
@@ -140,6 +142,8 @@ enum vmcs_field {
         VM_EXIT_MSR_LOAD_ADDR_HIGH      = 0x00002009,
         VM_ENTRY_MSR_LOAD_ADDR          = 0x0000200a,
         VM_ENTRY_MSR_LOAD_ADDR_HIGH     = 0x0000200b,
+        PML_ADDRESS                     = 0x0000200e,
+        PML_ADDRESS_HIGH                = 0x0000200f,
         TSC_OFFSET                      = 0x00002010,
         TSC_OFFSET_HIGH                 = 0x00002011,
         VIRTUAL_APIC_PAGE_ADDR          = 0x00002012,
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 5eea09915a15..358dcd338915 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -55,9 +55,8 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
                                    struct gnttab_map_grant_ref *kmap_ops,
                                    struct page **pages, unsigned int count);
 extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
-                                     struct gnttab_map_grant_ref *kmap_ops,
+                                     struct gnttab_unmap_grant_ref *kunmap_ops,
                                      struct page **pages, unsigned int count);
-extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn);
 
 /*
  * Helper functions to write or read unsigned long values to/from
@@ -154,21 +153,12 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn)
                 return mfn;
 
         pfn = mfn_to_pfn_no_overrides(mfn);
-        if (__pfn_to_mfn(pfn) != mfn) {
-                /*
-                 * If this appears to be a foreign mfn (because the pfn
-                 * doesn't map back to the mfn), then check the local override
-                 * table to see if there's a better pfn to use.
-                 *
-                 * m2p_find_override_pfn returns ~0 if it doesn't find anything.
-                 */
-                pfn = m2p_find_override_pfn(mfn, ~0);
-        }
+        if (__pfn_to_mfn(pfn) != mfn)
+                pfn = ~0;
 
         /*
-         * pfn is ~0 if there are no entries in the m2p for mfn or if the
-         * entry doesn't map back to the mfn and m2p_override doesn't have a
-         * valid entry for it.
+         * pfn is ~0 if there are no entries in the m2p for mfn or the
+         * entry doesn't map back to the mfn.
          */
         if (pfn == ~0 && __pfn_to_mfn(mfn) == IDENTITY_FRAME(mfn))
                 pfn = mfn;
diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h
index 5fa9770035dc..c9a6d68b8d62 100644
--- a/arch/x86/include/asm/xsave.h
+++ b/arch/x86/include/asm/xsave.h
@@ -82,18 +82,15 @@ static inline int xsave_state_booting(struct xsave_struct *fx, u64 mask)
         if (boot_cpu_has(X86_FEATURE_XSAVES))
                 asm volatile("1:"XSAVES"\n\t"
                         "2:\n\t"
-                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                             xstate_fault
+                        : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
                         :   "memory");
         else
                 asm volatile("1:"XSAVE"\n\t"
                         "2:\n\t"
-                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                             xstate_fault
+                        : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
                         :   "memory");
-
-        asm volatile(xstate_fault
-                     : "0" (0)
-                     : "memory");
-
         return err;
 }
 
@@ -112,18 +109,15 @@ static inline int xrstor_state_booting(struct xsave_struct *fx, u64 mask)
         if (boot_cpu_has(X86_FEATURE_XSAVES))
                 asm volatile("1:"XRSTORS"\n\t"
                         "2:\n\t"
-                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                             xstate_fault
+                        : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
                         :   "memory");
         else
                 asm volatile("1:"XRSTOR"\n\t"
                         "2:\n\t"
-                        : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
+                             xstate_fault
+                        : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
                         :   "memory");
-
-        asm volatile(xstate_fault
-                     : "0" (0)
-                     : "memory");
-
         return err;
 }
 
@@ -149,9 +143,9 @@ static inline int xsave_state(struct xsave_struct *fx, u64 mask)
          */
         alternative_input_2(
                 "1:"XSAVE,
-                "1:"XSAVEOPT,
+                XSAVEOPT,
                 X86_FEATURE_XSAVEOPT,
-                "1:"XSAVES,
+                XSAVES,
                 X86_FEATURE_XSAVES,
                 [fx] "D" (fx), "a" (lmask), "d" (hmask) :
                 "memory");
@@ -178,7 +172,7 @@ static inline int xrstor_state(struct xsave_struct *fx, u64 mask)
          */
         alternative_input(
                 "1: " XRSTOR,
-                "1: " XRSTORS,
+                XRSTORS,
                 X86_FEATURE_XSAVES,
                 "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
                 : "memory");
diff --git a/arch/x86/include/uapi/asm/hyperv.h b/arch/x86/include/uapi/asm/hyperv.h
index 462efe746d77..90c458e66e13 100644
--- a/arch/x86/include/uapi/asm/hyperv.h
+++ b/arch/x86/include/uapi/asm/hyperv.h
@@ -187,6 +187,17 @@
 #define HV_X64_MSR_SINT14                       0x4000009E
 #define HV_X64_MSR_SINT15                       0x4000009F
 
+/*
+ * Synthetic Timer MSRs. Four timers per vcpu.
+ */
+#define HV_X64_MSR_STIMER0_CONFIG               0x400000B0
+#define HV_X64_MSR_STIMER0_COUNT                0x400000B1
+#define HV_X64_MSR_STIMER1_CONFIG               0x400000B2
+#define HV_X64_MSR_STIMER1_COUNT                0x400000B3
+#define HV_X64_MSR_STIMER2_CONFIG               0x400000B4
+#define HV_X64_MSR_STIMER2_COUNT                0x400000B5
+#define HV_X64_MSR_STIMER3_CONFIG               0x400000B6
+#define HV_X64_MSR_STIMER3_COUNT                0x400000B7
 
 #define HV_X64_MSR_HYPERCALL_ENABLE             0x00000001
 #define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT 12
diff --git a/arch/x86/include/uapi/asm/msr-index.h b/arch/x86/include/uapi/asm/msr-index.h
index d979e5abae55..3ce079136c11 100644
--- a/arch/x86/include/uapi/asm/msr-index.h
+++ b/arch/x86/include/uapi/asm/msr-index.h
@@ -152,6 +152,10 @@
 #define MSR_CC6_DEMOTION_POLICY_CONFIG  0x00000668
 #define MSR_MC6_DEMOTION_POLICY_CONFIG  0x00000669
 
+#define MSR_CORE_PERF_LIMIT_REASONS     0x00000690
+#define MSR_GFX_PERF_LIMIT_REASONS      0x000006B0
+#define MSR_RING_PERF_LIMIT_REASONS     0x000006B1
+
 /* Hardware P state interface */
 #define MSR_PPERF                       0x0000064e
 #define MSR_PERF_LIMIT_REASONS          0x0000064f
@@ -360,8 +364,12 @@
 #define MSR_IA32_UCODE_WRITE            0x00000079
 #define MSR_IA32_UCODE_REV              0x0000008b
 
+#define MSR_IA32_SMM_MONITOR_CTL        0x0000009b
+#define MSR_IA32_SMBASE                 0x0000009e
+
 #define MSR_IA32_PERF_STATUS            0x00000198
 #define MSR_IA32_PERF_CTL               0x00000199
+#define INTEL_PERF_CTL_MASK             0xffff
 #define MSR_AMD_PSTATE_DEF_BASE         0xc0010064
 #define MSR_AMD_PERF_STATUS             0xc0010063
 #define MSR_AMD_PERF_CTL                0xc0010062
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index b813bf9da1e2..c5f1a1deb91a 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -56,6 +56,7 @@
 #define EXIT_REASON_MSR_READ            31
 #define EXIT_REASON_MSR_WRITE           32
 #define EXIT_REASON_INVALID_STATE       33
+#define EXIT_REASON_MSR_LOAD_FAIL       34
 #define EXIT_REASON_MWAIT_INSTRUCTION   36
 #define EXIT_REASON_MONITOR_INSTRUCTION 39
 #define EXIT_REASON_PAUSE_INSTRUCTION   40
@@ -72,6 +73,7 @@
 #define EXIT_REASON_XSETBV              55
 #define EXIT_REASON_APIC_WRITE          56
 #define EXIT_REASON_INVPCID             58
+#define EXIT_REASON_PML_FULL            62
 #define EXIT_REASON_XSAVES              63
 #define EXIT_REASON_XRSTORS             64
 
@@ -116,10 +118,14 @@
         { EXIT_REASON_APIC_WRITE,            "APIC_WRITE" }, \
         { EXIT_REASON_EOI_INDUCED,           "EOI_INDUCED" }, \
         { EXIT_REASON_INVALID_STATE,         "INVALID_STATE" }, \
+        { EXIT_REASON_MSR_LOAD_FAIL,         "MSR_LOAD_FAIL" }, \
         { EXIT_REASON_INVD,                  "INVD" }, \
         { EXIT_REASON_INVVPID,               "INVVPID" }, \
         { EXIT_REASON_INVPCID,               "INVPCID" }, \
         { EXIT_REASON_XSAVES,                "XSAVES" }, \
         { EXIT_REASON_XRSTORS,               "XRSTORS" }
 
+#define VMX_ABORT_SAVE_GUEST_MSR_FAIL        1
+#define VMX_ABORT_LOAD_HOST_MSR_FAIL         4
+
 #endif /* _UAPIVMX_H */