21 files changed, 112 insertions, 49 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 26387c7bf305..68261430fe6e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -446,12 +446,12 @@ config RETPOLINE
           branches. Requires a compiler with -mindirect-branch=thunk-extern
           support for full protection. The kernel may run slower.
 
-config X86_RESCTRL
-        bool "Resource Control support"
+config X86_CPU_RESCTRL
+        bool "x86 CPU resource control support"
         depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)
         select KERNFS
         help
-          Enable Resource Control support.
+          Enable x86 CPU resource control support.
 
           Provide support for the allocation and monitoring of system resources
           usage by the CPU.
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index 64037895b085..f62e347862cc 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -600,6 +600,16 @@ ENTRY(trampoline_32bit_src)
         leal    TRAMPOLINE_32BIT_PGTABLE_OFFSET(%ecx), %eax
         movl    %eax, %cr3
 3:
+        /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
+        pushl   %ecx
+        pushl   %edx
+        movl    $MSR_EFER, %ecx
+        rdmsr
+        btsl    $_EFER_LME, %eax
+        wrmsr
+        popl    %edx
+        popl    %ecx
+
         /* Enable PAE and LA57 (if required) paging modes */
         movl    $X86_CR4_PAE, %eax
         cmpl    $0, %edx
diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h
index 91f75638f6e6..6ff7e81b5628 100644
--- a/arch/x86/boot/compressed/pgtable.h
+++ b/arch/x86/boot/compressed/pgtable.h
@@ -6,7 +6,7 @@
 #define TRAMPOLINE_32BIT_PGTABLE_OFFSET 0
 
 #define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE
-#define TRAMPOLINE_32BIT_CODE_SIZE      0x60
+#define TRAMPOLINE_32BIT_CODE_SIZE      0x70
 
 #define TRAMPOLINE_32BIT_STACK_END      TRAMPOLINE_32BIT_SIZE
 
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 40e12cfc87f6..daafb893449b 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -3559,6 +3559,14 @@ static void free_excl_cntrs(int cpu)
 
 static void intel_pmu_cpu_dying(int cpu)
 {
+        fini_debug_store_on_cpu(cpu);
+
+        if (x86_pmu.counter_freezing)
+                disable_counter_freeze();
+}
+
+static void intel_pmu_cpu_dead(int cpu)
+{
         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
         struct intel_shared_regs *pc;
 
@@ -3570,11 +3578,6 @@ static void intel_pmu_cpu_dying(int cpu)
         }
 
         free_excl_cntrs(cpu);
-
-        fini_debug_store_on_cpu(cpu);
-
-        if (x86_pmu.counter_freezing)
-                disable_counter_freeze();
 }
 
 static void intel_pmu_sched_task(struct perf_event_context *ctx,
@@ -3663,6 +3666,7 @@ static __initconst const struct x86_pmu core_pmu = {
         .cpu_prepare            = intel_pmu_cpu_prepare,
         .cpu_starting           = intel_pmu_cpu_starting,
         .cpu_dying              = intel_pmu_cpu_dying,
+        .cpu_dead               = intel_pmu_cpu_dead,
 };
 
 static struct attribute *intel_pmu_attrs[];
@@ -3703,6 +3707,8 @@ static __initconst const struct x86_pmu intel_pmu = {
         .cpu_prepare            = intel_pmu_cpu_prepare,
         .cpu_starting           = intel_pmu_cpu_starting,
         .cpu_dying              = intel_pmu_cpu_dying,
+        .cpu_dead               = intel_pmu_cpu_dead,
+
         .guest_get_msrs         = intel_guest_get_msrs,
         .sched_task             = intel_pmu_sched_task,
 };
diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c
index c07bee31abe8..b10e04387f38 100644
--- a/arch/x86/events/intel/uncore_snbep.c
+++ b/arch/x86/events/intel/uncore_snbep.c
@@ -1222,6 +1222,8 @@ static struct pci_driver snbep_uncore_pci_driver = {
         .id_table       = snbep_uncore_pci_ids,
 };
 
+#define NODE_ID_MASK    0x7
+
 /*
  * build pci bus to socket mapping
  */
@@ -1243,7 +1245,7 @@ static int snbep_pci2phy_map_init(int devid, int nodeid_loc, int idmap_loc, bool
                 err = pci_read_config_dword(ubox_dev, nodeid_loc, &config);
                 if (err)
                         break;
-                nodeid = config;
+                nodeid = config & NODE_ID_MASK;
                 /* get the Node ID mapping */
                 err = pci_read_config_dword(ubox_dev, idmap_loc, &config);
                 if (err)
diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h
index 0dd6b0f4000e..d9a9993af882 100644
--- a/arch/x86/include/asm/intel-family.h
+++ b/arch/x86/include/asm/intel-family.h
@@ -6,7 +6,7 @@
  * "Big Core" Processors (Branded as Core, Xeon, etc...)
  *
  * The "_X" parts are generally the EP and EX Xeons, or the
- * "Extreme" ones, like Broadwell-E.
+ * "Extreme" ones, like Broadwell-E, or Atom microserver.
  *
  * While adding a new CPUID for a new microarchitecture, add a new
  * group to keep logically sorted out in chronological order. Within
@@ -71,6 +71,7 @@
 #define INTEL_FAM6_ATOM_GOLDMONT        0x5C /* Apollo Lake */
 #define INTEL_FAM6_ATOM_GOLDMONT_X      0x5F /* Denverton */
 #define INTEL_FAM6_ATOM_GOLDMONT_PLUS   0x7A /* Gemini Lake */
+#define INTEL_FAM6_ATOM_TREMONT_X       0x86 /* Jacobsville */
 
 /* Xeon Phi */
 
diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h
index 8f657286d599..0ce558a8150d 100644
--- a/arch/x86/include/asm/page_64_types.h
+++ b/arch/x86/include/asm/page_64_types.h
@@ -7,7 +7,11 @@
 #endif
 
 #ifdef CONFIG_KASAN
+#ifdef CONFIG_KASAN_EXTRA
+#define KASAN_STACK_ORDER 2
+#else
 #define KASAN_STACK_ORDER 1
+#endif
 #else
 #define KASAN_STACK_ORDER 0
 #endif
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 40616e805292..2779ace16d23 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -1065,7 +1065,7 @@ static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
                               pmd_t *pmdp, pmd_t pmd)
 {
-        native_set_pmd(pmdp, pmd);
+        set_pmd(pmdp, pmd);
 }
 
 static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
diff --git a/arch/x86/include/asm/resctrl_sched.h b/arch/x86/include/asm/resctrl_sched.h
index 40ebddde6ac2..f6b7fe2833cc 100644
--- a/arch/x86/include/asm/resctrl_sched.h
+++ b/arch/x86/include/asm/resctrl_sched.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_X86_RESCTRL_SCHED_H
 #define _ASM_X86_RESCTRL_SCHED_H
 
-#ifdef CONFIG_X86_RESCTRL
+#ifdef CONFIG_X86_CPU_RESCTRL
 
 #include <linux/sched.h>
 #include <linux/jump_label.h>
@@ -88,6 +88,6 @@ static inline void resctrl_sched_in(void)
 
 static inline void resctrl_sched_in(void) {}
 
-#endif /* CONFIG_X86_RESCTRL */
+#endif /* CONFIG_X86_CPU_RESCTRL */
 
 #endif /* _ASM_X86_RESCTRL_SCHED_H */
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index b6fa0869f7aa..cfd24f9f7614 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -39,7 +39,7 @@ obj-$(CONFIG_CPU_SUP_UMC_32)		+= umc.o
 obj-$(CONFIG_X86_MCE)                   += mce/
 obj-$(CONFIG_MTRR)                      += mtrr/
 obj-$(CONFIG_MICROCODE)                 += microcode/
-obj-$(CONFIG_X86_RESCTRL)               += resctrl/
+obj-$(CONFIG_X86_CPU_RESCTRL)           += resctrl/
 
 obj-$(CONFIG_X86_LOCAL_APIC)            += perfctr-watchdog.o
 
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 1de0f4170178..01874d54f4fd 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -71,7 +71,7 @@ void __init check_bugs(void)
          * identify_boot_cpu() initialized SMT support information, let the
          * core code know.
          */
-        cpu_smt_check_topology_early();
+        cpu_smt_check_topology();
 
         if (!IS_ENABLED(CONFIG_SMP)) {
                 pr_info("CPU: ");
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 672c7225cb1b..6ce290c506d9 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -784,6 +784,7 @@ static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
                         quirk_no_way_out(i, m, regs);
 
                 if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
+                        m->bank = i;
                         mce_read_aux(m, i);
                         *msg = tmp;
                         return 1;
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index 51adde0a0f1a..e1f3ba19ba54 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -855,7 +855,7 @@ load_microcode_amd(bool save, u8 family, const u8 *data, size_t size)
         if (!p) {
                 return ret;
         } else {
-                if (boot_cpu_data.microcode == p->patch_id)
+                if (boot_cpu_data.microcode >= p->patch_id)
                         return ret;
 
                 ret = UCODE_NEW;
diff --git a/arch/x86/kernel/cpu/resctrl/Makefile b/arch/x86/kernel/cpu/resctrl/Makefile
index 1cabe6fd8e11..4a06c37b9cf1 100644
--- a/arch/x86/kernel/cpu/resctrl/Makefile
+++ b/arch/x86/kernel/cpu/resctrl/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_X86_RESCTRL)       += core.o rdtgroup.o monitor.o
-obj-$(CONFIG_X86_RESCTRL)       += ctrlmondata.o pseudo_lock.o
+obj-$(CONFIG_X86_CPU_RESCTRL)   += core.o rdtgroup.o monitor.o
+obj-$(CONFIG_X86_CPU_RESCTRL)   += ctrlmondata.o pseudo_lock.o
 CFLAGS_pseudo_lock.o = -I$(src)
diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c
index 0d5efa34f359..53917a3ebf94 100644
--- a/arch/x86/kernel/kexec-bzimage64.c
+++ b/arch/x86/kernel/kexec-bzimage64.c
@@ -167,6 +167,9 @@ setup_efi_state(struct boot_params *params, unsigned long params_load_addr,
         struct efi_info *current_ei = &boot_params.efi_info;
         struct efi_info *ei = &params->efi_info;
 
+        if (!efi_enabled(EFI_RUNTIME_SERVICES))
+                return 0;
+
         if (!current_ei->efi_memmap_size)
                 return 0;
 
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 8ff20523661b..d8ea4ebd79e7 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -211,6 +211,7 @@ static void free_nested(struct kvm_vcpu *vcpu)
         if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
                 return;
 
+        hrtimer_cancel(&vmx->nested.preemption_timer);
         vmx->nested.vmxon = false;
         vmx->nested.smm.vmxon = false;
         free_vpid(vmx->nested.vpid02);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 4341175339f3..95d618045001 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -26,6 +26,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/mm.h>
 #include <linux/sched.h>
+#include <linux/sched/smt.h>
 #include <linux/slab.h>
 #include <linux/tboot.h>
 #include <linux/trace_events.h>
@@ -6823,7 +6824,7 @@ static int vmx_vm_init(struct kvm *kvm)
                          * Warn upon starting the first VM in a potentially
                          * insecure environment.
                          */
-                        if (cpu_smt_control == CPU_SMT_ENABLED)
+                        if (sched_smt_active())
                                 pr_warn_once(L1TF_MSG_SMT);
                         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER)
                                 pr_warn_once(L1TF_MSG_L1D);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3d27206f6c01..e67ecf25e690 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5116,6 +5116,13 @@ int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 
+        /*
+         * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
+         * is returned, but our callers are not ready for that and they blindly
+         * call kvm_inject_page_fault.  Ensure that they at least do not leak
+         * uninitialized kernel stack memory into cr2 and error code.
+         */
+        memset(exception, 0, sizeof(*exception));
         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
                                           exception);
 }
diff --git a/arch/x86/lib/iomem.c b/arch/x86/lib/iomem.c
index 66894675f3c8..df50451d94ef 100644
--- a/arch/x86/lib/iomem.c
+++ b/arch/x86/lib/iomem.c
@@ -2,8 +2,11 @@
 #include <linux/module.h>
 #include <linux/io.h>
 
+#define movs(type,to,from) \
+        asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory")
+
 /* Originally from i386/string.h */
-static __always_inline void __iomem_memcpy(void *to, const void *from, size_t n)
+static __always_inline void rep_movs(void *to, const void *from, size_t n)
 {
         unsigned long d0, d1, d2;
         asm volatile("rep ; movsl\n\t"
@@ -21,13 +24,37 @@ static __always_inline void __iomem_memcpy(void *to, const void *from, size_t n)
 
 void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
 {
-        __iomem_memcpy(to, (const void *)from, n);
+        if (unlikely(!n))
+                return;
+
+        /* Align any unaligned source IO */
+        if (unlikely(1 & (unsigned long)from)) {
+                movs("b", to, from);
+                n--;
+        }
+        if (n > 1 && unlikely(2 & (unsigned long)from)) {
+                movs("w", to, from);
+                n-=2;
+        }
+        rep_movs(to, (const void *)from, n);
 }
 EXPORT_SYMBOL(memcpy_fromio);
 
 void memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
 {
-        __iomem_memcpy((void *)to, (const void *) from, n);
+        if (unlikely(!n))
+                return;
+
+        /* Align any unaligned destination IO */
+        if (unlikely(1 & (unsigned long)to)) {
+                movs("b", to, from);
+                n--;
+        }
+        if (n > 1 && unlikely(2 & (unsigned long)to)) {
+                movs("w", to, from);
+                n-=2;
+        }
+        rep_movs((void *)to, (const void *) from, n);
 }
 EXPORT_SYMBOL(memcpy_toio);
 
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 2ff25ad33233..9d5c75f02295 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -595,7 +595,7 @@ static void show_ldttss(const struct desc_ptr *gdt, const char *name, u16 index)
                 return;
         }
 
-        addr = desc.base0 | (desc.base1 << 16) | (desc.base2 << 24);
+        addr = desc.base0 | (desc.base1 << 16) | ((unsigned long)desc.base2 << 24);
 #ifdef CONFIG_X86_64
         addr |= ((u64)desc.base3 << 32);
 #endif
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 4f8972311a77..14e6119838a6 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -230,6 +230,29 @@ static bool __cpa_pfn_in_highmap(unsigned long pfn)
 
 #endif
 
+/*
+ * See set_mce_nospec().
+ *
+ * Machine check recovery code needs to change cache mode of poisoned pages to
+ * UC to avoid speculative access logging another error. But passing the
+ * address of the 1:1 mapping to set_memory_uc() is a fine way to encourage a
+ * speculative access. So we cheat and flip the top bit of the address. This
+ * works fine for the code that updates the page tables. But at the end of the
+ * process we need to flush the TLB and cache and the non-canonical address
+ * causes a #GP fault when used by the INVLPG and CLFLUSH instructions.
+ *
+ * But in the common case we already have a canonical address. This code
+ * will fix the top bit if needed and is a no-op otherwise.
+ */
+static inline unsigned long fix_addr(unsigned long addr)
+{
+#ifdef CONFIG_X86_64
+        return (long)(addr << 1) >> 1;
+#else
+        return addr;
+#endif
+}
+
 static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
 {
         if (cpa->flags & CPA_PAGES_ARRAY) {
@@ -313,7 +336,7 @@ void __cpa_flush_tlb(void *data)
         unsigned int i;
 
         for (i = 0; i < cpa->numpages; i++)
-                __flush_tlb_one_kernel(__cpa_addr(cpa, i));
+                __flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
 }
 
 static void cpa_flush(struct cpa_data *data, int cache)
@@ -347,7 +370,7 @@ static void cpa_flush(struct cpa_data *data, int cache)
                  * Only flush present addresses:
                  */
                 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
-                        clflush_cache_range_opt((void *)addr, PAGE_SIZE);
+                        clflush_cache_range_opt((void *)fix_addr(addr), PAGE_SIZE);
         }
         mb();
 }
@@ -1627,29 +1650,6 @@ out:
         return ret;
 }
 
-/*
- * Machine check recovery code needs to change cache mode of poisoned
- * pages to UC to avoid speculative access logging another error. But
- * passing the address of the 1:1 mapping to set_memory_uc() is a fine
- * way to encourage a speculative access. So we cheat and flip the top
- * bit of the address. This works fine for the code that updates the
- * page tables. But at the end of the process we need to flush the cache
- * and the non-canonical address causes a #GP fault when used by the
- * CLFLUSH instruction.
- *
- * But in the common case we already have a canonical address. This code
- * will fix the top bit if needed and is a no-op otherwise.
- */
-static inline unsigned long make_addr_canonical_again(unsigned long addr)
-{
-#ifdef CONFIG_X86_64
-        return (long)(addr << 1) >> 1;
-#else
-        return addr;
-#endif
-}
-
-
 static int change_page_attr_set_clr(unsigned long *addr, int numpages,
                                     pgprot_t mask_set, pgprot_t mask_clr,
                                     int force_split, int in_flag,