aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r--arch/x86/kernel/acpi/Makefile1
-rw-r--r--arch/x86/kernel/acpi/boot.c35
-rw-r--r--arch/x86/kernel/acpi/cppc_msr.c58
-rw-r--r--arch/x86/kernel/apic/apic.c97
-rw-r--r--arch/x86/kernel/apic/io_apic.c4
-rw-r--r--arch/x86/kernel/apic/probe_32.c2
-rw-r--r--arch/x86/kernel/cpu/common.c23
-rw-r--r--arch/x86/kernel/cpu/hypervisor.c11
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce.c44
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce_amd.c204
-rw-r--r--arch/x86/kernel/kvm.c245
-rw-r--r--arch/x86/kernel/mpparse.c3
-rw-r--r--arch/x86/kernel/paravirt-spinlocks.c7
-rw-r--r--arch/x86/kernel/paravirt_patch_32.c4
-rw-r--r--arch/x86/kernel/paravirt_patch_64.c4
-rw-r--r--arch/x86/kernel/quirks.c31
-rw-r--r--arch/x86/kernel/setup.c21
-rw-r--r--arch/x86/kernel/smpboot.c59
-rw-r--r--arch/x86/kernel/x8664_ksyms_64.c2
19 files changed, 456 insertions, 399 deletions
diff --git a/arch/x86/kernel/acpi/Makefile b/arch/x86/kernel/acpi/Makefile
index 3242e591fa82..26b78d86f25a 100644
--- a/arch/x86/kernel/acpi/Makefile
+++ b/arch/x86/kernel/acpi/Makefile
@@ -1,6 +1,7 @@
1obj-$(CONFIG_ACPI) += boot.o 1obj-$(CONFIG_ACPI) += boot.o
2obj-$(CONFIG_ACPI_SLEEP) += sleep.o wakeup_$(BITS).o 2obj-$(CONFIG_ACPI_SLEEP) += sleep.o wakeup_$(BITS).o
3obj-$(CONFIG_ACPI_APEI) += apei.o 3obj-$(CONFIG_ACPI_APEI) += apei.o
4obj-$(CONFIG_ACPI_CPPC_LIB) += cppc_msr.o
4 5
5ifneq ($(CONFIG_ACPI_PROCESSOR),) 6ifneq ($(CONFIG_ACPI_PROCESSOR),)
6obj-y += cstate.o 7obj-y += cstate.o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 90d84c3eee53..32a7d70913ac 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -176,15 +176,10 @@ static int acpi_register_lapic(int id, u32 acpiid, u8 enabled)
176 return -EINVAL; 176 return -EINVAL;
177 } 177 }
178 178
179 if (!enabled) {
180 ++disabled_cpus;
181 return -EINVAL;
182 }
183
184 if (boot_cpu_physical_apicid != -1U) 179 if (boot_cpu_physical_apicid != -1U)
185 ver = apic_version[boot_cpu_physical_apicid]; 180 ver = boot_cpu_apic_version;
186 181
187 cpu = generic_processor_info(id, ver); 182 cpu = __generic_processor_info(id, ver, enabled);
188 if (cpu >= 0) 183 if (cpu >= 0)
189 early_per_cpu(x86_cpu_to_acpiid, cpu) = acpiid; 184 early_per_cpu(x86_cpu_to_acpiid, cpu) = acpiid;
190 185
@@ -282,6 +277,8 @@ acpi_parse_lapic_addr_ovr(struct acpi_subtable_header * header,
282 if (BAD_MADT_ENTRY(lapic_addr_ovr, end)) 277 if (BAD_MADT_ENTRY(lapic_addr_ovr, end))
283 return -EINVAL; 278 return -EINVAL;
284 279
280 acpi_table_print_madt_entry(header);
281
285 acpi_lapic_addr = lapic_addr_ovr->address; 282 acpi_lapic_addr = lapic_addr_ovr->address;
286 283
287 return 0; 284 return 0;
@@ -705,7 +702,7 @@ static void __init acpi_set_irq_model_ioapic(void)
705#ifdef CONFIG_ACPI_HOTPLUG_CPU 702#ifdef CONFIG_ACPI_HOTPLUG_CPU
706#include <acpi/processor.h> 703#include <acpi/processor.h>
707 704
708static void acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) 705int acpi_map_cpu2node(acpi_handle handle, int cpu, int physid)
709{ 706{
710#ifdef CONFIG_ACPI_NUMA 707#ifdef CONFIG_ACPI_NUMA
711 int nid; 708 int nid;
@@ -716,6 +713,7 @@ static void acpi_map_cpu2node(acpi_handle handle, int cpu, int physid)
716 numa_set_node(cpu, nid); 713 numa_set_node(cpu, nid);
717 } 714 }
718#endif 715#endif
716 return 0;
719} 717}
720 718
721int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, int *pcpu) 719int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, int *pcpu)
@@ -998,21 +996,6 @@ static int __init acpi_parse_madt_lapic_entries(void)
998 if (!boot_cpu_has(X86_FEATURE_APIC)) 996 if (!boot_cpu_has(X86_FEATURE_APIC))
999 return -ENODEV; 997 return -ENODEV;
1000 998
1001 /*
1002 * Note that the LAPIC address is obtained from the MADT (32-bit value)
1003 * and (optionally) overridden by a LAPIC_ADDR_OVR entry (64-bit value).
1004 */
1005
1006 count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_APIC_OVERRIDE,
1007 acpi_parse_lapic_addr_ovr, 0);
1008 if (count < 0) {
1009 printk(KERN_ERR PREFIX
1010 "Error parsing LAPIC address override entry\n");
1011 return count;
1012 }
1013
1014 register_lapic_address(acpi_lapic_addr);
1015
1016 count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_SAPIC, 999 count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_SAPIC,
1017 acpi_parse_sapic, MAX_LOCAL_APIC); 1000 acpi_parse_sapic, MAX_LOCAL_APIC);
1018 1001
@@ -1031,8 +1014,8 @@ static int __init acpi_parse_madt_lapic_entries(void)
1031 return ret; 1014 return ret;
1032 } 1015 }
1033 1016
1034 x2count = madt_proc[0].count; 1017 count = madt_proc[0].count;
1035 count = madt_proc[1].count; 1018 x2count = madt_proc[1].count;
1036 } 1019 }
1037 if (!count && !x2count) { 1020 if (!count && !x2count) {
1038 printk(KERN_ERR PREFIX "No LAPIC entries present\n"); 1021 printk(KERN_ERR PREFIX "No LAPIC entries present\n");
@@ -1513,7 +1496,7 @@ void __init acpi_boot_table_init(void)
1513 * If acpi_disabled, bail out 1496 * If acpi_disabled, bail out
1514 */ 1497 */
1515 if (acpi_disabled) 1498 if (acpi_disabled)
1516 return; 1499 return;
1517 1500
1518 /* 1501 /*
1519 * Initialize the ACPI boot-time table parser. 1502 * Initialize the ACPI boot-time table parser.
diff --git a/arch/x86/kernel/acpi/cppc_msr.c b/arch/x86/kernel/acpi/cppc_msr.c
new file mode 100644
index 000000000000..6fb478bf82fd
--- /dev/null
+++ b/arch/x86/kernel/acpi/cppc_msr.c
@@ -0,0 +1,58 @@
1/*
2 * cppc_msr.c: MSR Interface for CPPC
3 * Copyright (c) 2016, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16#include <acpi/cppc_acpi.h>
17#include <asm/msr.h>
18
19/* Refer to drivers/acpi/cppc_acpi.c for the description of functions */
20
21bool cpc_ffh_supported(void)
22{
23 return true;
24}
25
26int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val)
27{
28 int err;
29
30 err = rdmsrl_safe_on_cpu(cpunum, reg->address, val);
31 if (!err) {
32 u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
33 reg->bit_offset);
34
35 *val &= mask;
36 *val >>= reg->bit_offset;
37 }
38 return err;
39}
40
41int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
42{
43 u64 rd_val;
44 int err;
45
46 err = rdmsrl_safe_on_cpu(cpunum, reg->address, &rd_val);
47 if (!err) {
48 u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
49 reg->bit_offset);
50
51 val <<= reg->bit_offset;
52 val &= mask;
53 rd_val &= ~mask;
54 rd_val |= val;
55 err = wrmsrl_safe_on_cpu(cpunum, reg->address, rd_val);
56 }
57 return err;
58}
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index f3e9b2df4b16..f266b8a92a9e 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -64,6 +64,8 @@ unsigned disabled_cpus;
64unsigned int boot_cpu_physical_apicid = -1U; 64unsigned int boot_cpu_physical_apicid = -1U;
65EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); 65EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
66 66
67u8 boot_cpu_apic_version;
68
67/* 69/*
68 * The highest APIC ID seen during enumeration. 70 * The highest APIC ID seen during enumeration.
69 */ 71 */
@@ -1374,7 +1376,6 @@ void setup_local_APIC(void)
1374 * Actually disabling the focus CPU check just makes the hang less 1376 * Actually disabling the focus CPU check just makes the hang less
1375 * frequent as it makes the interrupt distributon model be more 1377 * frequent as it makes the interrupt distributon model be more
1376 * like LRU than MRU (the short-term load is more even across CPUs). 1378 * like LRU than MRU (the short-term load is more even across CPUs).
1377 * See also the comment in end_level_ioapic_irq(). --macro
1378 */ 1379 */
1379 1380
1380 /* 1381 /*
@@ -1816,8 +1817,7 @@ void __init init_apic_mappings(void)
1816 * since smp_sanity_check is prepared for such a case 1817 * since smp_sanity_check is prepared for such a case
1817 * and disable smp mode 1818 * and disable smp mode
1818 */ 1819 */
1819 apic_version[new_apicid] = 1820 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1820 GET_APIC_VERSION(apic_read(APIC_LVR));
1821 } 1821 }
1822} 1822}
1823 1823
@@ -1828,17 +1828,14 @@ void __init register_lapic_address(unsigned long address)
1828 if (!x2apic_mode) { 1828 if (!x2apic_mode) {
1829 set_fixmap_nocache(FIX_APIC_BASE, address); 1829 set_fixmap_nocache(FIX_APIC_BASE, address);
1830 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", 1830 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
1831 APIC_BASE, mp_lapic_addr); 1831 APIC_BASE, address);
1832 } 1832 }
1833 if (boot_cpu_physical_apicid == -1U) { 1833 if (boot_cpu_physical_apicid == -1U) {
1834 boot_cpu_physical_apicid = read_apic_id(); 1834 boot_cpu_physical_apicid = read_apic_id();
1835 apic_version[boot_cpu_physical_apicid] = 1835 boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1836 GET_APIC_VERSION(apic_read(APIC_LVR));
1837 } 1836 }
1838} 1837}
1839 1838
1840int apic_version[MAX_LOCAL_APIC];
1841
1842/* 1839/*
1843 * Local APIC interrupts 1840 * Local APIC interrupts
1844 */ 1841 */
@@ -2027,7 +2024,53 @@ void disconnect_bsp_APIC(int virt_wire_setup)
2027 apic_write(APIC_LVT1, value); 2024 apic_write(APIC_LVT1, value);
2028} 2025}
2029 2026
2030int generic_processor_info(int apicid, int version) 2027/*
2028 * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated
2029 * contiguously, it equals to current allocated max logical CPU ID plus 1.
2030 * All allocated CPU ID should be in [0, nr_logical_cpuidi), so the maximum of
2031 * nr_logical_cpuids is nr_cpu_ids.
2032 *
2033 * NOTE: Reserve 0 for BSP.
2034 */
2035static int nr_logical_cpuids = 1;
2036
2037/*
2038 * Used to store mapping between logical CPU IDs and APIC IDs.
2039 */
2040static int cpuid_to_apicid[] = {
2041 [0 ... NR_CPUS - 1] = -1,
2042};
2043
2044/*
2045 * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids
2046 * and cpuid_to_apicid[] synchronized.
2047 */
2048static int allocate_logical_cpuid(int apicid)
2049{
2050 int i;
2051
2052 /*
2053 * cpuid <-> apicid mapping is persistent, so when a cpu is up,
2054 * check if the kernel has allocated a cpuid for it.
2055 */
2056 for (i = 0; i < nr_logical_cpuids; i++) {
2057 if (cpuid_to_apicid[i] == apicid)
2058 return i;
2059 }
2060
2061 /* Allocate a new cpuid. */
2062 if (nr_logical_cpuids >= nr_cpu_ids) {
2063 WARN_ONCE(1, "Only %d processors supported."
2064 "Processor %d/0x%x and the rest are ignored.\n",
2065 nr_cpu_ids - 1, nr_logical_cpuids, apicid);
2066 return -1;
2067 }
2068
2069 cpuid_to_apicid[nr_logical_cpuids] = apicid;
2070 return nr_logical_cpuids++;
2071}
2072
2073int __generic_processor_info(int apicid, int version, bool enabled)
2031{ 2074{
2032 int cpu, max = nr_cpu_ids; 2075 int cpu, max = nr_cpu_ids;
2033 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, 2076 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
@@ -2102,8 +2145,16 @@ int generic_processor_info(int apicid, int version)
2102 * for BSP. 2145 * for BSP.
2103 */ 2146 */
2104 cpu = 0; 2147 cpu = 0;
2105 } else 2148
2106 cpu = cpumask_next_zero(-1, cpu_present_mask); 2149 /* Logical cpuid 0 is reserved for BSP. */
2150 cpuid_to_apicid[0] = apicid;
2151 } else {
2152 cpu = allocate_logical_cpuid(apicid);
2153 if (cpu < 0) {
2154 disabled_cpus++;
2155 return -EINVAL;
2156 }
2157 }
2107 2158
2108 /* 2159 /*
2109 * This can happen on physical hotplug. The sanity check at boot time 2160 * This can happen on physical hotplug. The sanity check at boot time
@@ -2120,8 +2171,6 @@ int generic_processor_info(int apicid, int version)
2120 return -ENOSPC; 2171 return -ENOSPC;
2121 } 2172 }
2122 2173
2123 num_processors++;
2124
2125 /* 2174 /*
2126 * Validate version 2175 * Validate version
2127 */ 2176 */
@@ -2130,14 +2179,12 @@ int generic_processor_info(int apicid, int version)
2130 cpu, apicid); 2179 cpu, apicid);
2131 version = 0x10; 2180 version = 0x10;
2132 } 2181 }
2133 apic_version[apicid] = version;
2134 2182
2135 if (version != apic_version[boot_cpu_physical_apicid]) { 2183 if (version != boot_cpu_apic_version) {
2136 pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n", 2184 pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2137 apic_version[boot_cpu_physical_apicid], cpu, version); 2185 boot_cpu_apic_version, cpu, version);
2138 } 2186 }
2139 2187
2140 physid_set(apicid, phys_cpu_present_map);
2141 if (apicid > max_physical_apicid) 2188 if (apicid > max_physical_apicid)
2142 max_physical_apicid = apicid; 2189 max_physical_apicid = apicid;
2143 2190
@@ -2150,11 +2197,23 @@ int generic_processor_info(int apicid, int version)
2150 apic->x86_32_early_logical_apicid(cpu); 2197 apic->x86_32_early_logical_apicid(cpu);
2151#endif 2198#endif
2152 set_cpu_possible(cpu, true); 2199 set_cpu_possible(cpu, true);
2153 set_cpu_present(cpu, true); 2200
2201 if (enabled) {
2202 num_processors++;
2203 physid_set(apicid, phys_cpu_present_map);
2204 set_cpu_present(cpu, true);
2205 } else {
2206 disabled_cpus++;
2207 }
2154 2208
2155 return cpu; 2209 return cpu;
2156} 2210}
2157 2211
2212int generic_processor_info(int apicid, int version)
2213{
2214 return __generic_processor_info(apicid, version, true);
2215}
2216
2158int hard_smp_processor_id(void) 2217int hard_smp_processor_id(void)
2159{ 2218{
2160 return read_apic_id(); 2219 return read_apic_id();
@@ -2277,7 +2336,7 @@ int __init APIC_init_uniprocessor(void)
2277 * Complain if the BIOS pretends there is one. 2336 * Complain if the BIOS pretends there is one.
2278 */ 2337 */
2279 if (!boot_cpu_has(X86_FEATURE_APIC) && 2338 if (!boot_cpu_has(X86_FEATURE_APIC) &&
2280 APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { 2339 APIC_INTEGRATED(boot_cpu_apic_version)) {
2281 pr_err("BIOS bug, local APIC 0x%x not detected!...\n", 2340 pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
2282 boot_cpu_physical_apicid); 2341 boot_cpu_physical_apicid);
2283 return -1; 2342 return -1;
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 7491f417a8e4..48e6d84f173e 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -1593,7 +1593,7 @@ void __init setup_ioapic_ids_from_mpc(void)
1593 * no meaning without the serial APIC bus. 1593 * no meaning without the serial APIC bus.
1594 */ 1594 */
1595 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1595 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1596 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid])) 1596 || APIC_XAPIC(boot_cpu_apic_version))
1597 return; 1597 return;
1598 setup_ioapic_ids_from_mpc_nocheck(); 1598 setup_ioapic_ids_from_mpc_nocheck();
1599} 1599}
@@ -2423,7 +2423,7 @@ static int io_apic_get_unique_id(int ioapic, int apic_id)
2423static u8 io_apic_unique_id(int idx, u8 id) 2423static u8 io_apic_unique_id(int idx, u8 id)
2424{ 2424{
2425 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 2425 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
2426 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid])) 2426 !APIC_XAPIC(boot_cpu_apic_version))
2427 return io_apic_get_unique_id(idx, id); 2427 return io_apic_get_unique_id(idx, id);
2428 else 2428 else
2429 return id; 2429 return id;
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index e5fb2f086460..c48264e202fd 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -152,7 +152,7 @@ early_param("apic", parse_apic);
152 152
153void __init default_setup_apic_routing(void) 153void __init default_setup_apic_routing(void)
154{ 154{
155 int version = apic_version[boot_cpu_physical_apicid]; 155 int version = boot_cpu_apic_version;
156 156
157 if (num_possible_cpus() > 8) { 157 if (num_possible_cpus() > 8) {
158 switch (boot_cpu_data.x86_vendor) { 158 switch (boot_cpu_data.x86_vendor) {
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 06919427d451..9bd910a7dd0a 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -804,21 +804,20 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
804 identify_cpu_without_cpuid(c); 804 identify_cpu_without_cpuid(c);
805 805
806 /* cyrix could have cpuid enabled via c_identify()*/ 806 /* cyrix could have cpuid enabled via c_identify()*/
807 if (!have_cpuid_p()) 807 if (have_cpuid_p()) {
808 return; 808 cpu_detect(c);
809 get_cpu_vendor(c);
810 get_cpu_cap(c);
809 811
810 cpu_detect(c); 812 if (this_cpu->c_early_init)
811 get_cpu_vendor(c); 813 this_cpu->c_early_init(c);
812 get_cpu_cap(c);
813
814 if (this_cpu->c_early_init)
815 this_cpu->c_early_init(c);
816 814
817 c->cpu_index = 0; 815 c->cpu_index = 0;
818 filter_cpuid_features(c, false); 816 filter_cpuid_features(c, false);
819 817
820 if (this_cpu->c_bsp_init) 818 if (this_cpu->c_bsp_init)
821 this_cpu->c_bsp_init(c); 819 this_cpu->c_bsp_init(c);
820 }
822 821
823 setup_force_cpu_cap(X86_FEATURE_ALWAYS); 822 setup_force_cpu_cap(X86_FEATURE_ALWAYS);
824 fpu__init_system(c); 823 fpu__init_system(c);
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index 27e46658ebe3..35691a6b0d32 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -86,3 +86,14 @@ bool __init hypervisor_x2apic_available(void)
86 x86_hyper->x2apic_available && 86 x86_hyper->x2apic_available &&
87 x86_hyper->x2apic_available(); 87 x86_hyper->x2apic_available();
88} 88}
89
90void hypervisor_pin_vcpu(int cpu)
91{
92 if (!x86_hyper)
93 return;
94
95 if (x86_hyper->pin_vcpu)
96 x86_hyper->pin_vcpu(cpu);
97 else
98 WARN_ONCE(1, "vcpu pinning requested but not supported!\n");
99}
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 79d8ec849468..a7fdf453d895 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -41,6 +41,7 @@
41#include <linux/debugfs.h> 41#include <linux/debugfs.h>
42#include <linux/irq_work.h> 42#include <linux/irq_work.h>
43#include <linux/export.h> 43#include <linux/export.h>
44#include <linux/jump_label.h>
44 45
45#include <asm/processor.h> 46#include <asm/processor.h>
46#include <asm/traps.h> 47#include <asm/traps.h>
@@ -292,6 +293,13 @@ static void print_mce(struct mce *m)
292 if (m->misc) 293 if (m->misc)
293 pr_cont("MISC %llx ", m->misc); 294 pr_cont("MISC %llx ", m->misc);
294 295
296 if (mce_flags.smca) {
297 if (m->synd)
298 pr_cont("SYND %llx ", m->synd);
299 if (m->ipid)
300 pr_cont("IPID %llx ", m->ipid);
301 }
302
295 pr_cont("\n"); 303 pr_cont("\n");
296 /* 304 /*
297 * Note this output is parsed by external tools and old fields 305 * Note this output is parsed by external tools and old fields
@@ -568,6 +576,7 @@ static void mce_read_aux(struct mce *m, int i)
568{ 576{
569 if (m->status & MCI_STATUS_MISCV) 577 if (m->status & MCI_STATUS_MISCV)
570 m->misc = mce_rdmsrl(msr_ops.misc(i)); 578 m->misc = mce_rdmsrl(msr_ops.misc(i));
579
571 if (m->status & MCI_STATUS_ADDRV) { 580 if (m->status & MCI_STATUS_ADDRV) {
572 m->addr = mce_rdmsrl(msr_ops.addr(i)); 581 m->addr = mce_rdmsrl(msr_ops.addr(i));
573 582
@@ -579,6 +588,23 @@ static void mce_read_aux(struct mce *m, int i)
579 m->addr >>= shift; 588 m->addr >>= shift;
580 m->addr <<= shift; 589 m->addr <<= shift;
581 } 590 }
591
592 /*
593 * Extract [55:<lsb>] where lsb is the least significant
594 * *valid* bit of the address bits.
595 */
596 if (mce_flags.smca) {
597 u8 lsb = (m->addr >> 56) & 0x3f;
598
599 m->addr &= GENMASK_ULL(55, lsb);
600 }
601 }
602
603 if (mce_flags.smca) {
604 m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));
605
606 if (m->status & MCI_STATUS_SYNDV)
607 m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
582 } 608 }
583} 609}
584 610
@@ -1633,17 +1659,6 @@ static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1633 1659
1634 if (c->x86 == 6 && c->x86_model == 45) 1660 if (c->x86 == 6 && c->x86_model == 45)
1635 quirk_no_way_out = quirk_sandybridge_ifu; 1661 quirk_no_way_out = quirk_sandybridge_ifu;
1636 /*
1637 * MCG_CAP.MCG_SER_P is necessary but not sufficient to know
1638 * whether this processor will actually generate recoverable
1639 * machine checks. Check to see if this is an E7 model Xeon.
1640 * We can't do a model number check because E5 and E7 use the
1641 * same model number. E5 doesn't support recovery, E7 does.
1642 */
1643 if (mca_cfg.recovery || (mca_cfg.ser &&
1644 !strncmp(c->x86_model_id,
1645 "Intel(R) Xeon(R) CPU E7-", 24)))
1646 set_cpu_cap(c, X86_FEATURE_MCE_RECOVERY);
1647 } 1662 }
1648 if (cfg->monarch_timeout < 0) 1663 if (cfg->monarch_timeout < 0)
1649 cfg->monarch_timeout = 0; 1664 cfg->monarch_timeout = 0;
@@ -2080,6 +2095,7 @@ void mce_disable_bank(int bank)
2080 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD. 2095 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
2081 * mce=nobootlog Don't log MCEs from before booting. 2096 * mce=nobootlog Don't log MCEs from before booting.
2082 * mce=bios_cmci_threshold Don't program the CMCI threshold 2097 * mce=bios_cmci_threshold Don't program the CMCI threshold
2098 * mce=recovery force enable memcpy_mcsafe()
2083 */ 2099 */
2084static int __init mcheck_enable(char *str) 2100static int __init mcheck_enable(char *str)
2085{ 2101{
@@ -2676,8 +2692,14 @@ static int __init mcheck_debugfs_init(void)
2676static int __init mcheck_debugfs_init(void) { return -EINVAL; } 2692static int __init mcheck_debugfs_init(void) { return -EINVAL; }
2677#endif 2693#endif
2678 2694
2695DEFINE_STATIC_KEY_FALSE(mcsafe_key);
2696EXPORT_SYMBOL_GPL(mcsafe_key);
2697
2679static int __init mcheck_late_init(void) 2698static int __init mcheck_late_init(void)
2680{ 2699{
2700 if (mca_cfg.recovery)
2701 static_branch_inc(&mcsafe_key);
2702
2681 mcheck_debugfs_init(); 2703 mcheck_debugfs_init();
2682 2704
2683 /* 2705 /*
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 7b7f3be783d4..9b5403462936 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -20,6 +20,7 @@
20#include <linux/init.h> 20#include <linux/init.h>
21#include <linux/cpu.h> 21#include <linux/cpu.h>
22#include <linux/smp.h> 22#include <linux/smp.h>
23#include <linux/string.h>
23 24
24#include <asm/amd_nb.h> 25#include <asm/amd_nb.h>
25#include <asm/apic.h> 26#include <asm/apic.h>
@@ -63,34 +64,71 @@ static const char * const th_names[] = {
63 "execution_unit", 64 "execution_unit",
64}; 65};
65 66
66/* Define HWID to IP type mappings for Scalable MCA */ 67static const char * const smca_umc_block_names[] = {
67struct amd_hwid amd_hwids[] = { 68 "dram_ecc",
68 [SMCA_F17H_CORE] = { "f17h_core", 0xB0 }, 69 "misc_umc"
69 [SMCA_DF] = { "data_fabric", 0x2E },
70 [SMCA_UMC] = { "umc", 0x96 },
71 [SMCA_PB] = { "param_block", 0x5 },
72 [SMCA_PSP] = { "psp", 0xFF },
73 [SMCA_SMU] = { "smu", 0x1 },
74}; 70};
75EXPORT_SYMBOL_GPL(amd_hwids); 71
76 72struct smca_bank_name smca_bank_names[] = {
77const char * const amd_core_mcablock_names[] = { 73 [SMCA_LS] = { "load_store", "Load Store Unit" },
78 [SMCA_LS] = "load_store", 74 [SMCA_IF] = { "insn_fetch", "Instruction Fetch Unit" },
79 [SMCA_IF] = "insn_fetch", 75 [SMCA_L2_CACHE] = { "l2_cache", "L2 Cache" },
80 [SMCA_L2_CACHE] = "l2_cache", 76 [SMCA_DE] = { "decode_unit", "Decode Unit" },
81 [SMCA_DE] = "decode_unit", 77 [SMCA_EX] = { "execution_unit", "Execution Unit" },
82 [RES] = "", 78 [SMCA_FP] = { "floating_point", "Floating Point Unit" },
83 [SMCA_EX] = "execution_unit", 79 [SMCA_L3_CACHE] = { "l3_cache", "L3 Cache" },
84 [SMCA_FP] = "floating_point", 80 [SMCA_CS] = { "coherent_slave", "Coherent Slave" },
85 [SMCA_L3_CACHE] = "l3_cache", 81 [SMCA_PIE] = { "pie", "Power, Interrupts, etc." },
82 [SMCA_UMC] = { "umc", "Unified Memory Controller" },
83 [SMCA_PB] = { "param_block", "Parameter Block" },
84 [SMCA_PSP] = { "psp", "Platform Security Processor" },
85 [SMCA_SMU] = { "smu", "System Management Unit" },
86}; 86};
87EXPORT_SYMBOL_GPL(amd_core_mcablock_names); 87EXPORT_SYMBOL_GPL(smca_bank_names);
88
89static struct smca_hwid_mcatype smca_hwid_mcatypes[] = {
90 /* { bank_type, hwid_mcatype, xec_bitmap } */
91
92 /* ZN Core (HWID=0xB0) MCA types */
93 { SMCA_LS, HWID_MCATYPE(0xB0, 0x0), 0x1FFFEF },
94 { SMCA_IF, HWID_MCATYPE(0xB0, 0x1), 0x3FFF },
95 { SMCA_L2_CACHE, HWID_MCATYPE(0xB0, 0x2), 0xF },
96 { SMCA_DE, HWID_MCATYPE(0xB0, 0x3), 0x1FF },
97 /* HWID 0xB0 MCATYPE 0x4 is Reserved */
98 { SMCA_EX, HWID_MCATYPE(0xB0, 0x5), 0x7FF },
99 { SMCA_FP, HWID_MCATYPE(0xB0, 0x6), 0x7F },
100 { SMCA_L3_CACHE, HWID_MCATYPE(0xB0, 0x7), 0xFF },
101
102 /* Data Fabric MCA types */
103 { SMCA_CS, HWID_MCATYPE(0x2E, 0x0), 0x1FF },
104 { SMCA_PIE, HWID_MCATYPE(0x2E, 0x1), 0xF },
105
106 /* Unified Memory Controller MCA type */
107 { SMCA_UMC, HWID_MCATYPE(0x96, 0x0), 0x3F },
108
109 /* Parameter Block MCA type */
110 { SMCA_PB, HWID_MCATYPE(0x05, 0x0), 0x1 },
111
112 /* Platform Security Processor MCA type */
113 { SMCA_PSP, HWID_MCATYPE(0xFF, 0x0), 0x1 },
88 114
89const char * const amd_df_mcablock_names[] = { 115 /* System Management Unit MCA type */
90 [SMCA_CS] = "coherent_slave", 116 { SMCA_SMU, HWID_MCATYPE(0x01, 0x0), 0x1 },
91 [SMCA_PIE] = "pie",
92}; 117};
93EXPORT_SYMBOL_GPL(amd_df_mcablock_names); 118
119struct smca_bank_info smca_banks[MAX_NR_BANKS];
120EXPORT_SYMBOL_GPL(smca_banks);
121
122/*
123 * In SMCA enabled processors, we can have multiple banks for a given IP type.
124 * So to define a unique name for each bank, we use a temp c-string to append
125 * the MCA_IPID[InstanceId] to type's name in get_name().
126 *
127 * InstanceId is 32 bits which is 8 characters. Make sure MAX_MCATYPE_NAME_LEN
128 * is greater than 8 plus 1 (for underscore) plus length of longest type name.
129 */
130#define MAX_MCATYPE_NAME_LEN 30
131static char buf_mcatype[MAX_MCATYPE_NAME_LEN];
94 132
95static DEFINE_PER_CPU(struct threshold_bank **, threshold_banks); 133static DEFINE_PER_CPU(struct threshold_bank **, threshold_banks);
96static DEFINE_PER_CPU(unsigned int, bank_map); /* see which banks are on */ 134static DEFINE_PER_CPU(unsigned int, bank_map); /* see which banks are on */
@@ -108,6 +146,36 @@ void (*deferred_error_int_vector)(void) = default_deferred_error_interrupt;
108 * CPU Initialization 146 * CPU Initialization
109 */ 147 */
110 148
149static void get_smca_bank_info(unsigned int bank)
150{
151 unsigned int i, hwid_mcatype, cpu = smp_processor_id();
152 struct smca_hwid_mcatype *type;
153 u32 high, instanceId;
154 u16 hwid, mcatype;
155
156 /* Collect bank_info using CPU 0 for now. */
157 if (cpu)
158 return;
159
160 if (rdmsr_safe_on_cpu(cpu, MSR_AMD64_SMCA_MCx_IPID(bank), &instanceId, &high)) {
161 pr_warn("Failed to read MCA_IPID for bank %d\n", bank);
162 return;
163 }
164
165 hwid = high & MCI_IPID_HWID;
166 mcatype = (high & MCI_IPID_MCATYPE) >> 16;
167 hwid_mcatype = HWID_MCATYPE(hwid, mcatype);
168
169 for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) {
170 type = &smca_hwid_mcatypes[i];
171 if (hwid_mcatype == type->hwid_mcatype) {
172 smca_banks[bank].type = type;
173 smca_banks[bank].type_instance = instanceId;
174 break;
175 }
176 }
177}
178
111struct thresh_restart { 179struct thresh_restart {
112 struct threshold_block *b; 180 struct threshold_block *b;
113 int reset; 181 int reset;
@@ -293,7 +361,7 @@ static void deferred_error_interrupt_enable(struct cpuinfo_x86 *c)
293 wrmsr(MSR_CU_DEF_ERR, low, high); 361 wrmsr(MSR_CU_DEF_ERR, low, high);
294} 362}
295 363
296static u32 get_block_address(u32 current_addr, u32 low, u32 high, 364static u32 get_block_address(unsigned int cpu, u32 current_addr, u32 low, u32 high,
297 unsigned int bank, unsigned int block) 365 unsigned int bank, unsigned int block)
298{ 366{
299 u32 addr = 0, offset = 0; 367 u32 addr = 0, offset = 0;
@@ -309,13 +377,13 @@ static u32 get_block_address(u32 current_addr, u32 low, u32 high,
309 */ 377 */
310 u32 low, high; 378 u32 low, high;
311 379
312 if (rdmsr_safe(MSR_AMD64_SMCA_MCx_CONFIG(bank), &low, &high)) 380 if (rdmsr_safe_on_cpu(cpu, MSR_AMD64_SMCA_MCx_CONFIG(bank), &low, &high))
313 return addr; 381 return addr;
314 382
315 if (!(low & MCI_CONFIG_MCAX)) 383 if (!(low & MCI_CONFIG_MCAX))
316 return addr; 384 return addr;
317 385
318 if (!rdmsr_safe(MSR_AMD64_SMCA_MCx_MISC(bank), &low, &high) && 386 if (!rdmsr_safe_on_cpu(cpu, MSR_AMD64_SMCA_MCx_MISC(bank), &low, &high) &&
319 (low & MASK_BLKPTR_LO)) 387 (low & MASK_BLKPTR_LO))
320 addr = MSR_AMD64_SMCA_MCx_MISCy(bank, block - 1); 388 addr = MSR_AMD64_SMCA_MCx_MISCy(bank, block - 1);
321 } 389 }
@@ -395,6 +463,20 @@ prepare_threshold_block(unsigned int bank, unsigned int block, u32 addr,
395 */ 463 */
396 smca_high &= ~BIT(2); 464 smca_high &= ~BIT(2);
397 465
466 /*
467 * SMCA sets the Deferred Error Interrupt type per bank.
468 *
469 * MCA_CONFIG[DeferredIntTypeSupported] is bit 5, and tells us
470 * if the DeferredIntType bit field is available.
471 *
472 * MCA_CONFIG[DeferredIntType] is bits [38:37] ([6:5] in the
473 * high portion of the MSR). OS should set this to 0x1 to enable
474 * APIC based interrupt. First, check that no interrupt has been
475 * set.
476 */
477 if ((smca_low & BIT(5)) && !((smca_high >> 5) & 0x3))
478 smca_high |= BIT(5);
479
398 wrmsr(smca_addr, smca_low, smca_high); 480 wrmsr(smca_addr, smca_low, smca_high);
399 } 481 }
400 482
@@ -421,12 +503,15 @@ out:
421void mce_amd_feature_init(struct cpuinfo_x86 *c) 503void mce_amd_feature_init(struct cpuinfo_x86 *c)
422{ 504{
423 u32 low = 0, high = 0, address = 0; 505 u32 low = 0, high = 0, address = 0;
424 unsigned int bank, block; 506 unsigned int bank, block, cpu = smp_processor_id();
425 int offset = -1; 507 int offset = -1;
426 508
427 for (bank = 0; bank < mca_cfg.banks; ++bank) { 509 for (bank = 0; bank < mca_cfg.banks; ++bank) {
510 if (mce_flags.smca)
511 get_smca_bank_info(bank);
512
428 for (block = 0; block < NR_BLOCKS; ++block) { 513 for (block = 0; block < NR_BLOCKS; ++block) {
429 address = get_block_address(address, low, high, bank, block); 514 address = get_block_address(cpu, address, low, high, bank, block);
430 if (!address) 515 if (!address)
431 break; 516 break;
432 517
@@ -476,9 +561,27 @@ __log_error(unsigned int bank, bool deferred_err, bool threshold_err, u64 misc)
476 if (threshold_err) 561 if (threshold_err)
477 m.misc = misc; 562 m.misc = misc;
478 563
479 if (m.status & MCI_STATUS_ADDRV) 564 if (m.status & MCI_STATUS_ADDRV) {
480 rdmsrl(msr_addr, m.addr); 565 rdmsrl(msr_addr, m.addr);
481 566
567 /*
568 * Extract [55:<lsb>] where lsb is the least significant
569 * *valid* bit of the address bits.
570 */
571 if (mce_flags.smca) {
572 u8 lsb = (m.addr >> 56) & 0x3f;
573
574 m.addr &= GENMASK_ULL(55, lsb);
575 }
576 }
577
578 if (mce_flags.smca) {
579 rdmsrl(MSR_AMD64_SMCA_MCx_IPID(bank), m.ipid);
580
581 if (m.status & MCI_STATUS_SYNDV)
582 rdmsrl(MSR_AMD64_SMCA_MCx_SYND(bank), m.synd);
583 }
584
482 mce_log(&m); 585 mce_log(&m);
483 586
484 wrmsrl(msr_status, 0); 587 wrmsrl(msr_status, 0);
@@ -541,15 +644,14 @@ static void amd_deferred_error_interrupt(void)
541static void amd_threshold_interrupt(void) 644static void amd_threshold_interrupt(void)
542{ 645{
543 u32 low = 0, high = 0, address = 0; 646 u32 low = 0, high = 0, address = 0;
544 int cpu = smp_processor_id(); 647 unsigned int bank, block, cpu = smp_processor_id();
545 unsigned int bank, block;
546 648
547 /* assume first bank caused it */ 649 /* assume first bank caused it */
548 for (bank = 0; bank < mca_cfg.banks; ++bank) { 650 for (bank = 0; bank < mca_cfg.banks; ++bank) {
549 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 651 if (!(per_cpu(bank_map, cpu) & (1 << bank)))
550 continue; 652 continue;
551 for (block = 0; block < NR_BLOCKS; ++block) { 653 for (block = 0; block < NR_BLOCKS; ++block) {
552 address = get_block_address(address, low, high, bank, block); 654 address = get_block_address(cpu, address, low, high, bank, block);
553 if (!address) 655 if (!address)
554 break; 656 break;
555 657
@@ -713,6 +815,34 @@ static struct kobj_type threshold_ktype = {
713 .default_attrs = default_attrs, 815 .default_attrs = default_attrs,
714}; 816};
715 817
818static const char *get_name(unsigned int bank, struct threshold_block *b)
819{
820 unsigned int bank_type;
821
822 if (!mce_flags.smca) {
823 if (b && bank == 4)
824 return bank4_names(b);
825
826 return th_names[bank];
827 }
828
829 if (!smca_banks[bank].type)
830 return NULL;
831
832 bank_type = smca_banks[bank].type->bank_type;
833
834 if (b && bank_type == SMCA_UMC) {
835 if (b->block < ARRAY_SIZE(smca_umc_block_names))
836 return smca_umc_block_names[b->block];
837 return NULL;
838 }
839
840 snprintf(buf_mcatype, MAX_MCATYPE_NAME_LEN,
841 "%s_%x", smca_bank_names[bank_type].name,
842 smca_banks[bank].type_instance);
843 return buf_mcatype;
844}
845
716static int allocate_threshold_blocks(unsigned int cpu, unsigned int bank, 846static int allocate_threshold_blocks(unsigned int cpu, unsigned int bank,
717 unsigned int block, u32 address) 847 unsigned int block, u32 address)
718{ 848{
@@ -767,11 +897,11 @@ static int allocate_threshold_blocks(unsigned int cpu, unsigned int bank,
767 897
768 err = kobject_init_and_add(&b->kobj, &threshold_ktype, 898 err = kobject_init_and_add(&b->kobj, &threshold_ktype,
769 per_cpu(threshold_banks, cpu)[bank]->kobj, 899 per_cpu(threshold_banks, cpu)[bank]->kobj,
770 (bank == 4 ? bank4_names(b) : th_names[bank])); 900 get_name(bank, b));
771 if (err) 901 if (err)
772 goto out_free; 902 goto out_free;
773recurse: 903recurse:
774 address = get_block_address(address, low, high, bank, ++block); 904 address = get_block_address(cpu, address, low, high, bank, ++block);
775 if (!address) 905 if (!address)
776 return 0; 906 return 0;
777 907
@@ -822,7 +952,7 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank)
822 struct device *dev = per_cpu(mce_device, cpu); 952 struct device *dev = per_cpu(mce_device, cpu);
823 struct amd_northbridge *nb = NULL; 953 struct amd_northbridge *nb = NULL;
824 struct threshold_bank *b = NULL; 954 struct threshold_bank *b = NULL;
825 const char *name = th_names[bank]; 955 const char *name = get_name(bank, NULL);
826 int err = 0; 956 int err = 0;
827 957
828 if (is_shared_bank(bank)) { 958 if (is_shared_bank(bank)) {
@@ -869,7 +999,7 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank)
869 } 999 }
870 } 1000 }
871 1001
872 err = allocate_threshold_blocks(cpu, bank, 0, MSR_IA32_MCx_MISC(bank)); 1002 err = allocate_threshold_blocks(cpu, bank, 0, msr_ops.misc(bank));
873 if (!err) 1003 if (!err)
874 goto out; 1004 goto out;
875 1005
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 1726c4c12336..865058d087ac 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -575,9 +575,6 @@ static void kvm_kick_cpu(int cpu)
575 kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid); 575 kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
576} 576}
577 577
578
579#ifdef CONFIG_QUEUED_SPINLOCKS
580
581#include <asm/qspinlock.h> 578#include <asm/qspinlock.h>
582 579
583static void kvm_wait(u8 *ptr, u8 val) 580static void kvm_wait(u8 *ptr, u8 val)
@@ -606,243 +603,6 @@ out:
606 local_irq_restore(flags); 603 local_irq_restore(flags);
607} 604}
608 605
609#else /* !CONFIG_QUEUED_SPINLOCKS */
610
611enum kvm_contention_stat {
612 TAKEN_SLOW,
613 TAKEN_SLOW_PICKUP,
614 RELEASED_SLOW,
615 RELEASED_SLOW_KICKED,
616 NR_CONTENTION_STATS
617};
618
619#ifdef CONFIG_KVM_DEBUG_FS
620#define HISTO_BUCKETS 30
621
622static struct kvm_spinlock_stats
623{
624 u32 contention_stats[NR_CONTENTION_STATS];
625 u32 histo_spin_blocked[HISTO_BUCKETS+1];
626 u64 time_blocked;
627} spinlock_stats;
628
629static u8 zero_stats;
630
631static inline void check_zero(void)
632{
633 u8 ret;
634 u8 old;
635
636 old = READ_ONCE(zero_stats);
637 if (unlikely(old)) {
638 ret = cmpxchg(&zero_stats, old, 0);
639 /* This ensures only one fellow resets the stat */
640 if (ret == old)
641 memset(&spinlock_stats, 0, sizeof(spinlock_stats));
642 }
643}
644
645static inline void add_stats(enum kvm_contention_stat var, u32 val)
646{
647 check_zero();
648 spinlock_stats.contention_stats[var] += val;
649}
650
651
652static inline u64 spin_time_start(void)
653{
654 return sched_clock();
655}
656
657static void __spin_time_accum(u64 delta, u32 *array)
658{
659 unsigned index;
660
661 index = ilog2(delta);
662 check_zero();
663
664 if (index < HISTO_BUCKETS)
665 array[index]++;
666 else
667 array[HISTO_BUCKETS]++;
668}
669
670static inline void spin_time_accum_blocked(u64 start)
671{
672 u32 delta;
673
674 delta = sched_clock() - start;
675 __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
676 spinlock_stats.time_blocked += delta;
677}
678
679static struct dentry *d_spin_debug;
680static struct dentry *d_kvm_debug;
681
682static struct dentry *kvm_init_debugfs(void)
683{
684 d_kvm_debug = debugfs_create_dir("kvm-guest", NULL);
685 if (!d_kvm_debug)
686 printk(KERN_WARNING "Could not create 'kvm' debugfs directory\n");
687
688 return d_kvm_debug;
689}
690
691static int __init kvm_spinlock_debugfs(void)
692{
693 struct dentry *d_kvm;
694
695 d_kvm = kvm_init_debugfs();
696 if (d_kvm == NULL)
697 return -ENOMEM;
698
699 d_spin_debug = debugfs_create_dir("spinlocks", d_kvm);
700
701 debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
702
703 debugfs_create_u32("taken_slow", 0444, d_spin_debug,
704 &spinlock_stats.contention_stats[TAKEN_SLOW]);
705 debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
706 &spinlock_stats.contention_stats[TAKEN_SLOW_PICKUP]);
707
708 debugfs_create_u32("released_slow", 0444, d_spin_debug,
709 &spinlock_stats.contention_stats[RELEASED_SLOW]);
710 debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
711 &spinlock_stats.contention_stats[RELEASED_SLOW_KICKED]);
712
713 debugfs_create_u64("time_blocked", 0444, d_spin_debug,
714 &spinlock_stats.time_blocked);
715
716 debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
717 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
718
719 return 0;
720}
721fs_initcall(kvm_spinlock_debugfs);
722#else /* !CONFIG_KVM_DEBUG_FS */
723static inline void add_stats(enum kvm_contention_stat var, u32 val)
724{
725}
726
727static inline u64 spin_time_start(void)
728{
729 return 0;
730}
731
732static inline void spin_time_accum_blocked(u64 start)
733{
734}
735#endif /* CONFIG_KVM_DEBUG_FS */
736
737struct kvm_lock_waiting {
738 struct arch_spinlock *lock;
739 __ticket_t want;
740};
741
742/* cpus 'waiting' on a spinlock to become available */
743static cpumask_t waiting_cpus;
744
745/* Track spinlock on which a cpu is waiting */
746static DEFINE_PER_CPU(struct kvm_lock_waiting, klock_waiting);
747
748__visible void kvm_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
749{
750 struct kvm_lock_waiting *w;
751 int cpu;
752 u64 start;
753 unsigned long flags;
754 __ticket_t head;
755
756 if (in_nmi())
757 return;
758
759 w = this_cpu_ptr(&klock_waiting);
760 cpu = smp_processor_id();
761 start = spin_time_start();
762
763 /*
764 * Make sure an interrupt handler can't upset things in a
765 * partially setup state.
766 */
767 local_irq_save(flags);
768
769 /*
770 * The ordering protocol on this is that the "lock" pointer
771 * may only be set non-NULL if the "want" ticket is correct.
772 * If we're updating "want", we must first clear "lock".
773 */
774 w->lock = NULL;
775 smp_wmb();
776 w->want = want;
777 smp_wmb();
778 w->lock = lock;
779
780 add_stats(TAKEN_SLOW, 1);
781
782 /*
783 * This uses set_bit, which is atomic but we should not rely on its
784 * reordering gurantees. So barrier is needed after this call.
785 */
786 cpumask_set_cpu(cpu, &waiting_cpus);
787
788 barrier();
789
790 /*
791 * Mark entry to slowpath before doing the pickup test to make
792 * sure we don't deadlock with an unlocker.
793 */
794 __ticket_enter_slowpath(lock);
795
796 /* make sure enter_slowpath, which is atomic does not cross the read */
797 smp_mb__after_atomic();
798
799 /*
800 * check again make sure it didn't become free while
801 * we weren't looking.
802 */
803 head = READ_ONCE(lock->tickets.head);
804 if (__tickets_equal(head, want)) {
805 add_stats(TAKEN_SLOW_PICKUP, 1);
806 goto out;
807 }
808
809 /*
810 * halt until it's our turn and kicked. Note that we do safe halt
811 * for irq enabled case to avoid hang when lock info is overwritten
812 * in irq spinlock slowpath and no spurious interrupt occur to save us.
813 */
814 if (arch_irqs_disabled_flags(flags))
815 halt();
816 else
817 safe_halt();
818
819out:
820 cpumask_clear_cpu(cpu, &waiting_cpus);
821 w->lock = NULL;
822 local_irq_restore(flags);
823 spin_time_accum_blocked(start);
824}
825PV_CALLEE_SAVE_REGS_THUNK(kvm_lock_spinning);
826
827/* Kick vcpu waiting on @lock->head to reach value @ticket */
828static void kvm_unlock_kick(struct arch_spinlock *lock, __ticket_t ticket)
829{
830 int cpu;
831
832 add_stats(RELEASED_SLOW, 1);
833 for_each_cpu(cpu, &waiting_cpus) {
834 const struct kvm_lock_waiting *w = &per_cpu(klock_waiting, cpu);
835 if (READ_ONCE(w->lock) == lock &&
836 READ_ONCE(w->want) == ticket) {
837 add_stats(RELEASED_SLOW_KICKED, 1);
838 kvm_kick_cpu(cpu);
839 break;
840 }
841 }
842}
843
844#endif /* !CONFIG_QUEUED_SPINLOCKS */
845
846/* 606/*
847 * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present. 607 * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
848 */ 608 */
@@ -854,16 +614,11 @@ void __init kvm_spinlock_init(void)
854 if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) 614 if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
855 return; 615 return;
856 616
857#ifdef CONFIG_QUEUED_SPINLOCKS
858 __pv_init_lock_hash(); 617 __pv_init_lock_hash();
859 pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 618 pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
860 pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); 619 pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock);
861 pv_lock_ops.wait = kvm_wait; 620 pv_lock_ops.wait = kvm_wait;
862 pv_lock_ops.kick = kvm_kick_cpu; 621 pv_lock_ops.kick = kvm_kick_cpu;
863#else /* !CONFIG_QUEUED_SPINLOCKS */
864 pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(kvm_lock_spinning);
865 pv_lock_ops.unlock_kick = kvm_unlock_kick;
866#endif
867} 622}
868 623
869static __init int kvm_spinlock_init_jump(void) 624static __init int kvm_spinlock_init_jump(void)
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index 068c4a929de6..0f8d20497383 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -499,6 +499,9 @@ void __init default_get_smp_config(unsigned int early)
499{ 499{
500 struct mpf_intel *mpf = mpf_found; 500 struct mpf_intel *mpf = mpf_found;
501 501
502 if (!smp_found_config)
503 return;
504
502 if (!mpf) 505 if (!mpf)
503 return; 506 return;
504 507
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index 1939a0269377..2c55a003b793 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -8,7 +8,6 @@
8 8
9#include <asm/paravirt.h> 9#include <asm/paravirt.h>
10 10
11#ifdef CONFIG_QUEUED_SPINLOCKS
12__visible void __native_queued_spin_unlock(struct qspinlock *lock) 11__visible void __native_queued_spin_unlock(struct qspinlock *lock)
13{ 12{
14 native_queued_spin_unlock(lock); 13 native_queued_spin_unlock(lock);
@@ -21,19 +20,13 @@ bool pv_is_native_spin_unlock(void)
21 return pv_lock_ops.queued_spin_unlock.func == 20 return pv_lock_ops.queued_spin_unlock.func ==
22 __raw_callee_save___native_queued_spin_unlock; 21 __raw_callee_save___native_queued_spin_unlock;
23} 22}
24#endif
25 23
26struct pv_lock_ops pv_lock_ops = { 24struct pv_lock_ops pv_lock_ops = {
27#ifdef CONFIG_SMP 25#ifdef CONFIG_SMP
28#ifdef CONFIG_QUEUED_SPINLOCKS
29 .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, 26 .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
30 .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), 27 .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock),
31 .wait = paravirt_nop, 28 .wait = paravirt_nop,
32 .kick = paravirt_nop, 29 .kick = paravirt_nop,
33#else /* !CONFIG_QUEUED_SPINLOCKS */
34 .lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
35 .unlock_kick = paravirt_nop,
36#endif /* !CONFIG_QUEUED_SPINLOCKS */
37#endif /* SMP */ 30#endif /* SMP */
38}; 31};
39EXPORT_SYMBOL(pv_lock_ops); 32EXPORT_SYMBOL(pv_lock_ops);
diff --git a/arch/x86/kernel/paravirt_patch_32.c b/arch/x86/kernel/paravirt_patch_32.c
index 158dc0650d5d..920c6ae08592 100644
--- a/arch/x86/kernel/paravirt_patch_32.c
+++ b/arch/x86/kernel/paravirt_patch_32.c
@@ -10,7 +10,7 @@ DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3");
10DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax"); 10DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax");
11DEF_NATIVE(pv_cpu_ops, clts, "clts"); 11DEF_NATIVE(pv_cpu_ops, clts, "clts");
12 12
13#if defined(CONFIG_PARAVIRT_SPINLOCKS) && defined(CONFIG_QUEUED_SPINLOCKS) 13#if defined(CONFIG_PARAVIRT_SPINLOCKS)
14DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%eax)"); 14DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%eax)");
15#endif 15#endif
16 16
@@ -49,7 +49,7 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
49 PATCH_SITE(pv_mmu_ops, read_cr3); 49 PATCH_SITE(pv_mmu_ops, read_cr3);
50 PATCH_SITE(pv_mmu_ops, write_cr3); 50 PATCH_SITE(pv_mmu_ops, write_cr3);
51 PATCH_SITE(pv_cpu_ops, clts); 51 PATCH_SITE(pv_cpu_ops, clts);
52#if defined(CONFIG_PARAVIRT_SPINLOCKS) && defined(CONFIG_QUEUED_SPINLOCKS) 52#if defined(CONFIG_PARAVIRT_SPINLOCKS)
53 case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock): 53 case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock):
54 if (pv_is_native_spin_unlock()) { 54 if (pv_is_native_spin_unlock()) {
55 start = start_pv_lock_ops_queued_spin_unlock; 55 start = start_pv_lock_ops_queued_spin_unlock;
diff --git a/arch/x86/kernel/paravirt_patch_64.c b/arch/x86/kernel/paravirt_patch_64.c
index e70087a04cc8..bb3840cedb4f 100644
--- a/arch/x86/kernel/paravirt_patch_64.c
+++ b/arch/x86/kernel/paravirt_patch_64.c
@@ -19,7 +19,7 @@ DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs");
19DEF_NATIVE(, mov32, "mov %edi, %eax"); 19DEF_NATIVE(, mov32, "mov %edi, %eax");
20DEF_NATIVE(, mov64, "mov %rdi, %rax"); 20DEF_NATIVE(, mov64, "mov %rdi, %rax");
21 21
22#if defined(CONFIG_PARAVIRT_SPINLOCKS) && defined(CONFIG_QUEUED_SPINLOCKS) 22#if defined(CONFIG_PARAVIRT_SPINLOCKS)
23DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%rdi)"); 23DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%rdi)");
24#endif 24#endif
25 25
@@ -61,7 +61,7 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
61 PATCH_SITE(pv_cpu_ops, clts); 61 PATCH_SITE(pv_cpu_ops, clts);
62 PATCH_SITE(pv_mmu_ops, flush_tlb_single); 62 PATCH_SITE(pv_mmu_ops, flush_tlb_single);
63 PATCH_SITE(pv_cpu_ops, wbinvd); 63 PATCH_SITE(pv_cpu_ops, wbinvd);
64#if defined(CONFIG_PARAVIRT_SPINLOCKS) && defined(CONFIG_QUEUED_SPINLOCKS) 64#if defined(CONFIG_PARAVIRT_SPINLOCKS)
65 case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock): 65 case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock):
66 if (pv_is_native_spin_unlock()) { 66 if (pv_is_native_spin_unlock()) {
67 start = start_pv_lock_ops_queued_spin_unlock; 67 start = start_pv_lock_ops_queued_spin_unlock;
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index cc457ff818ad..51402a7e4ca6 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -626,3 +626,34 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3,
626 amd_disable_seq_and_redirect_scrub); 626 amd_disable_seq_and_redirect_scrub);
627 627
628#endif 628#endif
629
630#if defined(CONFIG_X86_64) && defined(CONFIG_X86_MCE)
631#include <linux/jump_label.h>
632#include <asm/string_64.h>
633
634/* Ivy Bridge, Haswell, Broadwell */
635static void quirk_intel_brickland_xeon_ras_cap(struct pci_dev *pdev)
636{
637 u32 capid0;
638
639 pci_read_config_dword(pdev, 0x84, &capid0);
640
641 if (capid0 & 0x10)
642 static_branch_inc(&mcsafe_key);
643}
644
645/* Skylake */
646static void quirk_intel_purley_xeon_ras_cap(struct pci_dev *pdev)
647{
648 u32 capid0;
649
650 pci_read_config_dword(pdev, 0x84, &capid0);
651
652 if ((capid0 & 0xc0) == 0xc0)
653 static_branch_inc(&mcsafe_key);
654}
655DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x0ec3, quirk_intel_brickland_xeon_ras_cap);
656DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2fc0, quirk_intel_brickland_xeon_ras_cap);
657DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x6fc0, quirk_intel_brickland_xeon_ras_cap);
658DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2083, quirk_intel_purley_xeon_ras_cap);
659#endif
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 3aabfdcbcb52..eeb094ea794a 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1096,19 +1096,19 @@ void __init setup_arch(char **cmdline_p)
1096 memblock_set_current_limit(ISA_END_ADDRESS); 1096 memblock_set_current_limit(ISA_END_ADDRESS);
1097 memblock_x86_fill(); 1097 memblock_x86_fill();
1098 1098
1099 if (efi_enabled(EFI_BOOT)) { 1099 reserve_bios_regions();
1100
1101 if (efi_enabled(EFI_MEMMAP)) {
1100 efi_fake_memmap(); 1102 efi_fake_memmap();
1101 efi_find_mirror(); 1103 efi_find_mirror();
1102 } 1104 efi_esrt_init();
1103
1104 reserve_bios_regions();
1105 1105
1106 /* 1106 /*
1107 * The EFI specification says that boot service code won't be called 1107 * The EFI specification says that boot service code won't be
1108 * after ExitBootServices(). This is, in fact, a lie. 1108 * called after ExitBootServices(). This is, in fact, a lie.
1109 */ 1109 */
1110 if (efi_enabled(EFI_MEMMAP))
1111 efi_reserve_boot_services(); 1110 efi_reserve_boot_services();
1111 }
1112 1112
1113 /* preallocate 4k for mptable mpc */ 1113 /* preallocate 4k for mptable mpc */
1114 early_reserve_e820_mpc_new(); 1114 early_reserve_e820_mpc_new();
@@ -1219,8 +1219,7 @@ void __init setup_arch(char **cmdline_p)
1219 /* 1219 /*
1220 * get boot-time SMP configuration: 1220 * get boot-time SMP configuration:
1221 */ 1221 */
1222 if (smp_found_config) 1222 get_smp_config();
1223 get_smp_config();
1224 1223
1225 prefill_possible_map(); 1224 prefill_possible_map();
1226 1225
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 7e52f83d3a4b..7249dcf2cbcb 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -471,7 +471,7 @@ static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
471 return false; 471 return false;
472} 472}
473 473
474static struct sched_domain_topology_level numa_inside_package_topology[] = { 474static struct sched_domain_topology_level x86_numa_in_package_topology[] = {
475#ifdef CONFIG_SCHED_SMT 475#ifdef CONFIG_SCHED_SMT
476 { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, 476 { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
477#endif 477#endif
@@ -480,22 +480,23 @@ static struct sched_domain_topology_level numa_inside_package_topology[] = {
480#endif 480#endif
481 { NULL, }, 481 { NULL, },
482}; 482};
483
484static struct sched_domain_topology_level x86_topology[] = {
485#ifdef CONFIG_SCHED_SMT
486 { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
487#endif
488#ifdef CONFIG_SCHED_MC
489 { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
490#endif
491 { cpu_cpu_mask, SD_INIT_NAME(DIE) },
492 { NULL, },
493};
494
483/* 495/*
484 * set_sched_topology() sets the topology internal to a CPU. The 496 * Set if a package/die has multiple NUMA nodes inside.
485 * NUMA topologies are layered on top of it to build the full 497 * AMD Magny-Cours and Intel Cluster-on-Die have this.
486 * system topology.
487 *
488 * If NUMA nodes are observed to occur within a CPU package, this
489 * function should be called. It forces the sched domain code to
490 * only use the SMT level for the CPU portion of the topology.
491 * This essentially falls back to relying on NUMA information
492 * from the SRAT table to describe the entire system topology
493 * (except for hyperthreads).
494 */ 498 */
495static void primarily_use_numa_for_topology(void) 499static bool x86_has_numa_in_package;
496{
497 set_sched_topology(numa_inside_package_topology);
498}
499 500
500void set_cpu_sibling_map(int cpu) 501void set_cpu_sibling_map(int cpu)
501{ 502{
@@ -558,7 +559,7 @@ void set_cpu_sibling_map(int cpu)
558 c->booted_cores = cpu_data(i).booted_cores; 559 c->booted_cores = cpu_data(i).booted_cores;
559 } 560 }
560 if (match_die(c, o) && !topology_same_node(c, o)) 561 if (match_die(c, o) && !topology_same_node(c, o))
561 primarily_use_numa_for_topology(); 562 x86_has_numa_in_package = true;
562 } 563 }
563 564
564 threads = cpumask_weight(topology_sibling_cpumask(cpu)); 565 threads = cpumask_weight(topology_sibling_cpumask(cpu));
@@ -690,7 +691,7 @@ wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
690 * Give the other CPU some time to accept the IPI. 691 * Give the other CPU some time to accept the IPI.
691 */ 692 */
692 udelay(200); 693 udelay(200);
693 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { 694 if (APIC_INTEGRATED(boot_cpu_apic_version)) {
694 maxlvt = lapic_get_maxlvt(); 695 maxlvt = lapic_get_maxlvt();
695 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ 696 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
696 apic_write(APIC_ESR, 0); 697 apic_write(APIC_ESR, 0);
@@ -717,7 +718,7 @@ wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
717 /* 718 /*
718 * Be paranoid about clearing APIC errors. 719 * Be paranoid about clearing APIC errors.
719 */ 720 */
720 if (APIC_INTEGRATED(apic_version[phys_apicid])) { 721 if (APIC_INTEGRATED(boot_cpu_apic_version)) {
721 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ 722 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
722 apic_write(APIC_ESR, 0); 723 apic_write(APIC_ESR, 0);
723 apic_read(APIC_ESR); 724 apic_read(APIC_ESR);
@@ -756,7 +757,7 @@ wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
756 * Determine this based on the APIC version. 757 * Determine this based on the APIC version.
757 * If we don't have an integrated APIC, don't send the STARTUP IPIs. 758 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
758 */ 759 */
759 if (APIC_INTEGRATED(apic_version[phys_apicid])) 760 if (APIC_INTEGRATED(boot_cpu_apic_version))
760 num_starts = 2; 761 num_starts = 2;
761 else 762 else
762 num_starts = 0; 763 num_starts = 0;
@@ -993,7 +994,7 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
993 /* 994 /*
994 * Be paranoid about clearing APIC errors. 995 * Be paranoid about clearing APIC errors.
995 */ 996 */
996 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { 997 if (APIC_INTEGRATED(boot_cpu_apic_version)) {
997 apic_write(APIC_ESR, 0); 998 apic_write(APIC_ESR, 0);
998 apic_read(APIC_ESR); 999 apic_read(APIC_ESR);
999 } 1000 }
@@ -1248,7 +1249,7 @@ static int __init smp_sanity_check(unsigned max_cpus)
1248 /* 1249 /*
1249 * If we couldn't find a local APIC, then get out of here now! 1250 * If we couldn't find a local APIC, then get out of here now!
1250 */ 1251 */
1251 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && 1252 if (APIC_INTEGRATED(boot_cpu_apic_version) &&
1252 !boot_cpu_has(X86_FEATURE_APIC)) { 1253 !boot_cpu_has(X86_FEATURE_APIC)) {
1253 if (!disable_apic) { 1254 if (!disable_apic) {
1254 pr_err("BIOS bug, local APIC #%d not detected!...\n", 1255 pr_err("BIOS bug, local APIC #%d not detected!...\n",
@@ -1303,6 +1304,16 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
1303 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); 1304 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1304 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL); 1305 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1305 } 1306 }
1307
1308 /*
1309 * Set 'default' x86 topology, this matches default_topology() in that
1310 * it has NUMA nodes as a topology level. See also
1311 * native_smp_cpus_done().
1312 *
1313 * Must be done before set_cpus_sibling_map() is ran.
1314 */
1315 set_sched_topology(x86_topology);
1316
1306 set_cpu_sibling_map(0); 1317 set_cpu_sibling_map(0);
1307 1318
1308 switch (smp_sanity_check(max_cpus)) { 1319 switch (smp_sanity_check(max_cpus)) {
@@ -1322,14 +1333,13 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
1322 break; 1333 break;
1323 } 1334 }
1324 1335
1325 default_setup_apic_routing();
1326
1327 if (read_apic_id() != boot_cpu_physical_apicid) { 1336 if (read_apic_id() != boot_cpu_physical_apicid) {
1328 panic("Boot APIC ID in local APIC unexpected (%d vs %d)", 1337 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1329 read_apic_id(), boot_cpu_physical_apicid); 1338 read_apic_id(), boot_cpu_physical_apicid);
1330 /* Or can we switch back to PIC here? */ 1339 /* Or can we switch back to PIC here? */
1331 } 1340 }
1332 1341
1342 default_setup_apic_routing();
1333 cpu0_logical_apicid = apic_bsp_setup(false); 1343 cpu0_logical_apicid = apic_bsp_setup(false);
1334 1344
1335 pr_info("CPU%d: ", 0); 1345 pr_info("CPU%d: ", 0);
@@ -1369,6 +1379,9 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
1369{ 1379{
1370 pr_debug("Boot done\n"); 1380 pr_debug("Boot done\n");
1371 1381
1382 if (x86_has_numa_in_package)
1383 set_sched_topology(x86_numa_in_package_topology);
1384
1372 nmi_selftest(); 1385 nmi_selftest();
1373 impress_friends(); 1386 impress_friends();
1374 setup_ioapic_dest(); 1387 setup_ioapic_dest();
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 95e49f6e4fc3..b2cee3d19477 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -38,7 +38,7 @@ EXPORT_SYMBOL(__copy_user_nocache);
38EXPORT_SYMBOL(_copy_from_user); 38EXPORT_SYMBOL(_copy_from_user);
39EXPORT_SYMBOL(_copy_to_user); 39EXPORT_SYMBOL(_copy_to_user);
40 40
41EXPORT_SYMBOL_GPL(memcpy_mcsafe); 41EXPORT_SYMBOL_GPL(memcpy_mcsafe_unrolled);
42 42
43EXPORT_SYMBOL(copy_page); 43EXPORT_SYMBOL(copy_page);
44EXPORT_SYMBOL(clear_page); 44EXPORT_SYMBOL(clear_page);
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-14 08:47:53 -0400