aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/include/asm/apic.h29
-rw-r--r--arch/x86/include/asm/io_apic.h28
-rw-r--r--arch/x86/include/asm/x2apic.h62
-rw-r--r--arch/x86/kernel/acpi/boot.c8
-rw-r--r--arch/x86/kernel/apic/Makefile17
-rw-r--r--arch/x86/kernel/apic/apic.c48
-rw-r--r--arch/x86/kernel/apic/apic_flat_64.c26
-rw-r--r--arch/x86/kernel/apic/bigsmp_32.c12
-rw-r--r--arch/x86/kernel/apic/es7000_32.c10
-rw-r--r--arch/x86/kernel/apic/io_apic.c309
-rw-r--r--arch/x86/kernel/apic/numaq_32.c6
-rw-r--r--arch/x86/kernel/apic/probe_32.c117
-rw-r--r--arch/x86/kernel/apic/probe_64.c61
-rw-r--r--arch/x86/kernel/apic/summit_32.c4
-rw-r--r--arch/x86/kernel/apic/x2apic_cluster.c222
-rw-r--r--arch/x86/kernel/apic/x2apic_phys.c115
-rw-r--r--arch/x86/kernel/apic/x2apic_uv_x.c13
-rw-r--r--arch/x86/kernel/devicetree.c6
-rw-r--r--arch/x86/kernel/mpparse.c2
-rw-r--r--arch/x86/kernel/vmlinux.lds.S7
20 files changed, 539 insertions, 563 deletions
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index a0c46f061210..4a0b7c7e2cce 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -381,6 +381,26 @@ struct apic {
381extern struct apic *apic; 381extern struct apic *apic;
382 382
383/* 383/*
384 * APIC drivers are probed based on how they are listed in the .apicdrivers
385 * section. So the order is important and enforced by the ordering
386 * of different apic driver files in the Makefile.
387 *
388 * For the files having two apic drivers, we use apic_drivers()
389 * to enforce the order with in them.
390 */
391#define apic_driver(sym) \
392 static struct apic *__apicdrivers_##sym __used \
393 __aligned(sizeof(struct apic *)) \
394 __section(.apicdrivers) = { &sym }
395
396#define apic_drivers(sym1, sym2) \
397 static struct apic *__apicdrivers_##sym1##sym2[2] __used \
398 __aligned(sizeof(struct apic *)) \
399 __section(.apicdrivers) = { &sym1, &sym2 }
400
401extern struct apic *__apicdrivers[], *__apicdrivers_end[];
402
403/*
384 * APIC functionality to boot other CPUs - only used on SMP: 404 * APIC functionality to boot other CPUs - only used on SMP:
385 */ 405 */
386#ifdef CONFIG_SMP 406#ifdef CONFIG_SMP
@@ -458,15 +478,10 @@ static inline unsigned default_get_apic_id(unsigned long x)
458#define DEFAULT_TRAMPOLINE_PHYS_HIGH 0x469 478#define DEFAULT_TRAMPOLINE_PHYS_HIGH 0x469
459 479
460#ifdef CONFIG_X86_64 480#ifdef CONFIG_X86_64
461extern struct apic apic_flat;
462extern struct apic apic_physflat;
463extern struct apic apic_x2apic_cluster;
464extern struct apic apic_x2apic_phys;
465extern int default_acpi_madt_oem_check(char *, char *); 481extern int default_acpi_madt_oem_check(char *, char *);
466 482
467extern void apic_send_IPI_self(int vector); 483extern void apic_send_IPI_self(int vector);
468 484
469extern struct apic apic_x2apic_uv_x;
470DECLARE_PER_CPU(int, x2apic_extra_bits); 485DECLARE_PER_CPU(int, x2apic_extra_bits);
471 486
472extern int default_cpu_present_to_apicid(int mps_cpu); 487extern int default_cpu_present_to_apicid(int mps_cpu);
@@ -480,7 +495,7 @@ static inline void default_wait_for_init_deassert(atomic_t *deassert)
480 return; 495 return;
481} 496}
482 497
483extern void generic_bigsmp_probe(void); 498extern struct apic *generic_bigsmp_probe(void);
484 499
485 500
486#ifdef CONFIG_X86_LOCAL_APIC 501#ifdef CONFIG_X86_LOCAL_APIC
@@ -516,8 +531,6 @@ extern struct apic apic_noop;
516 531
517#ifdef CONFIG_X86_32 532#ifdef CONFIG_X86_32
518 533
519extern struct apic apic_default;
520
521static inline int noop_x86_32_early_logical_apicid(int cpu) 534static inline int noop_x86_32_early_logical_apicid(int cpu)
522{ 535{
523 return BAD_APICID; 536 return BAD_APICID;
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index a97a240f67f3..690d1cc9a877 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -105,12 +105,12 @@ struct IR_IO_APIC_route_entry {
105 * # of IO-APICs and # of IRQ routing registers 105 * # of IO-APICs and # of IRQ routing registers
106 */ 106 */
107extern int nr_ioapics; 107extern int nr_ioapics;
108extern int nr_ioapic_registers[MAX_IO_APICS];
109 108
110#define MP_MAX_IOAPIC_PIN 127 109extern int mpc_ioapic_id(int ioapic);
110extern unsigned int mpc_ioapic_addr(int ioapic);
111extern struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic);
111 112
112/* I/O APIC entries */ 113#define MP_MAX_IOAPIC_PIN 127
113extern struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
114 114
115/* # of MP IRQ source entries */ 115/* # of MP IRQ source entries */
116extern int mp_irq_entries; 116extern int mp_irq_entries;
@@ -152,11 +152,9 @@ extern void ioapic_insert_resources(void);
152 152
153int io_apic_setup_irq_pin_once(unsigned int irq, int node, struct io_apic_irq_attr *attr); 153int io_apic_setup_irq_pin_once(unsigned int irq, int node, struct io_apic_irq_attr *attr);
154 154
155extern struct IO_APIC_route_entry **alloc_ioapic_entries(void); 155extern int save_ioapic_entries(void);
156extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries); 156extern void mask_ioapic_entries(void);
157extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries); 157extern int restore_ioapic_entries(void);
158extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
159extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
160 158
161extern int get_nr_irqs_gsi(void); 159extern int get_nr_irqs_gsi(void);
162 160
@@ -192,19 +190,13 @@ struct io_apic_irq_attr;
192static inline int io_apic_set_pci_routing(struct device *dev, int irq, 190static inline int io_apic_set_pci_routing(struct device *dev, int irq,
193 struct io_apic_irq_attr *irq_attr) { return 0; } 191 struct io_apic_irq_attr *irq_attr) { return 0; }
194 192
195static inline struct IO_APIC_route_entry **alloc_ioapic_entries(void) 193static inline int save_ioapic_entries(void)
196{
197 return NULL;
198}
199
200static inline void free_ioapic_entries(struct IO_APIC_route_entry **ent) { }
201static inline int save_IO_APIC_setup(struct IO_APIC_route_entry **ent)
202{ 194{
203 return -ENOMEM; 195 return -ENOMEM;
204} 196}
205 197
206static inline void mask_IO_APIC_setup(struct IO_APIC_route_entry **ent) { } 198static inline void mask_ioapic_entries(void) { }
207static inline int restore_IO_APIC_setup(struct IO_APIC_route_entry **ent) 199static inline int restore_ioapic_entries(void)
208{ 200{
209 return -ENOMEM; 201 return -ENOMEM;
210} 202}
diff --git a/arch/x86/include/asm/x2apic.h b/arch/x86/include/asm/x2apic.h
new file mode 100644
index 000000000000..6bf5b8e478c0
--- /dev/null
+++ b/arch/x86/include/asm/x2apic.h
@@ -0,0 +1,62 @@
1/*
2 * Common bits for X2APIC cluster/physical modes.
3 */
4
5#ifndef _ASM_X86_X2APIC_H
6#define _ASM_X86_X2APIC_H
7
8#include <asm/apic.h>
9#include <asm/ipi.h>
10#include <linux/cpumask.h>
11
12/*
13 * Need to use more than cpu 0, because we need more vectors
14 * when MSI-X are used.
15 */
16static const struct cpumask *x2apic_target_cpus(void)
17{
18 return cpu_online_mask;
19}
20
21static int x2apic_apic_id_registered(void)
22{
23 return 1;
24}
25
26/*
27 * For now each logical cpu is in its own vector allocation domain.
28 */
29static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
30{
31 cpumask_clear(retmask);
32 cpumask_set_cpu(cpu, retmask);
33}
34
35static void
36__x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
37{
38 unsigned long cfg = __prepare_ICR(0, vector, dest);
39 native_x2apic_icr_write(cfg, apicid);
40}
41
42static unsigned int x2apic_get_apic_id(unsigned long id)
43{
44 return id;
45}
46
47static unsigned long x2apic_set_apic_id(unsigned int id)
48{
49 return id;
50}
51
52static int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
53{
54 return initial_apicid >> index_msb;
55}
56
57static void x2apic_send_IPI_self(int vector)
58{
59 apic_write(APIC_SELF_IPI, vector);
60}
61
62#endif /* _ASM_X86_X2APIC_H */
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 9a966c579af5..4558f0d0822d 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -970,7 +970,7 @@ void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
970 mp_irq.irqflag = (trigger << 2) | polarity; 970 mp_irq.irqflag = (trigger << 2) | polarity;
971 mp_irq.srcbus = MP_ISA_BUS; 971 mp_irq.srcbus = MP_ISA_BUS;
972 mp_irq.srcbusirq = bus_irq; /* IRQ */ 972 mp_irq.srcbusirq = bus_irq; /* IRQ */
973 mp_irq.dstapic = mp_ioapics[ioapic].apicid; /* APIC ID */ 973 mp_irq.dstapic = mpc_ioapic_id(ioapic); /* APIC ID */
974 mp_irq.dstirq = pin; /* INTIN# */ 974 mp_irq.dstirq = pin; /* INTIN# */
975 975
976 mp_save_irq(&mp_irq); 976 mp_save_irq(&mp_irq);
@@ -1021,7 +1021,7 @@ void __init mp_config_acpi_legacy_irqs(void)
1021 if (ioapic < 0) 1021 if (ioapic < 0)
1022 continue; 1022 continue;
1023 pin = mp_find_ioapic_pin(ioapic, gsi); 1023 pin = mp_find_ioapic_pin(ioapic, gsi);
1024 dstapic = mp_ioapics[ioapic].apicid; 1024 dstapic = mpc_ioapic_id(ioapic);
1025 1025
1026 for (idx = 0; idx < mp_irq_entries; idx++) { 1026 for (idx = 0; idx < mp_irq_entries; idx++) {
1027 struct mpc_intsrc *irq = mp_irqs + idx; 1027 struct mpc_intsrc *irq = mp_irqs + idx;
@@ -1082,7 +1082,7 @@ static int mp_config_acpi_gsi(struct device *dev, u32 gsi, int trigger,
1082 mp_irq.srcbus = number; 1082 mp_irq.srcbus = number;
1083 mp_irq.srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3); 1083 mp_irq.srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3);
1084 ioapic = mp_find_ioapic(gsi); 1084 ioapic = mp_find_ioapic(gsi);
1085 mp_irq.dstapic = mp_ioapics[ioapic].apicid; 1085 mp_irq.dstapic = mpc_ioapic_id(ioapic);
1086 mp_irq.dstirq = mp_find_ioapic_pin(ioapic, gsi); 1086 mp_irq.dstirq = mp_find_ioapic_pin(ioapic, gsi);
1087 1087
1088 mp_save_irq(&mp_irq); 1088 mp_save_irq(&mp_irq);
@@ -1113,7 +1113,7 @@ int mp_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity)
1113 1113
1114 if (ioapic_pin > MP_MAX_IOAPIC_PIN) { 1114 if (ioapic_pin > MP_MAX_IOAPIC_PIN) {
1115 printk(KERN_ERR "Invalid reference to IOAPIC pin " 1115 printk(KERN_ERR "Invalid reference to IOAPIC pin "
1116 "%d-%d\n", mp_ioapics[ioapic].apicid, 1116 "%d-%d\n", mpc_ioapic_id(ioapic),
1117 ioapic_pin); 1117 ioapic_pin);
1118 return gsi; 1118 return gsi;
1119 } 1119 }
diff --git a/arch/x86/kernel/apic/Makefile b/arch/x86/kernel/apic/Makefile
index 3966b564ea47..767fd04f2843 100644
--- a/arch/x86/kernel/apic/Makefile
+++ b/arch/x86/kernel/apic/Makefile
@@ -2,20 +2,25 @@
2# Makefile for local APIC drivers and for the IO-APIC code 2# Makefile for local APIC drivers and for the IO-APIC code
3# 3#
4 4
5obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o probe_$(BITS).o ipi.o 5obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o ipi.o
6obj-y += hw_nmi.o 6obj-y += hw_nmi.o
7 7
8obj-$(CONFIG_X86_IO_APIC) += io_apic.o 8obj-$(CONFIG_X86_IO_APIC) += io_apic.o
9obj-$(CONFIG_SMP) += ipi.o 9obj-$(CONFIG_SMP) += ipi.o
10 10
11ifeq ($(CONFIG_X86_64),y) 11ifeq ($(CONFIG_X86_64),y)
12obj-y += apic_flat_64.o 12# APIC probe will depend on the listing order here
13obj-$(CONFIG_X86_X2APIC) += x2apic_cluster.o
14obj-$(CONFIG_X86_X2APIC) += x2apic_phys.o
15obj-$(CONFIG_X86_UV) += x2apic_uv_x.o 13obj-$(CONFIG_X86_UV) += x2apic_uv_x.o
14obj-$(CONFIG_X86_X2APIC) += x2apic_phys.o
15obj-$(CONFIG_X86_X2APIC) += x2apic_cluster.o
16obj-y += apic_flat_64.o
16endif 17endif
17 18
18obj-$(CONFIG_X86_BIGSMP) += bigsmp_32.o 19# APIC probe will depend on the listing order here
19obj-$(CONFIG_X86_NUMAQ) += numaq_32.o 20obj-$(CONFIG_X86_NUMAQ) += numaq_32.o
20obj-$(CONFIG_X86_ES7000) += es7000_32.o
21obj-$(CONFIG_X86_SUMMIT) += summit_32.o 21obj-$(CONFIG_X86_SUMMIT) += summit_32.o
22obj-$(CONFIG_X86_BIGSMP) += bigsmp_32.o
23obj-$(CONFIG_X86_ES7000) += es7000_32.o
24
25# For 32bit, probe_32 need to be listed last
26obj-$(CONFIG_X86_LOCAL_APIC) += probe_$(BITS).o
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index f92a8e5d1e21..b961af86bfea 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1461,7 +1461,6 @@ int __init enable_IR(void)
1461void __init enable_IR_x2apic(void) 1461void __init enable_IR_x2apic(void)
1462{ 1462{
1463 unsigned long flags; 1463 unsigned long flags;
1464 struct IO_APIC_route_entry **ioapic_entries;
1465 int ret, x2apic_enabled = 0; 1464 int ret, x2apic_enabled = 0;
1466 int dmar_table_init_ret; 1465 int dmar_table_init_ret;
1467 1466
@@ -1469,13 +1468,7 @@ void __init enable_IR_x2apic(void)
1469 if (dmar_table_init_ret && !x2apic_supported()) 1468 if (dmar_table_init_ret && !x2apic_supported())
1470 return; 1469 return;
1471 1470
1472 ioapic_entries = alloc_ioapic_entries(); 1471 ret = save_ioapic_entries();
1473 if (!ioapic_entries) {
1474 pr_err("Allocate ioapic_entries failed\n");
1475 goto out;
1476 }
1477
1478 ret = save_IO_APIC_setup(ioapic_entries);
1479 if (ret) { 1472 if (ret) {
1480 pr_info("Saving IO-APIC state failed: %d\n", ret); 1473 pr_info("Saving IO-APIC state failed: %d\n", ret);
1481 goto out; 1474 goto out;
@@ -1483,7 +1476,7 @@ void __init enable_IR_x2apic(void)
1483 1476
1484 local_irq_save(flags); 1477 local_irq_save(flags);
1485 legacy_pic->mask_all(); 1478 legacy_pic->mask_all();
1486 mask_IO_APIC_setup(ioapic_entries); 1479 mask_ioapic_entries();
1487 1480
1488 if (dmar_table_init_ret) 1481 if (dmar_table_init_ret)
1489 ret = 0; 1482 ret = 0;
@@ -1514,14 +1507,11 @@ void __init enable_IR_x2apic(void)
1514 1507
1515nox2apic: 1508nox2apic:
1516 if (!ret) /* IR enabling failed */ 1509 if (!ret) /* IR enabling failed */
1517 restore_IO_APIC_setup(ioapic_entries); 1510 restore_ioapic_entries();
1518 legacy_pic->restore_mask(); 1511 legacy_pic->restore_mask();
1519 local_irq_restore(flags); 1512 local_irq_restore(flags);
1520 1513
1521out: 1514out:
1522 if (ioapic_entries)
1523 free_ioapic_entries(ioapic_entries);
1524
1525 if (x2apic_enabled) 1515 if (x2apic_enabled)
1526 return; 1516 return;
1527 1517
@@ -2095,28 +2085,20 @@ static void lapic_resume(void)
2095{ 2085{
2096 unsigned int l, h; 2086 unsigned int l, h;
2097 unsigned long flags; 2087 unsigned long flags;
2098 int maxlvt, ret; 2088 int maxlvt;
2099 struct IO_APIC_route_entry **ioapic_entries = NULL;
2100 2089
2101 if (!apic_pm_state.active) 2090 if (!apic_pm_state.active)
2102 return; 2091 return;
2103 2092
2104 local_irq_save(flags); 2093 local_irq_save(flags);
2105 if (intr_remapping_enabled) { 2094 if (intr_remapping_enabled) {
2106 ioapic_entries = alloc_ioapic_entries(); 2095 /*
2107 if (!ioapic_entries) { 2096 * IO-APIC and PIC have their own resume routines.
2108 WARN(1, "Alloc ioapic_entries in lapic resume failed."); 2097 * We just mask them here to make sure the interrupt
2109 goto restore; 2098 * subsystem is completely quiet while we enable x2apic
2110 } 2099 * and interrupt-remapping.
2111 2100 */
2112 ret = save_IO_APIC_setup(ioapic_entries); 2101 mask_ioapic_entries();
2113 if (ret) {
2114 WARN(1, "Saving IO-APIC state failed: %d\n", ret);
2115 free_ioapic_entries(ioapic_entries);
2116 goto restore;
2117 }
2118
2119 mask_IO_APIC_setup(ioapic_entries);
2120 legacy_pic->mask_all(); 2102 legacy_pic->mask_all();
2121 } 2103 }
2122 2104
@@ -2159,13 +2141,9 @@ static void lapic_resume(void)
2159 apic_write(APIC_ESR, 0); 2141 apic_write(APIC_ESR, 0);
2160 apic_read(APIC_ESR); 2142 apic_read(APIC_ESR);
2161 2143
2162 if (intr_remapping_enabled) { 2144 if (intr_remapping_enabled)
2163 reenable_intr_remapping(x2apic_mode); 2145 reenable_intr_remapping(x2apic_mode);
2164 legacy_pic->restore_mask(); 2146
2165 restore_IO_APIC_setup(ioapic_entries);
2166 free_ioapic_entries(ioapic_entries);
2167 }
2168restore:
2169 local_irq_restore(flags); 2147 local_irq_restore(flags);
2170} 2148}
2171 2149
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c
index 5652d31fe108..f7a41e4cae47 100644
--- a/arch/x86/kernel/apic/apic_flat_64.c
+++ b/arch/x86/kernel/apic/apic_flat_64.c
@@ -16,6 +16,7 @@
16#include <linux/ctype.h> 16#include <linux/ctype.h>
17#include <linux/init.h> 17#include <linux/init.h>
18#include <linux/hardirq.h> 18#include <linux/hardirq.h>
19#include <linux/module.h>
19#include <asm/smp.h> 20#include <asm/smp.h>
20#include <asm/apic.h> 21#include <asm/apic.h>
21#include <asm/ipi.h> 22#include <asm/ipi.h>
@@ -24,6 +25,12 @@
24#include <acpi/acpi_bus.h> 25#include <acpi/acpi_bus.h>
25#endif 26#endif
26 27
28static struct apic apic_physflat;
29static struct apic apic_flat;
30
31struct apic __read_mostly *apic = &apic_flat;
32EXPORT_SYMBOL_GPL(apic);
33
27static int flat_acpi_madt_oem_check(char *oem_id, char *oem_table_id) 34static int flat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
28{ 35{
29 return 1; 36 return 1;
@@ -164,7 +171,7 @@ static int flat_phys_pkg_id(int initial_apic_id, int index_msb)
164 return initial_apic_id >> index_msb; 171 return initial_apic_id >> index_msb;
165} 172}
166 173
167struct apic apic_flat = { 174static struct apic apic_flat = {
168 .name = "flat", 175 .name = "flat",
169 .probe = NULL, 176 .probe = NULL,
170 .acpi_madt_oem_check = flat_acpi_madt_oem_check, 177 .acpi_madt_oem_check = flat_acpi_madt_oem_check,
@@ -312,10 +319,18 @@ physflat_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
312 return per_cpu(x86_cpu_to_apicid, cpu); 319 return per_cpu(x86_cpu_to_apicid, cpu);
313} 320}
314 321
315struct apic apic_physflat = { 322static int physflat_probe(void)
323{
324 if (apic == &apic_physflat || num_possible_cpus() > 8)
325 return 1;
326
327 return 0;
328}
329
330static struct apic apic_physflat = {
316 331
317 .name = "physical flat", 332 .name = "physical flat",
318 .probe = NULL, 333 .probe = physflat_probe,
319 .acpi_madt_oem_check = physflat_acpi_madt_oem_check, 334 .acpi_madt_oem_check = physflat_acpi_madt_oem_check,
320 .apic_id_registered = flat_apic_id_registered, 335 .apic_id_registered = flat_apic_id_registered,
321 336
@@ -369,3 +384,8 @@ struct apic apic_physflat = {
369 .wait_icr_idle = native_apic_wait_icr_idle, 384 .wait_icr_idle = native_apic_wait_icr_idle,
370 .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, 385 .safe_wait_icr_idle = native_safe_apic_wait_icr_idle,
371}; 386};
387
388/*
389 * We need to check for physflat first, so this order is important.
390 */
391apic_drivers(apic_physflat, apic_flat);
diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c
index d84ac5a584b5..efd737e827f4 100644
--- a/arch/x86/kernel/apic/bigsmp_32.c
+++ b/arch/x86/kernel/apic/bigsmp_32.c
@@ -193,7 +193,7 @@ static int probe_bigsmp(void)
193 return dmi_bigsmp; 193 return dmi_bigsmp;
194} 194}
195 195
196struct apic apic_bigsmp = { 196static struct apic apic_bigsmp = {
197 197
198 .name = "bigsmp", 198 .name = "bigsmp",
199 .probe = probe_bigsmp, 199 .probe = probe_bigsmp,
@@ -254,3 +254,13 @@ struct apic apic_bigsmp = {
254 254
255 .x86_32_early_logical_apicid = bigsmp_early_logical_apicid, 255 .x86_32_early_logical_apicid = bigsmp_early_logical_apicid,
256}; 256};
257
258struct apic * __init generic_bigsmp_probe(void)
259{
260 if (probe_bigsmp())
261 return &apic_bigsmp;
262
263 return NULL;
264}
265
266apic_driver(apic_bigsmp);
diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c
index 70533de5bd29..9536b3fe43f8 100644
--- a/arch/x86/kernel/apic/es7000_32.c
+++ b/arch/x86/kernel/apic/es7000_32.c
@@ -620,7 +620,7 @@ static int es7000_mps_oem_check_cluster(struct mpc_table *mpc, char *oem,
620} 620}
621 621
622/* We've been warned by a false positive warning.Use __refdata to keep calm. */ 622/* We've been warned by a false positive warning.Use __refdata to keep calm. */
623struct apic __refdata apic_es7000_cluster = { 623static struct apic __refdata apic_es7000_cluster = {
624 624
625 .name = "es7000", 625 .name = "es7000",
626 .probe = probe_es7000, 626 .probe = probe_es7000,
@@ -685,7 +685,7 @@ struct apic __refdata apic_es7000_cluster = {
685 .x86_32_early_logical_apicid = es7000_early_logical_apicid, 685 .x86_32_early_logical_apicid = es7000_early_logical_apicid,
686}; 686};
687 687
688struct apic __refdata apic_es7000 = { 688static struct apic __refdata apic_es7000 = {
689 689
690 .name = "es7000", 690 .name = "es7000",
691 .probe = probe_es7000, 691 .probe = probe_es7000,
@@ -747,3 +747,9 @@ struct apic __refdata apic_es7000 = {
747 747
748 .x86_32_early_logical_apicid = es7000_early_logical_apicid, 748 .x86_32_early_logical_apicid = es7000_early_logical_apicid,
749}; 749};
750
751/*
752 * Need to check for es7000 followed by es7000_cluster, so this order
753 * in apic_drivers is important.
754 */
755apic_drivers(apic_es7000, apic_es7000_cluster);
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 45fd33d1fd3a..9488dcff7aec 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -76,17 +76,40 @@ int sis_apic_bug = -1;
76static DEFINE_RAW_SPINLOCK(ioapic_lock); 76static DEFINE_RAW_SPINLOCK(ioapic_lock);
77static DEFINE_RAW_SPINLOCK(vector_lock); 77static DEFINE_RAW_SPINLOCK(vector_lock);
78 78
79/* 79static struct ioapic {
80 * # of IRQ routing registers 80 /*
81 */ 81 * # of IRQ routing registers
82int nr_ioapic_registers[MAX_IO_APICS]; 82 */
83 int nr_registers;
84 /*
85 * Saved state during suspend/resume, or while enabling intr-remap.
86 */
87 struct IO_APIC_route_entry *saved_registers;
88 /* I/O APIC config */
89 struct mpc_ioapic mp_config;
90 /* IO APIC gsi routing info */
91 struct mp_ioapic_gsi gsi_config;
92 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
93} ioapics[MAX_IO_APICS];
83 94
84/* I/O APIC entries */ 95#define mpc_ioapic_ver(id) ioapics[id].mp_config.apicver
85struct mpc_ioapic mp_ioapics[MAX_IO_APICS]; 96
86int nr_ioapics; 97int mpc_ioapic_id(int id)
98{
99 return ioapics[id].mp_config.apicid;
100}
87 101
88/* IO APIC gsi routing info */ 102unsigned int mpc_ioapic_addr(int id)
89struct mp_ioapic_gsi mp_gsi_routing[MAX_IO_APICS]; 103{
104 return ioapics[id].mp_config.apicaddr;
105}
106
107struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int id)
108{
109 return &ioapics[id].gsi_config;
110}
111
112int nr_ioapics;
90 113
91/* The one past the highest gsi number used */ 114/* The one past the highest gsi number used */
92u32 gsi_top; 115u32 gsi_top;
@@ -179,6 +202,14 @@ int __init arch_early_irq_init(void)
179 io_apic_irqs = ~0UL; 202 io_apic_irqs = ~0UL;
180 } 203 }
181 204
205 for (i = 0; i < nr_ioapics; i++) {
206 ioapics[i].saved_registers =
207 kzalloc(sizeof(struct IO_APIC_route_entry) *
208 ioapics[i].nr_registers, GFP_KERNEL);
209 if (!ioapics[i].saved_registers)
210 pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
211 }
212
182 cfg = irq_cfgx; 213 cfg = irq_cfgx;
183 count = ARRAY_SIZE(irq_cfgx); 214 count = ARRAY_SIZE(irq_cfgx);
184 node = cpu_to_node(0); 215 node = cpu_to_node(0);
@@ -297,7 +328,7 @@ struct io_apic {
297static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) 328static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
298{ 329{
299 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx) 330 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
300 + (mp_ioapics[idx].apicaddr & ~PAGE_MASK); 331 + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
301} 332}
302 333
303static inline void io_apic_eoi(unsigned int apic, unsigned int vector) 334static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
@@ -573,7 +604,7 @@ static void clear_IO_APIC (void)
573 int apic, pin; 604 int apic, pin;
574 605
575 for (apic = 0; apic < nr_ioapics; apic++) 606 for (apic = 0; apic < nr_ioapics; apic++)
576 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) 607 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
577 clear_IO_APIC_pin(apic, pin); 608 clear_IO_APIC_pin(apic, pin);
578} 609}
579 610
@@ -615,74 +646,43 @@ static int __init ioapic_pirq_setup(char *str)
615__setup("pirq=", ioapic_pirq_setup); 646__setup("pirq=", ioapic_pirq_setup);
616#endif /* CONFIG_X86_32 */ 647#endif /* CONFIG_X86_32 */
617 648
618struct IO_APIC_route_entry **alloc_ioapic_entries(void)
619{
620 int apic;
621 struct IO_APIC_route_entry **ioapic_entries;
622
623 ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
624 GFP_KERNEL);
625 if (!ioapic_entries)
626 return 0;
627
628 for (apic = 0; apic < nr_ioapics; apic++) {
629 ioapic_entries[apic] =
630 kzalloc(sizeof(struct IO_APIC_route_entry) *
631 nr_ioapic_registers[apic], GFP_KERNEL);
632 if (!ioapic_entries[apic])
633 goto nomem;
634 }
635
636 return ioapic_entries;
637
638nomem:
639 while (--apic >= 0)
640 kfree(ioapic_entries[apic]);
641 kfree(ioapic_entries);
642
643 return 0;
644}
645
646/* 649/*
647 * Saves all the IO-APIC RTE's 650 * Saves all the IO-APIC RTE's
648 */ 651 */
649int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries) 652int save_ioapic_entries(void)
650{ 653{
651 int apic, pin; 654 int apic, pin;
652 655 int err = 0;
653 if (!ioapic_entries)
654 return -ENOMEM;
655 656
656 for (apic = 0; apic < nr_ioapics; apic++) { 657 for (apic = 0; apic < nr_ioapics; apic++) {
657 if (!ioapic_entries[apic]) 658 if (!ioapics[apic].saved_registers) {
658 return -ENOMEM; 659 err = -ENOMEM;
660 continue;
661 }
659 662
660 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) 663 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
661 ioapic_entries[apic][pin] = 664 ioapics[apic].saved_registers[pin] =
662 ioapic_read_entry(apic, pin); 665 ioapic_read_entry(apic, pin);
663 } 666 }
664 667
665 return 0; 668 return err;
666} 669}
667 670
668/* 671/*
669 * Mask all IO APIC entries. 672 * Mask all IO APIC entries.
670 */ 673 */
671void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries) 674void mask_ioapic_entries(void)
672{ 675{
673 int apic, pin; 676 int apic, pin;
674 677
675 if (!ioapic_entries)
676 return;
677
678 for (apic = 0; apic < nr_ioapics; apic++) { 678 for (apic = 0; apic < nr_ioapics; apic++) {
679 if (!ioapic_entries[apic]) 679 if (ioapics[apic].saved_registers)
680 break; 680 continue;
681 681
682 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { 682 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
683 struct IO_APIC_route_entry entry; 683 struct IO_APIC_route_entry entry;
684 684
685 entry = ioapic_entries[apic][pin]; 685 entry = ioapics[apic].saved_registers[pin];
686 if (!entry.mask) { 686 if (!entry.mask) {
687 entry.mask = 1; 687 entry.mask = 1;
688 ioapic_write_entry(apic, pin, entry); 688 ioapic_write_entry(apic, pin, entry);
@@ -692,36 +692,23 @@ void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
692} 692}
693 693
694/* 694/*
695 * Restore IO APIC entries which was saved in ioapic_entries. 695 * Restore IO APIC entries which was saved in the ioapic structure.
696 */ 696 */
697int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries) 697int restore_ioapic_entries(void)
698{ 698{
699 int apic, pin; 699 int apic, pin;
700 700
701 if (!ioapic_entries)
702 return -ENOMEM;
703
704 for (apic = 0; apic < nr_ioapics; apic++) { 701 for (apic = 0; apic < nr_ioapics; apic++) {
705 if (!ioapic_entries[apic]) 702 if (ioapics[apic].saved_registers)
706 return -ENOMEM; 703 continue;
707 704
708 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) 705 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
709 ioapic_write_entry(apic, pin, 706 ioapic_write_entry(apic, pin,
710 ioapic_entries[apic][pin]); 707 ioapics[apic].saved_registers[pin]);
711 } 708 }
712 return 0; 709 return 0;
713} 710}
714 711
715void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
716{
717 int apic;
718
719 for (apic = 0; apic < nr_ioapics; apic++)
720 kfree(ioapic_entries[apic]);
721
722 kfree(ioapic_entries);
723}
724
725/* 712/*
726 * Find the IRQ entry number of a certain pin. 713 * Find the IRQ entry number of a certain pin.
727 */ 714 */
@@ -731,7 +718,7 @@ static int find_irq_entry(int apic, int pin, int type)
731 718
732 for (i = 0; i < mp_irq_entries; i++) 719 for (i = 0; i < mp_irq_entries; i++)
733 if (mp_irqs[i].irqtype == type && 720 if (mp_irqs[i].irqtype == type &&
734 (mp_irqs[i].dstapic == mp_ioapics[apic].apicid || 721 (mp_irqs[i].dstapic == mpc_ioapic_id(apic) ||
735 mp_irqs[i].dstapic == MP_APIC_ALL) && 722 mp_irqs[i].dstapic == MP_APIC_ALL) &&
736 mp_irqs[i].dstirq == pin) 723 mp_irqs[i].dstirq == pin)
737 return i; 724 return i;
@@ -773,7 +760,7 @@ static int __init find_isa_irq_apic(int irq, int type)
773 if (i < mp_irq_entries) { 760 if (i < mp_irq_entries) {
774 int apic; 761 int apic;
775 for(apic = 0; apic < nr_ioapics; apic++) { 762 for(apic = 0; apic < nr_ioapics; apic++) {
776 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic) 763 if (mpc_ioapic_id(apic) == mp_irqs[i].dstapic)
777 return apic; 764 return apic;
778 } 765 }
779 } 766 }
@@ -942,6 +929,7 @@ static int pin_2_irq(int idx, int apic, int pin)
942{ 929{
943 int irq; 930 int irq;
944 int bus = mp_irqs[idx].srcbus; 931 int bus = mp_irqs[idx].srcbus;
932 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(apic);
945 933
946 /* 934 /*
947 * Debugging check, we are in big trouble if this message pops up! 935 * Debugging check, we are in big trouble if this message pops up!
@@ -952,7 +940,7 @@ static int pin_2_irq(int idx, int apic, int pin)
952 if (test_bit(bus, mp_bus_not_pci)) { 940 if (test_bit(bus, mp_bus_not_pci)) {
953 irq = mp_irqs[idx].srcbusirq; 941 irq = mp_irqs[idx].srcbusirq;
954 } else { 942 } else {
955 u32 gsi = mp_gsi_routing[apic].gsi_base + pin; 943 u32 gsi = gsi_cfg->gsi_base + pin;
956 944
957 if (gsi >= NR_IRQS_LEGACY) 945 if (gsi >= NR_IRQS_LEGACY)
958 irq = gsi; 946 irq = gsi;
@@ -1003,7 +991,7 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1003 int lbus = mp_irqs[i].srcbus; 991 int lbus = mp_irqs[i].srcbus;
1004 992
1005 for (apic = 0; apic < nr_ioapics; apic++) 993 for (apic = 0; apic < nr_ioapics; apic++)
1006 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic || 994 if (mpc_ioapic_id(apic) == mp_irqs[i].dstapic ||
1007 mp_irqs[i].dstapic == MP_APIC_ALL) 995 mp_irqs[i].dstapic == MP_APIC_ALL)
1008 break; 996 break;
1009 997
@@ -1222,7 +1210,7 @@ static inline int IO_APIC_irq_trigger(int irq)
1222 int apic, idx, pin; 1210 int apic, idx, pin;
1223 1211
1224 for (apic = 0; apic < nr_ioapics; apic++) { 1212 for (apic = 0; apic < nr_ioapics; apic++) {
1225 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { 1213 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1226 idx = find_irq_entry(apic, pin, mp_INT); 1214 idx = find_irq_entry(apic, pin, mp_INT);
1227 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin))) 1215 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1228 return irq_trigger(idx); 1216 return irq_trigger(idx);
@@ -1350,14 +1338,14 @@ static void setup_ioapic_irq(int apic_id, int pin, unsigned int irq,
1350 apic_printk(APIC_VERBOSE,KERN_DEBUG 1338 apic_printk(APIC_VERBOSE,KERN_DEBUG
1351 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> " 1339 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1352 "IRQ %d Mode:%i Active:%i)\n", 1340 "IRQ %d Mode:%i Active:%i)\n",
1353 apic_id, mp_ioapics[apic_id].apicid, pin, cfg->vector, 1341 apic_id, mpc_ioapic_id(apic_id), pin, cfg->vector,
1354 irq, trigger, polarity); 1342 irq, trigger, polarity);
1355 1343
1356 1344
1357 if (setup_ioapic_entry(mp_ioapics[apic_id].apicid, irq, &entry, 1345 if (setup_ioapic_entry(mpc_ioapic_id(apic_id), irq, &entry,
1358 dest, trigger, polarity, cfg->vector, pin)) { 1346 dest, trigger, polarity, cfg->vector, pin)) {
1359 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n", 1347 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1360 mp_ioapics[apic_id].apicid, pin); 1348 mpc_ioapic_id(apic_id), pin);
1361 __clear_irq_vector(irq, cfg); 1349 __clear_irq_vector(irq, cfg);
1362 return; 1350 return;
1363 } 1351 }
@@ -1369,17 +1357,13 @@ static void setup_ioapic_irq(int apic_id, int pin, unsigned int irq,
1369 ioapic_write_entry(apic_id, pin, entry); 1357 ioapic_write_entry(apic_id, pin, entry);
1370} 1358}
1371 1359
1372static struct {
1373 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
1374} mp_ioapic_routing[MAX_IO_APICS];
1375
1376static bool __init io_apic_pin_not_connected(int idx, int apic_id, int pin) 1360static bool __init io_apic_pin_not_connected(int idx, int apic_id, int pin)
1377{ 1361{
1378 if (idx != -1) 1362 if (idx != -1)
1379 return false; 1363 return false;
1380 1364
1381 apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n", 1365 apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n",
1382 mp_ioapics[apic_id].apicid, pin); 1366 mpc_ioapic_id(apic_id), pin);
1383 return true; 1367 return true;
1384} 1368}
1385 1369
@@ -1389,7 +1373,7 @@ static void __init __io_apic_setup_irqs(unsigned int apic_id)
1389 struct io_apic_irq_attr attr; 1373 struct io_apic_irq_attr attr;
1390 unsigned int pin, irq; 1374 unsigned int pin, irq;
1391 1375
1392 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) { 1376 for (pin = 0; pin < ioapics[apic_id].nr_registers; pin++) {
1393 idx = find_irq_entry(apic_id, pin, mp_INT); 1377 idx = find_irq_entry(apic_id, pin, mp_INT);
1394 if (io_apic_pin_not_connected(idx, apic_id, pin)) 1378 if (io_apic_pin_not_connected(idx, apic_id, pin))
1395 continue; 1379 continue;
@@ -1511,7 +1495,7 @@ __apicdebuginit(void) print_IO_APIC(void)
1511 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries); 1495 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1512 for (i = 0; i < nr_ioapics; i++) 1496 for (i = 0; i < nr_ioapics; i++)
1513 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n", 1497 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1514 mp_ioapics[i].apicid, nr_ioapic_registers[i]); 1498 mpc_ioapic_id(i), ioapics[i].nr_registers);
1515 1499
1516 /* 1500 /*
1517 * We are a bit conservative about what we expect. We have to 1501 * We are a bit conservative about what we expect. We have to
@@ -1531,7 +1515,7 @@ __apicdebuginit(void) print_IO_APIC(void)
1531 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1515 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1532 1516
1533 printk("\n"); 1517 printk("\n");
1534 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid); 1518 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(apic));
1535 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw); 1519 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1536 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID); 1520 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1537 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type); 1521 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
@@ -1825,7 +1809,7 @@ void __init enable_IO_APIC(void)
1825 for(apic = 0; apic < nr_ioapics; apic++) { 1809 for(apic = 0; apic < nr_ioapics; apic++) {
1826 int pin; 1810 int pin;
1827 /* See if any of the pins is in ExtINT mode */ 1811 /* See if any of the pins is in ExtINT mode */
1828 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { 1812 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1829 struct IO_APIC_route_entry entry; 1813 struct IO_APIC_route_entry entry;
1830 entry = ioapic_read_entry(apic, pin); 1814 entry = ioapic_read_entry(apic, pin);
1831 1815
@@ -1949,14 +1933,14 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void)
1949 reg_00.raw = io_apic_read(apic_id, 0); 1933 reg_00.raw = io_apic_read(apic_id, 0);
1950 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1934 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1951 1935
1952 old_id = mp_ioapics[apic_id].apicid; 1936 old_id = mpc_ioapic_id(apic_id);
1953 1937
1954 if (mp_ioapics[apic_id].apicid >= get_physical_broadcast()) { 1938 if (mpc_ioapic_id(apic_id) >= get_physical_broadcast()) {
1955 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n", 1939 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1956 apic_id, mp_ioapics[apic_id].apicid); 1940 apic_id, mpc_ioapic_id(apic_id));
1957 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1941 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1958 reg_00.bits.ID); 1942 reg_00.bits.ID);
1959 mp_ioapics[apic_id].apicid = reg_00.bits.ID; 1943 ioapics[apic_id].mp_config.apicid = reg_00.bits.ID;
1960 } 1944 }
1961 1945
1962 /* 1946 /*
@@ -1965,9 +1949,9 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void)
1965 * 'stuck on smp_invalidate_needed IPI wait' messages. 1949 * 'stuck on smp_invalidate_needed IPI wait' messages.
1966 */ 1950 */
1967 if (apic->check_apicid_used(&phys_id_present_map, 1951 if (apic->check_apicid_used(&phys_id_present_map,
1968 mp_ioapics[apic_id].apicid)) { 1952 mpc_ioapic_id(apic_id))) {
1969 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n", 1953 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1970 apic_id, mp_ioapics[apic_id].apicid); 1954 apic_id, mpc_ioapic_id(apic_id));
1971 for (i = 0; i < get_physical_broadcast(); i++) 1955 for (i = 0; i < get_physical_broadcast(); i++)
1972 if (!physid_isset(i, phys_id_present_map)) 1956 if (!physid_isset(i, phys_id_present_map))
1973 break; 1957 break;
@@ -1976,13 +1960,14 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void)
1976 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1960 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1977 i); 1961 i);
1978 physid_set(i, phys_id_present_map); 1962 physid_set(i, phys_id_present_map);
1979 mp_ioapics[apic_id].apicid = i; 1963 ioapics[apic_id].mp_config.apicid = i;
1980 } else { 1964 } else {
1981 physid_mask_t tmp; 1965 physid_mask_t tmp;
1982 apic->apicid_to_cpu_present(mp_ioapics[apic_id].apicid, &tmp); 1966 apic->apicid_to_cpu_present(mpc_ioapic_id(apic_id),
1967 &tmp);
1983 apic_printk(APIC_VERBOSE, "Setting %d in the " 1968 apic_printk(APIC_VERBOSE, "Setting %d in the "
1984 "phys_id_present_map\n", 1969 "phys_id_present_map\n",
1985 mp_ioapics[apic_id].apicid); 1970 mpc_ioapic_id(apic_id));
1986 physids_or(phys_id_present_map, phys_id_present_map, tmp); 1971 physids_or(phys_id_present_map, phys_id_present_map, tmp);
1987 } 1972 }
1988 1973
@@ -1990,24 +1975,24 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void)
1990 * We need to adjust the IRQ routing table 1975 * We need to adjust the IRQ routing table
1991 * if the ID changed. 1976 * if the ID changed.
1992 */ 1977 */
1993 if (old_id != mp_ioapics[apic_id].apicid) 1978 if (old_id != mpc_ioapic_id(apic_id))
1994 for (i = 0; i < mp_irq_entries; i++) 1979 for (i = 0; i < mp_irq_entries; i++)
1995 if (mp_irqs[i].dstapic == old_id) 1980 if (mp_irqs[i].dstapic == old_id)
1996 mp_irqs[i].dstapic 1981 mp_irqs[i].dstapic
1997 = mp_ioapics[apic_id].apicid; 1982 = mpc_ioapic_id(apic_id);
1998 1983
1999 /* 1984 /*
2000 * Update the ID register according to the right value 1985 * Update the ID register according to the right value
2001 * from the MPC table if they are different. 1986 * from the MPC table if they are different.
2002 */ 1987 */
2003 if (mp_ioapics[apic_id].apicid == reg_00.bits.ID) 1988 if (mpc_ioapic_id(apic_id) == reg_00.bits.ID)
2004 continue; 1989 continue;
2005 1990
2006 apic_printk(APIC_VERBOSE, KERN_INFO 1991 apic_printk(APIC_VERBOSE, KERN_INFO
2007 "...changing IO-APIC physical APIC ID to %d ...", 1992 "...changing IO-APIC physical APIC ID to %d ...",
2008 mp_ioapics[apic_id].apicid); 1993 mpc_ioapic_id(apic_id));
2009 1994
2010 reg_00.bits.ID = mp_ioapics[apic_id].apicid; 1995 reg_00.bits.ID = mpc_ioapic_id(apic_id);
2011 raw_spin_lock_irqsave(&ioapic_lock, flags); 1996 raw_spin_lock_irqsave(&ioapic_lock, flags);
2012 io_apic_write(apic_id, 0, reg_00.raw); 1997 io_apic_write(apic_id, 0, reg_00.raw);
2013 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1998 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -2018,7 +2003,7 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void)
2018 raw_spin_lock_irqsave(&ioapic_lock, flags); 2003 raw_spin_lock_irqsave(&ioapic_lock, flags);
2019 reg_00.raw = io_apic_read(apic_id, 0); 2004 reg_00.raw = io_apic_read(apic_id, 0);
2020 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2005 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2021 if (reg_00.bits.ID != mp_ioapics[apic_id].apicid) 2006 if (reg_00.bits.ID != mpc_ioapic_id(apic_id))
2022 printk("could not set ID!\n"); 2007 printk("could not set ID!\n");
2023 else 2008 else
2024 apic_printk(APIC_VERBOSE, " ok.\n"); 2009 apic_printk(APIC_VERBOSE, " ok.\n");
@@ -2404,7 +2389,7 @@ static void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2404 2389
2405 raw_spin_lock_irqsave(&ioapic_lock, flags); 2390 raw_spin_lock_irqsave(&ioapic_lock, flags);
2406 for_each_irq_pin(entry, cfg->irq_2_pin) { 2391 for_each_irq_pin(entry, cfg->irq_2_pin) {
2407 if (mp_ioapics[entry->apic].apicver >= 0x20) { 2392 if (mpc_ioapic_ver(entry->apic) >= 0x20) {
2408 /* 2393 /*
2409 * Intr-remapping uses pin number as the virtual vector 2394 * Intr-remapping uses pin number as the virtual vector
2410 * in the RTE. Actual vector is programmed in 2395 * in the RTE. Actual vector is programmed in
@@ -2918,49 +2903,19 @@ static int __init io_apic_bug_finalize(void)
2918 2903
2919late_initcall(io_apic_bug_finalize); 2904late_initcall(io_apic_bug_finalize);
2920 2905
2921static struct IO_APIC_route_entry *ioapic_saved_data[MAX_IO_APICS]; 2906static void resume_ioapic_id(int ioapic_id)
2922
2923static void suspend_ioapic(int ioapic_id)
2924{ 2907{
2925 struct IO_APIC_route_entry *saved_data = ioapic_saved_data[ioapic_id];
2926 int i;
2927
2928 if (!saved_data)
2929 return;
2930
2931 for (i = 0; i < nr_ioapic_registers[ioapic_id]; i++)
2932 saved_data[i] = ioapic_read_entry(ioapic_id, i);
2933}
2934
2935static int ioapic_suspend(void)
2936{
2937 int ioapic_id;
2938
2939 for (ioapic_id = 0; ioapic_id < nr_ioapics; ioapic_id++)
2940 suspend_ioapic(ioapic_id);
2941
2942 return 0;
2943}
2944
2945static void resume_ioapic(int ioapic_id)
2946{
2947 struct IO_APIC_route_entry *saved_data = ioapic_saved_data[ioapic_id];
2948 unsigned long flags; 2908 unsigned long flags;
2949 union IO_APIC_reg_00 reg_00; 2909 union IO_APIC_reg_00 reg_00;
2950 int i;
2951 2910
2952 if (!saved_data)
2953 return;
2954 2911
2955 raw_spin_lock_irqsave(&ioapic_lock, flags); 2912 raw_spin_lock_irqsave(&ioapic_lock, flags);
2956 reg_00.raw = io_apic_read(ioapic_id, 0); 2913 reg_00.raw = io_apic_read(ioapic_id, 0);
2957 if (reg_00.bits.ID != mp_ioapics[ioapic_id].apicid) { 2914 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_id)) {
2958 reg_00.bits.ID = mp_ioapics[ioapic_id].apicid; 2915 reg_00.bits.ID = mpc_ioapic_id(ioapic_id);
2959 io_apic_write(ioapic_id, 0, reg_00.raw); 2916 io_apic_write(ioapic_id, 0, reg_00.raw);
2960 } 2917 }
2961 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2918 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2962 for (i = 0; i < nr_ioapic_registers[ioapic_id]; i++)
2963 ioapic_write_entry(ioapic_id, i, saved_data[i]);
2964} 2919}
2965 2920
2966static void ioapic_resume(void) 2921static void ioapic_resume(void)
@@ -2968,28 +2923,18 @@ static void ioapic_resume(void)
2968 int ioapic_id; 2923 int ioapic_id;
2969 2924
2970 for (ioapic_id = nr_ioapics - 1; ioapic_id >= 0; ioapic_id--) 2925 for (ioapic_id = nr_ioapics - 1; ioapic_id >= 0; ioapic_id--)
2971 resume_ioapic(ioapic_id); 2926 resume_ioapic_id(ioapic_id);
2927
2928 restore_ioapic_entries();
2972} 2929}
2973 2930
2974static struct syscore_ops ioapic_syscore_ops = { 2931static struct syscore_ops ioapic_syscore_ops = {
2975 .suspend = ioapic_suspend, 2932 .suspend = save_ioapic_entries,
2976 .resume = ioapic_resume, 2933 .resume = ioapic_resume,
2977}; 2934};
2978 2935
2979static int __init ioapic_init_ops(void) 2936static int __init ioapic_init_ops(void)
2980{ 2937{
2981 int i;
2982
2983 for (i = 0; i < nr_ioapics; i++) {
2984 unsigned int size;
2985
2986 size = nr_ioapic_registers[i]
2987 * sizeof(struct IO_APIC_route_entry);
2988 ioapic_saved_data[i] = kzalloc(size, GFP_KERNEL);
2989 if (!ioapic_saved_data[i])
2990 pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
2991 }
2992
2993 register_syscore_ops(&ioapic_syscore_ops); 2938 register_syscore_ops(&ioapic_syscore_ops);
2994 2939
2995 return 0; 2940 return 0;
@@ -3592,14 +3537,14 @@ int io_apic_setup_irq_pin_once(unsigned int irq, int node,
3592 int ret; 3537 int ret;
3593 3538
3594 /* Avoid redundant programming */ 3539 /* Avoid redundant programming */
3595 if (test_bit(pin, mp_ioapic_routing[id].pin_programmed)) { 3540 if (test_bit(pin, ioapics[id].pin_programmed)) {
3596 pr_debug("Pin %d-%d already programmed\n", 3541 pr_debug("Pin %d-%d already programmed\n",
3597 mp_ioapics[id].apicid, pin); 3542 mpc_ioapic_id(id), pin);
3598 return 0; 3543 return 0;
3599 } 3544 }
3600 ret = io_apic_setup_irq_pin(irq, node, attr); 3545 ret = io_apic_setup_irq_pin(irq, node, attr);
3601 if (!ret) 3546 if (!ret)
3602 set_bit(pin, mp_ioapic_routing[id].pin_programmed); 3547 set_bit(pin, ioapics[id].pin_programmed);
3603 return ret; 3548 return ret;
3604} 3549}
3605 3550
@@ -3764,8 +3709,7 @@ static u8 __init io_apic_unique_id(u8 id)
3764 3709
3765 bitmap_zero(used, 256); 3710 bitmap_zero(used, 256);
3766 for (i = 0; i < nr_ioapics; i++) { 3711 for (i = 0; i < nr_ioapics; i++) {
3767 struct mpc_ioapic *ia = &mp_ioapics[i]; 3712 __set_bit(mpc_ioapic_id(i), used);
3768 __set_bit(ia->apicid, used);
3769 } 3713 }
3770 if (!test_bit(id, used)) 3714 if (!test_bit(id, used))
3771 return id; 3715 return id;
@@ -3825,7 +3769,7 @@ void __init setup_ioapic_dest(void)
3825 return; 3769 return;
3826 3770
3827 for (ioapic = 0; ioapic < nr_ioapics; ioapic++) 3771 for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
3828 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) { 3772 for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3829 irq_entry = find_irq_entry(ioapic, pin, mp_INT); 3773 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3830 if (irq_entry == -1) 3774 if (irq_entry == -1)
3831 continue; 3775 continue;
@@ -3896,7 +3840,7 @@ void __init ioapic_and_gsi_init(void)
3896 ioapic_res = ioapic_setup_resources(nr_ioapics); 3840 ioapic_res = ioapic_setup_resources(nr_ioapics);
3897 for (i = 0; i < nr_ioapics; i++) { 3841 for (i = 0; i < nr_ioapics; i++) {
3898 if (smp_found_config) { 3842 if (smp_found_config) {
3899 ioapic_phys = mp_ioapics[i].apicaddr; 3843 ioapic_phys = mpc_ioapic_addr(i);
3900#ifdef CONFIG_X86_32 3844#ifdef CONFIG_X86_32
3901 if (!ioapic_phys) { 3845 if (!ioapic_phys) {
3902 printk(KERN_ERR 3846 printk(KERN_ERR
@@ -3956,8 +3900,9 @@ int mp_find_ioapic(u32 gsi)
3956 3900
3957 /* Find the IOAPIC that manages this GSI. */ 3901 /* Find the IOAPIC that manages this GSI. */
3958 for (i = 0; i < nr_ioapics; i++) { 3902 for (i = 0; i < nr_ioapics; i++) {
3959 if ((gsi >= mp_gsi_routing[i].gsi_base) 3903 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
3960 && (gsi <= mp_gsi_routing[i].gsi_end)) 3904 if ((gsi >= gsi_cfg->gsi_base)
3905 && (gsi <= gsi_cfg->gsi_end))
3961 return i; 3906 return i;
3962 } 3907 }
3963 3908
@@ -3967,12 +3912,16 @@ int mp_find_ioapic(u32 gsi)
3967 3912
3968int mp_find_ioapic_pin(int ioapic, u32 gsi) 3913int mp_find_ioapic_pin(int ioapic, u32 gsi)
3969{ 3914{
3915 struct mp_ioapic_gsi *gsi_cfg;
3916
3970 if (WARN_ON(ioapic == -1)) 3917 if (WARN_ON(ioapic == -1))
3971 return -1; 3918 return -1;
3972 if (WARN_ON(gsi > mp_gsi_routing[ioapic].gsi_end)) 3919
3920 gsi_cfg = mp_ioapic_gsi_routing(ioapic);
3921 if (WARN_ON(gsi > gsi_cfg->gsi_end))
3973 return -1; 3922 return -1;
3974 3923
3975 return gsi - mp_gsi_routing[ioapic].gsi_base; 3924 return gsi - gsi_cfg->gsi_base;
3976} 3925}
3977 3926
3978static __init int bad_ioapic(unsigned long address) 3927static __init int bad_ioapic(unsigned long address)
@@ -3994,40 +3943,42 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
3994{ 3943{
3995 int idx = 0; 3944 int idx = 0;
3996 int entries; 3945 int entries;
3946 struct mp_ioapic_gsi *gsi_cfg;
3997 3947
3998 if (bad_ioapic(address)) 3948 if (bad_ioapic(address))
3999 return; 3949 return;
4000 3950
4001 idx = nr_ioapics; 3951 idx = nr_ioapics;
4002 3952
4003 mp_ioapics[idx].type = MP_IOAPIC; 3953 ioapics[idx].mp_config.type = MP_IOAPIC;
4004 mp_ioapics[idx].flags = MPC_APIC_USABLE; 3954 ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
4005 mp_ioapics[idx].apicaddr = address; 3955 ioapics[idx].mp_config.apicaddr = address;
4006 3956
4007 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address); 3957 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
4008 mp_ioapics[idx].apicid = io_apic_unique_id(id); 3958 ioapics[idx].mp_config.apicid = io_apic_unique_id(id);
4009 mp_ioapics[idx].apicver = io_apic_get_version(idx); 3959 ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
4010 3960
4011 /* 3961 /*
4012 * Build basic GSI lookup table to facilitate gsi->io_apic lookups 3962 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
4013 * and to prevent reprogramming of IOAPIC pins (PCI GSIs). 3963 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
4014 */ 3964 */
4015 entries = io_apic_get_redir_entries(idx); 3965 entries = io_apic_get_redir_entries(idx);
4016 mp_gsi_routing[idx].gsi_base = gsi_base; 3966 gsi_cfg = mp_ioapic_gsi_routing(idx);
4017 mp_gsi_routing[idx].gsi_end = gsi_base + entries - 1; 3967 gsi_cfg->gsi_base = gsi_base;
3968 gsi_cfg->gsi_end = gsi_base + entries - 1;
4018 3969
4019 /* 3970 /*
4020 * The number of IO-APIC IRQ registers (== #pins): 3971 * The number of IO-APIC IRQ registers (== #pins):
4021 */ 3972 */
4022 nr_ioapic_registers[idx] = entries; 3973 ioapics[idx].nr_registers = entries;
4023 3974
4024 if (mp_gsi_routing[idx].gsi_end >= gsi_top) 3975 if (gsi_cfg->gsi_end >= gsi_top)
4025 gsi_top = mp_gsi_routing[idx].gsi_end + 1; 3976 gsi_top = gsi_cfg->gsi_end + 1;
4026 3977
4027 printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, " 3978 printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
4028 "GSI %d-%d\n", idx, mp_ioapics[idx].apicid, 3979 "GSI %d-%d\n", idx, mpc_ioapic_id(idx),
4029 mp_ioapics[idx].apicver, mp_ioapics[idx].apicaddr, 3980 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
4030 mp_gsi_routing[idx].gsi_base, mp_gsi_routing[idx].gsi_end); 3981 gsi_cfg->gsi_base, gsi_cfg->gsi_end);
4031 3982
4032 nr_ioapics++; 3983 nr_ioapics++;
4033} 3984}
diff --git a/arch/x86/kernel/apic/numaq_32.c b/arch/x86/kernel/apic/numaq_32.c
index 30f13319e24b..c4a61ca1349a 100644
--- a/arch/x86/kernel/apic/numaq_32.c
+++ b/arch/x86/kernel/apic/numaq_32.c
@@ -472,8 +472,8 @@ static void numaq_setup_portio_remap(void)
472 (u_long) xquad_portio, (u_long) num_quads*XQUAD_PORTIO_QUAD); 472 (u_long) xquad_portio, (u_long) num_quads*XQUAD_PORTIO_QUAD);
473} 473}
474 474
475/* Use __refdata to keep false positive warning calm. */ 475/* Use __refdata to keep false positive warning calm. */
476struct apic __refdata apic_numaq = { 476static struct apic __refdata apic_numaq = {
477 477
478 .name = "NUMAQ", 478 .name = "NUMAQ",
479 .probe = probe_numaq, 479 .probe = probe_numaq,
@@ -537,3 +537,5 @@ struct apic __refdata apic_numaq = {
537 .x86_32_early_logical_apicid = noop_x86_32_early_logical_apicid, 537 .x86_32_early_logical_apicid = noop_x86_32_early_logical_apicid,
538 .x86_32_numa_cpu_node = numaq_numa_cpu_node, 538 .x86_32_numa_cpu_node = numaq_numa_cpu_node,
539}; 539};
540
541apic_driver(apic_numaq);
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index 6541e471fd91..b5254ad044ab 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -52,31 +52,6 @@ static int __init print_ipi_mode(void)
52} 52}
53late_initcall(print_ipi_mode); 53late_initcall(print_ipi_mode);
54 54
55void __init default_setup_apic_routing(void)
56{
57 int version = apic_version[boot_cpu_physical_apicid];
58
59 if (num_possible_cpus() > 8) {
60 switch (boot_cpu_data.x86_vendor) {
61 case X86_VENDOR_INTEL:
62 if (!APIC_XAPIC(version)) {
63 def_to_bigsmp = 0;
64 break;
65 }
66 /* If P4 and above fall through */
67 case X86_VENDOR_AMD:
68 def_to_bigsmp = 1;
69 }
70 }
71
72#ifdef CONFIG_X86_BIGSMP
73 generic_bigsmp_probe();
74#endif
75
76 if (apic->setup_apic_routing)
77 apic->setup_apic_routing();
78}
79
80static int default_x86_32_early_logical_apicid(int cpu) 55static int default_x86_32_early_logical_apicid(int cpu)
81{ 56{
82 return 1 << cpu; 57 return 1 << cpu;
@@ -112,7 +87,7 @@ static int probe_default(void)
112 return 1; 87 return 1;
113} 88}
114 89
115struct apic apic_default = { 90static struct apic apic_default = {
116 91
117 .name = "default", 92 .name = "default",
118 .probe = probe_default, 93 .probe = probe_default,
@@ -174,44 +149,22 @@ struct apic apic_default = {
174 .x86_32_early_logical_apicid = default_x86_32_early_logical_apicid, 149 .x86_32_early_logical_apicid = default_x86_32_early_logical_apicid,
175}; 150};
176 151
177extern struct apic apic_numaq; 152apic_driver(apic_default);
178extern struct apic apic_summit;
179extern struct apic apic_bigsmp;
180extern struct apic apic_es7000;
181extern struct apic apic_es7000_cluster;
182 153
183struct apic *apic = &apic_default; 154struct apic *apic = &apic_default;
184EXPORT_SYMBOL_GPL(apic); 155EXPORT_SYMBOL_GPL(apic);
185 156
186static struct apic *apic_probe[] __initdata = {
187#ifdef CONFIG_X86_NUMAQ
188 &apic_numaq,
189#endif
190#ifdef CONFIG_X86_SUMMIT
191 &apic_summit,
192#endif
193#ifdef CONFIG_X86_BIGSMP
194 &apic_bigsmp,
195#endif
196#ifdef CONFIG_X86_ES7000
197 &apic_es7000,
198 &apic_es7000_cluster,
199#endif
200 &apic_default, /* must be last */
201 NULL,
202};
203
204static int cmdline_apic __initdata; 157static int cmdline_apic __initdata;
205static int __init parse_apic(char *arg) 158static int __init parse_apic(char *arg)
206{ 159{
207 int i; 160 struct apic **drv;
208 161
209 if (!arg) 162 if (!arg)
210 return -EINVAL; 163 return -EINVAL;
211 164
212 for (i = 0; apic_probe[i]; i++) { 165 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
213 if (!strcmp(apic_probe[i]->name, arg)) { 166 if (!strcmp((*drv)->name, arg)) {
214 apic = apic_probe[i]; 167 apic = *drv;
215 cmdline_apic = 1; 168 cmdline_apic = 1;
216 return 0; 169 return 0;
217 } 170 }
@@ -222,38 +175,58 @@ static int __init parse_apic(char *arg)
222} 175}
223early_param("apic", parse_apic); 176early_param("apic", parse_apic);
224 177
225void __init generic_bigsmp_probe(void) 178void __init default_setup_apic_routing(void)
226{ 179{
180 int version = apic_version[boot_cpu_physical_apicid];
181
182 if (num_possible_cpus() > 8) {
183 switch (boot_cpu_data.x86_vendor) {
184 case X86_VENDOR_INTEL:
185 if (!APIC_XAPIC(version)) {
186 def_to_bigsmp = 0;
187 break;
188 }
189 /* If P4 and above fall through */
190 case X86_VENDOR_AMD:
191 def_to_bigsmp = 1;
192 }
193 }
194
227#ifdef CONFIG_X86_BIGSMP 195#ifdef CONFIG_X86_BIGSMP
228 /* 196 /*
229 * This routine is used to switch to bigsmp mode when 197 * This is used to switch to bigsmp mode when
230 * - There is no apic= option specified by the user 198 * - There is no apic= option specified by the user
231 * - generic_apic_probe() has chosen apic_default as the sub_arch 199 * - generic_apic_probe() has chosen apic_default as the sub_arch
232 * - we find more than 8 CPUs in acpi LAPIC listing with xAPIC support 200 * - we find more than 8 CPUs in acpi LAPIC listing with xAPIC support
233 */ 201 */
234 202
235 if (!cmdline_apic && apic == &apic_default) { 203 if (!cmdline_apic && apic == &apic_default) {
236 if (apic_bigsmp.probe()) { 204 struct apic *bigsmp = generic_bigsmp_probe();
237 apic = &apic_bigsmp; 205 if (bigsmp) {
206 apic = bigsmp;
238 printk(KERN_INFO "Overriding APIC driver with %s\n", 207 printk(KERN_INFO "Overriding APIC driver with %s\n",
239 apic->name); 208 apic->name);
240 } 209 }
241 } 210 }
242#endif 211#endif
212
213 if (apic->setup_apic_routing)
214 apic->setup_apic_routing();
243} 215}
244 216
245void __init generic_apic_probe(void) 217void __init generic_apic_probe(void)
246{ 218{
247 if (!cmdline_apic) { 219 if (!cmdline_apic) {
248 int i; 220 struct apic **drv;
249 for (i = 0; apic_probe[i]; i++) { 221
250 if (apic_probe[i]->probe()) { 222 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
251 apic = apic_probe[i]; 223 if ((*drv)->probe()) {
224 apic = *drv;
252 break; 225 break;
253 } 226 }
254 } 227 }
255 /* Not visible without early console */ 228 /* Not visible without early console */
256 if (!apic_probe[i]) 229 if (drv == __apicdrivers_end)
257 panic("Didn't find an APIC driver"); 230 panic("Didn't find an APIC driver");
258 } 231 }
259 printk(KERN_INFO "Using APIC driver %s\n", apic->name); 232 printk(KERN_INFO "Using APIC driver %s\n", apic->name);
@@ -264,16 +237,16 @@ void __init generic_apic_probe(void)
264int __init 237int __init
265generic_mps_oem_check(struct mpc_table *mpc, char *oem, char *productid) 238generic_mps_oem_check(struct mpc_table *mpc, char *oem, char *productid)
266{ 239{
267 int i; 240 struct apic **drv;
268 241
269 for (i = 0; apic_probe[i]; ++i) { 242 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
270 if (!apic_probe[i]->mps_oem_check) 243 if (!((*drv)->mps_oem_check))
271 continue; 244 continue;
272 if (!apic_probe[i]->mps_oem_check(mpc, oem, productid)) 245 if (!(*drv)->mps_oem_check(mpc, oem, productid))
273 continue; 246 continue;
274 247
275 if (!cmdline_apic) { 248 if (!cmdline_apic) {
276 apic = apic_probe[i]; 249 apic = *drv;
277 printk(KERN_INFO "Switched to APIC driver `%s'.\n", 250 printk(KERN_INFO "Switched to APIC driver `%s'.\n",
278 apic->name); 251 apic->name);
279 } 252 }
@@ -284,16 +257,16 @@ generic_mps_oem_check(struct mpc_table *mpc, char *oem, char *productid)
284 257
285int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id) 258int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
286{ 259{
287 int i; 260 struct apic **drv;
288 261
289 for (i = 0; apic_probe[i]; ++i) { 262 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
290 if (!apic_probe[i]->acpi_madt_oem_check) 263 if (!(*drv)->acpi_madt_oem_check)
291 continue; 264 continue;
292 if (!apic_probe[i]->acpi_madt_oem_check(oem_id, oem_table_id)) 265 if (!(*drv)->acpi_madt_oem_check(oem_id, oem_table_id))
293 continue; 266 continue;
294 267
295 if (!cmdline_apic) { 268 if (!cmdline_apic) {
296 apic = apic_probe[i]; 269 apic = *drv;
297 printk(KERN_INFO "Switched to APIC driver `%s'.\n", 270 printk(KERN_INFO "Switched to APIC driver `%s'.\n",
298 apic->name); 271 apic->name);
299 } 272 }
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index d8c4a6feb286..3fe986698929 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -23,27 +23,6 @@
23#include <asm/ipi.h> 23#include <asm/ipi.h>
24#include <asm/setup.h> 24#include <asm/setup.h>
25 25
26extern struct apic apic_flat;
27extern struct apic apic_physflat;
28extern struct apic apic_x2xpic_uv_x;
29extern struct apic apic_x2apic_phys;
30extern struct apic apic_x2apic_cluster;
31
32struct apic __read_mostly *apic = &apic_flat;
33EXPORT_SYMBOL_GPL(apic);
34
35static struct apic *apic_probe[] __initdata = {
36#ifdef CONFIG_X86_UV
37 &apic_x2apic_uv_x,
38#endif
39#ifdef CONFIG_X86_X2APIC
40 &apic_x2apic_phys,
41 &apic_x2apic_cluster,
42#endif
43 &apic_physflat,
44 NULL,
45};
46
47static int apicid_phys_pkg_id(int initial_apic_id, int index_msb) 26static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
48{ 27{
49 return hard_smp_processor_id() >> index_msb; 28 return hard_smp_processor_id() >> index_msb;
@@ -54,26 +33,20 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
54 */ 33 */
55void __init default_setup_apic_routing(void) 34void __init default_setup_apic_routing(void)
56{ 35{
36 struct apic **drv;
57 37
58 enable_IR_x2apic(); 38 enable_IR_x2apic();
59 39
60#ifdef CONFIG_X86_X2APIC 40 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
61 if (x2apic_mode 41 if ((*drv)->probe && (*drv)->probe()) {
62#ifdef CONFIG_X86_UV 42 if (apic != *drv) {
63 && apic != &apic_x2apic_uv_x 43 apic = *drv;
64#endif 44 pr_info("Switched APIC routing to %s.\n",
65 ) { 45 apic->name);
66 if (x2apic_phys) 46 }
67 apic = &apic_x2apic_phys; 47 break;
68 else 48 }
69 apic = &apic_x2apic_cluster;
70 } 49 }
71#endif
72
73 if (apic == &apic_flat && num_possible_cpus() > 8)
74 apic = &apic_physflat;
75
76 printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
77 50
78 if (is_vsmp_box()) { 51 if (is_vsmp_box()) {
79 /* need to update phys_pkg_id */ 52 /* need to update phys_pkg_id */
@@ -90,13 +63,15 @@ void apic_send_IPI_self(int vector)
90 63
91int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id) 64int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
92{ 65{
93 int i; 66 struct apic **drv;
94 67
95 for (i = 0; apic_probe[i]; ++i) { 68 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
96 if (apic_probe[i]->acpi_madt_oem_check(oem_id, oem_table_id)) { 69 if ((*drv)->acpi_madt_oem_check(oem_id, oem_table_id)) {
97 apic = apic_probe[i]; 70 if (apic != *drv) {
98 printk(KERN_INFO "Setting APIC routing to %s.\n", 71 apic = *drv;
99 apic->name); 72 pr_info("Setting APIC routing to %s.\n",
73 apic->name);
74 }
100 return 1; 75 return 1;
101 } 76 }
102 } 77 }
diff --git a/arch/x86/kernel/apic/summit_32.c b/arch/x86/kernel/apic/summit_32.c
index 35bcd7d995a1..19114423c58c 100644
--- a/arch/x86/kernel/apic/summit_32.c
+++ b/arch/x86/kernel/apic/summit_32.c
@@ -491,7 +491,7 @@ void setup_summit(void)
491} 491}
492#endif 492#endif
493 493
494struct apic apic_summit = { 494static struct apic apic_summit = {
495 495
496 .name = "summit", 496 .name = "summit",
497 .probe = probe_summit, 497 .probe = probe_summit,
@@ -552,3 +552,5 @@ struct apic apic_summit = {
552 552
553 .x86_32_early_logical_apicid = summit_early_logical_apicid, 553 .x86_32_early_logical_apicid = summit_early_logical_apicid,
554}; 554};
555
556apic_driver(apic_summit);
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index 90949bbd566d..500795875827 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -5,118 +5,95 @@
5#include <linux/ctype.h> 5#include <linux/ctype.h>
6#include <linux/init.h> 6#include <linux/init.h>
7#include <linux/dmar.h> 7#include <linux/dmar.h>
8#include <linux/cpu.h>
8 9
9#include <asm/smp.h> 10#include <asm/smp.h>
10#include <asm/apic.h> 11#include <asm/x2apic.h>
11#include <asm/ipi.h>
12 12
13static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid); 13static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
14static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
15static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
14 16
15static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id) 17static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
16{ 18{
17 return x2apic_enabled(); 19 return x2apic_enabled();
18} 20}
19 21
20/* 22static inline u32 x2apic_cluster(int cpu)
21 * need to use more than cpu 0, because we need more vectors when
22 * MSI-X are used.
23 */
24static const struct cpumask *x2apic_target_cpus(void)
25{ 23{
26 return cpu_online_mask; 24 return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
27}
28
29/*
30 * for now each logical cpu is in its own vector allocation domain.
31 */
32static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
33{
34 cpumask_clear(retmask);
35 cpumask_set_cpu(cpu, retmask);
36} 25}
37 26
38static void 27static void
39 __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest) 28__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
40{ 29{
41 unsigned long cfg; 30 struct cpumask *cpus_in_cluster_ptr;
31 struct cpumask *ipi_mask_ptr;
32 unsigned int cpu, this_cpu;
33 unsigned long flags;
34 u32 dest;
35
36 x2apic_wrmsr_fence();
37
38 local_irq_save(flags);
42 39
43 cfg = __prepare_ICR(0, vector, dest); 40 this_cpu = smp_processor_id();
44 41
45 /* 42 /*
46 * send the IPI. 43 * We are to modify mask, so we need an own copy
44 * and be sure it's manipulated with irq off.
47 */ 45 */
48 native_x2apic_icr_write(cfg, apicid); 46 ipi_mask_ptr = __raw_get_cpu_var(ipi_mask);
49} 47 cpumask_copy(ipi_mask_ptr, mask);
50 48
51/* 49 /*
52 * for now, we send the IPI's one by one in the cpumask. 50 * The idea is to send one IPI per cluster.
53 * TBD: Based on the cpu mask, we can send the IPI's to the cluster group 51 */
54 * at once. We have 16 cpu's in a cluster. This will minimize IPI register 52 for_each_cpu(cpu, ipi_mask_ptr) {
55 * writes. 53 unsigned long i;
56 */
57static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
58{
59 unsigned long query_cpu;
60 unsigned long flags;
61 54
62 x2apic_wrmsr_fence(); 55 cpus_in_cluster_ptr = per_cpu(cpus_in_cluster, cpu);
56 dest = 0;
63 57
64 local_irq_save(flags); 58 /* Collect cpus in cluster. */
65 for_each_cpu(query_cpu, mask) { 59 for_each_cpu_and(i, ipi_mask_ptr, cpus_in_cluster_ptr) {
66 __x2apic_send_IPI_dest( 60 if (apic_dest == APIC_DEST_ALLINC || i != this_cpu)
67 per_cpu(x86_cpu_to_logical_apicid, query_cpu), 61 dest |= per_cpu(x86_cpu_to_logical_apicid, i);
68 vector, apic->dest_logical); 62 }
63
64 if (!dest)
65 continue;
66
67 __x2apic_send_IPI_dest(dest, vector, apic->dest_logical);
68 /*
69 * Cluster sibling cpus should be discared now so
70 * we would not send IPI them second time.
71 */
72 cpumask_andnot(ipi_mask_ptr, ipi_mask_ptr, cpus_in_cluster_ptr);
69 } 73 }
74
70 local_irq_restore(flags); 75 local_irq_restore(flags);
71} 76}
72 77
78static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
79{
80 __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
81}
82
73static void 83static void
74 x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector) 84 x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
75{ 85{
76 unsigned long this_cpu = smp_processor_id(); 86 __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
77 unsigned long query_cpu;
78 unsigned long flags;
79
80 x2apic_wrmsr_fence();
81
82 local_irq_save(flags);
83 for_each_cpu(query_cpu, mask) {
84 if (query_cpu == this_cpu)
85 continue;
86 __x2apic_send_IPI_dest(
87 per_cpu(x86_cpu_to_logical_apicid, query_cpu),
88 vector, apic->dest_logical);
89 }
90 local_irq_restore(flags);
91} 87}
92 88
93static void x2apic_send_IPI_allbutself(int vector) 89static void x2apic_send_IPI_allbutself(int vector)
94{ 90{
95 unsigned long this_cpu = smp_processor_id(); 91 __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
96 unsigned long query_cpu;
97 unsigned long flags;
98
99 x2apic_wrmsr_fence();
100
101 local_irq_save(flags);
102 for_each_online_cpu(query_cpu) {
103 if (query_cpu == this_cpu)
104 continue;
105 __x2apic_send_IPI_dest(
106 per_cpu(x86_cpu_to_logical_apicid, query_cpu),
107 vector, apic->dest_logical);
108 }
109 local_irq_restore(flags);
110} 92}
111 93
112static void x2apic_send_IPI_all(int vector) 94static void x2apic_send_IPI_all(int vector)
113{ 95{
114 x2apic_send_IPI_mask(cpu_online_mask, vector); 96 __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
115}
116
117static int x2apic_apic_id_registered(void)
118{
119 return 1;
120} 97}
121 98
122static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask) 99static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
@@ -151,43 +128,90 @@ x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
151 return per_cpu(x86_cpu_to_logical_apicid, cpu); 128 return per_cpu(x86_cpu_to_logical_apicid, cpu);
152} 129}
153 130
154static unsigned int x2apic_cluster_phys_get_apic_id(unsigned long x) 131static void init_x2apic_ldr(void)
155{ 132{
156 unsigned int id; 133 unsigned int this_cpu = smp_processor_id();
134 unsigned int cpu;
157 135
158 id = x; 136 per_cpu(x86_cpu_to_logical_apicid, this_cpu) = apic_read(APIC_LDR);
159 return id; 137
138 __cpu_set(this_cpu, per_cpu(cpus_in_cluster, this_cpu));
139 for_each_online_cpu(cpu) {
140 if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
141 continue;
142 __cpu_set(this_cpu, per_cpu(cpus_in_cluster, cpu));
143 __cpu_set(cpu, per_cpu(cpus_in_cluster, this_cpu));
144 }
160} 145}
161 146
162static unsigned long set_apic_id(unsigned int id) 147 /*
148 * At CPU state changes, update the x2apic cluster sibling info.
149 */
150static int __cpuinit
151update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu)
163{ 152{
164 unsigned long x; 153 unsigned int this_cpu = (unsigned long)hcpu;
154 unsigned int cpu;
155 int err = 0;
156
157 switch (action) {
158 case CPU_UP_PREPARE:
159 if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, this_cpu),
160 GFP_KERNEL)) {
161 err = -ENOMEM;
162 } else if (!zalloc_cpumask_var(&per_cpu(ipi_mask, this_cpu),
163 GFP_KERNEL)) {
164 free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
165 err = -ENOMEM;
166 }
167 break;
168 case CPU_UP_CANCELED:
169 case CPU_UP_CANCELED_FROZEN:
170 case CPU_DEAD:
171 for_each_online_cpu(cpu) {
172 if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
173 continue;
174 __cpu_clear(this_cpu, per_cpu(cpus_in_cluster, cpu));
175 __cpu_clear(cpu, per_cpu(cpus_in_cluster, this_cpu));
176 }
177 free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
178 free_cpumask_var(per_cpu(ipi_mask, this_cpu));
179 break;
180 }
165 181
166 x = id; 182 return notifier_from_errno(err);
167 return x;
168} 183}
169 184
170static int x2apic_cluster_phys_pkg_id(int initial_apicid, int index_msb) 185static struct notifier_block __refdata x2apic_cpu_notifier = {
171{ 186 .notifier_call = update_clusterinfo,
172 return initial_apicid >> index_msb; 187};
173}
174 188
175static void x2apic_send_IPI_self(int vector) 189static int x2apic_init_cpu_notifier(void)
176{ 190{
177 apic_write(APIC_SELF_IPI, vector); 191 int cpu = smp_processor_id();
192
193 zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL);
194 zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL);
195
196 BUG_ON(!per_cpu(cpus_in_cluster, cpu) || !per_cpu(ipi_mask, cpu));
197
198 __cpu_set(cpu, per_cpu(cpus_in_cluster, cpu));
199 register_hotcpu_notifier(&x2apic_cpu_notifier);
200 return 1;
178} 201}
179 202
180static void init_x2apic_ldr(void) 203static int x2apic_cluster_probe(void)
181{ 204{
182 int cpu = smp_processor_id(); 205 if (x2apic_mode)
183 206 return x2apic_init_cpu_notifier();
184 per_cpu(x86_cpu_to_logical_apicid, cpu) = apic_read(APIC_LDR); 207 else
208 return 0;
185} 209}
186 210
187struct apic apic_x2apic_cluster = { 211static struct apic apic_x2apic_cluster = {
188 212
189 .name = "cluster x2apic", 213 .name = "cluster x2apic",
190 .probe = NULL, 214 .probe = x2apic_cluster_probe,
191 .acpi_madt_oem_check = x2apic_acpi_madt_oem_check, 215 .acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
192 .apic_id_registered = x2apic_apic_id_registered, 216 .apic_id_registered = x2apic_apic_id_registered,
193 217
@@ -211,11 +235,11 @@ struct apic apic_x2apic_cluster = {
211 .setup_portio_remap = NULL, 235 .setup_portio_remap = NULL,
212 .check_phys_apicid_present = default_check_phys_apicid_present, 236 .check_phys_apicid_present = default_check_phys_apicid_present,
213 .enable_apic_mode = NULL, 237 .enable_apic_mode = NULL,
214 .phys_pkg_id = x2apic_cluster_phys_pkg_id, 238 .phys_pkg_id = x2apic_phys_pkg_id,
215 .mps_oem_check = NULL, 239 .mps_oem_check = NULL,
216 240
217 .get_apic_id = x2apic_cluster_phys_get_apic_id, 241 .get_apic_id = x2apic_get_apic_id,
218 .set_apic_id = set_apic_id, 242 .set_apic_id = x2apic_set_apic_id,
219 .apic_id_mask = 0xFFFFFFFFu, 243 .apic_id_mask = 0xFFFFFFFFu,
220 244
221 .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid, 245 .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
@@ -240,3 +264,5 @@ struct apic apic_x2apic_cluster = {
240 .wait_icr_idle = native_x2apic_wait_icr_idle, 264 .wait_icr_idle = native_x2apic_wait_icr_idle,
241 .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, 265 .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
242}; 266};
267
268apic_driver(apic_x2apic_cluster);
diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c
index c7e6d6645bf4..f5373dfde21e 100644
--- a/arch/x86/kernel/apic/x2apic_phys.c
+++ b/arch/x86/kernel/apic/x2apic_phys.c
@@ -7,11 +7,12 @@
7#include <linux/dmar.h> 7#include <linux/dmar.h>
8 8
9#include <asm/smp.h> 9#include <asm/smp.h>
10#include <asm/apic.h> 10#include <asm/x2apic.h>
11#include <asm/ipi.h>
12 11
13int x2apic_phys; 12int x2apic_phys;
14 13
14static struct apic apic_x2apic_phys;
15
15static int set_x2apic_phys_mode(char *arg) 16static int set_x2apic_phys_mode(char *arg)
16{ 17{
17 x2apic_phys = 1; 18 x2apic_phys = 1;
@@ -27,94 +28,46 @@ static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
27 return 0; 28 return 0;
28} 29}
29 30
30/* 31static void
31 * need to use more than cpu 0, because we need more vectors when 32__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
32 * MSI-X are used.
33 */
34static const struct cpumask *x2apic_target_cpus(void)
35{
36 return cpu_online_mask;
37}
38
39static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
40{
41 cpumask_clear(retmask);
42 cpumask_set_cpu(cpu, retmask);
43}
44
45static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
46 unsigned int dest)
47{
48 unsigned long cfg;
49
50 cfg = __prepare_ICR(0, vector, dest);
51
52 /*
53 * send the IPI.
54 */
55 native_x2apic_icr_write(cfg, apicid);
56}
57
58static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
59{ 33{
60 unsigned long query_cpu; 34 unsigned long query_cpu;
35 unsigned long this_cpu;
61 unsigned long flags; 36 unsigned long flags;
62 37
63 x2apic_wrmsr_fence(); 38 x2apic_wrmsr_fence();
64 39
65 local_irq_save(flags); 40 local_irq_save(flags);
41
42 this_cpu = smp_processor_id();
66 for_each_cpu(query_cpu, mask) { 43 for_each_cpu(query_cpu, mask) {
44 if (apic_dest == APIC_DEST_ALLBUT && this_cpu == query_cpu)
45 continue;
67 __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu), 46 __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
68 vector, APIC_DEST_PHYSICAL); 47 vector, APIC_DEST_PHYSICAL);
69 } 48 }
70 local_irq_restore(flags); 49 local_irq_restore(flags);
71} 50}
72 51
52static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
53{
54 __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
55}
56
73static void 57static void
74 x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector) 58 x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
75{ 59{
76 unsigned long this_cpu = smp_processor_id(); 60 __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
77 unsigned long query_cpu;
78 unsigned long flags;
79
80 x2apic_wrmsr_fence();
81
82 local_irq_save(flags);
83 for_each_cpu(query_cpu, mask) {
84 if (query_cpu != this_cpu)
85 __x2apic_send_IPI_dest(
86 per_cpu(x86_cpu_to_apicid, query_cpu),
87 vector, APIC_DEST_PHYSICAL);
88 }
89 local_irq_restore(flags);
90} 61}
91 62
92static void x2apic_send_IPI_allbutself(int vector) 63static void x2apic_send_IPI_allbutself(int vector)
93{ 64{
94 unsigned long this_cpu = smp_processor_id(); 65 __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
95 unsigned long query_cpu;
96 unsigned long flags;
97
98 x2apic_wrmsr_fence();
99
100 local_irq_save(flags);
101 for_each_online_cpu(query_cpu) {
102 if (query_cpu == this_cpu)
103 continue;
104 __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
105 vector, APIC_DEST_PHYSICAL);
106 }
107 local_irq_restore(flags);
108} 66}
109 67
110static void x2apic_send_IPI_all(int vector) 68static void x2apic_send_IPI_all(int vector)
111{ 69{
112 x2apic_send_IPI_mask(cpu_online_mask, vector); 70 __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
113}
114
115static int x2apic_apic_id_registered(void)
116{
117 return 1;
118} 71}
119 72
120static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask) 73static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
@@ -149,34 +102,22 @@ x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
149 return per_cpu(x86_cpu_to_apicid, cpu); 102 return per_cpu(x86_cpu_to_apicid, cpu);
150} 103}
151 104
152static unsigned int x2apic_phys_get_apic_id(unsigned long x) 105static void init_x2apic_ldr(void)
153{
154 return x;
155}
156
157static unsigned long set_apic_id(unsigned int id)
158{
159 return id;
160}
161
162static int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
163{ 106{
164 return initial_apicid >> index_msb;
165} 107}
166 108
167static void x2apic_send_IPI_self(int vector) 109static int x2apic_phys_probe(void)
168{ 110{
169 apic_write(APIC_SELF_IPI, vector); 111 if (x2apic_mode && x2apic_phys)
170} 112 return 1;
171 113
172static void init_x2apic_ldr(void) 114 return apic == &apic_x2apic_phys;
173{
174} 115}
175 116
176struct apic apic_x2apic_phys = { 117static struct apic apic_x2apic_phys = {
177 118
178 .name = "physical x2apic", 119 .name = "physical x2apic",
179 .probe = NULL, 120 .probe = x2apic_phys_probe,
180 .acpi_madt_oem_check = x2apic_acpi_madt_oem_check, 121 .acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
181 .apic_id_registered = x2apic_apic_id_registered, 122 .apic_id_registered = x2apic_apic_id_registered,
182 123
@@ -203,8 +144,8 @@ struct apic apic_x2apic_phys = {
203 .phys_pkg_id = x2apic_phys_pkg_id, 144 .phys_pkg_id = x2apic_phys_pkg_id,
204 .mps_oem_check = NULL, 145 .mps_oem_check = NULL,
205 146
206 .get_apic_id = x2apic_phys_get_apic_id, 147 .get_apic_id = x2apic_get_apic_id,
207 .set_apic_id = set_apic_id, 148 .set_apic_id = x2apic_set_apic_id,
208 .apic_id_mask = 0xFFFFFFFFu, 149 .apic_id_mask = 0xFFFFFFFFu,
209 150
210 .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid, 151 .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
@@ -229,3 +170,5 @@ struct apic apic_x2apic_phys = {
229 .wait_icr_idle = native_x2apic_wait_icr_idle, 170 .wait_icr_idle = native_x2apic_wait_icr_idle,
230 .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, 171 .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
231}; 172};
173
174apic_driver(apic_x2apic_phys);
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 7acd2d2ac965..f450b683dfcf 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -58,6 +58,8 @@ unsigned int uv_apicid_hibits;
58EXPORT_SYMBOL_GPL(uv_apicid_hibits); 58EXPORT_SYMBOL_GPL(uv_apicid_hibits);
59static DEFINE_SPINLOCK(uv_nmi_lock); 59static DEFINE_SPINLOCK(uv_nmi_lock);
60 60
61static struct apic apic_x2apic_uv_x;
62
61static unsigned long __init uv_early_read_mmr(unsigned long addr) 63static unsigned long __init uv_early_read_mmr(unsigned long addr)
62{ 64{
63 unsigned long val, *mmr; 65 unsigned long val, *mmr;
@@ -326,10 +328,15 @@ static void uv_send_IPI_self(int vector)
326 apic_write(APIC_SELF_IPI, vector); 328 apic_write(APIC_SELF_IPI, vector);
327} 329}
328 330
329struct apic __refdata apic_x2apic_uv_x = { 331static int uv_probe(void)
332{
333 return apic == &apic_x2apic_uv_x;
334}
335
336static struct apic __refdata apic_x2apic_uv_x = {
330 337
331 .name = "UV large system", 338 .name = "UV large system",
332 .probe = NULL, 339 .probe = uv_probe,
333 .acpi_madt_oem_check = uv_acpi_madt_oem_check, 340 .acpi_madt_oem_check = uv_acpi_madt_oem_check,
334 .apic_id_registered = uv_apic_id_registered, 341 .apic_id_registered = uv_apic_id_registered,
335 342
@@ -859,3 +866,5 @@ void __init uv_system_init(void)
859 if (is_kdump_kernel()) 866 if (is_kdump_kernel())
860 reboot_type = BOOT_ACPI; 867 reboot_type = BOOT_ACPI;
861} 868}
869
870apic_driver(apic_x2apic_uv_x);
diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c
index e90f08458e6b..690bc8461835 100644
--- a/arch/x86/kernel/devicetree.c
+++ b/arch/x86/kernel/devicetree.c
@@ -369,6 +369,7 @@ static struct of_ioapic_type of_ioapic_type[] =
369static int ioapic_xlate(struct irq_domain *id, const u32 *intspec, u32 intsize, 369static int ioapic_xlate(struct irq_domain *id, const u32 *intspec, u32 intsize,
370 u32 *out_hwirq, u32 *out_type) 370 u32 *out_hwirq, u32 *out_type)
371{ 371{
372 struct mp_ioapic_gsi *gsi_cfg;
372 struct io_apic_irq_attr attr; 373 struct io_apic_irq_attr attr;
373 struct of_ioapic_type *it; 374 struct of_ioapic_type *it;
374 u32 line, idx, type; 375 u32 line, idx, type;
@@ -378,7 +379,8 @@ static int ioapic_xlate(struct irq_domain *id, const u32 *intspec, u32 intsize,
378 379
379 line = *intspec; 380 line = *intspec;
380 idx = (u32) id->priv; 381 idx = (u32) id->priv;
381 *out_hwirq = line + mp_gsi_routing[idx].gsi_base; 382 gsi_cfg = mp_ioapic_gsi_routing(idx);
383 *out_hwirq = line + gsi_cfg->gsi_base;
382 384
383 intspec++; 385 intspec++;
384 type = *intspec; 386 type = *intspec;
@@ -407,7 +409,7 @@ static void __init ioapic_add_ofnode(struct device_node *np)
407 } 409 }
408 410
409 for (i = 0; i < nr_ioapics; i++) { 411 for (i = 0; i < nr_ioapics; i++) {
410 if (r.start == mp_ioapics[i].apicaddr) { 412 if (r.start == mpc_ioapic_addr(i)) {
411 struct irq_domain *id; 413 struct irq_domain *id;
412 414
413 id = kzalloc(sizeof(*id), GFP_KERNEL); 415 id = kzalloc(sizeof(*id), GFP_KERNEL);
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index 6f9bfffb2720..9103b89c145a 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -285,7 +285,7 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
285 intsrc.type = MP_INTSRC; 285 intsrc.type = MP_INTSRC;
286 intsrc.irqflag = 0; /* conforming */ 286 intsrc.irqflag = 0; /* conforming */
287 intsrc.srcbus = 0; 287 intsrc.srcbus = 0;
288 intsrc.dstapic = mp_ioapics[0].apicid; 288 intsrc.dstapic = mpc_ioapic_id(0);
289 289
290 intsrc.irqtype = mp_INT; 290 intsrc.irqtype = mp_INT;
291 291
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 624a2016198e..49927a863cc1 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -306,6 +306,13 @@ SECTIONS
306 } 306 }
307 307
308 . = ALIGN(8); 308 . = ALIGN(8);
309 .apicdrivers : AT(ADDR(.apicdrivers) - LOAD_OFFSET) {
310 __apicdrivers = .;
311 *(.apicdrivers);
312 __apicdrivers_end = .;
313 }
314
315 . = ALIGN(8);
309 /* 316 /*
310 * .exit.text is discard at runtime, not link time, to deal with 317 * .exit.text is discard at runtime, not link time, to deal with
311 * references from .altinstructions and .eh_frame 318 * references from .altinstructions and .eh_frame
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-29 22:01:59 -0500