aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2008-07-18 15:13:20 -0400
committerIngo Molnar <mingo@elte.hu>2008-07-18 15:13:20 -0400
commitf6dc8ccaab6d8f63cbae1e6c73fe972b26f5376c (patch)
treec5643fcdc884a8d0bfc3f1bc28039cab7394e5bc /arch/x86/kernel
parent323ec001c6bb98eeabb5abbdbb8c8055d9496554 (diff)
parent5b664cb235e97afbf34db9c4d77f08ebd725335e (diff)
Merge branch 'linus' into core/generic-dma-coherent
Conflicts: kernel/Makefile Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r--arch/x86/kernel/.gitignore1
-rw-r--r--arch/x86/kernel/Makefile28
-rw-r--r--arch/x86/kernel/acpi/boot.c442
-rw-r--r--arch/x86/kernel/acpi/processor.c6
-rw-r--r--arch/x86/kernel/acpi/realmode/wakeup.S38
-rw-r--r--arch/x86/kernel/acpi/realmode/wakeup.h5
-rw-r--r--arch/x86/kernel/acpi/sleep.c40
-rw-r--r--arch/x86/kernel/alternative.c22
-rw-r--r--arch/x86/kernel/amd_iommu.c962
-rw-r--r--arch/x86/kernel/amd_iommu_init.c875
-rw-r--r--arch/x86/kernel/aperture_64.c313
-rw-r--r--arch/x86/kernel/apic_32.c127
-rw-r--r--arch/x86/kernel/apic_64.c68
-rw-r--r--arch/x86/kernel/apm_32.c37
-rw-r--r--arch/x86/kernel/asm-offsets_32.c2
-rw-r--r--arch/x86/kernel/asm-offsets_64.c7
-rw-r--r--arch/x86/kernel/cpu/Makefile4
-rw-r--r--arch/x86/kernel/cpu/addon_cpuid_features.c17
-rw-r--r--arch/x86/kernel/cpu/amd.c42
-rw-r--r--arch/x86/kernel/cpu/amd_64.c222
-rw-r--r--arch/x86/kernel/cpu/bugs.c27
-rw-r--r--arch/x86/kernel/cpu/bugs_64.c (renamed from arch/x86/kernel/bugs_64.c)0
-rw-r--r--arch/x86/kernel/cpu/centaur_64.c35
-rw-r--r--arch/x86/kernel/cpu/common.c2
-rw-r--r--arch/x86/kernel/cpu/common_64.c681
-rw-r--r--arch/x86/kernel/cpu/cpu.h5
-rw-r--r--arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c44
-rw-r--r--arch/x86/kernel/cpu/intel.c4
-rw-r--r--arch/x86/kernel/cpu/intel_64.c95
-rw-r--r--arch/x86/kernel/cpu/intel_cacheinfo.c1
-rw-r--r--arch/x86/kernel/cpu/mcheck/k7.c36
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce_64.c34
-rw-r--r--arch/x86/kernel/cpu/mcheck/non-fatal.c2
-rw-r--r--arch/x86/kernel/cpu/mcheck/p4.c90
-rw-r--r--arch/x86/kernel/cpu/mtrr/generic.c38
-rw-r--r--arch/x86/kernel/cpu/mtrr/main.c905
-rw-r--r--arch/x86/kernel/cpu/mtrr/mtrr.h3
-rw-r--r--arch/x86/kernel/cpu/perfctr-watchdog.c210
-rw-r--r--arch/x86/kernel/cpuid.c27
-rw-r--r--arch/x86/kernel/e820.c1390
-rw-r--r--arch/x86/kernel/e820_32.c775
-rw-r--r--arch/x86/kernel/e820_64.c952
-rw-r--r--arch/x86/kernel/early-quirks.c41
-rw-r--r--arch/x86/kernel/early_printk.c2
-rw-r--r--arch/x86/kernel/efi.c67
-rw-r--r--arch/x86/kernel/efi_32.c8
-rw-r--r--arch/x86/kernel/efi_64.c8
-rw-r--r--arch/x86/kernel/entry_32.S83
-rw-r--r--arch/x86/kernel/entry_64.S170
-rw-r--r--arch/x86/kernel/ftrace.c141
-rw-r--r--arch/x86/kernel/genapic_64.c2
-rw-r--r--arch/x86/kernel/genx2apic_uv_x.c214
-rw-r--r--arch/x86/kernel/head.c55
-rw-r--r--arch/x86/kernel/head32.c27
-rw-r--r--arch/x86/kernel/head64.c96
-rw-r--r--arch/x86/kernel/head_32.S13
-rw-r--r--arch/x86/kernel/head_64.S101
-rw-r--r--arch/x86/kernel/hpet.c63
-rw-r--r--arch/x86/kernel/i386_ksyms_32.c9
-rw-r--r--arch/x86/kernel/i8259.c (renamed from arch/x86/kernel/i8259_32.c)136
-rw-r--r--arch/x86/kernel/i8259_64.c512
-rw-r--r--arch/x86/kernel/io_apic_32.c678
-rw-r--r--arch/x86/kernel/io_apic_64.c282
-rw-r--r--arch/x86/kernel/ipi.c1
-rw-r--r--arch/x86/kernel/irq_32.c254
-rw-r--r--arch/x86/kernel/irq_64.c28
-rw-r--r--arch/x86/kernel/irqinit_32.c114
-rw-r--r--arch/x86/kernel/irqinit_64.c221
-rw-r--r--arch/x86/kernel/ldt.c6
-rw-r--r--arch/x86/kernel/machine_kexec_32.c8
-rw-r--r--arch/x86/kernel/machine_kexec_64.c6
-rw-r--r--arch/x86/kernel/microcode.c35
-rw-r--r--arch/x86/kernel/mmconf-fam10h_64.c1
-rw-r--r--arch/x86/kernel/mpparse.c847
-rw-r--r--arch/x86/kernel/msr.c16
-rw-r--r--arch/x86/kernel/nmi.c (renamed from arch/x86/kernel/nmi_32.c)233
-rw-r--r--arch/x86/kernel/nmi_64.c482
-rw-r--r--arch/x86/kernel/numaq_32.c32
-rw-r--r--arch/x86/kernel/paravirt.c36
-rw-r--r--arch/x86/kernel/paravirt_patch_32.c4
-rw-r--r--arch/x86/kernel/paravirt_patch_64.c9
-rw-r--r--arch/x86/kernel/pci-calgary_64.c4
-rw-r--r--arch/x86/kernel/pci-dma.c22
-rw-r--r--arch/x86/kernel/pci-gart_64.c95
-rw-r--r--arch/x86/kernel/pci-swiotlb_64.c2
-rw-r--r--arch/x86/kernel/probe_roms_32.c166
-rw-r--r--arch/x86/kernel/process.c220
-rw-r--r--arch/x86/kernel/process_32.c68
-rw-r--r--arch/x86/kernel/process_64.c88
-rw-r--r--arch/x86/kernel/ptrace.c4
-rw-r--r--arch/x86/kernel/quirks.c60
-rw-r--r--arch/x86/kernel/reboot.c18
-rw-r--r--arch/x86/kernel/reboot_fixups_32.c4
-rw-r--r--arch/x86/kernel/setup.c929
-rw-r--r--arch/x86/kernel/setup64.c287
-rw-r--r--arch/x86/kernel/setup_32.c964
-rw-r--r--arch/x86/kernel/setup_64.c1194
-rw-r--r--arch/x86/kernel/setup_percpu.c399
-rw-r--r--arch/x86/kernel/smp.c158
-rw-r--r--arch/x86/kernel/smpboot.c217
-rw-r--r--arch/x86/kernel/smpcommon.c56
-rw-r--r--arch/x86/kernel/srat_32.c358
-rw-r--r--arch/x86/kernel/stacktrace.c2
-rw-r--r--arch/x86/kernel/summit_32.c2
-rw-r--r--arch/x86/kernel/sys_i386_32.c64
-rw-r--r--arch/x86/kernel/time_32.c6
-rw-r--r--arch/x86/kernel/time_64.c16
-rw-r--r--arch/x86/kernel/tlb_32.c2
-rw-r--r--arch/x86/kernel/tlb_64.c7
-rw-r--r--arch/x86/kernel/tlb_uv.c792
-rw-r--r--arch/x86/kernel/trampoline.c2
-rw-r--r--arch/x86/kernel/traps_32.c190
-rw-r--r--arch/x86/kernel/traps_64.c541
-rw-r--r--arch/x86/kernel/tsc.c535
-rw-r--r--arch/x86/kernel/tsc_32.c451
-rw-r--r--arch/x86/kernel/tsc_64.c357
-rw-r--r--arch/x86/kernel/visws_quirks.c709
-rw-r--r--arch/x86/kernel/vmi_32.c6
-rw-r--r--arch/x86/kernel/vmiclock_32.c7
-rw-r--r--arch/x86/kernel/vmlinux_32.lds.S15
-rw-r--r--arch/x86/kernel/vmlinux_64.lds.S18
-rw-r--r--arch/x86/kernel/vsmp_64.c3
-rw-r--r--arch/x86/kernel/vsyscall_64.c19
-rw-r--r--arch/x86/kernel/x8664_ksyms_64.c16
124 files changed, 13347 insertions, 9381 deletions
diff --git a/arch/x86/kernel/.gitignore b/arch/x86/kernel/.gitignore
index 4ea38a39aed4..08f4fd731469 100644
--- a/arch/x86/kernel/.gitignore
+++ b/arch/x86/kernel/.gitignore
@@ -1,2 +1,3 @@
1vsyscall.lds 1vsyscall.lds
2vsyscall_32.lds 2vsyscall_32.lds
3vmlinux.lds
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 77807d4769c9..da140611bb57 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -2,10 +2,16 @@
2# Makefile for the linux kernel. 2# Makefile for the linux kernel.
3# 3#
4 4
5extra-y := head_$(BITS).o head$(BITS).o init_task.o vmlinux.lds 5extra-y := head_$(BITS).o head$(BITS).o head.o init_task.o vmlinux.lds
6 6
7CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE) 7CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE)
8 8
9ifdef CONFIG_FTRACE
10# Do not profile debug utilities
11CFLAGS_REMOVE_tsc.o = -pg
12CFLAGS_REMOVE_rtc.o = -pg
13endif
14
9# 15#
10# vsyscalls (which work on the user stack) should have 16# vsyscalls (which work on the user stack) should have
11# no stack-protector checks: 17# no stack-protector checks:
@@ -13,20 +19,21 @@ CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE)
13nostackp := $(call cc-option, -fno-stack-protector) 19nostackp := $(call cc-option, -fno-stack-protector)
14CFLAGS_vsyscall_64.o := $(PROFILING) -g0 $(nostackp) 20CFLAGS_vsyscall_64.o := $(PROFILING) -g0 $(nostackp)
15CFLAGS_hpet.o := $(nostackp) 21CFLAGS_hpet.o := $(nostackp)
16CFLAGS_tsc_64.o := $(nostackp) 22CFLAGS_tsc.o := $(nostackp)
17 23
18obj-y := process_$(BITS).o signal_$(BITS).o entry_$(BITS).o 24obj-y := process_$(BITS).o signal_$(BITS).o entry_$(BITS).o
19obj-y += traps_$(BITS).o irq_$(BITS).o 25obj-y += traps_$(BITS).o irq_$(BITS).o
20obj-y += time_$(BITS).o ioport.o ldt.o 26obj-y += time_$(BITS).o ioport.o ldt.o
21obj-y += setup_$(BITS).o i8259_$(BITS).o setup.o 27obj-y += setup.o i8259.o irqinit_$(BITS).o setup_percpu.o
28obj-$(CONFIG_X86_VISWS) += visws_quirks.o
29obj-$(CONFIG_X86_32) += probe_roms_32.o
22obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o 30obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
23obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o 31obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o
24obj-$(CONFIG_X86_64) += syscall_64.o vsyscall_64.o setup64.o 32obj-$(CONFIG_X86_64) += syscall_64.o vsyscall_64.o
25obj-y += bootflag.o e820_$(BITS).o 33obj-y += bootflag.o e820.o
26obj-y += pci-dma.o quirks.o i8237.o topology.o kdebugfs.o 34obj-y += pci-dma.o quirks.o i8237.o topology.o kdebugfs.o
27obj-y += alternative.o i8253.o pci-nommu.o 35obj-y += alternative.o i8253.o pci-nommu.o
28obj-$(CONFIG_X86_64) += bugs_64.o 36obj-y += tsc.o io_delay.o rtc.o
29obj-y += tsc_$(BITS).o io_delay.o rtc.o
30 37
31obj-$(CONFIG_X86_TRAMPOLINE) += trampoline.o 38obj-$(CONFIG_X86_TRAMPOLINE) += trampoline.o
32obj-y += process.o 39obj-y += process.o
@@ -53,9 +60,10 @@ obj-$(CONFIG_X86_32_SMP) += smpcommon.o
53obj-$(CONFIG_X86_64_SMP) += tsc_sync.o smpcommon.o 60obj-$(CONFIG_X86_64_SMP) += tsc_sync.o smpcommon.o
54obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o 61obj-$(CONFIG_X86_TRAMPOLINE) += trampoline_$(BITS).o
55obj-$(CONFIG_X86_MPPARSE) += mpparse.o 62obj-$(CONFIG_X86_MPPARSE) += mpparse.o
56obj-$(CONFIG_X86_LOCAL_APIC) += apic_$(BITS).o nmi_$(BITS).o 63obj-$(CONFIG_X86_LOCAL_APIC) += apic_$(BITS).o nmi.o
57obj-$(CONFIG_X86_IO_APIC) += io_apic_$(BITS).o 64obj-$(CONFIG_X86_IO_APIC) += io_apic_$(BITS).o
58obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o 65obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o
66obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o
59obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o 67obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o
60obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o 68obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o
61obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o 69obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o
@@ -64,7 +72,6 @@ obj-$(CONFIG_X86_SUMMIT_NUMA) += summit_32.o
64obj-y += vsmp_64.o 72obj-y += vsmp_64.o
65obj-$(CONFIG_KPROBES) += kprobes.o 73obj-$(CONFIG_KPROBES) += kprobes.o
66obj-$(CONFIG_MODULES) += module_$(BITS).o 74obj-$(CONFIG_MODULES) += module_$(BITS).o
67obj-$(CONFIG_ACPI_SRAT) += srat_32.o
68obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o 75obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o
69obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o 76obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o
70obj-$(CONFIG_KGDB) += kgdb.o 77obj-$(CONFIG_KGDB) += kgdb.o
@@ -94,12 +101,13 @@ obj-$(CONFIG_OLPC) += olpc.o
94### 101###
95# 64 bit specific files 102# 64 bit specific files
96ifeq ($(CONFIG_X86_64),y) 103ifeq ($(CONFIG_X86_64),y)
97 obj-y += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o 104 obj-y += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o
98 obj-$(CONFIG_X86_PM_TIMER) += pmtimer_64.o 105 obj-$(CONFIG_X86_PM_TIMER) += pmtimer_64.o
99 obj-$(CONFIG_AUDIT) += audit_64.o 106 obj-$(CONFIG_AUDIT) += audit_64.o
100 107
101 obj-$(CONFIG_GART_IOMMU) += pci-gart_64.o aperture_64.o 108 obj-$(CONFIG_GART_IOMMU) += pci-gart_64.o aperture_64.o
102 obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o 109 obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o
110 obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o
103 obj-$(CONFIG_SWIOTLB) += pci-swiotlb_64.o 111 obj-$(CONFIG_SWIOTLB) += pci-swiotlb_64.o
104 112
105 obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o 113 obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 33c5216fd3e1..f489d7a9be92 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -37,6 +37,7 @@
37#include <asm/pgtable.h> 37#include <asm/pgtable.h>
38#include <asm/io_apic.h> 38#include <asm/io_apic.h>
39#include <asm/apic.h> 39#include <asm/apic.h>
40#include <asm/genapic.h>
40#include <asm/io.h> 41#include <asm/io.h>
41#include <asm/mpspec.h> 42#include <asm/mpspec.h>
42#include <asm/smp.h> 43#include <asm/smp.h>
@@ -106,21 +107,6 @@ static u64 acpi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE;
106 */ 107 */
107enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_PIC; 108enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_PIC;
108 109
109#ifdef CONFIG_X86_64
110
111/* rely on all ACPI tables being in the direct mapping */
112char *__init __acpi_map_table(unsigned long phys_addr, unsigned long size)
113{
114 if (!phys_addr || !size)
115 return NULL;
116
117 if (phys_addr+size <= (max_pfn_mapped << PAGE_SHIFT) + PAGE_SIZE)
118 return __va(phys_addr);
119
120 return NULL;
121}
122
123#else
124 110
125/* 111/*
126 * Temporarily use the virtual area starting from FIX_IO_APIC_BASE_END, 112 * Temporarily use the virtual area starting from FIX_IO_APIC_BASE_END,
@@ -139,11 +125,15 @@ char *__init __acpi_map_table(unsigned long phys, unsigned long size)
139 unsigned long base, offset, mapped_size; 125 unsigned long base, offset, mapped_size;
140 int idx; 126 int idx;
141 127
142 if (phys + size < 8 * 1024 * 1024) 128 if (!phys || !size)
129 return NULL;
130
131 if (phys+size <= (max_low_pfn_mapped << PAGE_SHIFT))
143 return __va(phys); 132 return __va(phys);
144 133
145 offset = phys & (PAGE_SIZE - 1); 134 offset = phys & (PAGE_SIZE - 1);
146 mapped_size = PAGE_SIZE - offset; 135 mapped_size = PAGE_SIZE - offset;
136 clear_fixmap(FIX_ACPI_END);
147 set_fixmap(FIX_ACPI_END, phys); 137 set_fixmap(FIX_ACPI_END, phys);
148 base = fix_to_virt(FIX_ACPI_END); 138 base = fix_to_virt(FIX_ACPI_END);
149 139
@@ -155,13 +145,13 @@ char *__init __acpi_map_table(unsigned long phys, unsigned long size)
155 if (--idx < FIX_ACPI_BEGIN) 145 if (--idx < FIX_ACPI_BEGIN)
156 return NULL; /* cannot handle this */ 146 return NULL; /* cannot handle this */
157 phys += PAGE_SIZE; 147 phys += PAGE_SIZE;
148 clear_fixmap(idx);
158 set_fixmap(idx, phys); 149 set_fixmap(idx, phys);
159 mapped_size += PAGE_SIZE; 150 mapped_size += PAGE_SIZE;
160 } 151 }
161 152
162 return ((unsigned char *)base + offset); 153 return ((unsigned char *)base + offset);
163} 154}
164#endif
165 155
166#ifdef CONFIG_PCI_MMCONFIG 156#ifdef CONFIG_PCI_MMCONFIG
167/* The physical address of the MMCONFIG aperture. Set from ACPI tables. */ 157/* The physical address of the MMCONFIG aperture. Set from ACPI tables. */
@@ -338,8 +328,6 @@ acpi_parse_lapic_nmi(struct acpi_subtable_header * header, const unsigned long e
338 328
339#ifdef CONFIG_X86_IO_APIC 329#ifdef CONFIG_X86_IO_APIC
340 330
341struct mp_ioapic_routing mp_ioapic_routing[MAX_IO_APICS];
342
343static int __init 331static int __init
344acpi_parse_ioapic(struct acpi_subtable_header * header, const unsigned long end) 332acpi_parse_ioapic(struct acpi_subtable_header * header, const unsigned long end)
345{ 333{
@@ -514,8 +502,6 @@ int acpi_register_gsi(u32 gsi, int triggering, int polarity)
514 * Make sure all (legacy) PCI IRQs are set as level-triggered. 502 * Make sure all (legacy) PCI IRQs are set as level-triggered.
515 */ 503 */
516 if (acpi_irq_model == ACPI_IRQ_MODEL_PIC) { 504 if (acpi_irq_model == ACPI_IRQ_MODEL_PIC) {
517 extern void eisa_set_level_irq(unsigned int irq);
518
519 if (triggering == ACPI_LEVEL_SENSITIVE) 505 if (triggering == ACPI_LEVEL_SENSITIVE)
520 eisa_set_level_irq(gsi); 506 eisa_set_level_irq(gsi);
521 } 507 }
@@ -860,6 +846,364 @@ static int __init acpi_parse_madt_lapic_entries(void)
860#endif /* CONFIG_X86_LOCAL_APIC */ 846#endif /* CONFIG_X86_LOCAL_APIC */
861 847
862#ifdef CONFIG_X86_IO_APIC 848#ifdef CONFIG_X86_IO_APIC
849#define MP_ISA_BUS 0
850
851#ifdef CONFIG_X86_ES7000
852extern int es7000_plat;
853#endif
854
855static struct {
856 int apic_id;
857 int gsi_base;
858 int gsi_end;
859 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
860} mp_ioapic_routing[MAX_IO_APICS];
861
862static int mp_find_ioapic(int gsi)
863{
864 int i = 0;
865
866 /* Find the IOAPIC that manages this GSI. */
867 for (i = 0; i < nr_ioapics; i++) {
868 if ((gsi >= mp_ioapic_routing[i].gsi_base)
869 && (gsi <= mp_ioapic_routing[i].gsi_end))
870 return i;
871 }
872
873 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
874 return -1;
875}
876
877static u8 __init uniq_ioapic_id(u8 id)
878{
879#ifdef CONFIG_X86_32
880 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
881 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
882 return io_apic_get_unique_id(nr_ioapics, id);
883 else
884 return id;
885#else
886 int i;
887 DECLARE_BITMAP(used, 256);
888 bitmap_zero(used, 256);
889 for (i = 0; i < nr_ioapics; i++) {
890 struct mp_config_ioapic *ia = &mp_ioapics[i];
891 __set_bit(ia->mp_apicid, used);
892 }
893 if (!test_bit(id, used))
894 return id;
895 return find_first_zero_bit(used, 256);
896#endif
897}
898
899static int bad_ioapic(unsigned long address)
900{
901 if (nr_ioapics >= MAX_IO_APICS) {
902 printk(KERN_ERR "ERROR: Max # of I/O APICs (%d) exceeded "
903 "(found %d)\n", MAX_IO_APICS, nr_ioapics);
904 panic("Recompile kernel with bigger MAX_IO_APICS!\n");
905 }
906 if (!address) {
907 printk(KERN_ERR "WARNING: Bogus (zero) I/O APIC address"
908 " found in table, skipping!\n");
909 return 1;
910 }
911 return 0;
912}
913
914void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
915{
916 int idx = 0;
917
918 if (bad_ioapic(address))
919 return;
920
921 idx = nr_ioapics;
922
923 mp_ioapics[idx].mp_type = MP_IOAPIC;
924 mp_ioapics[idx].mp_flags = MPC_APIC_USABLE;
925 mp_ioapics[idx].mp_apicaddr = address;
926
927 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
928 mp_ioapics[idx].mp_apicid = uniq_ioapic_id(id);
929#ifdef CONFIG_X86_32
930 mp_ioapics[idx].mp_apicver = io_apic_get_version(idx);
931#else
932 mp_ioapics[idx].mp_apicver = 0;
933#endif
934 /*
935 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
936 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
937 */
938 mp_ioapic_routing[idx].apic_id = mp_ioapics[idx].mp_apicid;
939 mp_ioapic_routing[idx].gsi_base = gsi_base;
940 mp_ioapic_routing[idx].gsi_end = gsi_base +
941 io_apic_get_redir_entries(idx);
942
943 printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%lx, "
944 "GSI %d-%d\n", idx, mp_ioapics[idx].mp_apicid,
945 mp_ioapics[idx].mp_apicver, mp_ioapics[idx].mp_apicaddr,
946 mp_ioapic_routing[idx].gsi_base, mp_ioapic_routing[idx].gsi_end);
947
948 nr_ioapics++;
949}
950
951static void assign_to_mp_irq(struct mp_config_intsrc *m,
952 struct mp_config_intsrc *mp_irq)
953{
954 memcpy(mp_irq, m, sizeof(struct mp_config_intsrc));
955}
956
957static int mp_irq_cmp(struct mp_config_intsrc *mp_irq,
958 struct mp_config_intsrc *m)
959{
960 return memcmp(mp_irq, m, sizeof(struct mp_config_intsrc));
961}
962
963static void save_mp_irq(struct mp_config_intsrc *m)
964{
965 int i;
966
967 for (i = 0; i < mp_irq_entries; i++) {
968 if (!mp_irq_cmp(&mp_irqs[i], m))
969 return;
970 }
971
972 assign_to_mp_irq(m, &mp_irqs[mp_irq_entries]);
973 if (++mp_irq_entries == MAX_IRQ_SOURCES)
974 panic("Max # of irq sources exceeded!!\n");
975}
976
977void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
978{
979 int ioapic;
980 int pin;
981 struct mp_config_intsrc mp_irq;
982
983 /*
984 * Convert 'gsi' to 'ioapic.pin'.
985 */
986 ioapic = mp_find_ioapic(gsi);
987 if (ioapic < 0)
988 return;
989 pin = gsi - mp_ioapic_routing[ioapic].gsi_base;
990
991 /*
992 * TBD: This check is for faulty timer entries, where the override
993 * erroneously sets the trigger to level, resulting in a HUGE
994 * increase of timer interrupts!
995 */
996 if ((bus_irq == 0) && (trigger == 3))
997 trigger = 1;
998
999 mp_irq.mp_type = MP_INTSRC;
1000 mp_irq.mp_irqtype = mp_INT;
1001 mp_irq.mp_irqflag = (trigger << 2) | polarity;
1002 mp_irq.mp_srcbus = MP_ISA_BUS;
1003 mp_irq.mp_srcbusirq = bus_irq; /* IRQ */
1004 mp_irq.mp_dstapic = mp_ioapics[ioapic].mp_apicid; /* APIC ID */
1005 mp_irq.mp_dstirq = pin; /* INTIN# */
1006
1007 save_mp_irq(&mp_irq);
1008}
1009
1010void __init mp_config_acpi_legacy_irqs(void)
1011{
1012 int i;
1013 int ioapic;
1014 unsigned int dstapic;
1015 struct mp_config_intsrc mp_irq;
1016
1017#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
1018 /*
1019 * Fabricate the legacy ISA bus (bus #31).
1020 */
1021 mp_bus_id_to_type[MP_ISA_BUS] = MP_BUS_ISA;
1022#endif
1023 set_bit(MP_ISA_BUS, mp_bus_not_pci);
1024 Dprintk("Bus #%d is ISA\n", MP_ISA_BUS);
1025
1026#ifdef CONFIG_X86_ES7000
1027 /*
1028 * Older generations of ES7000 have no legacy identity mappings
1029 */
1030 if (es7000_plat == 1)
1031 return;
1032#endif
1033
1034 /*
1035 * Locate the IOAPIC that manages the ISA IRQs (0-15).
1036 */
1037 ioapic = mp_find_ioapic(0);
1038 if (ioapic < 0)
1039 return;
1040 dstapic = mp_ioapics[ioapic].mp_apicid;
1041
1042 /*
1043 * Use the default configuration for the IRQs 0-15. Unless
1044 * overridden by (MADT) interrupt source override entries.
1045 */
1046 for (i = 0; i < 16; i++) {
1047 int idx;
1048
1049 for (idx = 0; idx < mp_irq_entries; idx++) {
1050 struct mp_config_intsrc *irq = mp_irqs + idx;
1051
1052 /* Do we already have a mapping for this ISA IRQ? */
1053 if (irq->mp_srcbus == MP_ISA_BUS
1054 && irq->mp_srcbusirq == i)
1055 break;
1056
1057 /* Do we already have a mapping for this IOAPIC pin */
1058 if (irq->mp_dstapic == dstapic &&
1059 irq->mp_dstirq == i)
1060 break;
1061 }
1062
1063 if (idx != mp_irq_entries) {
1064 printk(KERN_DEBUG "ACPI: IRQ%d used by override.\n", i);
1065 continue; /* IRQ already used */
1066 }
1067
1068 mp_irq.mp_type = MP_INTSRC;
1069 mp_irq.mp_irqflag = 0; /* Conforming */
1070 mp_irq.mp_srcbus = MP_ISA_BUS;
1071 mp_irq.mp_dstapic = dstapic;
1072 mp_irq.mp_irqtype = mp_INT;
1073 mp_irq.mp_srcbusirq = i; /* Identity mapped */
1074 mp_irq.mp_dstirq = i;
1075
1076 save_mp_irq(&mp_irq);
1077 }
1078}
1079
1080int mp_register_gsi(u32 gsi, int triggering, int polarity)
1081{
1082 int ioapic;
1083 int ioapic_pin;
1084#ifdef CONFIG_X86_32
1085#define MAX_GSI_NUM 4096
1086#define IRQ_COMPRESSION_START 64
1087
1088 static int pci_irq = IRQ_COMPRESSION_START;
1089 /*
1090 * Mapping between Global System Interrupts, which
1091 * represent all possible interrupts, and IRQs
1092 * assigned to actual devices.
1093 */
1094 static int gsi_to_irq[MAX_GSI_NUM];
1095#else
1096
1097 if (acpi_irq_model != ACPI_IRQ_MODEL_IOAPIC)
1098 return gsi;
1099#endif
1100
1101 /* Don't set up the ACPI SCI because it's already set up */
1102 if (acpi_gbl_FADT.sci_interrupt == gsi)
1103 return gsi;
1104
1105 ioapic = mp_find_ioapic(gsi);
1106 if (ioapic < 0) {
1107 printk(KERN_WARNING "No IOAPIC for GSI %u\n", gsi);
1108 return gsi;
1109 }
1110
1111 ioapic_pin = gsi - mp_ioapic_routing[ioapic].gsi_base;
1112
1113#ifdef CONFIG_X86_32
1114 if (ioapic_renumber_irq)
1115 gsi = ioapic_renumber_irq(ioapic, gsi);
1116#endif
1117
1118 /*
1119 * Avoid pin reprogramming. PRTs typically include entries
1120 * with redundant pin->gsi mappings (but unique PCI devices);
1121 * we only program the IOAPIC on the first.
1122 */
1123 if (ioapic_pin > MP_MAX_IOAPIC_PIN) {
1124 printk(KERN_ERR "Invalid reference to IOAPIC pin "
1125 "%d-%d\n", mp_ioapic_routing[ioapic].apic_id,
1126 ioapic_pin);
1127 return gsi;
1128 }
1129 if (test_bit(ioapic_pin, mp_ioapic_routing[ioapic].pin_programmed)) {
1130 Dprintk(KERN_DEBUG "Pin %d-%d already programmed\n",
1131 mp_ioapic_routing[ioapic].apic_id, ioapic_pin);
1132#ifdef CONFIG_X86_32
1133 return (gsi < IRQ_COMPRESSION_START ? gsi : gsi_to_irq[gsi]);
1134#else
1135 return gsi;
1136#endif
1137 }
1138
1139 set_bit(ioapic_pin, mp_ioapic_routing[ioapic].pin_programmed);
1140#ifdef CONFIG_X86_32
1141 /*
1142 * For GSI >= 64, use IRQ compression
1143 */
1144 if ((gsi >= IRQ_COMPRESSION_START)
1145 && (triggering == ACPI_LEVEL_SENSITIVE)) {
1146 /*
1147 * For PCI devices assign IRQs in order, avoiding gaps
1148 * due to unused I/O APIC pins.
1149 */
1150 int irq = gsi;
1151 if (gsi < MAX_GSI_NUM) {
1152 /*
1153 * Retain the VIA chipset work-around (gsi > 15), but
1154 * avoid a problem where the 8254 timer (IRQ0) is setup
1155 * via an override (so it's not on pin 0 of the ioapic),
1156 * and at the same time, the pin 0 interrupt is a PCI
1157 * type. The gsi > 15 test could cause these two pins
1158 * to be shared as IRQ0, and they are not shareable.
1159 * So test for this condition, and if necessary, avoid
1160 * the pin collision.
1161 */
1162 gsi = pci_irq++;
1163 /*
1164 * Don't assign IRQ used by ACPI SCI
1165 */
1166 if (gsi == acpi_gbl_FADT.sci_interrupt)
1167 gsi = pci_irq++;
1168 gsi_to_irq[irq] = gsi;
1169 } else {
1170 printk(KERN_ERR "GSI %u is too high\n", gsi);
1171 return gsi;
1172 }
1173 }
1174#endif
1175 io_apic_set_pci_routing(ioapic, ioapic_pin, gsi,
1176 triggering == ACPI_EDGE_SENSITIVE ? 0 : 1,
1177 polarity == ACPI_ACTIVE_HIGH ? 0 : 1);
1178 return gsi;
1179}
1180
1181int mp_config_acpi_gsi(unsigned char number, unsigned int devfn, u8 pin,
1182 u32 gsi, int triggering, int polarity)
1183{
1184#ifdef CONFIG_X86_MPPARSE
1185 struct mp_config_intsrc mp_irq;
1186 int ioapic;
1187
1188 if (!acpi_ioapic)
1189 return 0;
1190
1191 /* print the entry should happen on mptable identically */
1192 mp_irq.mp_type = MP_INTSRC;
1193 mp_irq.mp_irqtype = mp_INT;
1194 mp_irq.mp_irqflag = (triggering == ACPI_EDGE_SENSITIVE ? 4 : 0x0c) |
1195 (polarity == ACPI_ACTIVE_HIGH ? 1 : 3);
1196 mp_irq.mp_srcbus = number;
1197 mp_irq.mp_srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3);
1198 ioapic = mp_find_ioapic(gsi);
1199 mp_irq.mp_dstapic = mp_ioapic_routing[ioapic].apic_id;
1200 mp_irq.mp_dstirq = gsi - mp_ioapic_routing[ioapic].gsi_base;
1201
1202 save_mp_irq(&mp_irq);
1203#endif
1204 return 0;
1205}
1206
863/* 1207/*
864 * Parse IOAPIC related entries in MADT 1208 * Parse IOAPIC related entries in MADT
865 * returns 0 on success, < 0 on error 1209 * returns 0 on success, < 0 on error
@@ -1009,8 +1353,6 @@ static void __init acpi_process_madt(void)
1009 return; 1353 return;
1010} 1354}
1011 1355
1012#ifdef __i386__
1013
1014static int __init disable_acpi_irq(const struct dmi_system_id *d) 1356static int __init disable_acpi_irq(const struct dmi_system_id *d)
1015{ 1357{
1016 if (!acpi_force) { 1358 if (!acpi_force) {
@@ -1061,6 +1403,16 @@ static int __init force_acpi_ht(const struct dmi_system_id *d)
1061} 1403}
1062 1404
1063/* 1405/*
1406 * Force ignoring BIOS IRQ0 pin2 override
1407 */
1408static int __init dmi_ignore_irq0_timer_override(const struct dmi_system_id *d)
1409{
1410 pr_notice("%s detected: Ignoring BIOS IRQ0 pin2 override\n", d->ident);
1411 acpi_skip_timer_override = 1;
1412 return 0;
1413}
1414
1415/*
1064 * If your system is blacklisted here, but you find that acpi=force 1416 * If your system is blacklisted here, but you find that acpi=force
1065 * works for you, please contact acpi-devel@sourceforge.net 1417 * works for you, please contact acpi-devel@sourceforge.net
1066 */ 1418 */
@@ -1227,11 +1579,35 @@ static struct dmi_system_id __initdata acpi_dmi_table[] = {
1227 DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 360"), 1579 DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 360"),
1228 }, 1580 },
1229 }, 1581 },
1582 /*
1583 * HP laptops which use a DSDT reporting as HP/SB400/10000,
1584 * which includes some code which overrides all temperature
1585 * trip points to 16C if the INTIN2 input of the I/O APIC
1586 * is enabled. This input is incorrectly designated the
1587 * ISA IRQ 0 via an interrupt source override even though
1588 * it is wired to the output of the master 8259A and INTIN0
1589 * is not connected at all. Force ignoring BIOS IRQ0 pin2
1590 * override in that cases.
1591 */
1592 {
1593 .callback = dmi_ignore_irq0_timer_override,
1594 .ident = "HP NX6125 laptop",
1595 .matches = {
1596 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1597 DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq nx6125"),
1598 },
1599 },
1600 {
1601 .callback = dmi_ignore_irq0_timer_override,
1602 .ident = "HP NX6325 laptop",
1603 .matches = {
1604 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1605 DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq nx6325"),
1606 },
1607 },
1230 {} 1608 {}
1231}; 1609};
1232 1610
1233#endif /* __i386__ */
1234
1235/* 1611/*
1236 * acpi_boot_table_init() and acpi_boot_init() 1612 * acpi_boot_table_init() and acpi_boot_init()
1237 * called from setup_arch(), always. 1613 * called from setup_arch(), always.
@@ -1259,9 +1635,7 @@ int __init acpi_boot_table_init(void)
1259{ 1635{
1260 int error; 1636 int error;
1261 1637
1262#ifdef __i386__
1263 dmi_check_system(acpi_dmi_table); 1638 dmi_check_system(acpi_dmi_table);
1264#endif
1265 1639
1266 /* 1640 /*
1267 * If acpi_disabled, bail out 1641 * If acpi_disabled, bail out
@@ -1386,6 +1760,20 @@ static int __init parse_pci(char *arg)
1386} 1760}
1387early_param("pci", parse_pci); 1761early_param("pci", parse_pci);
1388 1762
1763int __init acpi_mps_check(void)
1764{
1765#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_X86_MPPARSE)
1766/* mptable code is not built-in*/
1767 if (acpi_disabled || acpi_noirq) {
1768 printk(KERN_WARNING "MPS support code is not built-in.\n"
1769 "Using acpi=off or acpi=noirq or pci=noacpi "
1770 "may have problem\n");
1771 return 1;
1772 }
1773#endif
1774 return 0;
1775}
1776
1389#ifdef CONFIG_X86_IO_APIC 1777#ifdef CONFIG_X86_IO_APIC
1390static int __init parse_acpi_skip_timer_override(char *arg) 1778static int __init parse_acpi_skip_timer_override(char *arg)
1391{ 1779{
diff --git a/arch/x86/kernel/acpi/processor.c b/arch/x86/kernel/acpi/processor.c
index de2d2e4ebad9..7c074eec39fb 100644
--- a/arch/x86/kernel/acpi/processor.c
+++ b/arch/x86/kernel/acpi/processor.c
@@ -56,6 +56,12 @@ static void init_intel_pdc(struct acpi_processor *pr, struct cpuinfo_x86 *c)
56 if (cpu_has(c, X86_FEATURE_ACPI)) 56 if (cpu_has(c, X86_FEATURE_ACPI))
57 buf[2] |= ACPI_PDC_T_FFH; 57 buf[2] |= ACPI_PDC_T_FFH;
58 58
59 /*
60 * If mwait/monitor is unsupported, C2/C3_FFH will be disabled
61 */
62 if (!cpu_has(c, X86_FEATURE_MWAIT))
63 buf[2] &= ~(ACPI_PDC_C_C2C3_FFH);
64
59 obj->type = ACPI_TYPE_BUFFER; 65 obj->type = ACPI_TYPE_BUFFER;
60 obj->buffer.length = 12; 66 obj->buffer.length = 12;
61 obj->buffer.pointer = (u8 *) buf; 67 obj->buffer.pointer = (u8 *) buf;
diff --git a/arch/x86/kernel/acpi/realmode/wakeup.S b/arch/x86/kernel/acpi/realmode/wakeup.S
index f9b77fb37e5b..3355973b12ac 100644
--- a/arch/x86/kernel/acpi/realmode/wakeup.S
+++ b/arch/x86/kernel/acpi/realmode/wakeup.S
@@ -5,6 +5,7 @@
5#include <asm/msr-index.h> 5#include <asm/msr-index.h>
6#include <asm/page.h> 6#include <asm/page.h>
7#include <asm/pgtable.h> 7#include <asm/pgtable.h>
8#include <asm/processor-flags.h>
8 9
9 .code16 10 .code16
10 .section ".header", "a" 11 .section ".header", "a"
@@ -24,6 +25,11 @@ pmode_gdt: .quad 0
24realmode_flags: .long 0 25realmode_flags: .long 0
25real_magic: .long 0 26real_magic: .long 0
26trampoline_segment: .word 0 27trampoline_segment: .word 0
28_pad1: .byte 0
29wakeup_jmp: .byte 0xea /* ljmpw */
30wakeup_jmp_off: .word 3f
31wakeup_jmp_seg: .word 0
32wakeup_gdt: .quad 0, 0, 0
27signature: .long 0x51ee1111 33signature: .long 0x51ee1111
28 34
29 .text 35 .text
@@ -34,11 +40,34 @@ _start:
34 cli 40 cli
35 cld 41 cld
36 42
43 /* Apparently some dimwit BIOS programmers don't know how to
44 program a PM to RM transition, and we might end up here with
45 junk in the data segment descriptor registers. The only way
46 to repair that is to go into PM and fix it ourselves... */
47 movw $16, %cx
48 lgdtl %cs:wakeup_gdt
49 movl %cr0, %eax
50 orb $X86_CR0_PE, %al
51 movl %eax, %cr0
52 jmp 1f
531: ljmpw $8, $2f
542:
55 movw %cx, %ds
56 movw %cx, %es
57 movw %cx, %ss
58 movw %cx, %fs
59 movw %cx, %gs
60
61 andb $~X86_CR0_PE, %al
62 movl %eax, %cr0
63 jmp wakeup_jmp
643:
37 /* Set up segments */ 65 /* Set up segments */
38 movw %cs, %ax 66 movw %cs, %ax
39 movw %ax, %ds 67 movw %ax, %ds
40 movw %ax, %es 68 movw %ax, %es
41 movw %ax, %ss 69 movw %ax, %ss
70 lidtl wakeup_idt
42 71
43 movl $wakeup_stack_end, %esp 72 movl $wakeup_stack_end, %esp
44 73
@@ -98,7 +127,14 @@ bogus_real_magic:
98 jmp 1b 127 jmp 1b
99 128
100 .data 129 .data
101 .balign 4 130 .balign 8
131
132 /* This is the standard real-mode IDT */
133wakeup_idt:
134 .word 0xffff /* limit */
135 .long 0 /* address */
136 .word 0
137
102 .globl HEAP, heap_end 138 .globl HEAP, heap_end
103HEAP: 139HEAP:
104 .long wakeup_heap 140 .long wakeup_heap
diff --git a/arch/x86/kernel/acpi/realmode/wakeup.h b/arch/x86/kernel/acpi/realmode/wakeup.h
index ef8166fe8020..69d38d0b2b64 100644
--- a/arch/x86/kernel/acpi/realmode/wakeup.h
+++ b/arch/x86/kernel/acpi/realmode/wakeup.h
@@ -24,6 +24,11 @@ struct wakeup_header {
24 u32 realmode_flags; 24 u32 realmode_flags;
25 u32 real_magic; 25 u32 real_magic;
26 u16 trampoline_segment; /* segment with trampoline code, 64-bit only */ 26 u16 trampoline_segment; /* segment with trampoline code, 64-bit only */
27 u8 _pad1;
28 u8 wakeup_jmp;
29 u16 wakeup_jmp_off;
30 u16 wakeup_jmp_seg;
31 u64 wakeup_gdt[3];
27 u32 signature; /* To check we have correct structure */ 32 u32 signature; /* To check we have correct structure */
28} __attribute__((__packed__)); 33} __attribute__((__packed__));
29 34
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index afc25ee9964b..868de3d5c39d 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -23,6 +23,15 @@ static unsigned long acpi_realmode;
23static char temp_stack[10240]; 23static char temp_stack[10240];
24#endif 24#endif
25 25
26/* XXX: this macro should move to asm-x86/segment.h and be shared with the
27 boot code... */
28#define GDT_ENTRY(flags, base, limit) \
29 (((u64)(base & 0xff000000) << 32) | \
30 ((u64)flags << 40) | \
31 ((u64)(limit & 0x00ff0000) << 32) | \
32 ((u64)(base & 0x00ffffff) << 16) | \
33 ((u64)(limit & 0x0000ffff)))
34
26/** 35/**
27 * acpi_save_state_mem - save kernel state 36 * acpi_save_state_mem - save kernel state
28 * 37 *
@@ -50,6 +59,29 @@ int acpi_save_state_mem(void)
50 59
51 header->video_mode = saved_video_mode; 60 header->video_mode = saved_video_mode;
52 61
62 header->wakeup_jmp_seg = acpi_wakeup_address >> 4;
63
64 /*
65 * Set up the wakeup GDT. We set these up as Big Real Mode,
66 * that is, with limits set to 4 GB. At least the Lenovo
67 * Thinkpad X61 is known to need this for the video BIOS
68 * initialization quirk to work; this is likely to also
69 * be the case for other laptops or integrated video devices.
70 */
71
72 /* GDT[0]: GDT self-pointer */
73 header->wakeup_gdt[0] =
74 (u64)(sizeof(header->wakeup_gdt) - 1) +
75 ((u64)(acpi_wakeup_address +
76 ((char *)&header->wakeup_gdt - (char *)acpi_realmode))
77 << 16);
78 /* GDT[1]: big real mode-like code segment */
79 header->wakeup_gdt[1] =
80 GDT_ENTRY(0x809b, acpi_wakeup_address, 0xfffff);
81 /* GDT[2]: big real mode-like data segment */
82 header->wakeup_gdt[2] =
83 GDT_ENTRY(0x8093, acpi_wakeup_address, 0xfffff);
84
53#ifndef CONFIG_64BIT 85#ifndef CONFIG_64BIT
54 store_gdt((struct desc_ptr *)&header->pmode_gdt); 86 store_gdt((struct desc_ptr *)&header->pmode_gdt);
55 87
@@ -72,7 +104,9 @@ int acpi_save_state_mem(void)
72 saved_magic = 0x12345678; 104 saved_magic = 0x12345678;
73#else /* CONFIG_64BIT */ 105#else /* CONFIG_64BIT */
74 header->trampoline_segment = setup_trampoline() >> 4; 106 header->trampoline_segment = setup_trampoline() >> 4;
75 init_rsp = (unsigned long)temp_stack + 4096; 107#ifdef CONFIG_SMP
108 stack_start.sp = temp_stack + 4096;
109#endif
76 initial_code = (unsigned long)wakeup_long64; 110 initial_code = (unsigned long)wakeup_long64;
77 saved_magic = 0x123456789abcdef0; 111 saved_magic = 0x123456789abcdef0;
78#endif /* CONFIG_64BIT */ 112#endif /* CONFIG_64BIT */
@@ -111,7 +145,7 @@ void __init acpi_reserve_bootmem(void)
111 return; 145 return;
112 } 146 }
113 147
114 acpi_wakeup_address = acpi_realmode; 148 acpi_wakeup_address = virt_to_phys((void *)acpi_realmode);
115} 149}
116 150
117 151
@@ -124,6 +158,8 @@ static int __init acpi_sleep_setup(char *str)
124 acpi_realmode_flags |= 2; 158 acpi_realmode_flags |= 2;
125 if (strncmp(str, "s3_beep", 7) == 0) 159 if (strncmp(str, "s3_beep", 7) == 0)
126 acpi_realmode_flags |= 4; 160 acpi_realmode_flags |= 4;
161 if (strncmp(str, "old_ordering", 12) == 0)
162 acpi_old_suspend_ordering();
127 str = strchr(str, ','); 163 str = strchr(str, ',');
128 if (str != NULL) 164 if (str != NULL)
129 str += strspn(str, ", \t"); 165 str += strspn(str, ", \t");
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 65c7857a90dd..2763cb37b553 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1,6 +1,6 @@
1#include <linux/module.h> 1#include <linux/module.h>
2#include <linux/sched.h> 2#include <linux/sched.h>
3#include <linux/spinlock.h> 3#include <linux/mutex.h>
4#include <linux/list.h> 4#include <linux/list.h>
5#include <linux/kprobes.h> 5#include <linux/kprobes.h>
6#include <linux/mm.h> 6#include <linux/mm.h>
@@ -143,7 +143,7 @@ static const unsigned char *const p6_nops[ASM_NOP_MAX+1] = {
143#ifdef CONFIG_X86_64 143#ifdef CONFIG_X86_64
144 144
145extern char __vsyscall_0; 145extern char __vsyscall_0;
146static inline const unsigned char*const * find_nop_table(void) 146const unsigned char *const *find_nop_table(void)
147{ 147{
148 return boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || 148 return boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
149 boot_cpu_data.x86 < 6 ? k8_nops : p6_nops; 149 boot_cpu_data.x86 < 6 ? k8_nops : p6_nops;
@@ -162,7 +162,7 @@ static const struct nop {
162 { -1, NULL } 162 { -1, NULL }
163}; 163};
164 164
165static const unsigned char*const * find_nop_table(void) 165const unsigned char *const *find_nop_table(void)
166{ 166{
167 const unsigned char *const *noptable = intel_nops; 167 const unsigned char *const *noptable = intel_nops;
168 int i; 168 int i;
@@ -279,7 +279,7 @@ struct smp_alt_module {
279 struct list_head next; 279 struct list_head next;
280}; 280};
281static LIST_HEAD(smp_alt_modules); 281static LIST_HEAD(smp_alt_modules);
282static DEFINE_SPINLOCK(smp_alt); 282static DEFINE_MUTEX(smp_alt);
283static int smp_mode = 1; /* protected by smp_alt */ 283static int smp_mode = 1; /* protected by smp_alt */
284 284
285void alternatives_smp_module_add(struct module *mod, char *name, 285void alternatives_smp_module_add(struct module *mod, char *name,
@@ -312,12 +312,12 @@ void alternatives_smp_module_add(struct module *mod, char *name,
312 __func__, smp->locks, smp->locks_end, 312 __func__, smp->locks, smp->locks_end,
313 smp->text, smp->text_end, smp->name); 313 smp->text, smp->text_end, smp->name);
314 314
315 spin_lock(&smp_alt); 315 mutex_lock(&smp_alt);
316 list_add_tail(&smp->next, &smp_alt_modules); 316 list_add_tail(&smp->next, &smp_alt_modules);
317 if (boot_cpu_has(X86_FEATURE_UP)) 317 if (boot_cpu_has(X86_FEATURE_UP))
318 alternatives_smp_unlock(smp->locks, smp->locks_end, 318 alternatives_smp_unlock(smp->locks, smp->locks_end,
319 smp->text, smp->text_end); 319 smp->text, smp->text_end);
320 spin_unlock(&smp_alt); 320 mutex_unlock(&smp_alt);
321} 321}
322 322
323void alternatives_smp_module_del(struct module *mod) 323void alternatives_smp_module_del(struct module *mod)
@@ -327,17 +327,17 @@ void alternatives_smp_module_del(struct module *mod)
327 if (smp_alt_once || noreplace_smp) 327 if (smp_alt_once || noreplace_smp)
328 return; 328 return;
329 329
330 spin_lock(&smp_alt); 330 mutex_lock(&smp_alt);
331 list_for_each_entry(item, &smp_alt_modules, next) { 331 list_for_each_entry(item, &smp_alt_modules, next) {
332 if (mod != item->mod) 332 if (mod != item->mod)
333 continue; 333 continue;
334 list_del(&item->next); 334 list_del(&item->next);
335 spin_unlock(&smp_alt); 335 mutex_unlock(&smp_alt);
336 DPRINTK("%s: %s\n", __func__, item->name); 336 DPRINTK("%s: %s\n", __func__, item->name);
337 kfree(item); 337 kfree(item);
338 return; 338 return;
339 } 339 }
340 spin_unlock(&smp_alt); 340 mutex_unlock(&smp_alt);
341} 341}
342 342
343void alternatives_smp_switch(int smp) 343void alternatives_smp_switch(int smp)
@@ -359,7 +359,7 @@ void alternatives_smp_switch(int smp)
359 return; 359 return;
360 BUG_ON(!smp && (num_online_cpus() > 1)); 360 BUG_ON(!smp && (num_online_cpus() > 1));
361 361
362 spin_lock(&smp_alt); 362 mutex_lock(&smp_alt);
363 363
364 /* 364 /*
365 * Avoid unnecessary switches because it forces JIT based VMs to 365 * Avoid unnecessary switches because it forces JIT based VMs to
@@ -383,7 +383,7 @@ void alternatives_smp_switch(int smp)
383 mod->text, mod->text_end); 383 mod->text, mod->text_end);
384 } 384 }
385 smp_mode = smp; 385 smp_mode = smp;
386 spin_unlock(&smp_alt); 386 mutex_unlock(&smp_alt);
387} 387}
388 388
389#endif 389#endif
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
new file mode 100644
index 000000000000..f2766d84c7a0
--- /dev/null
+++ b/arch/x86/kernel/amd_iommu.c
@@ -0,0 +1,962 @@
1/*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/pci.h>
21#include <linux/gfp.h>
22#include <linux/bitops.h>
23#include <linux/scatterlist.h>
24#include <linux/iommu-helper.h>
25#include <asm/proto.h>
26#include <asm/gart.h>
27#include <asm/amd_iommu_types.h>
28#include <asm/amd_iommu.h>
29
30#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
31
32#define to_pages(addr, size) \
33 (round_up(((addr) & ~PAGE_MASK) + (size), PAGE_SIZE) >> PAGE_SHIFT)
34
35static DEFINE_RWLOCK(amd_iommu_devtable_lock);
36
37struct command {
38 u32 data[4];
39};
40
41static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
42 struct unity_map_entry *e);
43
44static int iommu_has_npcache(struct amd_iommu *iommu)
45{
46 return iommu->cap & IOMMU_CAP_NPCACHE;
47}
48
49static int __iommu_queue_command(struct amd_iommu *iommu, struct command *cmd)
50{
51 u32 tail, head;
52 u8 *target;
53
54 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
55 target = (iommu->cmd_buf + tail);
56 memcpy_toio(target, cmd, sizeof(*cmd));
57 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
58 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
59 if (tail == head)
60 return -ENOMEM;
61 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
62
63 return 0;
64}
65
66static int iommu_queue_command(struct amd_iommu *iommu, struct command *cmd)
67{
68 unsigned long flags;
69 int ret;
70
71 spin_lock_irqsave(&iommu->lock, flags);
72 ret = __iommu_queue_command(iommu, cmd);
73 spin_unlock_irqrestore(&iommu->lock, flags);
74
75 return ret;
76}
77
78static int iommu_completion_wait(struct amd_iommu *iommu)
79{
80 int ret;
81 struct command cmd;
82 volatile u64 ready = 0;
83 unsigned long ready_phys = virt_to_phys(&ready);
84
85 memset(&cmd, 0, sizeof(cmd));
86 cmd.data[0] = LOW_U32(ready_phys) | CMD_COMPL_WAIT_STORE_MASK;
87 cmd.data[1] = HIGH_U32(ready_phys);
88 cmd.data[2] = 1; /* value written to 'ready' */
89 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
90
91 iommu->need_sync = 0;
92
93 ret = iommu_queue_command(iommu, &cmd);
94
95 if (ret)
96 return ret;
97
98 while (!ready)
99 cpu_relax();
100
101 return 0;
102}
103
104static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
105{
106 struct command cmd;
107
108 BUG_ON(iommu == NULL);
109
110 memset(&cmd, 0, sizeof(cmd));
111 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
112 cmd.data[0] = devid;
113
114 iommu->need_sync = 1;
115
116 return iommu_queue_command(iommu, &cmd);
117}
118
119static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
120 u64 address, u16 domid, int pde, int s)
121{
122 struct command cmd;
123
124 memset(&cmd, 0, sizeof(cmd));
125 address &= PAGE_MASK;
126 CMD_SET_TYPE(&cmd, CMD_INV_IOMMU_PAGES);
127 cmd.data[1] |= domid;
128 cmd.data[2] = LOW_U32(address);
129 cmd.data[3] = HIGH_U32(address);
130 if (s)
131 cmd.data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
132 if (pde)
133 cmd.data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
134
135 iommu->need_sync = 1;
136
137 return iommu_queue_command(iommu, &cmd);
138}
139
140static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
141 u64 address, size_t size)
142{
143 int s = 0;
144 unsigned pages = to_pages(address, size);
145
146 address &= PAGE_MASK;
147
148 if (pages > 1) {
149 /*
150 * If we have to flush more than one page, flush all
151 * TLB entries for this domain
152 */
153 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
154 s = 1;
155 }
156
157 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, s);
158
159 return 0;
160}
161
162static int iommu_map(struct protection_domain *dom,
163 unsigned long bus_addr,
164 unsigned long phys_addr,
165 int prot)
166{
167 u64 __pte, *pte, *page;
168
169 bus_addr = PAGE_ALIGN(bus_addr);
170 phys_addr = PAGE_ALIGN(bus_addr);
171
172 /* only support 512GB address spaces for now */
173 if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
174 return -EINVAL;
175
176 pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
177
178 if (!IOMMU_PTE_PRESENT(*pte)) {
179 page = (u64 *)get_zeroed_page(GFP_KERNEL);
180 if (!page)
181 return -ENOMEM;
182 *pte = IOMMU_L2_PDE(virt_to_phys(page));
183 }
184
185 pte = IOMMU_PTE_PAGE(*pte);
186 pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
187
188 if (!IOMMU_PTE_PRESENT(*pte)) {
189 page = (u64 *)get_zeroed_page(GFP_KERNEL);
190 if (!page)
191 return -ENOMEM;
192 *pte = IOMMU_L1_PDE(virt_to_phys(page));
193 }
194
195 pte = IOMMU_PTE_PAGE(*pte);
196 pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];
197
198 if (IOMMU_PTE_PRESENT(*pte))
199 return -EBUSY;
200
201 __pte = phys_addr | IOMMU_PTE_P;
202 if (prot & IOMMU_PROT_IR)
203 __pte |= IOMMU_PTE_IR;
204 if (prot & IOMMU_PROT_IW)
205 __pte |= IOMMU_PTE_IW;
206
207 *pte = __pte;
208
209 return 0;
210}
211
212static int iommu_for_unity_map(struct amd_iommu *iommu,
213 struct unity_map_entry *entry)
214{
215 u16 bdf, i;
216
217 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
218 bdf = amd_iommu_alias_table[i];
219 if (amd_iommu_rlookup_table[bdf] == iommu)
220 return 1;
221 }
222
223 return 0;
224}
225
226static int iommu_init_unity_mappings(struct amd_iommu *iommu)
227{
228 struct unity_map_entry *entry;
229 int ret;
230
231 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
232 if (!iommu_for_unity_map(iommu, entry))
233 continue;
234 ret = dma_ops_unity_map(iommu->default_dom, entry);
235 if (ret)
236 return ret;
237 }
238
239 return 0;
240}
241
242static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
243 struct unity_map_entry *e)
244{
245 u64 addr;
246 int ret;
247
248 for (addr = e->address_start; addr < e->address_end;
249 addr += PAGE_SIZE) {
250 ret = iommu_map(&dma_dom->domain, addr, addr, e->prot);
251 if (ret)
252 return ret;
253 /*
254 * if unity mapping is in aperture range mark the page
255 * as allocated in the aperture
256 */
257 if (addr < dma_dom->aperture_size)
258 __set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);
259 }
260
261 return 0;
262}
263
264static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
265 u16 devid)
266{
267 struct unity_map_entry *e;
268 int ret;
269
270 list_for_each_entry(e, &amd_iommu_unity_map, list) {
271 if (!(devid >= e->devid_start && devid <= e->devid_end))
272 continue;
273 ret = dma_ops_unity_map(dma_dom, e);
274 if (ret)
275 return ret;
276 }
277
278 return 0;
279}
280
281static unsigned long dma_mask_to_pages(unsigned long mask)
282{
283 return (mask >> PAGE_SHIFT) +
284 (PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT);
285}
286
287static unsigned long dma_ops_alloc_addresses(struct device *dev,
288 struct dma_ops_domain *dom,
289 unsigned int pages)
290{
291 unsigned long limit = dma_mask_to_pages(*dev->dma_mask);
292 unsigned long address;
293 unsigned long size = dom->aperture_size >> PAGE_SHIFT;
294 unsigned long boundary_size;
295
296 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
297 PAGE_SIZE) >> PAGE_SHIFT;
298 limit = limit < size ? limit : size;
299
300 if (dom->next_bit >= limit)
301 dom->next_bit = 0;
302
303 address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
304 0 , boundary_size, 0);
305 if (address == -1)
306 address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
307 0, boundary_size, 0);
308
309 if (likely(address != -1)) {
310 dom->next_bit = address + pages;
311 address <<= PAGE_SHIFT;
312 } else
313 address = bad_dma_address;
314
315 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
316
317 return address;
318}
319
320static void dma_ops_free_addresses(struct dma_ops_domain *dom,
321 unsigned long address,
322 unsigned int pages)
323{
324 address >>= PAGE_SHIFT;
325 iommu_area_free(dom->bitmap, address, pages);
326}
327
328static u16 domain_id_alloc(void)
329{
330 unsigned long flags;
331 int id;
332
333 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
334 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
335 BUG_ON(id == 0);
336 if (id > 0 && id < MAX_DOMAIN_ID)
337 __set_bit(id, amd_iommu_pd_alloc_bitmap);
338 else
339 id = 0;
340 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
341
342 return id;
343}
344
345static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
346 unsigned long start_page,
347 unsigned int pages)
348{
349 unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
350
351 if (start_page + pages > last_page)
352 pages = last_page - start_page;
353
354 set_bit_string(dom->bitmap, start_page, pages);
355}
356
357static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
358{
359 int i, j;
360 u64 *p1, *p2, *p3;
361
362 p1 = dma_dom->domain.pt_root;
363
364 if (!p1)
365 return;
366
367 for (i = 0; i < 512; ++i) {
368 if (!IOMMU_PTE_PRESENT(p1[i]))
369 continue;
370
371 p2 = IOMMU_PTE_PAGE(p1[i]);
372 for (j = 0; j < 512; ++i) {
373 if (!IOMMU_PTE_PRESENT(p2[j]))
374 continue;
375 p3 = IOMMU_PTE_PAGE(p2[j]);
376 free_page((unsigned long)p3);
377 }
378
379 free_page((unsigned long)p2);
380 }
381
382 free_page((unsigned long)p1);
383}
384
385static void dma_ops_domain_free(struct dma_ops_domain *dom)
386{
387 if (!dom)
388 return;
389
390 dma_ops_free_pagetable(dom);
391
392 kfree(dom->pte_pages);
393
394 kfree(dom->bitmap);
395
396 kfree(dom);
397}
398
399static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
400 unsigned order)
401{
402 struct dma_ops_domain *dma_dom;
403 unsigned i, num_pte_pages;
404 u64 *l2_pde;
405 u64 address;
406
407 /*
408 * Currently the DMA aperture must be between 32 MB and 1GB in size
409 */
410 if ((order < 25) || (order > 30))
411 return NULL;
412
413 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
414 if (!dma_dom)
415 return NULL;
416
417 spin_lock_init(&dma_dom->domain.lock);
418
419 dma_dom->domain.id = domain_id_alloc();
420 if (dma_dom->domain.id == 0)
421 goto free_dma_dom;
422 dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
423 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
424 dma_dom->domain.priv = dma_dom;
425 if (!dma_dom->domain.pt_root)
426 goto free_dma_dom;
427 dma_dom->aperture_size = (1ULL << order);
428 dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),
429 GFP_KERNEL);
430 if (!dma_dom->bitmap)
431 goto free_dma_dom;
432 /*
433 * mark the first page as allocated so we never return 0 as
434 * a valid dma-address. So we can use 0 as error value
435 */
436 dma_dom->bitmap[0] = 1;
437 dma_dom->next_bit = 0;
438
439 if (iommu->exclusion_start &&
440 iommu->exclusion_start < dma_dom->aperture_size) {
441 unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
442 int pages = to_pages(iommu->exclusion_start,
443 iommu->exclusion_length);
444 dma_ops_reserve_addresses(dma_dom, startpage, pages);
445 }
446
447 num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
448 dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),
449 GFP_KERNEL);
450 if (!dma_dom->pte_pages)
451 goto free_dma_dom;
452
453 l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
454 if (l2_pde == NULL)
455 goto free_dma_dom;
456
457 dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
458
459 for (i = 0; i < num_pte_pages; ++i) {
460 dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);
461 if (!dma_dom->pte_pages[i])
462 goto free_dma_dom;
463 address = virt_to_phys(dma_dom->pte_pages[i]);
464 l2_pde[i] = IOMMU_L1_PDE(address);
465 }
466
467 return dma_dom;
468
469free_dma_dom:
470 dma_ops_domain_free(dma_dom);
471
472 return NULL;
473}
474
475static struct protection_domain *domain_for_device(u16 devid)
476{
477 struct protection_domain *dom;
478 unsigned long flags;
479
480 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
481 dom = amd_iommu_pd_table[devid];
482 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
483
484 return dom;
485}
486
487static void set_device_domain(struct amd_iommu *iommu,
488 struct protection_domain *domain,
489 u16 devid)
490{
491 unsigned long flags;
492
493 u64 pte_root = virt_to_phys(domain->pt_root);
494
495 pte_root |= (domain->mode & 0x07) << 9;
496 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | 2;
497
498 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
499 amd_iommu_dev_table[devid].data[0] = pte_root;
500 amd_iommu_dev_table[devid].data[1] = pte_root >> 32;
501 amd_iommu_dev_table[devid].data[2] = domain->id;
502
503 amd_iommu_pd_table[devid] = domain;
504 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
505
506 iommu_queue_inv_dev_entry(iommu, devid);
507
508 iommu->need_sync = 1;
509}
510
511static int get_device_resources(struct device *dev,
512 struct amd_iommu **iommu,
513 struct protection_domain **domain,
514 u16 *bdf)
515{
516 struct dma_ops_domain *dma_dom;
517 struct pci_dev *pcidev;
518 u16 _bdf;
519
520 BUG_ON(!dev || dev->bus != &pci_bus_type || !dev->dma_mask);
521
522 pcidev = to_pci_dev(dev);
523 _bdf = (pcidev->bus->number << 8) | pcidev->devfn;
524
525 if (_bdf >= amd_iommu_last_bdf) {
526 *iommu = NULL;
527 *domain = NULL;
528 *bdf = 0xffff;
529 return 0;
530 }
531
532 *bdf = amd_iommu_alias_table[_bdf];
533
534 *iommu = amd_iommu_rlookup_table[*bdf];
535 if (*iommu == NULL)
536 return 0;
537 dma_dom = (*iommu)->default_dom;
538 *domain = domain_for_device(*bdf);
539 if (*domain == NULL) {
540 *domain = &dma_dom->domain;
541 set_device_domain(*iommu, *domain, *bdf);
542 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
543 "device ", (*domain)->id);
544 print_devid(_bdf, 1);
545 }
546
547 return 1;
548}
549
550static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
551 struct dma_ops_domain *dom,
552 unsigned long address,
553 phys_addr_t paddr,
554 int direction)
555{
556 u64 *pte, __pte;
557
558 WARN_ON(address > dom->aperture_size);
559
560 paddr &= PAGE_MASK;
561
562 pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
563 pte += IOMMU_PTE_L0_INDEX(address);
564
565 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
566
567 if (direction == DMA_TO_DEVICE)
568 __pte |= IOMMU_PTE_IR;
569 else if (direction == DMA_FROM_DEVICE)
570 __pte |= IOMMU_PTE_IW;
571 else if (direction == DMA_BIDIRECTIONAL)
572 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
573
574 WARN_ON(*pte);
575
576 *pte = __pte;
577
578 return (dma_addr_t)address;
579}
580
581static void dma_ops_domain_unmap(struct amd_iommu *iommu,
582 struct dma_ops_domain *dom,
583 unsigned long address)
584{
585 u64 *pte;
586
587 if (address >= dom->aperture_size)
588 return;
589
590 WARN_ON(address & 0xfffULL || address > dom->aperture_size);
591
592 pte = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
593 pte += IOMMU_PTE_L0_INDEX(address);
594
595 WARN_ON(!*pte);
596
597 *pte = 0ULL;
598}
599
600static dma_addr_t __map_single(struct device *dev,
601 struct amd_iommu *iommu,
602 struct dma_ops_domain *dma_dom,
603 phys_addr_t paddr,
604 size_t size,
605 int dir)
606{
607 dma_addr_t offset = paddr & ~PAGE_MASK;
608 dma_addr_t address, start;
609 unsigned int pages;
610 int i;
611
612 pages = to_pages(paddr, size);
613 paddr &= PAGE_MASK;
614
615 address = dma_ops_alloc_addresses(dev, dma_dom, pages);
616 if (unlikely(address == bad_dma_address))
617 goto out;
618
619 start = address;
620 for (i = 0; i < pages; ++i) {
621 dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
622 paddr += PAGE_SIZE;
623 start += PAGE_SIZE;
624 }
625 address += offset;
626
627out:
628 return address;
629}
630
631static void __unmap_single(struct amd_iommu *iommu,
632 struct dma_ops_domain *dma_dom,
633 dma_addr_t dma_addr,
634 size_t size,
635 int dir)
636{
637 dma_addr_t i, start;
638 unsigned int pages;
639
640 if ((dma_addr == 0) || (dma_addr + size > dma_dom->aperture_size))
641 return;
642
643 pages = to_pages(dma_addr, size);
644 dma_addr &= PAGE_MASK;
645 start = dma_addr;
646
647 for (i = 0; i < pages; ++i) {
648 dma_ops_domain_unmap(iommu, dma_dom, start);
649 start += PAGE_SIZE;
650 }
651
652 dma_ops_free_addresses(dma_dom, dma_addr, pages);
653}
654
655static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
656 size_t size, int dir)
657{
658 unsigned long flags;
659 struct amd_iommu *iommu;
660 struct protection_domain *domain;
661 u16 devid;
662 dma_addr_t addr;
663
664 get_device_resources(dev, &iommu, &domain, &devid);
665
666 if (iommu == NULL || domain == NULL)
667 return (dma_addr_t)paddr;
668
669 spin_lock_irqsave(&domain->lock, flags);
670 addr = __map_single(dev, iommu, domain->priv, paddr, size, dir);
671 if (addr == bad_dma_address)
672 goto out;
673
674 if (iommu_has_npcache(iommu))
675 iommu_flush_pages(iommu, domain->id, addr, size);
676
677 if (iommu->need_sync)
678 iommu_completion_wait(iommu);
679
680out:
681 spin_unlock_irqrestore(&domain->lock, flags);
682
683 return addr;
684}
685
686static void unmap_single(struct device *dev, dma_addr_t dma_addr,
687 size_t size, int dir)
688{
689 unsigned long flags;
690 struct amd_iommu *iommu;
691 struct protection_domain *domain;
692 u16 devid;
693
694 if (!get_device_resources(dev, &iommu, &domain, &devid))
695 return;
696
697 spin_lock_irqsave(&domain->lock, flags);
698
699 __unmap_single(iommu, domain->priv, dma_addr, size, dir);
700
701 iommu_flush_pages(iommu, domain->id, dma_addr, size);
702
703 if (iommu->need_sync)
704 iommu_completion_wait(iommu);
705
706 spin_unlock_irqrestore(&domain->lock, flags);
707}
708
709static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
710 int nelems, int dir)
711{
712 struct scatterlist *s;
713 int i;
714
715 for_each_sg(sglist, s, nelems, i) {
716 s->dma_address = (dma_addr_t)sg_phys(s);
717 s->dma_length = s->length;
718 }
719
720 return nelems;
721}
722
723static int map_sg(struct device *dev, struct scatterlist *sglist,
724 int nelems, int dir)
725{
726 unsigned long flags;
727 struct amd_iommu *iommu;
728 struct protection_domain *domain;
729 u16 devid;
730 int i;
731 struct scatterlist *s;
732 phys_addr_t paddr;
733 int mapped_elems = 0;
734
735 get_device_resources(dev, &iommu, &domain, &devid);
736
737 if (!iommu || !domain)
738 return map_sg_no_iommu(dev, sglist, nelems, dir);
739
740 spin_lock_irqsave(&domain->lock, flags);
741
742 for_each_sg(sglist, s, nelems, i) {
743 paddr = sg_phys(s);
744
745 s->dma_address = __map_single(dev, iommu, domain->priv,
746 paddr, s->length, dir);
747
748 if (s->dma_address) {
749 s->dma_length = s->length;
750 mapped_elems++;
751 } else
752 goto unmap;
753 if (iommu_has_npcache(iommu))
754 iommu_flush_pages(iommu, domain->id, s->dma_address,
755 s->dma_length);
756 }
757
758 if (iommu->need_sync)
759 iommu_completion_wait(iommu);
760
761out:
762 spin_unlock_irqrestore(&domain->lock, flags);
763
764 return mapped_elems;
765unmap:
766 for_each_sg(sglist, s, mapped_elems, i) {
767 if (s->dma_address)
768 __unmap_single(iommu, domain->priv, s->dma_address,
769 s->dma_length, dir);
770 s->dma_address = s->dma_length = 0;
771 }
772
773 mapped_elems = 0;
774
775 goto out;
776}
777
778static void unmap_sg(struct device *dev, struct scatterlist *sglist,
779 int nelems, int dir)
780{
781 unsigned long flags;
782 struct amd_iommu *iommu;
783 struct protection_domain *domain;
784 struct scatterlist *s;
785 u16 devid;
786 int i;
787
788 if (!get_device_resources(dev, &iommu, &domain, &devid))
789 return;
790
791 spin_lock_irqsave(&domain->lock, flags);
792
793 for_each_sg(sglist, s, nelems, i) {
794 __unmap_single(iommu, domain->priv, s->dma_address,
795 s->dma_length, dir);
796 iommu_flush_pages(iommu, domain->id, s->dma_address,
797 s->dma_length);
798 s->dma_address = s->dma_length = 0;
799 }
800
801 if (iommu->need_sync)
802 iommu_completion_wait(iommu);
803
804 spin_unlock_irqrestore(&domain->lock, flags);
805}
806
807static void *alloc_coherent(struct device *dev, size_t size,
808 dma_addr_t *dma_addr, gfp_t flag)
809{
810 unsigned long flags;
811 void *virt_addr;
812 struct amd_iommu *iommu;
813 struct protection_domain *domain;
814 u16 devid;
815 phys_addr_t paddr;
816
817 virt_addr = (void *)__get_free_pages(flag, get_order(size));
818 if (!virt_addr)
819 return 0;
820
821 memset(virt_addr, 0, size);
822 paddr = virt_to_phys(virt_addr);
823
824 get_device_resources(dev, &iommu, &domain, &devid);
825
826 if (!iommu || !domain) {
827 *dma_addr = (dma_addr_t)paddr;
828 return virt_addr;
829 }
830
831 spin_lock_irqsave(&domain->lock, flags);
832
833 *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
834 size, DMA_BIDIRECTIONAL);
835
836 if (*dma_addr == bad_dma_address) {
837 free_pages((unsigned long)virt_addr, get_order(size));
838 virt_addr = NULL;
839 goto out;
840 }
841
842 if (iommu_has_npcache(iommu))
843 iommu_flush_pages(iommu, domain->id, *dma_addr, size);
844
845 if (iommu->need_sync)
846 iommu_completion_wait(iommu);
847
848out:
849 spin_unlock_irqrestore(&domain->lock, flags);
850
851 return virt_addr;
852}
853
854static void free_coherent(struct device *dev, size_t size,
855 void *virt_addr, dma_addr_t dma_addr)
856{
857 unsigned long flags;
858 struct amd_iommu *iommu;
859 struct protection_domain *domain;
860 u16 devid;
861
862 get_device_resources(dev, &iommu, &domain, &devid);
863
864 if (!iommu || !domain)
865 goto free_mem;
866
867 spin_lock_irqsave(&domain->lock, flags);
868
869 __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
870 iommu_flush_pages(iommu, domain->id, dma_addr, size);
871
872 if (iommu->need_sync)
873 iommu_completion_wait(iommu);
874
875 spin_unlock_irqrestore(&domain->lock, flags);
876
877free_mem:
878 free_pages((unsigned long)virt_addr, get_order(size));
879}
880
881/*
882 * If the driver core informs the DMA layer if a driver grabs a device
883 * we don't need to preallocate the protection domains anymore.
884 * For now we have to.
885 */
886void prealloc_protection_domains(void)
887{
888 struct pci_dev *dev = NULL;
889 struct dma_ops_domain *dma_dom;
890 struct amd_iommu *iommu;
891 int order = amd_iommu_aperture_order;
892 u16 devid;
893
894 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
895 devid = (dev->bus->number << 8) | dev->devfn;
896 if (devid >= amd_iommu_last_bdf)
897 continue;
898 devid = amd_iommu_alias_table[devid];
899 if (domain_for_device(devid))
900 continue;
901 iommu = amd_iommu_rlookup_table[devid];
902 if (!iommu)
903 continue;
904 dma_dom = dma_ops_domain_alloc(iommu, order);
905 if (!dma_dom)
906 continue;
907 init_unity_mappings_for_device(dma_dom, devid);
908 set_device_domain(iommu, &dma_dom->domain, devid);
909 printk(KERN_INFO "AMD IOMMU: Allocated domain %d for device ",
910 dma_dom->domain.id);
911 print_devid(devid, 1);
912 }
913}
914
915static struct dma_mapping_ops amd_iommu_dma_ops = {
916 .alloc_coherent = alloc_coherent,
917 .free_coherent = free_coherent,
918 .map_single = map_single,
919 .unmap_single = unmap_single,
920 .map_sg = map_sg,
921 .unmap_sg = unmap_sg,
922};
923
924int __init amd_iommu_init_dma_ops(void)
925{
926 struct amd_iommu *iommu;
927 int order = amd_iommu_aperture_order;
928 int ret;
929
930 list_for_each_entry(iommu, &amd_iommu_list, list) {
931 iommu->default_dom = dma_ops_domain_alloc(iommu, order);
932 if (iommu->default_dom == NULL)
933 return -ENOMEM;
934 ret = iommu_init_unity_mappings(iommu);
935 if (ret)
936 goto free_domains;
937 }
938
939 if (amd_iommu_isolate)
940 prealloc_protection_domains();
941
942 iommu_detected = 1;
943 force_iommu = 1;
944 bad_dma_address = 0;
945#ifdef CONFIG_GART_IOMMU
946 gart_iommu_aperture_disabled = 1;
947 gart_iommu_aperture = 0;
948#endif
949
950 dma_ops = &amd_iommu_dma_ops;
951
952 return 0;
953
954free_domains:
955
956 list_for_each_entry(iommu, &amd_iommu_list, list) {
957 if (iommu->default_dom)
958 dma_ops_domain_free(iommu->default_dom);
959 }
960
961 return ret;
962}
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
new file mode 100644
index 000000000000..2a13e430437d
--- /dev/null
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -0,0 +1,875 @@
1/*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/pci.h>
21#include <linux/acpi.h>
22#include <linux/gfp.h>
23#include <linux/list.h>
24#include <linux/sysdev.h>
25#include <asm/pci-direct.h>
26#include <asm/amd_iommu_types.h>
27#include <asm/amd_iommu.h>
28#include <asm/gart.h>
29
30/*
31 * definitions for the ACPI scanning code
32 */
33#define UPDATE_LAST_BDF(x) do {\
34 if ((x) > amd_iommu_last_bdf) \
35 amd_iommu_last_bdf = (x); \
36 } while (0);
37
38#define DEVID(bus, devfn) (((bus) << 8) | (devfn))
39#define PCI_BUS(x) (((x) >> 8) & 0xff)
40#define IVRS_HEADER_LENGTH 48
41#define TBL_SIZE(x) (1 << (PAGE_SHIFT + get_order(amd_iommu_last_bdf * (x))))
42
43#define ACPI_IVHD_TYPE 0x10
44#define ACPI_IVMD_TYPE_ALL 0x20
45#define ACPI_IVMD_TYPE 0x21
46#define ACPI_IVMD_TYPE_RANGE 0x22
47
48#define IVHD_DEV_ALL 0x01
49#define IVHD_DEV_SELECT 0x02
50#define IVHD_DEV_SELECT_RANGE_START 0x03
51#define IVHD_DEV_RANGE_END 0x04
52#define IVHD_DEV_ALIAS 0x42
53#define IVHD_DEV_ALIAS_RANGE 0x43
54#define IVHD_DEV_EXT_SELECT 0x46
55#define IVHD_DEV_EXT_SELECT_RANGE 0x47
56
57#define IVHD_FLAG_HT_TUN_EN 0x00
58#define IVHD_FLAG_PASSPW_EN 0x01
59#define IVHD_FLAG_RESPASSPW_EN 0x02
60#define IVHD_FLAG_ISOC_EN 0x03
61
62#define IVMD_FLAG_EXCL_RANGE 0x08
63#define IVMD_FLAG_UNITY_MAP 0x01
64
65#define ACPI_DEVFLAG_INITPASS 0x01
66#define ACPI_DEVFLAG_EXTINT 0x02
67#define ACPI_DEVFLAG_NMI 0x04
68#define ACPI_DEVFLAG_SYSMGT1 0x10
69#define ACPI_DEVFLAG_SYSMGT2 0x20
70#define ACPI_DEVFLAG_LINT0 0x40
71#define ACPI_DEVFLAG_LINT1 0x80
72#define ACPI_DEVFLAG_ATSDIS 0x10000000
73
74struct ivhd_header {
75 u8 type;
76 u8 flags;
77 u16 length;
78 u16 devid;
79 u16 cap_ptr;
80 u64 mmio_phys;
81 u16 pci_seg;
82 u16 info;
83 u32 reserved;
84} __attribute__((packed));
85
86struct ivhd_entry {
87 u8 type;
88 u16 devid;
89 u8 flags;
90 u32 ext;
91} __attribute__((packed));
92
93struct ivmd_header {
94 u8 type;
95 u8 flags;
96 u16 length;
97 u16 devid;
98 u16 aux;
99 u64 resv;
100 u64 range_start;
101 u64 range_length;
102} __attribute__((packed));
103
104static int __initdata amd_iommu_detected;
105
106u16 amd_iommu_last_bdf;
107struct list_head amd_iommu_unity_map;
108unsigned amd_iommu_aperture_order = 26;
109int amd_iommu_isolate;
110
111struct list_head amd_iommu_list;
112struct dev_table_entry *amd_iommu_dev_table;
113u16 *amd_iommu_alias_table;
114struct amd_iommu **amd_iommu_rlookup_table;
115struct protection_domain **amd_iommu_pd_table;
116unsigned long *amd_iommu_pd_alloc_bitmap;
117
118static u32 dev_table_size;
119static u32 alias_table_size;
120static u32 rlookup_table_size;
121
122static void __init iommu_set_exclusion_range(struct amd_iommu *iommu)
123{
124 u64 start = iommu->exclusion_start & PAGE_MASK;
125 u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
126 u64 entry;
127
128 if (!iommu->exclusion_start)
129 return;
130
131 entry = start | MMIO_EXCL_ENABLE_MASK;
132 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
133 &entry, sizeof(entry));
134
135 entry = limit;
136 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
137 &entry, sizeof(entry));
138}
139
140static void __init iommu_set_device_table(struct amd_iommu *iommu)
141{
142 u32 entry;
143
144 BUG_ON(iommu->mmio_base == NULL);
145
146 entry = virt_to_phys(amd_iommu_dev_table);
147 entry |= (dev_table_size >> 12) - 1;
148 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
149 &entry, sizeof(entry));
150}
151
152static void __init iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
153{
154 u32 ctrl;
155
156 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
157 ctrl |= (1 << bit);
158 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
159}
160
161static void __init iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
162{
163 u32 ctrl;
164
165 ctrl = (u64)readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
166 ctrl &= ~(1 << bit);
167 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
168}
169
170void __init iommu_enable(struct amd_iommu *iommu)
171{
172 printk(KERN_INFO "AMD IOMMU: Enabling IOMMU at ");
173 print_devid(iommu->devid, 0);
174 printk(" cap 0x%hx\n", iommu->cap_ptr);
175
176 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
177}
178
179static u8 * __init iommu_map_mmio_space(u64 address)
180{
181 u8 *ret;
182
183 if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu"))
184 return NULL;
185
186 ret = ioremap_nocache(address, MMIO_REGION_LENGTH);
187 if (ret != NULL)
188 return ret;
189
190 release_mem_region(address, MMIO_REGION_LENGTH);
191
192 return NULL;
193}
194
195static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
196{
197 if (iommu->mmio_base)
198 iounmap(iommu->mmio_base);
199 release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);
200}
201
202static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
203{
204 u32 cap;
205
206 cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
207 UPDATE_LAST_BDF(DEVID(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
208
209 return 0;
210}
211
212static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
213{
214 u8 *p = (void *)h, *end = (void *)h;
215 struct ivhd_entry *dev;
216
217 p += sizeof(*h);
218 end += h->length;
219
220 find_last_devid_on_pci(PCI_BUS(h->devid),
221 PCI_SLOT(h->devid),
222 PCI_FUNC(h->devid),
223 h->cap_ptr);
224
225 while (p < end) {
226 dev = (struct ivhd_entry *)p;
227 switch (dev->type) {
228 case IVHD_DEV_SELECT:
229 case IVHD_DEV_RANGE_END:
230 case IVHD_DEV_ALIAS:
231 case IVHD_DEV_EXT_SELECT:
232 UPDATE_LAST_BDF(dev->devid);
233 break;
234 default:
235 break;
236 }
237 p += 0x04 << (*p >> 6);
238 }
239
240 WARN_ON(p != end);
241
242 return 0;
243}
244
245static int __init find_last_devid_acpi(struct acpi_table_header *table)
246{
247 int i;
248 u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;
249 struct ivhd_header *h;
250
251 /*
252 * Validate checksum here so we don't need to do it when
253 * we actually parse the table
254 */
255 for (i = 0; i < table->length; ++i)
256 checksum += p[i];
257 if (checksum != 0)
258 /* ACPI table corrupt */
259 return -ENODEV;
260
261 p += IVRS_HEADER_LENGTH;
262
263 end += table->length;
264 while (p < end) {
265 h = (struct ivhd_header *)p;
266 switch (h->type) {
267 case ACPI_IVHD_TYPE:
268 find_last_devid_from_ivhd(h);
269 break;
270 default:
271 break;
272 }
273 p += h->length;
274 }
275 WARN_ON(p != end);
276
277 return 0;
278}
279
280static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
281{
282 u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL,
283 get_order(CMD_BUFFER_SIZE));
284 u64 entry = 0;
285
286 if (cmd_buf == NULL)
287 return NULL;
288
289 iommu->cmd_buf_size = CMD_BUFFER_SIZE;
290
291 memset(cmd_buf, 0, CMD_BUFFER_SIZE);
292
293 entry = (u64)virt_to_phys(cmd_buf);
294 entry |= MMIO_CMD_SIZE_512;
295 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
296 &entry, sizeof(entry));
297
298 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
299
300 return cmd_buf;
301}
302
303static void __init free_command_buffer(struct amd_iommu *iommu)
304{
305 if (iommu->cmd_buf)
306 free_pages((unsigned long)iommu->cmd_buf,
307 get_order(CMD_BUFFER_SIZE));
308}
309
310static void set_dev_entry_bit(u16 devid, u8 bit)
311{
312 int i = (bit >> 5) & 0x07;
313 int _bit = bit & 0x1f;
314
315 amd_iommu_dev_table[devid].data[i] |= (1 << _bit);
316}
317
318static void __init set_dev_entry_from_acpi(u16 devid, u32 flags, u32 ext_flags)
319{
320 if (flags & ACPI_DEVFLAG_INITPASS)
321 set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
322 if (flags & ACPI_DEVFLAG_EXTINT)
323 set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
324 if (flags & ACPI_DEVFLAG_NMI)
325 set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
326 if (flags & ACPI_DEVFLAG_SYSMGT1)
327 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
328 if (flags & ACPI_DEVFLAG_SYSMGT2)
329 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
330 if (flags & ACPI_DEVFLAG_LINT0)
331 set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
332 if (flags & ACPI_DEVFLAG_LINT1)
333 set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
334}
335
336static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
337{
338 amd_iommu_rlookup_table[devid] = iommu;
339}
340
341static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
342{
343 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
344
345 if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
346 return;
347
348 if (iommu) {
349 set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
350 iommu->exclusion_start = m->range_start;
351 iommu->exclusion_length = m->range_length;
352 }
353}
354
355static void __init init_iommu_from_pci(struct amd_iommu *iommu)
356{
357 int bus = PCI_BUS(iommu->devid);
358 int dev = PCI_SLOT(iommu->devid);
359 int fn = PCI_FUNC(iommu->devid);
360 int cap_ptr = iommu->cap_ptr;
361 u32 range;
362
363 iommu->cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_CAP_HDR_OFFSET);
364
365 range = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
366 iommu->first_device = DEVID(MMIO_GET_BUS(range), MMIO_GET_FD(range));
367 iommu->last_device = DEVID(MMIO_GET_BUS(range), MMIO_GET_LD(range));
368}
369
370static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
371 struct ivhd_header *h)
372{
373 u8 *p = (u8 *)h;
374 u8 *end = p, flags = 0;
375 u16 dev_i, devid = 0, devid_start = 0, devid_to = 0;
376 u32 ext_flags = 0;
377 bool alias = 0;
378 struct ivhd_entry *e;
379
380 /*
381 * First set the recommended feature enable bits from ACPI
382 * into the IOMMU control registers
383 */
384 h->flags & IVHD_FLAG_HT_TUN_EN ?
385 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
386 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
387
388 h->flags & IVHD_FLAG_PASSPW_EN ?
389 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
390 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
391
392 h->flags & IVHD_FLAG_RESPASSPW_EN ?
393 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
394 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
395
396 h->flags & IVHD_FLAG_ISOC_EN ?
397 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
398 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
399
400 /*
401 * make IOMMU memory accesses cache coherent
402 */
403 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
404
405 /*
406 * Done. Now parse the device entries
407 */
408 p += sizeof(struct ivhd_header);
409 end += h->length;
410
411 while (p < end) {
412 e = (struct ivhd_entry *)p;
413 switch (e->type) {
414 case IVHD_DEV_ALL:
415 for (dev_i = iommu->first_device;
416 dev_i <= iommu->last_device; ++dev_i)
417 set_dev_entry_from_acpi(dev_i, e->flags, 0);
418 break;
419 case IVHD_DEV_SELECT:
420 devid = e->devid;
421 set_dev_entry_from_acpi(devid, e->flags, 0);
422 break;
423 case IVHD_DEV_SELECT_RANGE_START:
424 devid_start = e->devid;
425 flags = e->flags;
426 ext_flags = 0;
427 alias = 0;
428 break;
429 case IVHD_DEV_ALIAS:
430 devid = e->devid;
431 devid_to = e->ext >> 8;
432 set_dev_entry_from_acpi(devid, e->flags, 0);
433 amd_iommu_alias_table[devid] = devid_to;
434 break;
435 case IVHD_DEV_ALIAS_RANGE:
436 devid_start = e->devid;
437 flags = e->flags;
438 devid_to = e->ext >> 8;
439 ext_flags = 0;
440 alias = 1;
441 break;
442 case IVHD_DEV_EXT_SELECT:
443 devid = e->devid;
444 set_dev_entry_from_acpi(devid, e->flags, e->ext);
445 break;
446 case IVHD_DEV_EXT_SELECT_RANGE:
447 devid_start = e->devid;
448 flags = e->flags;
449 ext_flags = e->ext;
450 alias = 0;
451 break;
452 case IVHD_DEV_RANGE_END:
453 devid = e->devid;
454 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
455 if (alias)
456 amd_iommu_alias_table[dev_i] = devid_to;
457 set_dev_entry_from_acpi(
458 amd_iommu_alias_table[dev_i],
459 flags, ext_flags);
460 }
461 break;
462 default:
463 break;
464 }
465
466 p += 0x04 << (e->type >> 6);
467 }
468}
469
470static int __init init_iommu_devices(struct amd_iommu *iommu)
471{
472 u16 i;
473
474 for (i = iommu->first_device; i <= iommu->last_device; ++i)
475 set_iommu_for_device(iommu, i);
476
477 return 0;
478}
479
480static void __init free_iommu_one(struct amd_iommu *iommu)
481{
482 free_command_buffer(iommu);
483 iommu_unmap_mmio_space(iommu);
484}
485
486static void __init free_iommu_all(void)
487{
488 struct amd_iommu *iommu, *next;
489
490 list_for_each_entry_safe(iommu, next, &amd_iommu_list, list) {
491 list_del(&iommu->list);
492 free_iommu_one(iommu);
493 kfree(iommu);
494 }
495}
496
497static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
498{
499 spin_lock_init(&iommu->lock);
500 list_add_tail(&iommu->list, &amd_iommu_list);
501
502 /*
503 * Copy data from ACPI table entry to the iommu struct
504 */
505 iommu->devid = h->devid;
506 iommu->cap_ptr = h->cap_ptr;
507 iommu->mmio_phys = h->mmio_phys;
508 iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);
509 if (!iommu->mmio_base)
510 return -ENOMEM;
511
512 iommu_set_device_table(iommu);
513 iommu->cmd_buf = alloc_command_buffer(iommu);
514 if (!iommu->cmd_buf)
515 return -ENOMEM;
516
517 init_iommu_from_pci(iommu);
518 init_iommu_from_acpi(iommu, h);
519 init_iommu_devices(iommu);
520
521 return 0;
522}
523
524static int __init init_iommu_all(struct acpi_table_header *table)
525{
526 u8 *p = (u8 *)table, *end = (u8 *)table;
527 struct ivhd_header *h;
528 struct amd_iommu *iommu;
529 int ret;
530
531 INIT_LIST_HEAD(&amd_iommu_list);
532
533 end += table->length;
534 p += IVRS_HEADER_LENGTH;
535
536 while (p < end) {
537 h = (struct ivhd_header *)p;
538 switch (*p) {
539 case ACPI_IVHD_TYPE:
540 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
541 if (iommu == NULL)
542 return -ENOMEM;
543 ret = init_iommu_one(iommu, h);
544 if (ret)
545 return ret;
546 break;
547 default:
548 break;
549 }
550 p += h->length;
551
552 }
553 WARN_ON(p != end);
554
555 return 0;
556}
557
558static void __init free_unity_maps(void)
559{
560 struct unity_map_entry *entry, *next;
561
562 list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
563 list_del(&entry->list);
564 kfree(entry);
565 }
566}
567
568static int __init init_exclusion_range(struct ivmd_header *m)
569{
570 int i;
571
572 switch (m->type) {
573 case ACPI_IVMD_TYPE:
574 set_device_exclusion_range(m->devid, m);
575 break;
576 case ACPI_IVMD_TYPE_ALL:
577 for (i = 0; i < amd_iommu_last_bdf; ++i)
578 set_device_exclusion_range(i, m);
579 break;
580 case ACPI_IVMD_TYPE_RANGE:
581 for (i = m->devid; i <= m->aux; ++i)
582 set_device_exclusion_range(i, m);
583 break;
584 default:
585 break;
586 }
587
588 return 0;
589}
590
591static int __init init_unity_map_range(struct ivmd_header *m)
592{
593 struct unity_map_entry *e = 0;
594
595 e = kzalloc(sizeof(*e), GFP_KERNEL);
596 if (e == NULL)
597 return -ENOMEM;
598
599 switch (m->type) {
600 default:
601 case ACPI_IVMD_TYPE:
602 e->devid_start = e->devid_end = m->devid;
603 break;
604 case ACPI_IVMD_TYPE_ALL:
605 e->devid_start = 0;
606 e->devid_end = amd_iommu_last_bdf;
607 break;
608 case ACPI_IVMD_TYPE_RANGE:
609 e->devid_start = m->devid;
610 e->devid_end = m->aux;
611 break;
612 }
613 e->address_start = PAGE_ALIGN(m->range_start);
614 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
615 e->prot = m->flags >> 1;
616
617 list_add_tail(&e->list, &amd_iommu_unity_map);
618
619 return 0;
620}
621
622static int __init init_memory_definitions(struct acpi_table_header *table)
623{
624 u8 *p = (u8 *)table, *end = (u8 *)table;
625 struct ivmd_header *m;
626
627 INIT_LIST_HEAD(&amd_iommu_unity_map);
628
629 end += table->length;
630 p += IVRS_HEADER_LENGTH;
631
632 while (p < end) {
633 m = (struct ivmd_header *)p;
634 if (m->flags & IVMD_FLAG_EXCL_RANGE)
635 init_exclusion_range(m);
636 else if (m->flags & IVMD_FLAG_UNITY_MAP)
637 init_unity_map_range(m);
638
639 p += m->length;
640 }
641
642 return 0;
643}
644
645static void __init enable_iommus(void)
646{
647 struct amd_iommu *iommu;
648
649 list_for_each_entry(iommu, &amd_iommu_list, list) {
650 iommu_set_exclusion_range(iommu);
651 iommu_enable(iommu);
652 }
653}
654
655/*
656 * Suspend/Resume support
657 * disable suspend until real resume implemented
658 */
659
660static int amd_iommu_resume(struct sys_device *dev)
661{
662 return 0;
663}
664
665static int amd_iommu_suspend(struct sys_device *dev, pm_message_t state)
666{
667 return -EINVAL;
668}
669
670static struct sysdev_class amd_iommu_sysdev_class = {
671 .name = "amd_iommu",
672 .suspend = amd_iommu_suspend,
673 .resume = amd_iommu_resume,
674};
675
676static struct sys_device device_amd_iommu = {
677 .id = 0,
678 .cls = &amd_iommu_sysdev_class,
679};
680
681int __init amd_iommu_init(void)
682{
683 int i, ret = 0;
684
685
686 if (no_iommu) {
687 printk(KERN_INFO "AMD IOMMU disabled by kernel command line\n");
688 return 0;
689 }
690
691 if (!amd_iommu_detected)
692 return -ENODEV;
693
694 /*
695 * First parse ACPI tables to find the largest Bus/Dev/Func
696 * we need to handle. Upon this information the shared data
697 * structures for the IOMMUs in the system will be allocated
698 */
699 if (acpi_table_parse("IVRS", find_last_devid_acpi) != 0)
700 return -ENODEV;
701
702 dev_table_size = TBL_SIZE(DEV_TABLE_ENTRY_SIZE);
703 alias_table_size = TBL_SIZE(ALIAS_TABLE_ENTRY_SIZE);
704 rlookup_table_size = TBL_SIZE(RLOOKUP_TABLE_ENTRY_SIZE);
705
706 ret = -ENOMEM;
707
708 /* Device table - directly used by all IOMMUs */
709 amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL,
710 get_order(dev_table_size));
711 if (amd_iommu_dev_table == NULL)
712 goto out;
713
714 /*
715 * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
716 * IOMMU see for that device
717 */
718 amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
719 get_order(alias_table_size));
720 if (amd_iommu_alias_table == NULL)
721 goto free;
722
723 /* IOMMU rlookup table - find the IOMMU for a specific device */
724 amd_iommu_rlookup_table = (void *)__get_free_pages(GFP_KERNEL,
725 get_order(rlookup_table_size));
726 if (amd_iommu_rlookup_table == NULL)
727 goto free;
728
729 /*
730 * Protection Domain table - maps devices to protection domains
731 * This table has the same size as the rlookup_table
732 */
733 amd_iommu_pd_table = (void *)__get_free_pages(GFP_KERNEL,
734 get_order(rlookup_table_size));
735 if (amd_iommu_pd_table == NULL)
736 goto free;
737
738 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(GFP_KERNEL,
739 get_order(MAX_DOMAIN_ID/8));
740 if (amd_iommu_pd_alloc_bitmap == NULL)
741 goto free;
742
743 /*
744 * memory is allocated now; initialize the device table with all zeroes
745 * and let all alias entries point to itself
746 */
747 memset(amd_iommu_dev_table, 0, dev_table_size);
748 for (i = 0; i < amd_iommu_last_bdf; ++i)
749 amd_iommu_alias_table[i] = i;
750
751 memset(amd_iommu_pd_table, 0, rlookup_table_size);
752 memset(amd_iommu_pd_alloc_bitmap, 0, MAX_DOMAIN_ID / 8);
753
754 /*
755 * never allocate domain 0 because its used as the non-allocated and
756 * error value placeholder
757 */
758 amd_iommu_pd_alloc_bitmap[0] = 1;
759
760 /*
761 * now the data structures are allocated and basically initialized
762 * start the real acpi table scan
763 */
764 ret = -ENODEV;
765 if (acpi_table_parse("IVRS", init_iommu_all) != 0)
766 goto free;
767
768 if (acpi_table_parse("IVRS", init_memory_definitions) != 0)
769 goto free;
770
771 ret = amd_iommu_init_dma_ops();
772 if (ret)
773 goto free;
774
775 ret = sysdev_class_register(&amd_iommu_sysdev_class);
776 if (ret)
777 goto free;
778
779 ret = sysdev_register(&device_amd_iommu);
780 if (ret)
781 goto free;
782
783 enable_iommus();
784
785 printk(KERN_INFO "AMD IOMMU: aperture size is %d MB\n",
786 (1 << (amd_iommu_aperture_order-20)));
787
788 printk(KERN_INFO "AMD IOMMU: device isolation ");
789 if (amd_iommu_isolate)
790 printk("enabled\n");
791 else
792 printk("disabled\n");
793
794out:
795 return ret;
796
797free:
798 if (amd_iommu_pd_alloc_bitmap)
799 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, 1);
800
801 if (amd_iommu_pd_table)
802 free_pages((unsigned long)amd_iommu_pd_table,
803 get_order(rlookup_table_size));
804
805 if (amd_iommu_rlookup_table)
806 free_pages((unsigned long)amd_iommu_rlookup_table,
807 get_order(rlookup_table_size));
808
809 if (amd_iommu_alias_table)
810 free_pages((unsigned long)amd_iommu_alias_table,
811 get_order(alias_table_size));
812
813 if (amd_iommu_dev_table)
814 free_pages((unsigned long)amd_iommu_dev_table,
815 get_order(dev_table_size));
816
817 free_iommu_all();
818
819 free_unity_maps();
820
821 goto out;
822}
823
824static int __init early_amd_iommu_detect(struct acpi_table_header *table)
825{
826 return 0;
827}
828
829void __init amd_iommu_detect(void)
830{
831 if (swiotlb || no_iommu || iommu_detected)
832 return;
833
834 if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {
835 iommu_detected = 1;
836 amd_iommu_detected = 1;
837#ifdef CONFIG_GART_IOMMU
838 gart_iommu_aperture_disabled = 1;
839 gart_iommu_aperture = 0;
840#endif
841 }
842}
843
844static int __init parse_amd_iommu_options(char *str)
845{
846 for (; *str; ++str) {
847 if (strcmp(str, "isolate") == 0)
848 amd_iommu_isolate = 1;
849 }
850
851 return 1;
852}
853
854static int __init parse_amd_iommu_size_options(char *str)
855{
856 for (; *str; ++str) {
857 if (strcmp(str, "32M") == 0)
858 amd_iommu_aperture_order = 25;
859 if (strcmp(str, "64M") == 0)
860 amd_iommu_aperture_order = 26;
861 if (strcmp(str, "128M") == 0)
862 amd_iommu_aperture_order = 27;
863 if (strcmp(str, "256M") == 0)
864 amd_iommu_aperture_order = 28;
865 if (strcmp(str, "512M") == 0)
866 amd_iommu_aperture_order = 29;
867 if (strcmp(str, "1G") == 0)
868 amd_iommu_aperture_order = 30;
869 }
870
871 return 1;
872}
873
874__setup("amd_iommu=", parse_amd_iommu_options);
875__setup("amd_iommu_size=", parse_amd_iommu_size_options);
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 479926d9e004..9f907806c1a5 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -35,6 +35,18 @@ int fallback_aper_force __initdata;
35 35
36int fix_aperture __initdata = 1; 36int fix_aperture __initdata = 1;
37 37
38struct bus_dev_range {
39 int bus;
40 int dev_base;
41 int dev_limit;
42};
43
44static struct bus_dev_range bus_dev_ranges[] __initdata = {
45 { 0x00, 0x18, 0x20},
46 { 0xff, 0x00, 0x20},
47 { 0xfe, 0x00, 0x20}
48};
49
38static struct resource gart_resource = { 50static struct resource gart_resource = {
39 .name = "GART", 51 .name = "GART",
40 .flags = IORESOURCE_MEM, 52 .flags = IORESOURCE_MEM,
@@ -55,8 +67,9 @@ static u32 __init allocate_aperture(void)
55 u32 aper_size; 67 u32 aper_size;
56 void *p; 68 void *p;
57 69
58 if (fallback_aper_order > 7) 70 /* aper_size should <= 1G */
59 fallback_aper_order = 7; 71 if (fallback_aper_order > 5)
72 fallback_aper_order = 5;
60 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 73 aper_size = (32 * 1024 * 1024) << fallback_aper_order;
61 74
62 /* 75 /*
@@ -65,7 +78,20 @@ static u32 __init allocate_aperture(void)
65 * memory. Unfortunately we cannot move it up because that would 78 * memory. Unfortunately we cannot move it up because that would
66 * make the IOMMU useless. 79 * make the IOMMU useless.
67 */ 80 */
68 p = __alloc_bootmem_nopanic(aper_size, aper_size, 0); 81 /*
82 * using 512M as goal, in case kexec will load kernel_big
83 * that will do the on position decompress, and could overlap with
84 * that positon with gart that is used.
85 * sequende:
86 * kernel_small
87 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
88 * ==> kernel_small(gart area become e820_reserved)
89 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
90 * ==> kerne_big (uncompressed size will be big than 64M or 128M)
91 * so don't use 512M below as gart iommu, leave the space for kernel
92 * code for safe
93 */
94 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
69 if (!p || __pa(p)+aper_size > 0xffffffff) { 95 if (!p || __pa(p)+aper_size > 0xffffffff) {
70 printk(KERN_ERR 96 printk(KERN_ERR
71 "Cannot allocate aperture memory hole (%p,%uK)\n", 97 "Cannot allocate aperture memory hole (%p,%uK)\n",
@@ -83,69 +109,53 @@ static u32 __init allocate_aperture(void)
83 return (u32)__pa(p); 109 return (u32)__pa(p);
84} 110}
85 111
86static int __init aperture_valid(u64 aper_base, u32 aper_size)
87{
88 if (!aper_base)
89 return 0;
90
91 if (aper_base + aper_size > 0x100000000UL) {
92 printk(KERN_ERR "Aperture beyond 4GB. Ignoring.\n");
93 return 0;
94 }
95 if (e820_any_mapped(aper_base, aper_base + aper_size, E820_RAM)) {
96 printk(KERN_ERR "Aperture pointing to e820 RAM. Ignoring.\n");
97 return 0;
98 }
99 if (aper_size < 64*1024*1024) {
100 printk(KERN_ERR "Aperture too small (%d MB)\n", aper_size>>20);
101 return 0;
102 }
103
104 return 1;
105}
106 112
107/* Find a PCI capability */ 113/* Find a PCI capability */
108static __u32 __init find_cap(int num, int slot, int func, int cap) 114static u32 __init find_cap(int bus, int slot, int func, int cap)
109{ 115{
110 int bytes; 116 int bytes;
111 u8 pos; 117 u8 pos;
112 118
113 if (!(read_pci_config_16(num, slot, func, PCI_STATUS) & 119 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) &
114 PCI_STATUS_CAP_LIST)) 120 PCI_STATUS_CAP_LIST))
115 return 0; 121 return 0;
116 122
117 pos = read_pci_config_byte(num, slot, func, PCI_CAPABILITY_LIST); 123 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST);
118 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 124 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
119 u8 id; 125 u8 id;
120 126
121 pos &= ~3; 127 pos &= ~3;
122 id = read_pci_config_byte(num, slot, func, pos+PCI_CAP_LIST_ID); 128 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID);
123 if (id == 0xff) 129 if (id == 0xff)
124 break; 130 break;
125 if (id == cap) 131 if (id == cap)
126 return pos; 132 return pos;
127 pos = read_pci_config_byte(num, slot, func, 133 pos = read_pci_config_byte(bus, slot, func,
128 pos+PCI_CAP_LIST_NEXT); 134 pos+PCI_CAP_LIST_NEXT);
129 } 135 }
130 return 0; 136 return 0;
131} 137}
132 138
133/* Read a standard AGPv3 bridge header */ 139/* Read a standard AGPv3 bridge header */
134static __u32 __init read_agp(int num, int slot, int func, int cap, u32 *order) 140static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order)
135{ 141{
136 u32 apsize; 142 u32 apsize;
137 u32 apsizereg; 143 u32 apsizereg;
138 int nbits; 144 int nbits;
139 u32 aper_low, aper_hi; 145 u32 aper_low, aper_hi;
140 u64 aper; 146 u64 aper;
147 u32 old_order;
141 148
142 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", num, slot, func); 149 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func);
143 apsizereg = read_pci_config_16(num, slot, func, cap + 0x14); 150 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14);
144 if (apsizereg == 0xffffffff) { 151 if (apsizereg == 0xffffffff) {
145 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 152 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n");
146 return 0; 153 return 0;
147 } 154 }
148 155
156 /* old_order could be the value from NB gart setting */
157 old_order = *order;
158
149 apsize = apsizereg & 0xfff; 159 apsize = apsizereg & 0xfff;
150 /* Some BIOS use weird encodings not in the AGPv3 table. */ 160 /* Some BIOS use weird encodings not in the AGPv3 table. */
151 if (apsize & 0xff) 161 if (apsize & 0xff)
@@ -155,14 +165,26 @@ static __u32 __init read_agp(int num, int slot, int func, int cap, u32 *order)
155 if ((int)*order < 0) /* < 32MB */ 165 if ((int)*order < 0) /* < 32MB */
156 *order = 0; 166 *order = 0;
157 167
158 aper_low = read_pci_config(num, slot, func, 0x10); 168 aper_low = read_pci_config(bus, slot, func, 0x10);
159 aper_hi = read_pci_config(num, slot, func, 0x14); 169 aper_hi = read_pci_config(bus, slot, func, 0x14);
160 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 170 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32);
161 171
172 /*
173 * On some sick chips, APSIZE is 0. It means it wants 4G
174 * so let double check that order, and lets trust AMD NB settings:
175 */
176 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n",
177 aper, 32 << old_order);
178 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) {
179 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n",
180 32 << *order, apsizereg);
181 *order = old_order;
182 }
183
162 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 184 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n",
163 aper, 32 << *order, apsizereg); 185 aper, 32 << *order, apsizereg);
164 186
165 if (!aperture_valid(aper, (32*1024*1024) << *order)) 187 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20))
166 return 0; 188 return 0;
167 return (u32)aper; 189 return (u32)aper;
168} 190}
@@ -180,17 +202,17 @@ static __u32 __init read_agp(int num, int slot, int func, int cap, u32 *order)
180 * the AGP bridges should be always an own bus on the HT hierarchy, 202 * the AGP bridges should be always an own bus on the HT hierarchy,
181 * but do it here for future safety. 203 * but do it here for future safety.
182 */ 204 */
183static __u32 __init search_agp_bridge(u32 *order, int *valid_agp) 205static u32 __init search_agp_bridge(u32 *order, int *valid_agp)
184{ 206{
185 int num, slot, func; 207 int bus, slot, func;
186 208
187 /* Poor man's PCI discovery */ 209 /* Poor man's PCI discovery */
188 for (num = 0; num < 256; num++) { 210 for (bus = 0; bus < 256; bus++) {
189 for (slot = 0; slot < 32; slot++) { 211 for (slot = 0; slot < 32; slot++) {
190 for (func = 0; func < 8; func++) { 212 for (func = 0; func < 8; func++) {
191 u32 class, cap; 213 u32 class, cap;
192 u8 type; 214 u8 type;
193 class = read_pci_config(num, slot, func, 215 class = read_pci_config(bus, slot, func,
194 PCI_CLASS_REVISION); 216 PCI_CLASS_REVISION);
195 if (class == 0xffffffff) 217 if (class == 0xffffffff)
196 break; 218 break;
@@ -199,17 +221,17 @@ static __u32 __init search_agp_bridge(u32 *order, int *valid_agp)
199 case PCI_CLASS_BRIDGE_HOST: 221 case PCI_CLASS_BRIDGE_HOST:
200 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 222 case PCI_CLASS_BRIDGE_OTHER: /* needed? */
201 /* AGP bridge? */ 223 /* AGP bridge? */
202 cap = find_cap(num, slot, func, 224 cap = find_cap(bus, slot, func,
203 PCI_CAP_ID_AGP); 225 PCI_CAP_ID_AGP);
204 if (!cap) 226 if (!cap)
205 break; 227 break;
206 *valid_agp = 1; 228 *valid_agp = 1;
207 return read_agp(num, slot, func, cap, 229 return read_agp(bus, slot, func, cap,
208 order); 230 order);
209 } 231 }
210 232
211 /* No multi-function device? */ 233 /* No multi-function device? */
212 type = read_pci_config_byte(num, slot, func, 234 type = read_pci_config_byte(bus, slot, func,
213 PCI_HEADER_TYPE); 235 PCI_HEADER_TYPE);
214 if (!(type & 0x80)) 236 if (!(type & 0x80))
215 break; 237 break;
@@ -249,36 +271,50 @@ void __init early_gart_iommu_check(void)
249 * or BIOS forget to put that in reserved. 271 * or BIOS forget to put that in reserved.
250 * try to update e820 to make that region as reserved. 272 * try to update e820 to make that region as reserved.
251 */ 273 */
252 int fix, num; 274 int i, fix, slot;
253 u32 ctl; 275 u32 ctl;
254 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 276 u32 aper_size = 0, aper_order = 0, last_aper_order = 0;
255 u64 aper_base = 0, last_aper_base = 0; 277 u64 aper_base = 0, last_aper_base = 0;
256 int aper_enabled = 0, last_aper_enabled = 0; 278 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0;
257 279
258 if (!early_pci_allowed()) 280 if (!early_pci_allowed())
259 return; 281 return;
260 282
283 /* This is mostly duplicate of iommu_hole_init */
261 fix = 0; 284 fix = 0;
262 for (num = 24; num < 32; num++) { 285 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
263 if (!early_is_k8_nb(read_pci_config(0, num, 3, 0x00))) 286 int bus;
264 continue; 287 int dev_base, dev_limit;
265 288
266 ctl = read_pci_config(0, num, 3, 0x90); 289 bus = bus_dev_ranges[i].bus;
267 aper_enabled = ctl & 1; 290 dev_base = bus_dev_ranges[i].dev_base;
268 aper_order = (ctl >> 1) & 7; 291 dev_limit = bus_dev_ranges[i].dev_limit;
269 aper_size = (32 * 1024 * 1024) << aper_order; 292
270 aper_base = read_pci_config(0, num, 3, 0x94) & 0x7fff; 293 for (slot = dev_base; slot < dev_limit; slot++) {
271 aper_base <<= 25; 294 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
272 295 continue;
273 if ((last_aper_order && aper_order != last_aper_order) || 296
274 (last_aper_base && aper_base != last_aper_base) || 297 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
275 (last_aper_enabled && aper_enabled != last_aper_enabled)) { 298 aper_enabled = ctl & AMD64_GARTEN;
276 fix = 1; 299 aper_order = (ctl >> 1) & 7;
277 break; 300 aper_size = (32 * 1024 * 1024) << aper_order;
301 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
302 aper_base <<= 25;
303
304 if (last_valid) {
305 if ((aper_order != last_aper_order) ||
306 (aper_base != last_aper_base) ||
307 (aper_enabled != last_aper_enabled)) {
308 fix = 1;
309 break;
310 }
311 }
312
313 last_aper_order = aper_order;
314 last_aper_base = aper_base;
315 last_aper_enabled = aper_enabled;
316 last_valid = 1;
278 } 317 }
279 last_aper_order = aper_order;
280 last_aper_base = aper_base;
281 last_aper_enabled = aper_enabled;
282 } 318 }
283 319
284 if (!fix && !aper_enabled) 320 if (!fix && !aper_enabled)
@@ -290,32 +326,46 @@ void __init early_gart_iommu_check(void)
290 if (gart_fix_e820 && !fix && aper_enabled) { 326 if (gart_fix_e820 && !fix && aper_enabled) {
291 if (e820_any_mapped(aper_base, aper_base + aper_size, 327 if (e820_any_mapped(aper_base, aper_base + aper_size,
292 E820_RAM)) { 328 E820_RAM)) {
293 /* reserved it, so we can resuse it in second kernel */ 329 /* reserve it, so we can reuse it in second kernel */
294 printk(KERN_INFO "update e820 for GART\n"); 330 printk(KERN_INFO "update e820 for GART\n");
295 add_memory_region(aper_base, aper_size, E820_RESERVED); 331 e820_add_region(aper_base, aper_size, E820_RESERVED);
296 update_e820(); 332 update_e820();
297 } 333 }
298 return;
299 } 334 }
300 335
336 if (!fix)
337 return;
338
301 /* different nodes have different setting, disable them all at first*/ 339 /* different nodes have different setting, disable them all at first*/
302 for (num = 24; num < 32; num++) { 340 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
303 if (!early_is_k8_nb(read_pci_config(0, num, 3, 0x00))) 341 int bus;
304 continue; 342 int dev_base, dev_limit;
343
344 bus = bus_dev_ranges[i].bus;
345 dev_base = bus_dev_ranges[i].dev_base;
346 dev_limit = bus_dev_ranges[i].dev_limit;
347
348 for (slot = dev_base; slot < dev_limit; slot++) {
349 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
350 continue;
305 351
306 ctl = read_pci_config(0, num, 3, 0x90); 352 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
307 ctl &= ~1; 353 ctl &= ~AMD64_GARTEN;
308 write_pci_config(0, num, 3, 0x90, ctl); 354 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
355 }
309 } 356 }
310 357
311} 358}
312 359
360static int __initdata printed_gart_size_msg;
361
313void __init gart_iommu_hole_init(void) 362void __init gart_iommu_hole_init(void)
314{ 363{
364 u32 agp_aper_base = 0, agp_aper_order = 0;
315 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 365 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0;
316 u64 aper_base, last_aper_base = 0; 366 u64 aper_base, last_aper_base = 0;
317 int fix, num, valid_agp = 0; 367 int fix, slot, valid_agp = 0;
318 int node; 368 int i, node;
319 369
320 if (gart_iommu_aperture_disabled || !fix_aperture || 370 if (gart_iommu_aperture_disabled || !fix_aperture ||
321 !early_pci_allowed()) 371 !early_pci_allowed())
@@ -323,38 +373,65 @@ void __init gart_iommu_hole_init(void)
323 373
324 printk(KERN_INFO "Checking aperture...\n"); 374 printk(KERN_INFO "Checking aperture...\n");
325 375
376 if (!fallback_aper_force)
377 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp);
378
326 fix = 0; 379 fix = 0;
327 node = 0; 380 node = 0;
328 for (num = 24; num < 32; num++) { 381 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
329 if (!early_is_k8_nb(read_pci_config(0, num, 3, 0x00))) 382 int bus;
330 continue; 383 int dev_base, dev_limit;
331 384
332 iommu_detected = 1; 385 bus = bus_dev_ranges[i].bus;
333 gart_iommu_aperture = 1; 386 dev_base = bus_dev_ranges[i].dev_base;
334 387 dev_limit = bus_dev_ranges[i].dev_limit;
335 aper_order = (read_pci_config(0, num, 3, 0x90) >> 1) & 7; 388
336 aper_size = (32 * 1024 * 1024) << aper_order; 389 for (slot = dev_base; slot < dev_limit; slot++) {
337 aper_base = read_pci_config(0, num, 3, 0x94) & 0x7fff; 390 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
338 aper_base <<= 25; 391 continue;
339 392
340 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 393 iommu_detected = 1;
341 node, aper_base, aper_size >> 20); 394 gart_iommu_aperture = 1;
342 node++; 395
343 396 aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7;
344 if (!aperture_valid(aper_base, aper_size)) { 397 aper_size = (32 * 1024 * 1024) << aper_order;
345 fix = 1; 398 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
346 break; 399 aper_base <<= 25;
347 } 400
401 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n",
402 node, aper_base, aper_size >> 20);
403 node++;
404
405 if (!aperture_valid(aper_base, aper_size, 64<<20)) {
406 if (valid_agp && agp_aper_base &&
407 agp_aper_base == aper_base &&
408 agp_aper_order == aper_order) {
409 /* the same between two setting from NB and agp */
410 if (!no_iommu &&
411 max_pfn > MAX_DMA32_PFN &&
412 !printed_gart_size_msg) {
413 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n");
414 printk(KERN_ERR "please increase GART size in your BIOS setup\n");
415 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n");
416 printed_gart_size_msg = 1;
417 }
418 } else {
419 fix = 1;
420 goto out;
421 }
422 }
348 423
349 if ((last_aper_order && aper_order != last_aper_order) || 424 if ((last_aper_order && aper_order != last_aper_order) ||
350 (last_aper_base && aper_base != last_aper_base)) { 425 (last_aper_base && aper_base != last_aper_base)) {
351 fix = 1; 426 fix = 1;
352 break; 427 goto out;
428 }
429 last_aper_order = aper_order;
430 last_aper_base = aper_base;
353 } 431 }
354 last_aper_order = aper_order;
355 last_aper_base = aper_base;
356 } 432 }
357 433
434out:
358 if (!fix && !fallback_aper_force) { 435 if (!fix && !fallback_aper_force) {
359 if (last_aper_base) { 436 if (last_aper_base) {
360 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 437 unsigned long n = (32 * 1024 * 1024) << last_aper_order;
@@ -364,14 +441,16 @@ void __init gart_iommu_hole_init(void)
364 return; 441 return;
365 } 442 }
366 443
367 if (!fallback_aper_force) 444 if (!fallback_aper_force) {
368 aper_alloc = search_agp_bridge(&aper_order, &valid_agp); 445 aper_alloc = agp_aper_base;
446 aper_order = agp_aper_order;
447 }
369 448
370 if (aper_alloc) { 449 if (aper_alloc) {
371 /* Got the aperture from the AGP bridge */ 450 /* Got the aperture from the AGP bridge */
372 } else if (swiotlb && !valid_agp) { 451 } else if (swiotlb && !valid_agp) {
373 /* Do nothing */ 452 /* Do nothing */
374 } else if ((!no_iommu && end_pfn > MAX_DMA32_PFN) || 453 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) ||
375 force_iommu || 454 force_iommu ||
376 valid_agp || 455 valid_agp ||
377 fallback_aper_force) { 456 fallback_aper_force) {
@@ -401,16 +480,24 @@ void __init gart_iommu_hole_init(void)
401 } 480 }
402 481
403 /* Fix up the north bridges */ 482 /* Fix up the north bridges */
404 for (num = 24; num < 32; num++) { 483 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
405 if (!early_is_k8_nb(read_pci_config(0, num, 3, 0x00))) 484 int bus;
406 continue; 485 int dev_base, dev_limit;
407 486
408 /* 487 bus = bus_dev_ranges[i].bus;
409 * Don't enable translation yet. That is done later. 488 dev_base = bus_dev_ranges[i].dev_base;
410 * Assume this BIOS didn't initialise the GART so 489 dev_limit = bus_dev_ranges[i].dev_limit;
411 * just overwrite all previous bits 490 for (slot = dev_base; slot < dev_limit; slot++) {
412 */ 491 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
413 write_pci_config(0, num, 3, 0x90, aper_order<<1); 492 continue;
414 write_pci_config(0, num, 3, 0x94, aper_alloc>>25); 493
494 /* Don't enable translation yet. That is done later.
495 Assume this BIOS didn't initialise the GART so
496 just overwrite all previous bits */
497 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1);
498 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
499 }
415 } 500 }
501
502 set_up_gart_resume(aper_order, aper_alloc);
416} 503}
diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c
index 4b99b1bdeb6c..a437d027f20b 100644
--- a/arch/x86/kernel/apic_32.c
+++ b/arch/x86/kernel/apic_32.c
@@ -52,30 +52,41 @@
52 52
53unsigned long mp_lapic_addr; 53unsigned long mp_lapic_addr;
54 54
55DEFINE_PER_CPU(u16, x86_bios_cpu_apicid) = BAD_APICID;
56EXPORT_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
57
58/* 55/*
59 * Knob to control our willingness to enable the local APIC. 56 * Knob to control our willingness to enable the local APIC.
60 * 57 *
61 * -1=force-disable, +1=force-enable 58 * +1=force-enable
62 */ 59 */
63static int enable_local_apic __initdata; 60static int force_enable_local_apic;
61int disable_apic;
64 62
65/* Local APIC timer verification ok */ 63/* Local APIC timer verification ok */
66static int local_apic_timer_verify_ok; 64static int local_apic_timer_verify_ok;
67/* Disable local APIC timer from the kernel commandline or via dmi quirk 65/* Disable local APIC timer from the kernel commandline or via dmi quirk */
68 or using CPU MSR check */ 66static int local_apic_timer_disabled;
69int local_apic_timer_disabled;
70/* Local APIC timer works in C2 */ 67/* Local APIC timer works in C2 */
71int local_apic_timer_c2_ok; 68int local_apic_timer_c2_ok;
72EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 69EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
73 70
71int first_system_vector = 0xfe;
72
73char system_vectors[NR_VECTORS] = { [0 ... NR_VECTORS-1] = SYS_VECTOR_FREE};
74
74/* 75/*
75 * Debug level, exported for io_apic.c 76 * Debug level, exported for io_apic.c
76 */ 77 */
77int apic_verbosity; 78int apic_verbosity;
78 79
80int pic_mode;
81
82/* Have we found an MP table */
83int smp_found_config;
84
85static struct resource lapic_resource = {
86 .name = "Local APIC",
87 .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
88};
89
79static unsigned int calibration_result; 90static unsigned int calibration_result;
80 91
81static int lapic_next_event(unsigned long delta, 92static int lapic_next_event(unsigned long delta,
@@ -545,7 +556,7 @@ void __init setup_boot_APIC_clock(void)
545 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 556 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
546 else 557 else
547 printk(KERN_WARNING "APIC timer registered as dummy," 558 printk(KERN_WARNING "APIC timer registered as dummy,"
548 " due to nmi_watchdog=1!\n"); 559 " due to nmi_watchdog=%d!\n", nmi_watchdog);
549 } 560 }
550 561
551 /* Setup the lapic or request the broadcast */ 562 /* Setup the lapic or request the broadcast */
@@ -963,7 +974,7 @@ void __cpuinit setup_local_APIC(void)
963 * Double-check whether this APIC is really registered. 974 * Double-check whether this APIC is really registered.
964 */ 975 */
965 if (!apic_id_registered()) 976 if (!apic_id_registered())
966 BUG(); 977 WARN_ON_ONCE(1);
967 978
968 /* 979 /*
969 * Intel recommends to set DFR, LDR and TPR before enabling 980 * Intel recommends to set DFR, LDR and TPR before enabling
@@ -1094,7 +1105,7 @@ static int __init detect_init_APIC(void)
1094 u32 h, l, features; 1105 u32 h, l, features;
1095 1106
1096 /* Disabled by kernel option? */ 1107 /* Disabled by kernel option? */
1097 if (enable_local_apic < 0) 1108 if (disable_apic)
1098 return -1; 1109 return -1;
1099 1110
1100 switch (boot_cpu_data.x86_vendor) { 1111 switch (boot_cpu_data.x86_vendor) {
@@ -1117,7 +1128,7 @@ static int __init detect_init_APIC(void)
1117 * Over-ride BIOS and try to enable the local APIC only if 1128 * Over-ride BIOS and try to enable the local APIC only if
1118 * "lapic" specified. 1129 * "lapic" specified.
1119 */ 1130 */
1120 if (enable_local_apic <= 0) { 1131 if (!force_enable_local_apic) {
1121 printk(KERN_INFO "Local APIC disabled by BIOS -- " 1132 printk(KERN_INFO "Local APIC disabled by BIOS -- "
1122 "you can enable it with \"lapic\"\n"); 1133 "you can enable it with \"lapic\"\n");
1123 return -1; 1134 return -1;
@@ -1154,9 +1165,6 @@ static int __init detect_init_APIC(void)
1154 if (l & MSR_IA32_APICBASE_ENABLE) 1165 if (l & MSR_IA32_APICBASE_ENABLE)
1155 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE; 1166 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
1156 1167
1157 if (nmi_watchdog != NMI_NONE && nmi_watchdog != NMI_DISABLED)
1158 nmi_watchdog = NMI_LOCAL_APIC;
1159
1160 printk(KERN_INFO "Found and enabled local APIC!\n"); 1168 printk(KERN_INFO "Found and enabled local APIC!\n");
1161 1169
1162 apic_pm_activate(); 1170 apic_pm_activate();
@@ -1195,36 +1203,6 @@ void __init init_apic_mappings(void)
1195 if (boot_cpu_physical_apicid == -1U) 1203 if (boot_cpu_physical_apicid == -1U)
1196 boot_cpu_physical_apicid = GET_APIC_ID(read_apic_id()); 1204 boot_cpu_physical_apicid = GET_APIC_ID(read_apic_id());
1197 1205
1198#ifdef CONFIG_X86_IO_APIC
1199 {
1200 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
1201 int i;
1202
1203 for (i = 0; i < nr_ioapics; i++) {
1204 if (smp_found_config) {
1205 ioapic_phys = mp_ioapics[i].mpc_apicaddr;
1206 if (!ioapic_phys) {
1207 printk(KERN_ERR
1208 "WARNING: bogus zero IO-APIC "
1209 "address found in MPTABLE, "
1210 "disabling IO/APIC support!\n");
1211 smp_found_config = 0;
1212 skip_ioapic_setup = 1;
1213 goto fake_ioapic_page;
1214 }
1215 } else {
1216fake_ioapic_page:
1217 ioapic_phys = (unsigned long)
1218 alloc_bootmem_pages(PAGE_SIZE);
1219 ioapic_phys = __pa(ioapic_phys);
1220 }
1221 set_fixmap_nocache(idx, ioapic_phys);
1222 printk(KERN_DEBUG "mapped IOAPIC to %08lx (%08lx)\n",
1223 __fix_to_virt(idx), ioapic_phys);
1224 idx++;
1225 }
1226 }
1227#endif
1228} 1206}
1229 1207
1230/* 1208/*
@@ -1236,7 +1214,7 @@ int apic_version[MAX_APICS];
1236 1214
1237int __init APIC_init_uniprocessor(void) 1215int __init APIC_init_uniprocessor(void)
1238{ 1216{
1239 if (enable_local_apic < 0) 1217 if (disable_apic)
1240 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); 1218 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1241 1219
1242 if (!smp_found_config && !cpu_has_apic) 1220 if (!smp_found_config && !cpu_has_apic)
@@ -1265,10 +1243,14 @@ int __init APIC_init_uniprocessor(void)
1265#ifdef CONFIG_CRASH_DUMP 1243#ifdef CONFIG_CRASH_DUMP
1266 boot_cpu_physical_apicid = GET_APIC_ID(read_apic_id()); 1244 boot_cpu_physical_apicid = GET_APIC_ID(read_apic_id());
1267#endif 1245#endif
1268 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid); 1246 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1269 1247
1270 setup_local_APIC(); 1248 setup_local_APIC();
1271 1249
1250#ifdef CONFIG_X86_IO_APIC
1251 if (!smp_found_config || skip_ioapic_setup || !nr_ioapics)
1252#endif
1253 localise_nmi_watchdog();
1272 end_local_APIC_setup(); 1254 end_local_APIC_setup();
1273#ifdef CONFIG_X86_IO_APIC 1255#ifdef CONFIG_X86_IO_APIC
1274 if (smp_found_config) 1256 if (smp_found_config)
@@ -1351,13 +1333,17 @@ void __init smp_intr_init(void)
1351 * The reschedule interrupt is a CPU-to-CPU reschedule-helper 1333 * The reschedule interrupt is a CPU-to-CPU reschedule-helper
1352 * IPI, driven by wakeup. 1334 * IPI, driven by wakeup.
1353 */ 1335 */
1354 set_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt); 1336 alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
1355 1337
1356 /* IPI for invalidation */ 1338 /* IPI for invalidation */
1357 set_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt); 1339 alloc_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
1358 1340
1359 /* IPI for generic function call */ 1341 /* IPI for generic function call */
1360 set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); 1342 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
1343
1344 /* IPI for single call function */
1345 set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
1346 call_function_single_interrupt);
1361} 1347}
1362#endif 1348#endif
1363 1349
@@ -1370,15 +1356,15 @@ void __init apic_intr_init(void)
1370 smp_intr_init(); 1356 smp_intr_init();
1371#endif 1357#endif
1372 /* self generated IPI for local APIC timer */ 1358 /* self generated IPI for local APIC timer */
1373 set_intr_gate(LOCAL_TIMER_VECTOR, apic_timer_interrupt); 1359 alloc_intr_gate(LOCAL_TIMER_VECTOR, apic_timer_interrupt);
1374 1360
1375 /* IPI vectors for APIC spurious and error interrupts */ 1361 /* IPI vectors for APIC spurious and error interrupts */
1376 set_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt); 1362 alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
1377 set_intr_gate(ERROR_APIC_VECTOR, error_interrupt); 1363 alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
1378 1364
1379 /* thermal monitor LVT interrupt */ 1365 /* thermal monitor LVT interrupt */
1380#ifdef CONFIG_X86_MCE_P4THERMAL 1366#ifdef CONFIG_X86_MCE_P4THERMAL
1381 set_intr_gate(THERMAL_APIC_VECTOR, thermal_interrupt); 1367 alloc_intr_gate(THERMAL_APIC_VECTOR, thermal_interrupt);
1382#endif 1368#endif
1383} 1369}
1384 1370
@@ -1513,6 +1499,9 @@ void __cpuinit generic_processor_info(int apicid, int version)
1513 */ 1499 */
1514 cpu = 0; 1500 cpu = 0;
1515 1501
1502 if (apicid > max_physical_apicid)
1503 max_physical_apicid = apicid;
1504
1516 /* 1505 /*
1517 * Would be preferable to switch to bigsmp when CONFIG_HOTPLUG_CPU=y 1506 * Would be preferable to switch to bigsmp when CONFIG_HOTPLUG_CPU=y
1518 * but we need to work other dependencies like SMP_SUSPEND etc 1507 * but we need to work other dependencies like SMP_SUSPEND etc
@@ -1520,7 +1509,7 @@ void __cpuinit generic_processor_info(int apicid, int version)
1520 * if (CPU_HOTPLUG_ENABLED || num_processors > 8) 1509 * if (CPU_HOTPLUG_ENABLED || num_processors > 8)
1521 * - Ashok Raj <ashok.raj@intel.com> 1510 * - Ashok Raj <ashok.raj@intel.com>
1522 */ 1511 */
1523 if (num_processors > 8) { 1512 if (max_physical_apicid >= 8) {
1524 switch (boot_cpu_data.x86_vendor) { 1513 switch (boot_cpu_data.x86_vendor) {
1525 case X86_VENDOR_INTEL: 1514 case X86_VENDOR_INTEL:
1526 if (!APIC_XAPIC(version)) { 1515 if (!APIC_XAPIC(version)) {
@@ -1534,9 +1523,9 @@ void __cpuinit generic_processor_info(int apicid, int version)
1534 } 1523 }
1535#ifdef CONFIG_SMP 1524#ifdef CONFIG_SMP
1536 /* are we being called early in kernel startup? */ 1525 /* are we being called early in kernel startup? */
1537 if (x86_cpu_to_apicid_early_ptr) { 1526 if (early_per_cpu_ptr(x86_cpu_to_apicid)) {
1538 u16 *cpu_to_apicid = x86_cpu_to_apicid_early_ptr; 1527 u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
1539 u16 *bios_cpu_apicid = x86_bios_cpu_apicid_early_ptr; 1528 u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
1540 1529
1541 cpu_to_apicid[cpu] = apicid; 1530 cpu_to_apicid[cpu] = apicid;
1542 bios_cpu_apicid[cpu] = apicid; 1531 bios_cpu_apicid[cpu] = apicid;
@@ -1703,14 +1692,14 @@ static void apic_pm_activate(void) { }
1703 */ 1692 */
1704static int __init parse_lapic(char *arg) 1693static int __init parse_lapic(char *arg)
1705{ 1694{
1706 enable_local_apic = 1; 1695 force_enable_local_apic = 1;
1707 return 0; 1696 return 0;
1708} 1697}
1709early_param("lapic", parse_lapic); 1698early_param("lapic", parse_lapic);
1710 1699
1711static int __init parse_nolapic(char *arg) 1700static int __init parse_nolapic(char *arg)
1712{ 1701{
1713 enable_local_apic = -1; 1702 disable_apic = 1;
1714 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); 1703 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1715 return 0; 1704 return 0;
1716} 1705}
@@ -1740,3 +1729,21 @@ static int __init apic_set_verbosity(char *str)
1740} 1729}
1741__setup("apic=", apic_set_verbosity); 1730__setup("apic=", apic_set_verbosity);
1742 1731
1732static int __init lapic_insert_resource(void)
1733{
1734 if (!apic_phys)
1735 return -1;
1736
1737 /* Put local APIC into the resource map. */
1738 lapic_resource.start = apic_phys;
1739 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
1740 insert_resource(&iomem_resource, &lapic_resource);
1741
1742 return 0;
1743}
1744
1745/*
1746 * need call insert after e820_reserve_resources()
1747 * that is using request_resource
1748 */
1749late_initcall(lapic_insert_resource);
diff --git a/arch/x86/kernel/apic_64.c b/arch/x86/kernel/apic_64.c
index 0633cfd0dc29..1e3d32e27c14 100644
--- a/arch/x86/kernel/apic_64.c
+++ b/arch/x86/kernel/apic_64.c
@@ -43,7 +43,7 @@
43#include <mach_ipi.h> 43#include <mach_ipi.h>
44#include <mach_apic.h> 44#include <mach_apic.h>
45 45
46int disable_apic_timer __cpuinitdata; 46static int disable_apic_timer __cpuinitdata;
47static int apic_calibrate_pmtmr __initdata; 47static int apic_calibrate_pmtmr __initdata;
48int disable_apic; 48int disable_apic;
49 49
@@ -56,6 +56,9 @@ EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
56 */ 56 */
57int apic_verbosity; 57int apic_verbosity;
58 58
59/* Have we found an MP table */
60int smp_found_config;
61
59static struct resource lapic_resource = { 62static struct resource lapic_resource = {
60 .name = "Local APIC", 63 .name = "Local APIC",
61 .flags = IORESOURCE_MEM | IORESOURCE_BUSY, 64 .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
@@ -87,9 +90,6 @@ static unsigned long apic_phys;
87 90
88unsigned long mp_lapic_addr; 91unsigned long mp_lapic_addr;
89 92
90DEFINE_PER_CPU(u16, x86_bios_cpu_apicid) = BAD_APICID;
91EXPORT_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
92
93unsigned int __cpuinitdata maxcpus = NR_CPUS; 93unsigned int __cpuinitdata maxcpus = NR_CPUS;
94/* 94/*
95 * Get the LAPIC version 95 * Get the LAPIC version
@@ -417,37 +417,13 @@ void __init setup_boot_APIC_clock(void)
417 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY; 417 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
418 else 418 else
419 printk(KERN_WARNING "APIC timer registered as dummy," 419 printk(KERN_WARNING "APIC timer registered as dummy,"
420 " due to nmi_watchdog=1!\n"); 420 " due to nmi_watchdog=%d!\n", nmi_watchdog);
421 421
422 setup_APIC_timer(); 422 setup_APIC_timer();
423} 423}
424 424
425/*
426 * AMD C1E enabled CPUs have a real nasty problem: Some BIOSes set the
427 * C1E flag only in the secondary CPU, so when we detect the wreckage
428 * we already have enabled the boot CPU local apic timer. Check, if
429 * disable_apic_timer is set and the DUMMY flag is cleared. If yes,
430 * set the DUMMY flag again and force the broadcast mode in the
431 * clockevents layer.
432 */
433static void __cpuinit check_boot_apic_timer_broadcast(void)
434{
435 if (!disable_apic_timer ||
436 (lapic_clockevent.features & CLOCK_EVT_FEAT_DUMMY))
437 return;
438
439 printk(KERN_INFO "AMD C1E detected late. Force timer broadcast.\n");
440 lapic_clockevent.features |= CLOCK_EVT_FEAT_DUMMY;
441
442 local_irq_enable();
443 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
444 &boot_cpu_physical_apicid);
445 local_irq_disable();
446}
447
448void __cpuinit setup_secondary_APIC_clock(void) 425void __cpuinit setup_secondary_APIC_clock(void)
449{ 426{
450 check_boot_apic_timer_broadcast();
451 setup_APIC_timer(); 427 setup_APIC_timer();
452} 428}
453 429
@@ -850,7 +826,6 @@ static void __cpuinit lapic_setup_esr(void)
850void __cpuinit end_local_APIC_setup(void) 826void __cpuinit end_local_APIC_setup(void)
851{ 827{
852 lapic_setup_esr(); 828 lapic_setup_esr();
853 nmi_watchdog_default();
854 setup_apic_nmi_watchdog(NULL); 829 setup_apic_nmi_watchdog(NULL);
855 apic_pm_activate(); 830 apic_pm_activate();
856} 831}
@@ -875,7 +850,7 @@ static int __init detect_init_APIC(void)
875 850
876void __init early_init_lapic_mapping(void) 851void __init early_init_lapic_mapping(void)
877{ 852{
878 unsigned long apic_phys; 853 unsigned long phys_addr;
879 854
880 /* 855 /*
881 * If no local APIC can be found then go out 856 * If no local APIC can be found then go out
@@ -884,11 +859,11 @@ void __init early_init_lapic_mapping(void)
884 if (!smp_found_config) 859 if (!smp_found_config)
885 return; 860 return;
886 861
887 apic_phys = mp_lapic_addr; 862 phys_addr = mp_lapic_addr;
888 863
889 set_fixmap_nocache(FIX_APIC_BASE, apic_phys); 864 set_fixmap_nocache(FIX_APIC_BASE, phys_addr);
890 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", 865 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
891 APIC_BASE, apic_phys); 866 APIC_BASE, phys_addr);
892 867
893 /* 868 /*
894 * Fetch the APIC ID of the BSP in case we have a 869 * Fetch the APIC ID of the BSP in case we have a
@@ -942,7 +917,9 @@ int __init APIC_init_uniprocessor(void)
942 917
943 verify_local_APIC(); 918 verify_local_APIC();
944 919
945 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid); 920 connect_bsp_APIC();
921
922 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
946 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid)); 923 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
947 924
948 setup_local_APIC(); 925 setup_local_APIC();
@@ -954,6 +931,8 @@ int __init APIC_init_uniprocessor(void)
954 if (!skip_ioapic_setup && nr_ioapics) 931 if (!skip_ioapic_setup && nr_ioapics)
955 enable_IO_APIC(); 932 enable_IO_APIC();
956 933
934 if (!smp_found_config || skip_ioapic_setup || !nr_ioapics)
935 localise_nmi_watchdog();
957 end_local_APIC_setup(); 936 end_local_APIC_setup();
958 937
959 if (smp_found_config && !skip_ioapic_setup && nr_ioapics) 938 if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
@@ -1021,6 +1000,14 @@ asmlinkage void smp_error_interrupt(void)
1021 irq_exit(); 1000 irq_exit();
1022} 1001}
1023 1002
1003/**
1004 * * connect_bsp_APIC - attach the APIC to the interrupt system
1005 * */
1006void __init connect_bsp_APIC(void)
1007{
1008 enable_apic_mode();
1009}
1010
1024void disconnect_bsp_APIC(int virt_wire_setup) 1011void disconnect_bsp_APIC(int virt_wire_setup)
1025{ 1012{
1026 /* Go back to Virtual Wire compatibility mode */ 1013 /* Go back to Virtual Wire compatibility mode */
@@ -1090,10 +1077,13 @@ void __cpuinit generic_processor_info(int apicid, int version)
1090 */ 1077 */
1091 cpu = 0; 1078 cpu = 0;
1092 } 1079 }
1080 if (apicid > max_physical_apicid)
1081 max_physical_apicid = apicid;
1082
1093 /* are we being called early in kernel startup? */ 1083 /* are we being called early in kernel startup? */
1094 if (x86_cpu_to_apicid_early_ptr) { 1084 if (early_per_cpu_ptr(x86_cpu_to_apicid)) {
1095 u16 *cpu_to_apicid = x86_cpu_to_apicid_early_ptr; 1085 u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
1096 u16 *bios_cpu_apicid = x86_bios_cpu_apicid_early_ptr; 1086 u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
1097 1087
1098 cpu_to_apicid[cpu] = apicid; 1088 cpu_to_apicid[cpu] = apicid;
1099 bios_cpu_apicid[cpu] = apicid; 1089 bios_cpu_apicid[cpu] = apicid;
@@ -1269,7 +1259,7 @@ __cpuinit int apic_is_clustered_box(void)
1269 if ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && !is_vsmp_box()) 1259 if ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && !is_vsmp_box())
1270 return 0; 1260 return 0;
1271 1261
1272 bios_cpu_apicid = x86_bios_cpu_apicid_early_ptr; 1262 bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
1273 bitmap_zero(clustermap, NUM_APIC_CLUSTERS); 1263 bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
1274 1264
1275 for (i = 0; i < NR_CPUS; i++) { 1265 for (i = 0; i < NR_CPUS; i++) {
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index bf9290e29013..bf9b441331e9 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -204,6 +204,7 @@
204#include <linux/module.h> 204#include <linux/module.h>
205 205
206#include <linux/poll.h> 206#include <linux/poll.h>
207#include <linux/smp_lock.h>
207#include <linux/types.h> 208#include <linux/types.h>
208#include <linux/stddef.h> 209#include <linux/stddef.h>
209#include <linux/timer.h> 210#include <linux/timer.h>
@@ -228,6 +229,7 @@
228#include <linux/suspend.h> 229#include <linux/suspend.h>
229#include <linux/kthread.h> 230#include <linux/kthread.h>
230#include <linux/jiffies.h> 231#include <linux/jiffies.h>
232#include <linux/smp_lock.h>
231 233
232#include <asm/system.h> 234#include <asm/system.h>
233#include <asm/uaccess.h> 235#include <asm/uaccess.h>
@@ -1149,7 +1151,7 @@ static void queue_event(apm_event_t event, struct apm_user *sender)
1149 as->event_tail = 0; 1151 as->event_tail = 0;
1150 } 1152 }
1151 as->events[as->event_head] = event; 1153 as->events[as->event_head] = event;
1152 if ((!as->suser) || (!as->writer)) 1154 if (!as->suser || !as->writer)
1153 continue; 1155 continue;
1154 switch (event) { 1156 switch (event) {
1155 case APM_SYS_SUSPEND: 1157 case APM_SYS_SUSPEND:
@@ -1211,9 +1213,9 @@ static int suspend(int vetoable)
1211 if (err != APM_SUCCESS) 1213 if (err != APM_SUCCESS)
1212 apm_error("suspend", err); 1214 apm_error("suspend", err);
1213 err = (err == APM_SUCCESS) ? 0 : -EIO; 1215 err = (err == APM_SUCCESS) ? 0 : -EIO;
1214 device_power_up(); 1216 device_power_up(PMSG_RESUME);
1215 local_irq_enable(); 1217 local_irq_enable();
1216 device_resume(); 1218 device_resume(PMSG_RESUME);
1217 queue_event(APM_NORMAL_RESUME, NULL); 1219 queue_event(APM_NORMAL_RESUME, NULL);
1218 spin_lock(&user_list_lock); 1220 spin_lock(&user_list_lock);
1219 for (as = user_list; as != NULL; as = as->next) { 1221 for (as = user_list; as != NULL; as = as->next) {
@@ -1238,7 +1240,7 @@ static void standby(void)
1238 apm_error("standby", err); 1240 apm_error("standby", err);
1239 1241
1240 local_irq_disable(); 1242 local_irq_disable();
1241 device_power_up(); 1243 device_power_up(PMSG_RESUME);
1242 local_irq_enable(); 1244 local_irq_enable();
1243} 1245}
1244 1246
@@ -1324,7 +1326,7 @@ static void check_events(void)
1324 ignore_bounce = 1; 1326 ignore_bounce = 1;
1325 if ((event != APM_NORMAL_RESUME) 1327 if ((event != APM_NORMAL_RESUME)
1326 || (ignore_normal_resume == 0)) { 1328 || (ignore_normal_resume == 0)) {
1327 device_resume(); 1329 device_resume(PMSG_RESUME);
1328 queue_event(event, NULL); 1330 queue_event(event, NULL);
1329 } 1331 }
1330 ignore_normal_resume = 0; 1332 ignore_normal_resume = 0;
@@ -1396,7 +1398,7 @@ static void apm_mainloop(void)
1396 1398
1397static int check_apm_user(struct apm_user *as, const char *func) 1399static int check_apm_user(struct apm_user *as, const char *func)
1398{ 1400{
1399 if ((as == NULL) || (as->magic != APM_BIOS_MAGIC)) { 1401 if (as == NULL || as->magic != APM_BIOS_MAGIC) {
1400 printk(KERN_ERR "apm: %s passed bad filp\n", func); 1402 printk(KERN_ERR "apm: %s passed bad filp\n", func);
1401 return 1; 1403 return 1;
1402 } 1404 }
@@ -1459,18 +1461,19 @@ static unsigned int do_poll(struct file *fp, poll_table *wait)
1459 return 0; 1461 return 0;
1460} 1462}
1461 1463
1462static int do_ioctl(struct inode *inode, struct file *filp, 1464static long do_ioctl(struct file *filp, u_int cmd, u_long arg)
1463 u_int cmd, u_long arg)
1464{ 1465{
1465 struct apm_user *as; 1466 struct apm_user *as;
1467 int ret;
1466 1468
1467 as = filp->private_data; 1469 as = filp->private_data;
1468 if (check_apm_user(as, "ioctl")) 1470 if (check_apm_user(as, "ioctl"))
1469 return -EIO; 1471 return -EIO;
1470 if ((!as->suser) || (!as->writer)) 1472 if (!as->suser || !as->writer)
1471 return -EPERM; 1473 return -EPERM;
1472 switch (cmd) { 1474 switch (cmd) {
1473 case APM_IOC_STANDBY: 1475 case APM_IOC_STANDBY:
1476 lock_kernel();
1474 if (as->standbys_read > 0) { 1477 if (as->standbys_read > 0) {
1475 as->standbys_read--; 1478 as->standbys_read--;
1476 as->standbys_pending--; 1479 as->standbys_pending--;
@@ -1479,8 +1482,10 @@ static int do_ioctl(struct inode *inode, struct file *filp,
1479 queue_event(APM_USER_STANDBY, as); 1482 queue_event(APM_USER_STANDBY, as);
1480 if (standbys_pending <= 0) 1483 if (standbys_pending <= 0)
1481 standby(); 1484 standby();
1485 unlock_kernel();
1482 break; 1486 break;
1483 case APM_IOC_SUSPEND: 1487 case APM_IOC_SUSPEND:
1488 lock_kernel();
1484 if (as->suspends_read > 0) { 1489 if (as->suspends_read > 0) {
1485 as->suspends_read--; 1490 as->suspends_read--;
1486 as->suspends_pending--; 1491 as->suspends_pending--;
@@ -1488,16 +1493,17 @@ static int do_ioctl(struct inode *inode, struct file *filp,
1488 } else 1493 } else
1489 queue_event(APM_USER_SUSPEND, as); 1494 queue_event(APM_USER_SUSPEND, as);
1490 if (suspends_pending <= 0) { 1495 if (suspends_pending <= 0) {
1491 return suspend(1); 1496 ret = suspend(1);
1492 } else { 1497 } else {
1493 as->suspend_wait = 1; 1498 as->suspend_wait = 1;
1494 wait_event_interruptible(apm_suspend_waitqueue, 1499 wait_event_interruptible(apm_suspend_waitqueue,
1495 as->suspend_wait == 0); 1500 as->suspend_wait == 0);
1496 return as->suspend_result; 1501 ret = as->suspend_result;
1497 } 1502 }
1498 break; 1503 unlock_kernel();
1504 return ret;
1499 default: 1505 default:
1500 return -EINVAL; 1506 return -ENOTTY;
1501 } 1507 }
1502 return 0; 1508 return 0;
1503} 1509}
@@ -1544,10 +1550,12 @@ static int do_open(struct inode *inode, struct file *filp)
1544{ 1550{
1545 struct apm_user *as; 1551 struct apm_user *as;
1546 1552
1553 lock_kernel();
1547 as = kmalloc(sizeof(*as), GFP_KERNEL); 1554 as = kmalloc(sizeof(*as), GFP_KERNEL);
1548 if (as == NULL) { 1555 if (as == NULL) {
1549 printk(KERN_ERR "apm: cannot allocate struct of size %d bytes\n", 1556 printk(KERN_ERR "apm: cannot allocate struct of size %d bytes\n",
1550 sizeof(*as)); 1557 sizeof(*as));
1558 unlock_kernel();
1551 return -ENOMEM; 1559 return -ENOMEM;
1552 } 1560 }
1553 as->magic = APM_BIOS_MAGIC; 1561 as->magic = APM_BIOS_MAGIC;
@@ -1569,6 +1577,7 @@ static int do_open(struct inode *inode, struct file *filp)
1569 user_list = as; 1577 user_list = as;
1570 spin_unlock(&user_list_lock); 1578 spin_unlock(&user_list_lock);
1571 filp->private_data = as; 1579 filp->private_data = as;
1580 unlock_kernel();
1572 return 0; 1581 return 0;
1573} 1582}
1574 1583
@@ -1860,7 +1869,7 @@ static const struct file_operations apm_bios_fops = {
1860 .owner = THIS_MODULE, 1869 .owner = THIS_MODULE,
1861 .read = do_read, 1870 .read = do_read,
1862 .poll = do_poll, 1871 .poll = do_poll,
1863 .ioctl = do_ioctl, 1872 .unlocked_ioctl = do_ioctl,
1864 .open = do_open, 1873 .open = do_open,
1865 .release = do_release, 1874 .release = do_release,
1866}; 1875};
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 92588083950f..6649d09ad88f 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -111,7 +111,7 @@ void foo(void)
111 OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable); 111 OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable);
112 OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable); 112 OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable);
113 OFFSET(PV_CPU_iret, pv_cpu_ops, iret); 113 OFFSET(PV_CPU_iret, pv_cpu_ops, iret);
114 OFFSET(PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret); 114 OFFSET(PV_CPU_irq_enable_sysexit, pv_cpu_ops, irq_enable_sysexit);
115 OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0); 115 OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0);
116#endif 116#endif
117 117
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index f126c05d6170..bacf5deeec2d 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -34,7 +34,7 @@ int main(void)
34 ENTRY(pid); 34 ENTRY(pid);
35 BLANK(); 35 BLANK();
36#undef ENTRY 36#undef ENTRY
37#define ENTRY(entry) DEFINE(threadinfo_ ## entry, offsetof(struct thread_info, entry)) 37#define ENTRY(entry) DEFINE(TI_ ## entry, offsetof(struct thread_info, entry))
38 ENTRY(flags); 38 ENTRY(flags);
39 ENTRY(addr_limit); 39 ENTRY(addr_limit);
40 ENTRY(preempt_count); 40 ENTRY(preempt_count);
@@ -61,8 +61,11 @@ int main(void)
61 OFFSET(PARAVIRT_PATCH_pv_irq_ops, paravirt_patch_template, pv_irq_ops); 61 OFFSET(PARAVIRT_PATCH_pv_irq_ops, paravirt_patch_template, pv_irq_ops);
62 OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable); 62 OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable);
63 OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable); 63 OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable);
64 OFFSET(PV_IRQ_adjust_exception_frame, pv_irq_ops, adjust_exception_frame);
64 OFFSET(PV_CPU_iret, pv_cpu_ops, iret); 65 OFFSET(PV_CPU_iret, pv_cpu_ops, iret);
65 OFFSET(PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret); 66 OFFSET(PV_CPU_usergs_sysret32, pv_cpu_ops, usergs_sysret32);
67 OFFSET(PV_CPU_usergs_sysret64, pv_cpu_ops, usergs_sysret64);
68 OFFSET(PV_CPU_irq_enable_sysexit, pv_cpu_ops, irq_enable_sysexit);
66 OFFSET(PV_CPU_swapgs, pv_cpu_ops, swapgs); 69 OFFSET(PV_CPU_swapgs, pv_cpu_ops, swapgs);
67 OFFSET(PV_MMU_read_cr2, pv_mmu_ops, read_cr2); 70 OFFSET(PV_MMU_read_cr2, pv_mmu_ops, read_cr2);
68#endif 71#endif
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index a0c6f8190887..ee76eaad3001 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -6,11 +6,15 @@ obj-y := intel_cacheinfo.o addon_cpuid_features.o
6obj-y += proc.o feature_names.o 6obj-y += proc.o feature_names.o
7 7
8obj-$(CONFIG_X86_32) += common.o bugs.o 8obj-$(CONFIG_X86_32) += common.o bugs.o
9obj-$(CONFIG_X86_64) += common_64.o bugs_64.o
9obj-$(CONFIG_X86_32) += amd.o 10obj-$(CONFIG_X86_32) += amd.o
11obj-$(CONFIG_X86_64) += amd_64.o
10obj-$(CONFIG_X86_32) += cyrix.o 12obj-$(CONFIG_X86_32) += cyrix.o
11obj-$(CONFIG_X86_32) += centaur.o 13obj-$(CONFIG_X86_32) += centaur.o
14obj-$(CONFIG_X86_64) += centaur_64.o
12obj-$(CONFIG_X86_32) += transmeta.o 15obj-$(CONFIG_X86_32) += transmeta.o
13obj-$(CONFIG_X86_32) += intel.o 16obj-$(CONFIG_X86_32) += intel.o
17obj-$(CONFIG_X86_64) += intel_64.o
14obj-$(CONFIG_X86_32) += umc.o 18obj-$(CONFIG_X86_32) += umc.o
15 19
16obj-$(CONFIG_X86_MCE) += mcheck/ 20obj-$(CONFIG_X86_MCE) += mcheck/
diff --git a/arch/x86/kernel/cpu/addon_cpuid_features.c b/arch/x86/kernel/cpu/addon_cpuid_features.c
index c2e1ce33c7cb..84a8220a6072 100644
--- a/arch/x86/kernel/cpu/addon_cpuid_features.c
+++ b/arch/x86/kernel/cpu/addon_cpuid_features.c
@@ -1,9 +1,7 @@
1
2/* 1/*
3 * Routines to indentify additional cpu features that are scattered in 2 * Routines to indentify additional cpu features that are scattered in
4 * cpuid space. 3 * cpuid space.
5 */ 4 */
6
7#include <linux/cpu.h> 5#include <linux/cpu.h>
8 6
9#include <asm/pat.h> 7#include <asm/pat.h>
@@ -53,19 +51,20 @@ void __cpuinit init_scattered_cpuid_features(struct cpuinfo_x86 *c)
53#ifdef CONFIG_X86_PAT 51#ifdef CONFIG_X86_PAT
54void __cpuinit validate_pat_support(struct cpuinfo_x86 *c) 52void __cpuinit validate_pat_support(struct cpuinfo_x86 *c)
55{ 53{
54 if (!cpu_has_pat)
55 pat_disable("PAT not supported by CPU.");
56
56 switch (c->x86_vendor) { 57 switch (c->x86_vendor) {
57 case X86_VENDOR_AMD:
58 if (c->x86 >= 0xf && c->x86 <= 0x11)
59 return;
60 break;
61 case X86_VENDOR_INTEL: 58 case X86_VENDOR_INTEL:
62 if (c->x86 == 0xF || (c->x86 == 6 && c->x86_model >= 15)) 59 if (c->x86 == 0xF || (c->x86 == 6 && c->x86_model >= 15))
63 return; 60 return;
64 break; 61 break;
62 case X86_VENDOR_AMD:
63 case X86_VENDOR_CENTAUR:
64 case X86_VENDOR_TRANSMETA:
65 return;
65 } 66 }
66 67
67 pat_disable(cpu_has_pat ? 68 pat_disable("PAT disabled. Not yet verified on this CPU type.");
68 "PAT disabled. Not yet verified on this CPU type." :
69 "PAT not supported by CPU.");
70} 69}
71#endif 70#endif
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 245866828294..81a07ca65d44 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -24,43 +24,6 @@
24extern void vide(void); 24extern void vide(void);
25__asm__(".align 4\nvide: ret"); 25__asm__(".align 4\nvide: ret");
26 26
27#ifdef CONFIG_X86_LOCAL_APIC
28#define ENABLE_C1E_MASK 0x18000000
29#define CPUID_PROCESSOR_SIGNATURE 1
30#define CPUID_XFAM 0x0ff00000
31#define CPUID_XFAM_K8 0x00000000
32#define CPUID_XFAM_10H 0x00100000
33#define CPUID_XFAM_11H 0x00200000
34#define CPUID_XMOD 0x000f0000
35#define CPUID_XMOD_REV_F 0x00040000
36
37/* AMD systems with C1E don't have a working lAPIC timer. Check for that. */
38static __cpuinit int amd_apic_timer_broken(void)
39{
40 u32 lo, hi;
41 u32 eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
42 switch (eax & CPUID_XFAM) {
43 case CPUID_XFAM_K8:
44 if ((eax & CPUID_XMOD) < CPUID_XMOD_REV_F)
45 break;
46 case CPUID_XFAM_10H:
47 case CPUID_XFAM_11H:
48 rdmsr(MSR_K8_ENABLE_C1E, lo, hi);
49 if (lo & ENABLE_C1E_MASK) {
50 if (smp_processor_id() != boot_cpu_physical_apicid)
51 printk(KERN_INFO "AMD C1E detected late. "
52 " Force timer broadcast.\n");
53 return 1;
54 }
55 break;
56 default:
57 /* err on the side of caution */
58 return 1;
59 }
60 return 0;
61}
62#endif
63
64int force_mwait __cpuinitdata; 27int force_mwait __cpuinitdata;
65 28
66static void __cpuinit early_init_amd(struct cpuinfo_x86 *c) 29static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
@@ -297,11 +260,6 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
297 num_cache_leaves = 3; 260 num_cache_leaves = 3;
298 } 261 }
299 262
300#ifdef CONFIG_X86_LOCAL_APIC
301 if (amd_apic_timer_broken())
302 local_apic_timer_disabled = 1;
303#endif
304
305 /* K6s reports MCEs but don't actually have all the MSRs */ 263 /* K6s reports MCEs but don't actually have all the MSRs */
306 if (c->x86 < 6) 264 if (c->x86 < 6)
307 clear_cpu_cap(c, X86_FEATURE_MCE); 265 clear_cpu_cap(c, X86_FEATURE_MCE);
diff --git a/arch/x86/kernel/cpu/amd_64.c b/arch/x86/kernel/cpu/amd_64.c
new file mode 100644
index 000000000000..7c36fb8a28d4
--- /dev/null
+++ b/arch/x86/kernel/cpu/amd_64.c
@@ -0,0 +1,222 @@
1#include <linux/init.h>
2#include <linux/mm.h>
3
4#include <asm/numa_64.h>
5#include <asm/mmconfig.h>
6#include <asm/cacheflush.h>
7
8#include <mach_apic.h>
9
10#include "cpu.h"
11
12int force_mwait __cpuinitdata;
13
14#ifdef CONFIG_NUMA
15static int __cpuinit nearby_node(int apicid)
16{
17 int i, node;
18
19 for (i = apicid - 1; i >= 0; i--) {
20 node = apicid_to_node[i];
21 if (node != NUMA_NO_NODE && node_online(node))
22 return node;
23 }
24 for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
25 node = apicid_to_node[i];
26 if (node != NUMA_NO_NODE && node_online(node))
27 return node;
28 }
29 return first_node(node_online_map); /* Shouldn't happen */
30}
31#endif
32
33/*
34 * On a AMD dual core setup the lower bits of the APIC id distingush the cores.
35 * Assumes number of cores is a power of two.
36 */
37static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
38{
39#ifdef CONFIG_SMP
40 unsigned bits;
41#ifdef CONFIG_NUMA
42 int cpu = smp_processor_id();
43 int node = 0;
44 unsigned apicid = hard_smp_processor_id();
45#endif
46 bits = c->x86_coreid_bits;
47
48 /* Low order bits define the core id (index of core in socket) */
49 c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
50 /* Convert the initial APIC ID into the socket ID */
51 c->phys_proc_id = c->initial_apicid >> bits;
52
53#ifdef CONFIG_NUMA
54 node = c->phys_proc_id;
55 if (apicid_to_node[apicid] != NUMA_NO_NODE)
56 node = apicid_to_node[apicid];
57 if (!node_online(node)) {
58 /* Two possibilities here:
59 - The CPU is missing memory and no node was created.
60 In that case try picking one from a nearby CPU
61 - The APIC IDs differ from the HyperTransport node IDs
62 which the K8 northbridge parsing fills in.
63 Assume they are all increased by a constant offset,
64 but in the same order as the HT nodeids.
65 If that doesn't result in a usable node fall back to the
66 path for the previous case. */
67
68 int ht_nodeid = c->initial_apicid;
69
70 if (ht_nodeid >= 0 &&
71 apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
72 node = apicid_to_node[ht_nodeid];
73 /* Pick a nearby node */
74 if (!node_online(node))
75 node = nearby_node(apicid);
76 }
77 numa_set_node(cpu, node);
78
79 printk(KERN_INFO "CPU %d/%x -> Node %d\n", cpu, apicid, node);
80#endif
81#endif
82}
83
84static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c)
85{
86#ifdef CONFIG_SMP
87 unsigned bits, ecx;
88
89 /* Multi core CPU? */
90 if (c->extended_cpuid_level < 0x80000008)
91 return;
92
93 ecx = cpuid_ecx(0x80000008);
94
95 c->x86_max_cores = (ecx & 0xff) + 1;
96
97 /* CPU telling us the core id bits shift? */
98 bits = (ecx >> 12) & 0xF;
99
100 /* Otherwise recompute */
101 if (bits == 0) {
102 while ((1 << bits) < c->x86_max_cores)
103 bits++;
104 }
105
106 c->x86_coreid_bits = bits;
107
108#endif
109}
110
111static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
112{
113 early_init_amd_mc(c);
114
115 /* c->x86_power is 8000_0007 edx. Bit 8 is constant TSC */
116 if (c->x86_power & (1<<8))
117 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
118}
119
120static void __cpuinit init_amd(struct cpuinfo_x86 *c)
121{
122 unsigned level;
123
124#ifdef CONFIG_SMP
125 unsigned long value;
126
127 /*
128 * Disable TLB flush filter by setting HWCR.FFDIS on K8
129 * bit 6 of msr C001_0015
130 *
131 * Errata 63 for SH-B3 steppings
132 * Errata 122 for all steppings (F+ have it disabled by default)
133 */
134 if (c->x86 == 0xf) {
135 rdmsrl(MSR_K8_HWCR, value);
136 value |= 1 << 6;
137 wrmsrl(MSR_K8_HWCR, value);
138 }
139#endif
140
141 /* Bit 31 in normal CPUID used for nonstandard 3DNow ID;
142 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway */
143 clear_cpu_cap(c, 0*32+31);
144
145 /* On C+ stepping K8 rep microcode works well for copy/memset */
146 if (c->x86 == 0xf) {
147 level = cpuid_eax(1);
148 if((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
149 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
150 }
151 if (c->x86 == 0x10 || c->x86 == 0x11)
152 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
153
154 /* Enable workaround for FXSAVE leak */
155 if (c->x86 >= 6)
156 set_cpu_cap(c, X86_FEATURE_FXSAVE_LEAK);
157
158 level = get_model_name(c);
159 if (!level) {
160 switch (c->x86) {
161 case 0xf:
162 /* Should distinguish Models here, but this is only
163 a fallback anyways. */
164 strcpy(c->x86_model_id, "Hammer");
165 break;
166 }
167 }
168 display_cacheinfo(c);
169
170 /* Multi core CPU? */
171 if (c->extended_cpuid_level >= 0x80000008)
172 amd_detect_cmp(c);
173
174 if (c->extended_cpuid_level >= 0x80000006 &&
175 (cpuid_edx(0x80000006) & 0xf000))
176 num_cache_leaves = 4;
177 else
178 num_cache_leaves = 3;
179
180 if (c->x86 >= 0xf && c->x86 <= 0x11)
181 set_cpu_cap(c, X86_FEATURE_K8);
182
183 /* MFENCE stops RDTSC speculation */
184 set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
185
186 if (c->x86 == 0x10) {
187 /* do this for boot cpu */
188 if (c == &boot_cpu_data)
189 check_enable_amd_mmconf_dmi();
190
191 fam10h_check_enable_mmcfg();
192 }
193
194 if (c == &boot_cpu_data && c->x86 >= 0xf && c->x86 <= 0x11) {
195 unsigned long long tseg;
196
197 /*
198 * Split up direct mapping around the TSEG SMM area.
199 * Don't do it for gbpages because there seems very little
200 * benefit in doing so.
201 */
202 if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
203 printk(KERN_DEBUG "tseg: %010llx\n", tseg);
204 if ((tseg>>PMD_SHIFT) <
205 (max_low_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) ||
206 ((tseg>>PMD_SHIFT) <
207 (max_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) &&
208 (tseg>>PMD_SHIFT) >= (1ULL<<(32 - PMD_SHIFT))))
209 set_memory_4k((unsigned long)__va(tseg), 1);
210 }
211 }
212}
213
214static struct cpu_dev amd_cpu_dev __cpuinitdata = {
215 .c_vendor = "AMD",
216 .c_ident = { "AuthenticAMD" },
217 .c_early_init = early_init_amd,
218 .c_init = init_amd,
219};
220
221cpu_vendor_dev_register(X86_VENDOR_AMD, &amd_cpu_dev);
222
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 170d2f5523b2..1b1c56bb338f 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -59,8 +59,12 @@ static void __init check_fpu(void)
59 return; 59 return;
60 } 60 }
61 61
62/* trap_init() enabled FXSR and company _before_ testing for FP problems here. */ 62 /*
63 /* Test for the divl bug.. */ 63 * trap_init() enabled FXSR and company _before_ testing for FP
64 * problems here.
65 *
66 * Test for the divl bug..
67 */
64 __asm__("fninit\n\t" 68 __asm__("fninit\n\t"
65 "fldl %1\n\t" 69 "fldl %1\n\t"
66 "fdivl %2\n\t" 70 "fdivl %2\n\t"
@@ -108,10 +112,15 @@ static void __init check_popad(void)
108 "movl $12345678,%%eax; movl $0,%%edi; pusha; popa; movl (%%edx,%%edi),%%ecx " 112 "movl $12345678,%%eax; movl $0,%%edi; pusha; popa; movl (%%edx,%%edi),%%ecx "
109 : "=&a" (res) 113 : "=&a" (res)
110 : "d" (inp) 114 : "d" (inp)
111 : "ecx", "edi" ); 115 : "ecx", "edi");
112 /* If this fails, it means that any user program may lock the CPU hard. Too bad. */ 116 /*
113 if (res != 12345678) printk( "Buggy.\n" ); 117 * If this fails, it means that any user program may lock the
114 else printk( "OK.\n" ); 118 * CPU hard. Too bad.
119 */
120 if (res != 12345678)
121 printk("Buggy.\n");
122 else
123 printk("OK.\n");
115#endif 124#endif
116} 125}
117 126
@@ -137,7 +146,8 @@ static void __init check_config(void)
137 * i486+ only features! (WP works in supervisor mode and the 146 * i486+ only features! (WP works in supervisor mode and the
138 * new "invlpg" and "bswap" instructions) 147 * new "invlpg" and "bswap" instructions)
139 */ 148 */
140#if defined(CONFIG_X86_WP_WORKS_OK) || defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_BSWAP) 149#if defined(CONFIG_X86_WP_WORKS_OK) || defined(CONFIG_X86_INVLPG) || \
150 defined(CONFIG_X86_BSWAP)
141 if (boot_cpu_data.x86 == 3) 151 if (boot_cpu_data.x86 == 3)
142 panic("Kernel requires i486+ for 'invlpg' and other features"); 152 panic("Kernel requires i486+ for 'invlpg' and other features");
143#endif 153#endif
@@ -170,6 +180,7 @@ void __init check_bugs(void)
170 check_fpu(); 180 check_fpu();
171 check_hlt(); 181 check_hlt();
172 check_popad(); 182 check_popad();
173 init_utsname()->machine[1] = '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 183 init_utsname()->machine[1] =
184 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
174 alternative_instructions(); 185 alternative_instructions();
175} 186}
diff --git a/arch/x86/kernel/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index 9a3ed0649d4e..9a3ed0649d4e 100644
--- a/arch/x86/kernel/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
diff --git a/arch/x86/kernel/cpu/centaur_64.c b/arch/x86/kernel/cpu/centaur_64.c
new file mode 100644
index 000000000000..1d181c40e2e1
--- /dev/null
+++ b/arch/x86/kernel/cpu/centaur_64.c
@@ -0,0 +1,35 @@
1#include <linux/init.h>
2#include <linux/smp.h>
3
4#include <asm/cpufeature.h>
5#include <asm/processor.h>
6
7#include "cpu.h"
8
9static void __cpuinit early_init_centaur(struct cpuinfo_x86 *c)
10{
11 if (c->x86 == 0x6 && c->x86_model >= 0xf)
12 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
13
14 set_cpu_cap(c, X86_FEATURE_SYSENTER32);
15}
16
17static void __cpuinit init_centaur(struct cpuinfo_x86 *c)
18{
19 if (c->x86 == 0x6 && c->x86_model >= 0xf) {
20 c->x86_cache_alignment = c->x86_clflush_size * 2;
21 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
22 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
23 }
24 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
25}
26
27static struct cpu_dev centaur_cpu_dev __cpuinitdata = {
28 .c_vendor = "Centaur",
29 .c_ident = { "CentaurHauls" },
30 .c_early_init = early_init_centaur,
31 .c_init = init_centaur,
32};
33
34cpu_vendor_dev_register(X86_VENDOR_CENTAUR, &centaur_cpu_dev);
35
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index d0463a946247..80ab20d4fa39 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -427,7 +427,7 @@ __setup("serialnumber", x86_serial_nr_setup);
427/* 427/*
428 * This does the hard work of actually picking apart the CPU stuff... 428 * This does the hard work of actually picking apart the CPU stuff...
429 */ 429 */
430void __cpuinit identify_cpu(struct cpuinfo_x86 *c) 430static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
431{ 431{
432 int i; 432 int i;
433 433
diff --git a/arch/x86/kernel/cpu/common_64.c b/arch/x86/kernel/cpu/common_64.c
new file mode 100644
index 000000000000..7b8cc72feb40
--- /dev/null
+++ b/arch/x86/kernel/cpu/common_64.c
@@ -0,0 +1,681 @@
1#include <linux/init.h>
2#include <linux/kernel.h>
3#include <linux/sched.h>
4#include <linux/string.h>
5#include <linux/bootmem.h>
6#include <linux/bitops.h>
7#include <linux/module.h>
8#include <linux/kgdb.h>
9#include <linux/topology.h>
10#include <linux/string.h>
11#include <linux/delay.h>
12#include <linux/smp.h>
13#include <linux/module.h>
14#include <linux/percpu.h>
15#include <asm/processor.h>
16#include <asm/i387.h>
17#include <asm/msr.h>
18#include <asm/io.h>
19#include <asm/mmu_context.h>
20#include <asm/mtrr.h>
21#include <asm/mce.h>
22#include <asm/pat.h>
23#include <asm/numa.h>
24#ifdef CONFIG_X86_LOCAL_APIC
25#include <asm/mpspec.h>
26#include <asm/apic.h>
27#include <mach_apic.h>
28#endif
29#include <asm/pda.h>
30#include <asm/pgtable.h>
31#include <asm/processor.h>
32#include <asm/desc.h>
33#include <asm/atomic.h>
34#include <asm/proto.h>
35#include <asm/sections.h>
36#include <asm/setup.h>
37#include <asm/genapic.h>
38
39#include "cpu.h"
40
41/* We need valid kernel segments for data and code in long mode too
42 * IRET will check the segment types kkeil 2000/10/28
43 * Also sysret mandates a special GDT layout
44 */
45/* The TLS descriptors are currently at a different place compared to i386.
46 Hopefully nobody expects them at a fixed place (Wine?) */
47DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = {
48 [GDT_ENTRY_KERNEL32_CS] = { { { 0x0000ffff, 0x00cf9b00 } } },
49 [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00af9b00 } } },
50 [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9300 } } },
51 [GDT_ENTRY_DEFAULT_USER32_CS] = { { { 0x0000ffff, 0x00cffb00 } } },
52 [GDT_ENTRY_DEFAULT_USER_DS] = { { { 0x0000ffff, 0x00cff300 } } },
53 [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00affb00 } } },
54} };
55EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
56
57__u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;
58
59/* Current gdt points %fs at the "master" per-cpu area: after this,
60 * it's on the real one. */
61void switch_to_new_gdt(void)
62{
63 struct desc_ptr gdt_descr;
64
65 gdt_descr.address = (long)get_cpu_gdt_table(smp_processor_id());
66 gdt_descr.size = GDT_SIZE - 1;
67 load_gdt(&gdt_descr);
68}
69
70struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
71
72static void __cpuinit default_init(struct cpuinfo_x86 *c)
73{
74 display_cacheinfo(c);
75}
76
77static struct cpu_dev __cpuinitdata default_cpu = {
78 .c_init = default_init,
79 .c_vendor = "Unknown",
80};
81static struct cpu_dev *this_cpu __cpuinitdata = &default_cpu;
82
83int __cpuinit get_model_name(struct cpuinfo_x86 *c)
84{
85 unsigned int *v;
86
87 if (c->extended_cpuid_level < 0x80000004)
88 return 0;
89
90 v = (unsigned int *) c->x86_model_id;
91 cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
92 cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
93 cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
94 c->x86_model_id[48] = 0;
95 return 1;
96}
97
98
99void __cpuinit display_cacheinfo(struct cpuinfo_x86 *c)
100{
101 unsigned int n, dummy, ebx, ecx, edx;
102
103 n = c->extended_cpuid_level;
104
105 if (n >= 0x80000005) {
106 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
107 printk(KERN_INFO "CPU: L1 I Cache: %dK (%d bytes/line), "
108 "D cache %dK (%d bytes/line)\n",
109 edx>>24, edx&0xFF, ecx>>24, ecx&0xFF);
110 c->x86_cache_size = (ecx>>24) + (edx>>24);
111 /* On K8 L1 TLB is inclusive, so don't count it */
112 c->x86_tlbsize = 0;
113 }
114
115 if (n >= 0x80000006) {
116 cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
117 ecx = cpuid_ecx(0x80000006);
118 c->x86_cache_size = ecx >> 16;
119 c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
120
121 printk(KERN_INFO "CPU: L2 Cache: %dK (%d bytes/line)\n",
122 c->x86_cache_size, ecx & 0xFF);
123 }
124}
125
126void __cpuinit detect_ht(struct cpuinfo_x86 *c)
127{
128#ifdef CONFIG_SMP
129 u32 eax, ebx, ecx, edx;
130 int index_msb, core_bits;
131
132 cpuid(1, &eax, &ebx, &ecx, &edx);
133
134
135 if (!cpu_has(c, X86_FEATURE_HT))
136 return;
137 if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
138 goto out;
139
140 smp_num_siblings = (ebx & 0xff0000) >> 16;
141
142 if (smp_num_siblings == 1) {
143 printk(KERN_INFO "CPU: Hyper-Threading is disabled\n");
144 } else if (smp_num_siblings > 1) {
145
146 if (smp_num_siblings > NR_CPUS) {
147 printk(KERN_WARNING "CPU: Unsupported number of "
148 "siblings %d", smp_num_siblings);
149 smp_num_siblings = 1;
150 return;
151 }
152
153 index_msb = get_count_order(smp_num_siblings);
154 c->phys_proc_id = phys_pkg_id(index_msb);
155
156 smp_num_siblings = smp_num_siblings / c->x86_max_cores;
157
158 index_msb = get_count_order(smp_num_siblings);
159
160 core_bits = get_count_order(c->x86_max_cores);
161
162 c->cpu_core_id = phys_pkg_id(index_msb) &
163 ((1 << core_bits) - 1);
164 }
165out:
166 if ((c->x86_max_cores * smp_num_siblings) > 1) {
167 printk(KERN_INFO "CPU: Physical Processor ID: %d\n",
168 c->phys_proc_id);
169 printk(KERN_INFO "CPU: Processor Core ID: %d\n",
170 c->cpu_core_id);
171 }
172
173#endif
174}
175
176static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
177{
178 char *v = c->x86_vendor_id;
179 int i;
180 static int printed;
181
182 for (i = 0; i < X86_VENDOR_NUM; i++) {
183 if (cpu_devs[i]) {
184 if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
185 (cpu_devs[i]->c_ident[1] &&
186 !strcmp(v, cpu_devs[i]->c_ident[1]))) {
187 c->x86_vendor = i;
188 this_cpu = cpu_devs[i];
189 return;
190 }
191 }
192 }
193 if (!printed) {
194 printed++;
195 printk(KERN_ERR "CPU: Vendor unknown, using generic init.\n");
196 printk(KERN_ERR "CPU: Your system may be unstable.\n");
197 }
198 c->x86_vendor = X86_VENDOR_UNKNOWN;
199}
200
201static void __init early_cpu_support_print(void)
202{
203 int i,j;
204 struct cpu_dev *cpu_devx;
205
206 printk("KERNEL supported cpus:\n");
207 for (i = 0; i < X86_VENDOR_NUM; i++) {
208 cpu_devx = cpu_devs[i];
209 if (!cpu_devx)
210 continue;
211 for (j = 0; j < 2; j++) {
212 if (!cpu_devx->c_ident[j])
213 continue;
214 printk(" %s %s\n", cpu_devx->c_vendor,
215 cpu_devx->c_ident[j]);
216 }
217 }
218}
219
220static void __cpuinit early_identify_cpu(struct cpuinfo_x86 *c);
221
222void __init early_cpu_init(void)
223{
224 struct cpu_vendor_dev *cvdev;
225
226 for (cvdev = __x86cpuvendor_start ;
227 cvdev < __x86cpuvendor_end ;
228 cvdev++)
229 cpu_devs[cvdev->vendor] = cvdev->cpu_dev;
230 early_cpu_support_print();
231 early_identify_cpu(&boot_cpu_data);
232}
233
234/* Do some early cpuid on the boot CPU to get some parameter that are
235 needed before check_bugs. Everything advanced is in identify_cpu
236 below. */
237static void __cpuinit early_identify_cpu(struct cpuinfo_x86 *c)
238{
239 u32 tfms, xlvl;
240
241 c->loops_per_jiffy = loops_per_jiffy;
242 c->x86_cache_size = -1;
243 c->x86_vendor = X86_VENDOR_UNKNOWN;
244 c->x86_model = c->x86_mask = 0; /* So far unknown... */
245 c->x86_vendor_id[0] = '\0'; /* Unset */
246 c->x86_model_id[0] = '\0'; /* Unset */
247 c->x86_clflush_size = 64;
248 c->x86_cache_alignment = c->x86_clflush_size;
249 c->x86_max_cores = 1;
250 c->x86_coreid_bits = 0;
251 c->extended_cpuid_level = 0;
252 memset(&c->x86_capability, 0, sizeof c->x86_capability);
253
254 /* Get vendor name */
255 cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
256 (unsigned int *)&c->x86_vendor_id[0],
257 (unsigned int *)&c->x86_vendor_id[8],
258 (unsigned int *)&c->x86_vendor_id[4]);
259
260 get_cpu_vendor(c);
261
262 /* Initialize the standard set of capabilities */
263 /* Note that the vendor-specific code below might override */
264
265 /* Intel-defined flags: level 0x00000001 */
266 if (c->cpuid_level >= 0x00000001) {
267 __u32 misc;
268 cpuid(0x00000001, &tfms, &misc, &c->x86_capability[4],
269 &c->x86_capability[0]);
270 c->x86 = (tfms >> 8) & 0xf;
271 c->x86_model = (tfms >> 4) & 0xf;
272 c->x86_mask = tfms & 0xf;
273 if (c->x86 == 0xf)
274 c->x86 += (tfms >> 20) & 0xff;
275 if (c->x86 >= 0x6)
276 c->x86_model += ((tfms >> 16) & 0xF) << 4;
277 if (test_cpu_cap(c, X86_FEATURE_CLFLSH))
278 c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
279 } else {
280 /* Have CPUID level 0 only - unheard of */
281 c->x86 = 4;
282 }
283
284 c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xff;
285#ifdef CONFIG_SMP
286 c->phys_proc_id = c->initial_apicid;
287#endif
288 /* AMD-defined flags: level 0x80000001 */
289 xlvl = cpuid_eax(0x80000000);
290 c->extended_cpuid_level = xlvl;
291 if ((xlvl & 0xffff0000) == 0x80000000) {
292 if (xlvl >= 0x80000001) {
293 c->x86_capability[1] = cpuid_edx(0x80000001);
294 c->x86_capability[6] = cpuid_ecx(0x80000001);
295 }
296 if (xlvl >= 0x80000004)
297 get_model_name(c); /* Default name */
298 }
299
300 /* Transmeta-defined flags: level 0x80860001 */
301 xlvl = cpuid_eax(0x80860000);
302 if ((xlvl & 0xffff0000) == 0x80860000) {
303 /* Don't set x86_cpuid_level here for now to not confuse. */
304 if (xlvl >= 0x80860001)
305 c->x86_capability[2] = cpuid_edx(0x80860001);
306 }
307
308 c->extended_cpuid_level = cpuid_eax(0x80000000);
309 if (c->extended_cpuid_level >= 0x80000007)
310 c->x86_power = cpuid_edx(0x80000007);
311
312 if (c->extended_cpuid_level >= 0x80000008) {
313 u32 eax = cpuid_eax(0x80000008);
314
315 c->x86_virt_bits = (eax >> 8) & 0xff;
316 c->x86_phys_bits = eax & 0xff;
317 }
318
319 /* Assume all 64-bit CPUs support 32-bit syscall */
320 set_cpu_cap(c, X86_FEATURE_SYSCALL32);
321
322 if (c->x86_vendor != X86_VENDOR_UNKNOWN &&
323 cpu_devs[c->x86_vendor]->c_early_init)
324 cpu_devs[c->x86_vendor]->c_early_init(c);
325
326 validate_pat_support(c);
327
328 /* early_param could clear that, but recall get it set again */
329 if (disable_apic)
330 clear_cpu_cap(c, X86_FEATURE_APIC);
331}
332
333/*
334 * This does the hard work of actually picking apart the CPU stuff...
335 */
336static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
337{
338 int i;
339
340 early_identify_cpu(c);
341
342 init_scattered_cpuid_features(c);
343
344 c->apicid = phys_pkg_id(0);
345
346 /*
347 * Vendor-specific initialization. In this section we
348 * canonicalize the feature flags, meaning if there are
349 * features a certain CPU supports which CPUID doesn't
350 * tell us, CPUID claiming incorrect flags, or other bugs,
351 * we handle them here.
352 *
353 * At the end of this section, c->x86_capability better
354 * indicate the features this CPU genuinely supports!
355 */
356 if (this_cpu->c_init)
357 this_cpu->c_init(c);
358
359 detect_ht(c);
360
361 /*
362 * On SMP, boot_cpu_data holds the common feature set between
363 * all CPUs; so make sure that we indicate which features are
364 * common between the CPUs. The first time this routine gets
365 * executed, c == &boot_cpu_data.
366 */
367 if (c != &boot_cpu_data) {
368 /* AND the already accumulated flags with these */
369 for (i = 0; i < NCAPINTS; i++)
370 boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
371 }
372
373 /* Clear all flags overriden by options */
374 for (i = 0; i < NCAPINTS; i++)
375 c->x86_capability[i] &= ~cleared_cpu_caps[i];
376
377#ifdef CONFIG_X86_MCE
378 mcheck_init(c);
379#endif
380 select_idle_routine(c);
381
382#ifdef CONFIG_NUMA
383 numa_add_cpu(smp_processor_id());
384#endif
385
386}
387
388void __cpuinit identify_boot_cpu(void)
389{
390 identify_cpu(&boot_cpu_data);
391}
392
393void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
394{
395 BUG_ON(c == &boot_cpu_data);
396 identify_cpu(c);
397 mtrr_ap_init();
398}
399
400static __init int setup_noclflush(char *arg)
401{
402 setup_clear_cpu_cap(X86_FEATURE_CLFLSH);
403 return 1;
404}
405__setup("noclflush", setup_noclflush);
406
407void __cpuinit print_cpu_info(struct cpuinfo_x86 *c)
408{
409 if (c->x86_model_id[0])
410 printk(KERN_CONT "%s", c->x86_model_id);
411
412 if (c->x86_mask || c->cpuid_level >= 0)
413 printk(KERN_CONT " stepping %02x\n", c->x86_mask);
414 else
415 printk(KERN_CONT "\n");
416}
417
418static __init int setup_disablecpuid(char *arg)
419{
420 int bit;
421 if (get_option(&arg, &bit) && bit < NCAPINTS*32)
422 setup_clear_cpu_cap(bit);
423 else
424 return 0;
425 return 1;
426}
427__setup("clearcpuid=", setup_disablecpuid);
428
429cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE;
430
431struct x8664_pda **_cpu_pda __read_mostly;
432EXPORT_SYMBOL(_cpu_pda);
433
434struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
435
436char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss;
437
438unsigned long __supported_pte_mask __read_mostly = ~0UL;
439EXPORT_SYMBOL_GPL(__supported_pte_mask);
440
441static int do_not_nx __cpuinitdata;
442
443/* noexec=on|off
444Control non executable mappings for 64bit processes.
445
446on Enable(default)
447off Disable
448*/
449static int __init nonx_setup(char *str)
450{
451 if (!str)
452 return -EINVAL;
453 if (!strncmp(str, "on", 2)) {
454 __supported_pte_mask |= _PAGE_NX;
455 do_not_nx = 0;
456 } else if (!strncmp(str, "off", 3)) {
457 do_not_nx = 1;
458 __supported_pte_mask &= ~_PAGE_NX;
459 }
460 return 0;
461}
462early_param("noexec", nonx_setup);
463
464int force_personality32;
465
466/* noexec32=on|off
467Control non executable heap for 32bit processes.
468To control the stack too use noexec=off
469
470on PROT_READ does not imply PROT_EXEC for 32bit processes (default)
471off PROT_READ implies PROT_EXEC
472*/
473static int __init nonx32_setup(char *str)
474{
475 if (!strcmp(str, "on"))
476 force_personality32 &= ~READ_IMPLIES_EXEC;
477 else if (!strcmp(str, "off"))
478 force_personality32 |= READ_IMPLIES_EXEC;
479 return 1;
480}
481__setup("noexec32=", nonx32_setup);
482
483void pda_init(int cpu)
484{
485 struct x8664_pda *pda = cpu_pda(cpu);
486
487 /* Setup up data that may be needed in __get_free_pages early */
488 loadsegment(fs, 0);
489 loadsegment(gs, 0);
490 /* Memory clobbers used to order PDA accessed */
491 mb();
492 wrmsrl(MSR_GS_BASE, pda);
493 mb();
494
495 pda->cpunumber = cpu;
496 pda->irqcount = -1;
497 pda->kernelstack = (unsigned long)stack_thread_info() -
498 PDA_STACKOFFSET + THREAD_SIZE;
499 pda->active_mm = &init_mm;
500 pda->mmu_state = 0;
501
502 if (cpu == 0) {
503 /* others are initialized in smpboot.c */
504 pda->pcurrent = &init_task;
505 pda->irqstackptr = boot_cpu_stack;
506 } else {
507 pda->irqstackptr = (char *)
508 __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
509 if (!pda->irqstackptr)
510 panic("cannot allocate irqstack for cpu %d", cpu);
511
512 if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE)
513 pda->nodenumber = cpu_to_node(cpu);
514 }
515
516 pda->irqstackptr += IRQSTACKSIZE-64;
517}
518
519char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ +
520 DEBUG_STKSZ]
521__attribute__((section(".bss.page_aligned")));
522
523extern asmlinkage void ignore_sysret(void);
524
525/* May not be marked __init: used by software suspend */
526void syscall_init(void)
527{
528 /*
529 * LSTAR and STAR live in a bit strange symbiosis.
530 * They both write to the same internal register. STAR allows to
531 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
532 */
533 wrmsrl(MSR_STAR, ((u64)__USER32_CS)<<48 | ((u64)__KERNEL_CS)<<32);
534 wrmsrl(MSR_LSTAR, system_call);
535 wrmsrl(MSR_CSTAR, ignore_sysret);
536
537#ifdef CONFIG_IA32_EMULATION
538 syscall32_cpu_init();
539#endif
540
541 /* Flags to clear on syscall */
542 wrmsrl(MSR_SYSCALL_MASK,
543 X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL);
544}
545
546void __cpuinit check_efer(void)
547{
548 unsigned long efer;
549
550 rdmsrl(MSR_EFER, efer);
551 if (!(efer & EFER_NX) || do_not_nx)
552 __supported_pte_mask &= ~_PAGE_NX;
553}
554
555unsigned long kernel_eflags;
556
557/*
558 * Copies of the original ist values from the tss are only accessed during
559 * debugging, no special alignment required.
560 */
561DEFINE_PER_CPU(struct orig_ist, orig_ist);
562
563/*
564 * cpu_init() initializes state that is per-CPU. Some data is already
565 * initialized (naturally) in the bootstrap process, such as the GDT
566 * and IDT. We reload them nevertheless, this function acts as a
567 * 'CPU state barrier', nothing should get across.
568 * A lot of state is already set up in PDA init.
569 */
570void __cpuinit cpu_init(void)
571{
572 int cpu = stack_smp_processor_id();
573 struct tss_struct *t = &per_cpu(init_tss, cpu);
574 struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu);
575 unsigned long v;
576 char *estacks = NULL;
577 struct task_struct *me;
578 int i;
579
580 /* CPU 0 is initialised in head64.c */
581 if (cpu != 0)
582 pda_init(cpu);
583 else
584 estacks = boot_exception_stacks;
585
586 me = current;
587
588 if (cpu_test_and_set(cpu, cpu_initialized))
589 panic("CPU#%d already initialized!\n", cpu);
590
591 printk(KERN_INFO "Initializing CPU#%d\n", cpu);
592
593 clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
594
595 /*
596 * Initialize the per-CPU GDT with the boot GDT,
597 * and set up the GDT descriptor:
598 */
599
600 switch_to_new_gdt();
601 load_idt((const struct desc_ptr *)&idt_descr);
602
603 memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
604 syscall_init();
605
606 wrmsrl(MSR_FS_BASE, 0);
607 wrmsrl(MSR_KERNEL_GS_BASE, 0);
608 barrier();
609
610 check_efer();
611
612 /*
613 * set up and load the per-CPU TSS
614 */
615 for (v = 0; v < N_EXCEPTION_STACKS; v++) {
616 static const unsigned int order[N_EXCEPTION_STACKS] = {
617 [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
618 [DEBUG_STACK - 1] = DEBUG_STACK_ORDER
619 };
620 if (cpu) {
621 estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
622 if (!estacks)
623 panic("Cannot allocate exception stack %ld %d\n",
624 v, cpu);
625 }
626 estacks += PAGE_SIZE << order[v];
627 orig_ist->ist[v] = t->x86_tss.ist[v] = (unsigned long)estacks;
628 }
629
630 t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
631 /*
632 * <= is required because the CPU will access up to
633 * 8 bits beyond the end of the IO permission bitmap.
634 */
635 for (i = 0; i <= IO_BITMAP_LONGS; i++)
636 t->io_bitmap[i] = ~0UL;
637
638 atomic_inc(&init_mm.mm_count);
639 me->active_mm = &init_mm;
640 if (me->mm)
641 BUG();
642 enter_lazy_tlb(&init_mm, me);
643
644 load_sp0(t, &current->thread);
645 set_tss_desc(cpu, t);
646 load_TR_desc();
647 load_LDT(&init_mm.context);
648
649#ifdef CONFIG_KGDB
650 /*
651 * If the kgdb is connected no debug regs should be altered. This
652 * is only applicable when KGDB and a KGDB I/O module are built
653 * into the kernel and you are using early debugging with
654 * kgdbwait. KGDB will control the kernel HW breakpoint registers.
655 */
656 if (kgdb_connected && arch_kgdb_ops.correct_hw_break)
657 arch_kgdb_ops.correct_hw_break();
658 else {
659#endif
660 /*
661 * Clear all 6 debug registers:
662 */
663
664 set_debugreg(0UL, 0);
665 set_debugreg(0UL, 1);
666 set_debugreg(0UL, 2);
667 set_debugreg(0UL, 3);
668 set_debugreg(0UL, 6);
669 set_debugreg(0UL, 7);
670#ifdef CONFIG_KGDB
671 /* If the kgdb is connected no debug regs should be altered. */
672 }
673#endif
674
675 fpu_init();
676
677 raw_local_save_flags(kernel_eflags);
678
679 if (is_uv_system())
680 uv_cpu_init();
681}
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 783691b2a738..4d894e8565fe 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -1,3 +1,6 @@
1#ifndef ARCH_X86_CPU_H
2
3#define ARCH_X86_CPU_H
1 4
2struct cpu_model_info { 5struct cpu_model_info {
3 int vendor; 6 int vendor;
@@ -36,3 +39,5 @@ extern struct cpu_vendor_dev __x86cpuvendor_start[], __x86cpuvendor_end[];
36 39
37extern int get_model_name(struct cpuinfo_x86 *c); 40extern int get_model_name(struct cpuinfo_x86 *c);
38extern void display_cacheinfo(struct cpuinfo_x86 *c); 41extern void display_cacheinfo(struct cpuinfo_x86 *c);
42
43#endif
diff --git a/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c b/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c
index f03e9153618e..965ea52767ac 100644
--- a/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c
+++ b/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c
@@ -26,9 +26,10 @@
26#define NFORCE2_SAFE_DISTANCE 50 26#define NFORCE2_SAFE_DISTANCE 50
27 27
28/* Delay in ms between FSB changes */ 28/* Delay in ms between FSB changes */
29//#define NFORCE2_DELAY 10 29/* #define NFORCE2_DELAY 10 */
30 30
31/* nforce2_chipset: 31/*
32 * nforce2_chipset:
32 * FSB is changed using the chipset 33 * FSB is changed using the chipset
33 */ 34 */
34static struct pci_dev *nforce2_chipset_dev; 35static struct pci_dev *nforce2_chipset_dev;
@@ -36,13 +37,13 @@ static struct pci_dev *nforce2_chipset_dev;
36/* fid: 37/* fid:
37 * multiplier * 10 38 * multiplier * 10
38 */ 39 */
39static int fid = 0; 40static int fid;
40 41
41/* min_fsb, max_fsb: 42/* min_fsb, max_fsb:
42 * minimum and maximum FSB (= FSB at boot time) 43 * minimum and maximum FSB (= FSB at boot time)
43 */ 44 */
44static int min_fsb = 0; 45static int min_fsb;
45static int max_fsb = 0; 46static int max_fsb;
46 47
47MODULE_AUTHOR("Sebastian Witt <se.witt@gmx.net>"); 48MODULE_AUTHOR("Sebastian Witt <se.witt@gmx.net>");
48MODULE_DESCRIPTION("nForce2 FSB changing cpufreq driver"); 49MODULE_DESCRIPTION("nForce2 FSB changing cpufreq driver");
@@ -53,7 +54,7 @@ module_param(min_fsb, int, 0444);
53 54
54MODULE_PARM_DESC(fid, "CPU multiplier to use (11.5 = 115)"); 55MODULE_PARM_DESC(fid, "CPU multiplier to use (11.5 = 115)");
55MODULE_PARM_DESC(min_fsb, 56MODULE_PARM_DESC(min_fsb,
56 "Minimum FSB to use, if not defined: current FSB - 50"); 57 "Minimum FSB to use, if not defined: current FSB - 50");
57 58
58#define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "cpufreq-nforce2", msg) 59#define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "cpufreq-nforce2", msg)
59 60
@@ -139,7 +140,7 @@ static unsigned int nforce2_fsb_read(int bootfsb)
139 140
140 /* Get chipset boot FSB from subdevice 5 (FSB at boot-time) */ 141 /* Get chipset boot FSB from subdevice 5 (FSB at boot-time) */
141 nforce2_sub5 = pci_get_subsys(PCI_VENDOR_ID_NVIDIA, 142 nforce2_sub5 = pci_get_subsys(PCI_VENDOR_ID_NVIDIA,
142 0x01EF,PCI_ANY_ID,PCI_ANY_ID,NULL); 143 0x01EF, PCI_ANY_ID, PCI_ANY_ID, NULL);
143 if (!nforce2_sub5) 144 if (!nforce2_sub5)
144 return 0; 145 return 0;
145 146
@@ -147,13 +148,13 @@ static unsigned int nforce2_fsb_read(int bootfsb)
147 fsb /= 1000000; 148 fsb /= 1000000;
148 149
149 /* Check if PLL register is already set */ 150 /* Check if PLL register is already set */
150 pci_read_config_byte(nforce2_chipset_dev,NFORCE2_PLLENABLE, (u8 *)&temp); 151 pci_read_config_byte(nforce2_chipset_dev, NFORCE2_PLLENABLE, (u8 *)&temp);
151 152
152 if(bootfsb || !temp) 153 if (bootfsb || !temp)
153 return fsb; 154 return fsb;
154 155
155 /* Use PLL register FSB value */ 156 /* Use PLL register FSB value */
156 pci_read_config_dword(nforce2_chipset_dev,NFORCE2_PLLREG, &temp); 157 pci_read_config_dword(nforce2_chipset_dev, NFORCE2_PLLREG, &temp);
157 fsb = nforce2_calc_fsb(temp); 158 fsb = nforce2_calc_fsb(temp);
158 159
159 return fsb; 160 return fsb;
@@ -184,7 +185,7 @@ static int nforce2_set_fsb(unsigned int fsb)
184 } 185 }
185 186
186 /* First write? Then set actual value */ 187 /* First write? Then set actual value */
187 pci_read_config_byte(nforce2_chipset_dev,NFORCE2_PLLENABLE, (u8 *)&temp); 188 pci_read_config_byte(nforce2_chipset_dev, NFORCE2_PLLENABLE, (u8 *)&temp);
188 if (!temp) { 189 if (!temp) {
189 pll = nforce2_calc_pll(tfsb); 190 pll = nforce2_calc_pll(tfsb);
190 191
@@ -210,7 +211,8 @@ static int nforce2_set_fsb(unsigned int fsb)
210 tfsb--; 211 tfsb--;
211 212
212 /* Calculate the PLL reg. value */ 213 /* Calculate the PLL reg. value */
213 if ((pll = nforce2_calc_pll(tfsb)) == -1) 214 pll = nforce2_calc_pll(tfsb);
215 if (pll == -1)
214 return -EINVAL; 216 return -EINVAL;
215 217
216 nforce2_write_pll(pll); 218 nforce2_write_pll(pll);
@@ -249,7 +251,7 @@ static unsigned int nforce2_get(unsigned int cpu)
249static int nforce2_target(struct cpufreq_policy *policy, 251static int nforce2_target(struct cpufreq_policy *policy,
250 unsigned int target_freq, unsigned int relation) 252 unsigned int target_freq, unsigned int relation)
251{ 253{
252// unsigned long flags; 254/* unsigned long flags; */
253 struct cpufreq_freqs freqs; 255 struct cpufreq_freqs freqs;
254 unsigned int target_fsb; 256 unsigned int target_fsb;
255 257
@@ -271,17 +273,17 @@ static int nforce2_target(struct cpufreq_policy *policy,
271 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); 273 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
272 274
273 /* Disable IRQs */ 275 /* Disable IRQs */
274 //local_irq_save(flags); 276 /* local_irq_save(flags); */
275 277
276 if (nforce2_set_fsb(target_fsb) < 0) 278 if (nforce2_set_fsb(target_fsb) < 0)
277 printk(KERN_ERR "cpufreq: Changing FSB to %d failed\n", 279 printk(KERN_ERR "cpufreq: Changing FSB to %d failed\n",
278 target_fsb); 280 target_fsb);
279 else 281 else
280 dprintk("Changed FSB successfully to %d\n", 282 dprintk("Changed FSB successfully to %d\n",
281 target_fsb); 283 target_fsb);
282 284
283 /* Enable IRQs */ 285 /* Enable IRQs */
284 //local_irq_restore(flags); 286 /* local_irq_restore(flags); */
285 287
286 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); 288 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
287 289
@@ -302,8 +304,8 @@ static int nforce2_verify(struct cpufreq_policy *policy)
302 policy->max = (fsb_pol_max + 1) * fid * 100; 304 policy->max = (fsb_pol_max + 1) * fid * 100;
303 305
304 cpufreq_verify_within_limits(policy, 306 cpufreq_verify_within_limits(policy,
305 policy->cpuinfo.min_freq, 307 policy->cpuinfo.min_freq,
306 policy->cpuinfo.max_freq); 308 policy->cpuinfo.max_freq);
307 return 0; 309 return 0;
308} 310}
309 311
@@ -347,7 +349,7 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy)
347 /* Set maximum FSB to FSB at boot time */ 349 /* Set maximum FSB to FSB at boot time */
348 max_fsb = nforce2_fsb_read(1); 350 max_fsb = nforce2_fsb_read(1);
349 351
350 if(!max_fsb) 352 if (!max_fsb)
351 return -EIO; 353 return -EIO;
352 354
353 if (!min_fsb) 355 if (!min_fsb)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index fe9224c51d37..70609efdf1da 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -226,6 +226,10 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
226 226
227 if (cpu_has_bts) 227 if (cpu_has_bts)
228 ds_init_intel(c); 228 ds_init_intel(c);
229
230#ifdef CONFIG_X86_NUMAQ
231 numaq_tsc_disable();
232#endif
229} 233}
230 234
231static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) 235static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned int size)
diff --git a/arch/x86/kernel/cpu/intel_64.c b/arch/x86/kernel/cpu/intel_64.c
new file mode 100644
index 000000000000..1019c58d39f0
--- /dev/null
+++ b/arch/x86/kernel/cpu/intel_64.c
@@ -0,0 +1,95 @@
1#include <linux/init.h>
2#include <linux/smp.h>
3#include <asm/processor.h>
4#include <asm/ptrace.h>
5#include <asm/topology.h>
6#include <asm/numa_64.h>
7
8#include "cpu.h"
9
10static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
11{
12 if ((c->x86 == 0xf && c->x86_model >= 0x03) ||
13 (c->x86 == 0x6 && c->x86_model >= 0x0e))
14 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
15
16 set_cpu_cap(c, X86_FEATURE_SYSENTER32);
17}
18
19/*
20 * find out the number of processor cores on the die
21 */
22static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c)
23{
24 unsigned int eax, t;
25
26 if (c->cpuid_level < 4)
27 return 1;
28
29 cpuid_count(4, 0, &eax, &t, &t, &t);
30
31 if (eax & 0x1f)
32 return ((eax >> 26) + 1);
33 else
34 return 1;
35}
36
37static void __cpuinit srat_detect_node(void)
38{
39#ifdef CONFIG_NUMA
40 unsigned node;
41 int cpu = smp_processor_id();
42 int apicid = hard_smp_processor_id();
43
44 /* Don't do the funky fallback heuristics the AMD version employs
45 for now. */
46 node = apicid_to_node[apicid];
47 if (node == NUMA_NO_NODE || !node_online(node))
48 node = first_node(node_online_map);
49 numa_set_node(cpu, node);
50
51 printk(KERN_INFO "CPU %d/%x -> Node %d\n", cpu, apicid, node);
52#endif
53}
54
55static void __cpuinit init_intel(struct cpuinfo_x86 *c)
56{
57 init_intel_cacheinfo(c);
58 if (c->cpuid_level > 9) {
59 unsigned eax = cpuid_eax(10);
60 /* Check for version and the number of counters */
61 if ((eax & 0xff) && (((eax>>8) & 0xff) > 1))
62 set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON);
63 }
64
65 if (cpu_has_ds) {
66 unsigned int l1, l2;
67 rdmsr(MSR_IA32_MISC_ENABLE, l1, l2);
68 if (!(l1 & (1<<11)))
69 set_cpu_cap(c, X86_FEATURE_BTS);
70 if (!(l1 & (1<<12)))
71 set_cpu_cap(c, X86_FEATURE_PEBS);
72 }
73
74
75 if (cpu_has_bts)
76 ds_init_intel(c);
77
78 if (c->x86 == 15)
79 c->x86_cache_alignment = c->x86_clflush_size * 2;
80 if (c->x86 == 6)
81 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
82 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
83 c->x86_max_cores = intel_num_cpu_cores(c);
84
85 srat_detect_node();
86}
87
88static struct cpu_dev intel_cpu_dev __cpuinitdata = {
89 .c_vendor = "Intel",
90 .c_ident = { "GenuineIntel" },
91 .c_early_init = early_init_intel,
92 .c_init = init_intel,
93};
94cpu_vendor_dev_register(X86_VENDOR_INTEL, &intel_cpu_dev);
95
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 26d615dcb149..2c8afafa18e8 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -62,6 +62,7 @@ static struct _cache_table cache_table[] __cpuinitdata =
62 { 0x4b, LVL_3, 8192 }, /* 16-way set assoc, 64 byte line size */ 62 { 0x4b, LVL_3, 8192 }, /* 16-way set assoc, 64 byte line size */
63 { 0x4c, LVL_3, 12288 }, /* 12-way set assoc, 64 byte line size */ 63 { 0x4c, LVL_3, 12288 }, /* 12-way set assoc, 64 byte line size */
64 { 0x4d, LVL_3, 16384 }, /* 16-way set assoc, 64 byte line size */ 64 { 0x4d, LVL_3, 16384 }, /* 16-way set assoc, 64 byte line size */
65 { 0x4e, LVL_2, 6144 }, /* 24-way set assoc, 64 byte line size */
65 { 0x60, LVL_1_DATA, 16 }, /* 8-way set assoc, sectored cache, 64 byte line size */ 66 { 0x60, LVL_1_DATA, 16 }, /* 8-way set assoc, sectored cache, 64 byte line size */
66 { 0x66, LVL_1_DATA, 8 }, /* 4-way set assoc, sectored cache, 64 byte line size */ 67 { 0x66, LVL_1_DATA, 8 }, /* 4-way set assoc, sectored cache, 64 byte line size */
67 { 0x67, LVL_1_DATA, 16 }, /* 4-way set assoc, sectored cache, 64 byte line size */ 68 { 0x67, LVL_1_DATA, 16 }, /* 4-way set assoc, sectored cache, 64 byte line size */
diff --git a/arch/x86/kernel/cpu/mcheck/k7.c b/arch/x86/kernel/cpu/mcheck/k7.c
index e633c9c2b764..f390c9f66351 100644
--- a/arch/x86/kernel/cpu/mcheck/k7.c
+++ b/arch/x86/kernel/cpu/mcheck/k7.c
@@ -9,23 +9,23 @@
9#include <linux/interrupt.h> 9#include <linux/interrupt.h>
10#include <linux/smp.h> 10#include <linux/smp.h>
11 11
12#include <asm/processor.h> 12#include <asm/processor.h>
13#include <asm/system.h> 13#include <asm/system.h>
14#include <asm/msr.h> 14#include <asm/msr.h>
15 15
16#include "mce.h" 16#include "mce.h"
17 17
18/* Machine Check Handler For AMD Athlon/Duron */ 18/* Machine Check Handler For AMD Athlon/Duron */
19static void k7_machine_check(struct pt_regs * regs, long error_code) 19static void k7_machine_check(struct pt_regs *regs, long error_code)
20{ 20{
21 int recover=1; 21 int recover = 1;
22 u32 alow, ahigh, high, low; 22 u32 alow, ahigh, high, low;
23 u32 mcgstl, mcgsth; 23 u32 mcgstl, mcgsth;
24 int i; 24 int i;
25 25
26 rdmsr (MSR_IA32_MCG_STATUS, mcgstl, mcgsth); 26 rdmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
27 if (mcgstl & (1<<0)) /* Recoverable ? */ 27 if (mcgstl & (1<<0)) /* Recoverable ? */
28 recover=0; 28 recover = 0;
29 29
30 printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n", 30 printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n",
31 smp_processor_id(), mcgsth, mcgstl); 31 smp_processor_id(), mcgsth, mcgstl);
@@ -60,12 +60,12 @@ static void k7_machine_check(struct pt_regs * regs, long error_code)
60 } 60 }
61 61
62 if (recover&2) 62 if (recover&2)
63 panic ("CPU context corrupt"); 63 panic("CPU context corrupt");
64 if (recover&1) 64 if (recover&1)
65 panic ("Unable to continue"); 65 panic("Unable to continue");
66 printk (KERN_EMERG "Attempting to continue.\n"); 66 printk(KERN_EMERG "Attempting to continue.\n");
67 mcgstl &= ~(1<<2); 67 mcgstl &= ~(1<<2);
68 wrmsr (MSR_IA32_MCG_STATUS,mcgstl, mcgsth); 68 wrmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
69} 69}
70 70
71 71
@@ -81,25 +81,25 @@ void amd_mcheck_init(struct cpuinfo_x86 *c)
81 machine_check_vector = k7_machine_check; 81 machine_check_vector = k7_machine_check;
82 wmb(); 82 wmb();
83 83
84 printk (KERN_INFO "Intel machine check architecture supported.\n"); 84 printk(KERN_INFO "Intel machine check architecture supported.\n");
85 rdmsr (MSR_IA32_MCG_CAP, l, h); 85 rdmsr(MSR_IA32_MCG_CAP, l, h);
86 if (l & (1<<8)) /* Control register present ? */ 86 if (l & (1<<8)) /* Control register present ? */
87 wrmsr (MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 87 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
88 nr_mce_banks = l & 0xff; 88 nr_mce_banks = l & 0xff;
89 89
90 /* Clear status for MC index 0 separately, we don't touch CTL, 90 /* Clear status for MC index 0 separately, we don't touch CTL,
91 * as some K7 Athlons cause spurious MCEs when its enabled. */ 91 * as some K7 Athlons cause spurious MCEs when its enabled. */
92 if (boot_cpu_data.x86 == 6) { 92 if (boot_cpu_data.x86 == 6) {
93 wrmsr (MSR_IA32_MC0_STATUS, 0x0, 0x0); 93 wrmsr(MSR_IA32_MC0_STATUS, 0x0, 0x0);
94 i = 1; 94 i = 1;
95 } else 95 } else
96 i = 0; 96 i = 0;
97 for (; i<nr_mce_banks; i++) { 97 for (; i < nr_mce_banks; i++) {
98 wrmsr (MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff); 98 wrmsr(MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff);
99 wrmsr (MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0); 99 wrmsr(MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0);
100 } 100 }
101 101
102 set_in_cr4 (X86_CR4_MCE); 102 set_in_cr4(X86_CR4_MCE);
103 printk (KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n", 103 printk(KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n",
104 smp_processor_id()); 104 smp_processor_id());
105} 105}
diff --git a/arch/x86/kernel/cpu/mcheck/mce_64.c b/arch/x86/kernel/cpu/mcheck/mce_64.c
index e07e8c068ae0..c4a7ec31394c 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_64.c
@@ -9,6 +9,7 @@
9#include <linux/types.h> 9#include <linux/types.h>
10#include <linux/kernel.h> 10#include <linux/kernel.h>
11#include <linux/sched.h> 11#include <linux/sched.h>
12#include <linux/smp_lock.h>
12#include <linux/string.h> 13#include <linux/string.h>
13#include <linux/rcupdate.h> 14#include <linux/rcupdate.h>
14#include <linux/kallsyms.h> 15#include <linux/kallsyms.h>
@@ -31,7 +32,7 @@
31#include <asm/idle.h> 32#include <asm/idle.h>
32 33
33#define MISC_MCELOG_MINOR 227 34#define MISC_MCELOG_MINOR 227
34#define NR_BANKS 6 35#define NR_SYSFS_BANKS 6
35 36
36atomic_t mce_entry; 37atomic_t mce_entry;
37 38
@@ -46,7 +47,7 @@ static int mce_dont_init;
46 */ 47 */
47static int tolerant = 1; 48static int tolerant = 1;
48static int banks; 49static int banks;
49static unsigned long bank[NR_BANKS] = { [0 ... NR_BANKS-1] = ~0UL }; 50static unsigned long bank[NR_SYSFS_BANKS] = { [0 ... NR_SYSFS_BANKS-1] = ~0UL };
50static unsigned long notify_user; 51static unsigned long notify_user;
51static int rip_msr; 52static int rip_msr;
52static int mce_bootlog = -1; 53static int mce_bootlog = -1;
@@ -209,7 +210,7 @@ void do_machine_check(struct pt_regs * regs, long error_code)
209 barrier(); 210 barrier();
210 211
211 for (i = 0; i < banks; i++) { 212 for (i = 0; i < banks; i++) {
212 if (!bank[i]) 213 if (i < NR_SYSFS_BANKS && !bank[i])
213 continue; 214 continue;
214 215
215 m.misc = 0; 216 m.misc = 0;
@@ -363,7 +364,7 @@ static void mcheck_check_cpu(void *info)
363 364
364static void mcheck_timer(struct work_struct *work) 365static void mcheck_timer(struct work_struct *work)
365{ 366{
366 on_each_cpu(mcheck_check_cpu, NULL, 1, 1); 367 on_each_cpu(mcheck_check_cpu, NULL, 1);
367 368
368 /* 369 /*
369 * Alert userspace if needed. If we logged an MCE, reduce the 370 * Alert userspace if needed. If we logged an MCE, reduce the
@@ -444,9 +445,10 @@ static void mce_init(void *dummy)
444 445
445 rdmsrl(MSR_IA32_MCG_CAP, cap); 446 rdmsrl(MSR_IA32_MCG_CAP, cap);
446 banks = cap & 0xff; 447 banks = cap & 0xff;
447 if (banks > NR_BANKS) { 448 if (banks > MCE_EXTENDED_BANK) {
448 printk(KERN_INFO "MCE: warning: using only %d banks\n", banks); 449 banks = MCE_EXTENDED_BANK;
449 banks = NR_BANKS; 450 printk(KERN_INFO "MCE: warning: using only %d banks\n",
451 MCE_EXTENDED_BANK);
450 } 452 }
451 /* Use accurate RIP reporting if available. */ 453 /* Use accurate RIP reporting if available. */
452 if ((cap & (1<<9)) && ((cap >> 16) & 0xff) >= 9) 454 if ((cap & (1<<9)) && ((cap >> 16) & 0xff) >= 9)
@@ -462,7 +464,11 @@ static void mce_init(void *dummy)
462 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 464 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
463 465
464 for (i = 0; i < banks; i++) { 466 for (i = 0; i < banks; i++) {
465 wrmsrl(MSR_IA32_MC0_CTL+4*i, bank[i]); 467 if (i < NR_SYSFS_BANKS)
468 wrmsrl(MSR_IA32_MC0_CTL+4*i, bank[i]);
469 else
470 wrmsrl(MSR_IA32_MC0_CTL+4*i, ~0UL);
471
466 wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0); 472 wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
467 } 473 }
468} 474}
@@ -527,10 +533,12 @@ static int open_exclu; /* already open exclusive? */
527 533
528static int mce_open(struct inode *inode, struct file *file) 534static int mce_open(struct inode *inode, struct file *file)
529{ 535{
536 lock_kernel();
530 spin_lock(&mce_state_lock); 537 spin_lock(&mce_state_lock);
531 538
532 if (open_exclu || (open_count && (file->f_flags & O_EXCL))) { 539 if (open_exclu || (open_count && (file->f_flags & O_EXCL))) {
533 spin_unlock(&mce_state_lock); 540 spin_unlock(&mce_state_lock);
541 unlock_kernel();
534 return -EBUSY; 542 return -EBUSY;
535 } 543 }
536 544
@@ -539,6 +547,7 @@ static int mce_open(struct inode *inode, struct file *file)
539 open_count++; 547 open_count++;
540 548
541 spin_unlock(&mce_state_lock); 549 spin_unlock(&mce_state_lock);
550 unlock_kernel();
542 551
543 return nonseekable_open(inode, file); 552 return nonseekable_open(inode, file);
544} 553}
@@ -612,7 +621,7 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,
612 * Collect entries that were still getting written before the 621 * Collect entries that were still getting written before the
613 * synchronize. 622 * synchronize.
614 */ 623 */
615 on_each_cpu(collect_tscs, cpu_tsc, 1, 1); 624 on_each_cpu(collect_tscs, cpu_tsc, 1);
616 for (i = next; i < MCE_LOG_LEN; i++) { 625 for (i = next; i < MCE_LOG_LEN; i++) {
617 if (mcelog.entry[i].finished && 626 if (mcelog.entry[i].finished &&
618 mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) { 627 mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) {
@@ -737,7 +746,7 @@ static void mce_restart(void)
737 if (next_interval) 746 if (next_interval)
738 cancel_delayed_work(&mcheck_work); 747 cancel_delayed_work(&mcheck_work);
739 /* Timer race is harmless here */ 748 /* Timer race is harmless here */
740 on_each_cpu(mce_init, NULL, 1, 1); 749 on_each_cpu(mce_init, NULL, 1);
741 next_interval = check_interval * HZ; 750 next_interval = check_interval * HZ;
742 if (next_interval) 751 if (next_interval)
743 schedule_delayed_work(&mcheck_work, 752 schedule_delayed_work(&mcheck_work,
@@ -766,7 +775,10 @@ DEFINE_PER_CPU(struct sys_device, device_mce);
766 } \ 775 } \
767 static SYSDEV_ATTR(name, 0644, show_ ## name, set_ ## name); 776 static SYSDEV_ATTR(name, 0644, show_ ## name, set_ ## name);
768 777
769/* TBD should generate these dynamically based on number of available banks */ 778/*
779 * TBD should generate these dynamically based on number of available banks.
780 * Have only 6 contol banks in /sysfs until then.
781 */
770ACCESSOR(bank0ctl,bank[0],mce_restart()) 782ACCESSOR(bank0ctl,bank[0],mce_restart())
771ACCESSOR(bank1ctl,bank[1],mce_restart()) 783ACCESSOR(bank1ctl,bank[1],mce_restart())
772ACCESSOR(bank2ctl,bank[2],mce_restart()) 784ACCESSOR(bank2ctl,bank[2],mce_restart())
diff --git a/arch/x86/kernel/cpu/mcheck/non-fatal.c b/arch/x86/kernel/cpu/mcheck/non-fatal.c
index 00ccb6c14ec2..cc1fccdd31e0 100644
--- a/arch/x86/kernel/cpu/mcheck/non-fatal.c
+++ b/arch/x86/kernel/cpu/mcheck/non-fatal.c
@@ -59,7 +59,7 @@ static DECLARE_DELAYED_WORK(mce_work, mce_work_fn);
59 59
60static void mce_work_fn(struct work_struct *work) 60static void mce_work_fn(struct work_struct *work)
61{ 61{
62 on_each_cpu(mce_checkregs, NULL, 1, 1); 62 on_each_cpu(mce_checkregs, NULL, 1);
63 schedule_delayed_work(&mce_work, round_jiffies_relative(MCE_RATE)); 63 schedule_delayed_work(&mce_work, round_jiffies_relative(MCE_RATE));
64} 64}
65 65
diff --git a/arch/x86/kernel/cpu/mcheck/p4.c b/arch/x86/kernel/cpu/mcheck/p4.c
index cb03345554a5..eef001ad3bde 100644
--- a/arch/x86/kernel/cpu/mcheck/p4.c
+++ b/arch/x86/kernel/cpu/mcheck/p4.c
@@ -8,7 +8,7 @@
8#include <linux/interrupt.h> 8#include <linux/interrupt.h>
9#include <linux/smp.h> 9#include <linux/smp.h>
10 10
11#include <asm/processor.h> 11#include <asm/processor.h>
12#include <asm/system.h> 12#include <asm/system.h>
13#include <asm/msr.h> 13#include <asm/msr.h>
14#include <asm/apic.h> 14#include <asm/apic.h>
@@ -32,12 +32,12 @@ struct intel_mce_extended_msrs {
32 /* u32 *reserved[]; */ 32 /* u32 *reserved[]; */
33}; 33};
34 34
35static int mce_num_extended_msrs = 0; 35static int mce_num_extended_msrs;
36 36
37 37
38#ifdef CONFIG_X86_MCE_P4THERMAL 38#ifdef CONFIG_X86_MCE_P4THERMAL
39static void unexpected_thermal_interrupt(struct pt_regs *regs) 39static void unexpected_thermal_interrupt(struct pt_regs *regs)
40{ 40{
41 printk(KERN_ERR "CPU%d: Unexpected LVT TMR interrupt!\n", 41 printk(KERN_ERR "CPU%d: Unexpected LVT TMR interrupt!\n",
42 smp_processor_id()); 42 smp_processor_id());
43 add_taint(TAINT_MACHINE_CHECK); 43 add_taint(TAINT_MACHINE_CHECK);
@@ -83,7 +83,7 @@ static void intel_init_thermal(struct cpuinfo_x86 *c)
83 * be some SMM goo which handles it, so we can't even put a handler 83 * be some SMM goo which handles it, so we can't even put a handler
84 * since it might be delivered via SMI already -zwanem. 84 * since it might be delivered via SMI already -zwanem.
85 */ 85 */
86 rdmsr (MSR_IA32_MISC_ENABLE, l, h); 86 rdmsr(MSR_IA32_MISC_ENABLE, l, h);
87 h = apic_read(APIC_LVTTHMR); 87 h = apic_read(APIC_LVTTHMR);
88 if ((l & (1<<3)) && (h & APIC_DM_SMI)) { 88 if ((l & (1<<3)) && (h & APIC_DM_SMI)) {
89 printk(KERN_DEBUG "CPU%d: Thermal monitoring handled by SMI\n", 89 printk(KERN_DEBUG "CPU%d: Thermal monitoring handled by SMI\n",
@@ -91,7 +91,7 @@ static void intel_init_thermal(struct cpuinfo_x86 *c)
91 return; /* -EBUSY */ 91 return; /* -EBUSY */
92 } 92 }
93 93
94 /* check whether a vector already exists, temporarily masked? */ 94 /* check whether a vector already exists, temporarily masked? */
95 if (h & APIC_VECTOR_MASK) { 95 if (h & APIC_VECTOR_MASK) {
96 printk(KERN_DEBUG "CPU%d: Thermal LVT vector (%#x) already " 96 printk(KERN_DEBUG "CPU%d: Thermal LVT vector (%#x) already "
97 "installed\n", 97 "installed\n",
@@ -104,18 +104,18 @@ static void intel_init_thermal(struct cpuinfo_x86 *c)
104 h |= (APIC_DM_FIXED | APIC_LVT_MASKED); /* we'll mask till we're ready */ 104 h |= (APIC_DM_FIXED | APIC_LVT_MASKED); /* we'll mask till we're ready */
105 apic_write_around(APIC_LVTTHMR, h); 105 apic_write_around(APIC_LVTTHMR, h);
106 106
107 rdmsr (MSR_IA32_THERM_INTERRUPT, l, h); 107 rdmsr(MSR_IA32_THERM_INTERRUPT, l, h);
108 wrmsr (MSR_IA32_THERM_INTERRUPT, l | 0x03 , h); 108 wrmsr(MSR_IA32_THERM_INTERRUPT, l | 0x03 , h);
109 109
110 /* ok we're good to go... */ 110 /* ok we're good to go... */
111 vendor_thermal_interrupt = intel_thermal_interrupt; 111 vendor_thermal_interrupt = intel_thermal_interrupt;
112
113 rdmsr (MSR_IA32_MISC_ENABLE, l, h);
114 wrmsr (MSR_IA32_MISC_ENABLE, l | (1<<3), h);
115 112
116 l = apic_read (APIC_LVTTHMR); 113 rdmsr(MSR_IA32_MISC_ENABLE, l, h);
117 apic_write_around (APIC_LVTTHMR, l & ~APIC_LVT_MASKED); 114 wrmsr(MSR_IA32_MISC_ENABLE, l | (1<<3), h);
118 printk (KERN_INFO "CPU%d: Thermal monitoring enabled\n", cpu); 115
116 l = apic_read(APIC_LVTTHMR);
117 apic_write_around(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
118 printk(KERN_INFO "CPU%d: Thermal monitoring enabled\n", cpu);
119 119
120 /* enable thermal throttle processing */ 120 /* enable thermal throttle processing */
121 atomic_set(&therm_throt_en, 1); 121 atomic_set(&therm_throt_en, 1);
@@ -129,28 +129,28 @@ static inline void intel_get_extended_msrs(struct intel_mce_extended_msrs *r)
129{ 129{
130 u32 h; 130 u32 h;
131 131
132 rdmsr (MSR_IA32_MCG_EAX, r->eax, h); 132 rdmsr(MSR_IA32_MCG_EAX, r->eax, h);
133 rdmsr (MSR_IA32_MCG_EBX, r->ebx, h); 133 rdmsr(MSR_IA32_MCG_EBX, r->ebx, h);
134 rdmsr (MSR_IA32_MCG_ECX, r->ecx, h); 134 rdmsr(MSR_IA32_MCG_ECX, r->ecx, h);
135 rdmsr (MSR_IA32_MCG_EDX, r->edx, h); 135 rdmsr(MSR_IA32_MCG_EDX, r->edx, h);
136 rdmsr (MSR_IA32_MCG_ESI, r->esi, h); 136 rdmsr(MSR_IA32_MCG_ESI, r->esi, h);
137 rdmsr (MSR_IA32_MCG_EDI, r->edi, h); 137 rdmsr(MSR_IA32_MCG_EDI, r->edi, h);
138 rdmsr (MSR_IA32_MCG_EBP, r->ebp, h); 138 rdmsr(MSR_IA32_MCG_EBP, r->ebp, h);
139 rdmsr (MSR_IA32_MCG_ESP, r->esp, h); 139 rdmsr(MSR_IA32_MCG_ESP, r->esp, h);
140 rdmsr (MSR_IA32_MCG_EFLAGS, r->eflags, h); 140 rdmsr(MSR_IA32_MCG_EFLAGS, r->eflags, h);
141 rdmsr (MSR_IA32_MCG_EIP, r->eip, h); 141 rdmsr(MSR_IA32_MCG_EIP, r->eip, h);
142} 142}
143 143
144static void intel_machine_check(struct pt_regs * regs, long error_code) 144static void intel_machine_check(struct pt_regs *regs, long error_code)
145{ 145{
146 int recover=1; 146 int recover = 1;
147 u32 alow, ahigh, high, low; 147 u32 alow, ahigh, high, low;
148 u32 mcgstl, mcgsth; 148 u32 mcgstl, mcgsth;
149 int i; 149 int i;
150 150
151 rdmsr (MSR_IA32_MCG_STATUS, mcgstl, mcgsth); 151 rdmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
152 if (mcgstl & (1<<0)) /* Recoverable ? */ 152 if (mcgstl & (1<<0)) /* Recoverable ? */
153 recover=0; 153 recover = 0;
154 154
155 printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n", 155 printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n",
156 smp_processor_id(), mcgsth, mcgstl); 156 smp_processor_id(), mcgsth, mcgstl);
@@ -191,20 +191,20 @@ static void intel_machine_check(struct pt_regs * regs, long error_code)
191 } 191 }
192 192
193 if (recover & 2) 193 if (recover & 2)
194 panic ("CPU context corrupt"); 194 panic("CPU context corrupt");
195 if (recover & 1) 195 if (recover & 1)
196 panic ("Unable to continue"); 196 panic("Unable to continue");
197 197
198 printk(KERN_EMERG "Attempting to continue.\n"); 198 printk(KERN_EMERG "Attempting to continue.\n");
199 /* 199 /*
200 * Do not clear the MSR_IA32_MCi_STATUS if the error is not 200 * Do not clear the MSR_IA32_MCi_STATUS if the error is not
201 * recoverable/continuable.This will allow BIOS to look at the MSRs 201 * recoverable/continuable.This will allow BIOS to look at the MSRs
202 * for errors if the OS could not log the error. 202 * for errors if the OS could not log the error.
203 */ 203 */
204 for (i=0; i<nr_mce_banks; i++) { 204 for (i = 0; i < nr_mce_banks; i++) {
205 u32 msr; 205 u32 msr;
206 msr = MSR_IA32_MC0_STATUS+i*4; 206 msr = MSR_IA32_MC0_STATUS+i*4;
207 rdmsr (msr, low, high); 207 rdmsr(msr, low, high);
208 if (high&(1<<31)) { 208 if (high&(1<<31)) {
209 /* Clear it */ 209 /* Clear it */
210 wrmsr(msr, 0UL, 0UL); 210 wrmsr(msr, 0UL, 0UL);
@@ -214,7 +214,7 @@ static void intel_machine_check(struct pt_regs * regs, long error_code)
214 } 214 }
215 } 215 }
216 mcgstl &= ~(1<<2); 216 mcgstl &= ~(1<<2);
217 wrmsr (MSR_IA32_MCG_STATUS,mcgstl, mcgsth); 217 wrmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
218} 218}
219 219
220 220
@@ -222,30 +222,30 @@ void intel_p4_mcheck_init(struct cpuinfo_x86 *c)
222{ 222{
223 u32 l, h; 223 u32 l, h;
224 int i; 224 int i;
225 225
226 machine_check_vector = intel_machine_check; 226 machine_check_vector = intel_machine_check;
227 wmb(); 227 wmb();
228 228
229 printk (KERN_INFO "Intel machine check architecture supported.\n"); 229 printk(KERN_INFO "Intel machine check architecture supported.\n");
230 rdmsr (MSR_IA32_MCG_CAP, l, h); 230 rdmsr(MSR_IA32_MCG_CAP, l, h);
231 if (l & (1<<8)) /* Control register present ? */ 231 if (l & (1<<8)) /* Control register present ? */
232 wrmsr (MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 232 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
233 nr_mce_banks = l & 0xff; 233 nr_mce_banks = l & 0xff;
234 234
235 for (i=0; i<nr_mce_banks; i++) { 235 for (i = 0; i < nr_mce_banks; i++) {
236 wrmsr (MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff); 236 wrmsr(MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff);
237 wrmsr (MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0); 237 wrmsr(MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0);
238 } 238 }
239 239
240 set_in_cr4 (X86_CR4_MCE); 240 set_in_cr4(X86_CR4_MCE);
241 printk (KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n", 241 printk(KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n",
242 smp_processor_id()); 242 smp_processor_id());
243 243
244 /* Check for P4/Xeon extended MCE MSRs */ 244 /* Check for P4/Xeon extended MCE MSRs */
245 rdmsr (MSR_IA32_MCG_CAP, l, h); 245 rdmsr(MSR_IA32_MCG_CAP, l, h);
246 if (l & (1<<9)) {/* MCG_EXT_P */ 246 if (l & (1<<9)) {/* MCG_EXT_P */
247 mce_num_extended_msrs = (l >> 16) & 0xff; 247 mce_num_extended_msrs = (l >> 16) & 0xff;
248 printk (KERN_INFO "CPU%d: Intel P4/Xeon Extended MCE MSRs (%d)" 248 printk(KERN_INFO "CPU%d: Intel P4/Xeon Extended MCE MSRs (%d)"
249 " available\n", 249 " available\n",
250 smp_processor_id(), mce_num_extended_msrs); 250 smp_processor_id(), mce_num_extended_msrs);
251 251
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 5d241ce94a44..509bd3d9eacd 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -37,7 +37,7 @@ static struct fixed_range_block fixed_range_blocks[] = {
37static unsigned long smp_changes_mask; 37static unsigned long smp_changes_mask;
38static struct mtrr_state mtrr_state = {}; 38static struct mtrr_state mtrr_state = {};
39static int mtrr_state_set; 39static int mtrr_state_set;
40static u64 tom2; 40u64 mtrr_tom2;
41 41
42#undef MODULE_PARAM_PREFIX 42#undef MODULE_PARAM_PREFIX
43#define MODULE_PARAM_PREFIX "mtrr." 43#define MODULE_PARAM_PREFIX "mtrr."
@@ -139,8 +139,8 @@ u8 mtrr_type_lookup(u64 start, u64 end)
139 } 139 }
140 } 140 }
141 141
142 if (tom2) { 142 if (mtrr_tom2) {
143 if (start >= (1ULL<<32) && (end < tom2)) 143 if (start >= (1ULL<<32) && (end < mtrr_tom2))
144 return MTRR_TYPE_WRBACK; 144 return MTRR_TYPE_WRBACK;
145 } 145 }
146 146
@@ -158,6 +158,20 @@ get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
158 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); 158 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
159} 159}
160 160
161/* fill the MSR pair relating to a var range */
162void fill_mtrr_var_range(unsigned int index,
163 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
164{
165 struct mtrr_var_range *vr;
166
167 vr = mtrr_state.var_ranges;
168
169 vr[index].base_lo = base_lo;
170 vr[index].base_hi = base_hi;
171 vr[index].mask_lo = mask_lo;
172 vr[index].mask_hi = mask_hi;
173}
174
161static void 175static void
162get_fixed_ranges(mtrr_type * frs) 176get_fixed_ranges(mtrr_type * frs)
163{ 177{
@@ -213,13 +227,13 @@ void __init get_mtrr_state(void)
213 mtrr_state.enabled = (lo & 0xc00) >> 10; 227 mtrr_state.enabled = (lo & 0xc00) >> 10;
214 228
215 if (amd_special_default_mtrr()) { 229 if (amd_special_default_mtrr()) {
216 unsigned lo, hi; 230 unsigned low, high;
217 /* TOP_MEM2 */ 231 /* TOP_MEM2 */
218 rdmsr(MSR_K8_TOP_MEM2, lo, hi); 232 rdmsr(MSR_K8_TOP_MEM2, low, high);
219 tom2 = hi; 233 mtrr_tom2 = high;
220 tom2 <<= 32; 234 mtrr_tom2 <<= 32;
221 tom2 |= lo; 235 mtrr_tom2 |= low;
222 tom2 &= 0xffffff8000000ULL; 236 mtrr_tom2 &= 0xffffff800000ULL;
223 } 237 }
224 if (mtrr_show) { 238 if (mtrr_show) {
225 int high_width; 239 int high_width;
@@ -251,9 +265,9 @@ void __init get_mtrr_state(void)
251 else 265 else
252 printk(KERN_INFO "MTRR %u disabled\n", i); 266 printk(KERN_INFO "MTRR %u disabled\n", i);
253 } 267 }
254 if (tom2) { 268 if (mtrr_tom2) {
255 printk(KERN_INFO "TOM2: %016llx aka %lldM\n", 269 printk(KERN_INFO "TOM2: %016llx aka %lldM\n",
256 tom2, tom2>>20); 270 mtrr_tom2, mtrr_tom2>>20);
257 } 271 }
258 } 272 }
259 mtrr_state_set = 1; 273 mtrr_state_set = 1;
@@ -328,7 +342,7 @@ static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
328 342
329 if (lo != msrwords[0] || hi != msrwords[1]) { 343 if (lo != msrwords[0] || hi != msrwords[1]) {
330 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 344 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
331 boot_cpu_data.x86 == 15 && 345 (boot_cpu_data.x86 >= 0x0f && boot_cpu_data.x86 <= 0x11) &&
332 ((msrwords[0] | msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK)) 346 ((msrwords[0] | msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK))
333 k8_enable_fixed_iorrs(); 347 k8_enable_fixed_iorrs();
334 mtrr_wrmsr(msr, msrwords[0], msrwords[1]); 348 mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 6a1e278d9323..6f23969c8faf 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -37,6 +37,7 @@
37#include <linux/smp.h> 37#include <linux/smp.h>
38#include <linux/cpu.h> 38#include <linux/cpu.h>
39#include <linux/mutex.h> 39#include <linux/mutex.h>
40#include <linux/sort.h>
40 41
41#include <asm/e820.h> 42#include <asm/e820.h>
42#include <asm/mtrr.h> 43#include <asm/mtrr.h>
@@ -222,7 +223,7 @@ static void set_mtrr(unsigned int reg, unsigned long base,
222 atomic_set(&data.gate,0); 223 atomic_set(&data.gate,0);
223 224
224 /* Start the ball rolling on other CPUs */ 225 /* Start the ball rolling on other CPUs */
225 if (smp_call_function(ipi_handler, &data, 1, 0) != 0) 226 if (smp_call_function(ipi_handler, &data, 0) != 0)
226 panic("mtrr: timed out waiting for other CPUs\n"); 227 panic("mtrr: timed out waiting for other CPUs\n");
227 228
228 local_irq_save(flags); 229 local_irq_save(flags);
@@ -609,6 +610,787 @@ static struct sysdev_driver mtrr_sysdev_driver = {
609 .resume = mtrr_restore, 610 .resume = mtrr_restore,
610}; 611};
611 612
613/* should be related to MTRR_VAR_RANGES nums */
614#define RANGE_NUM 256
615
616struct res_range {
617 unsigned long start;
618 unsigned long end;
619};
620
621static int __init
622add_range(struct res_range *range, int nr_range, unsigned long start,
623 unsigned long end)
624{
625 /* out of slots */
626 if (nr_range >= RANGE_NUM)
627 return nr_range;
628
629 range[nr_range].start = start;
630 range[nr_range].end = end;
631
632 nr_range++;
633
634 return nr_range;
635}
636
637static int __init
638add_range_with_merge(struct res_range *range, int nr_range, unsigned long start,
639 unsigned long end)
640{
641 int i;
642
643 /* try to merge it with old one */
644 for (i = 0; i < nr_range; i++) {
645 unsigned long final_start, final_end;
646 unsigned long common_start, common_end;
647
648 if (!range[i].end)
649 continue;
650
651 common_start = max(range[i].start, start);
652 common_end = min(range[i].end, end);
653 if (common_start > common_end + 1)
654 continue;
655
656 final_start = min(range[i].start, start);
657 final_end = max(range[i].end, end);
658
659 range[i].start = final_start;
660 range[i].end = final_end;
661 return nr_range;
662 }
663
664 /* need to add that */
665 return add_range(range, nr_range, start, end);
666}
667
668static void __init
669subtract_range(struct res_range *range, unsigned long start, unsigned long end)
670{
671 int i, j;
672
673 for (j = 0; j < RANGE_NUM; j++) {
674 if (!range[j].end)
675 continue;
676
677 if (start <= range[j].start && end >= range[j].end) {
678 range[j].start = 0;
679 range[j].end = 0;
680 continue;
681 }
682
683 if (start <= range[j].start && end < range[j].end &&
684 range[j].start < end + 1) {
685 range[j].start = end + 1;
686 continue;
687 }
688
689
690 if (start > range[j].start && end >= range[j].end &&
691 range[j].end > start - 1) {
692 range[j].end = start - 1;
693 continue;
694 }
695
696 if (start > range[j].start && end < range[j].end) {
697 /* find the new spare */
698 for (i = 0; i < RANGE_NUM; i++) {
699 if (range[i].end == 0)
700 break;
701 }
702 if (i < RANGE_NUM) {
703 range[i].end = range[j].end;
704 range[i].start = end + 1;
705 } else {
706 printk(KERN_ERR "run of slot in ranges\n");
707 }
708 range[j].end = start - 1;
709 continue;
710 }
711 }
712}
713
714static int __init cmp_range(const void *x1, const void *x2)
715{
716 const struct res_range *r1 = x1;
717 const struct res_range *r2 = x2;
718 long start1, start2;
719
720 start1 = r1->start;
721 start2 = r2->start;
722
723 return start1 - start2;
724}
725
726struct var_mtrr_range_state {
727 unsigned long base_pfn;
728 unsigned long size_pfn;
729 mtrr_type type;
730};
731
732struct var_mtrr_range_state __initdata range_state[RANGE_NUM];
733static int __initdata debug_print;
734
735static int __init
736x86_get_mtrr_mem_range(struct res_range *range, int nr_range,
737 unsigned long extra_remove_base,
738 unsigned long extra_remove_size)
739{
740 unsigned long i, base, size;
741 mtrr_type type;
742
743 for (i = 0; i < num_var_ranges; i++) {
744 type = range_state[i].type;
745 if (type != MTRR_TYPE_WRBACK)
746 continue;
747 base = range_state[i].base_pfn;
748 size = range_state[i].size_pfn;
749 nr_range = add_range_with_merge(range, nr_range, base,
750 base + size - 1);
751 }
752 if (debug_print) {
753 printk(KERN_DEBUG "After WB checking\n");
754 for (i = 0; i < nr_range; i++)
755 printk(KERN_DEBUG "MTRR MAP PFN: %016lx - %016lx\n",
756 range[i].start, range[i].end + 1);
757 }
758
759 /* take out UC ranges */
760 for (i = 0; i < num_var_ranges; i++) {
761 type = range_state[i].type;
762 if (type != MTRR_TYPE_UNCACHABLE)
763 continue;
764 size = range_state[i].size_pfn;
765 if (!size)
766 continue;
767 base = range_state[i].base_pfn;
768 subtract_range(range, base, base + size - 1);
769 }
770 if (extra_remove_size)
771 subtract_range(range, extra_remove_base,
772 extra_remove_base + extra_remove_size - 1);
773
774 /* get new range num */
775 nr_range = 0;
776 for (i = 0; i < RANGE_NUM; i++) {
777 if (!range[i].end)
778 continue;
779 nr_range++;
780 }
781 if (debug_print) {
782 printk(KERN_DEBUG "After UC checking\n");
783 for (i = 0; i < nr_range; i++)
784 printk(KERN_DEBUG "MTRR MAP PFN: %016lx - %016lx\n",
785 range[i].start, range[i].end + 1);
786 }
787
788 /* sort the ranges */
789 sort(range, nr_range, sizeof(struct res_range), cmp_range, NULL);
790 if (debug_print) {
791 printk(KERN_DEBUG "After sorting\n");
792 for (i = 0; i < nr_range; i++)
793 printk(KERN_DEBUG "MTRR MAP PFN: %016lx - %016lx\n",
794 range[i].start, range[i].end + 1);
795 }
796
797 /* clear those is not used */
798 for (i = nr_range; i < RANGE_NUM; i++)
799 memset(&range[i], 0, sizeof(range[i]));
800
801 return nr_range;
802}
803
804static struct res_range __initdata range[RANGE_NUM];
805
806#ifdef CONFIG_MTRR_SANITIZER
807
808static unsigned long __init sum_ranges(struct res_range *range, int nr_range)
809{
810 unsigned long sum;
811 int i;
812
813 sum = 0;
814 for (i = 0; i < nr_range; i++)
815 sum += range[i].end + 1 - range[i].start;
816
817 return sum;
818}
819
820static int enable_mtrr_cleanup __initdata =
821 CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT;
822
823static int __init disable_mtrr_cleanup_setup(char *str)
824{
825 if (enable_mtrr_cleanup != -1)
826 enable_mtrr_cleanup = 0;
827 return 0;
828}
829early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup);
830
831static int __init enable_mtrr_cleanup_setup(char *str)
832{
833 if (enable_mtrr_cleanup != -1)
834 enable_mtrr_cleanup = 1;
835 return 0;
836}
837early_param("enble_mtrr_cleanup", enable_mtrr_cleanup_setup);
838
839struct var_mtrr_state {
840 unsigned long range_startk;
841 unsigned long range_sizek;
842 unsigned long chunk_sizek;
843 unsigned long gran_sizek;
844 unsigned int reg;
845};
846
847static void __init
848set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
849 unsigned char type, unsigned int address_bits)
850{
851 u32 base_lo, base_hi, mask_lo, mask_hi;
852 u64 base, mask;
853
854 if (!sizek) {
855 fill_mtrr_var_range(reg, 0, 0, 0, 0);
856 return;
857 }
858
859 mask = (1ULL << address_bits) - 1;
860 mask &= ~((((u64)sizek) << 10) - 1);
861
862 base = ((u64)basek) << 10;
863
864 base |= type;
865 mask |= 0x800;
866
867 base_lo = base & ((1ULL<<32) - 1);
868 base_hi = base >> 32;
869
870 mask_lo = mask & ((1ULL<<32) - 1);
871 mask_hi = mask >> 32;
872
873 fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi);
874}
875
876static void __init
877save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
878 unsigned char type)
879{
880 range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10);
881 range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10);
882 range_state[reg].type = type;
883}
884
885static void __init
886set_var_mtrr_all(unsigned int address_bits)
887{
888 unsigned long basek, sizek;
889 unsigned char type;
890 unsigned int reg;
891
892 for (reg = 0; reg < num_var_ranges; reg++) {
893 basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10);
894 sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10);
895 type = range_state[reg].type;
896
897 set_var_mtrr(reg, basek, sizek, type, address_bits);
898 }
899}
900
901static unsigned int __init
902range_to_mtrr(unsigned int reg, unsigned long range_startk,
903 unsigned long range_sizek, unsigned char type)
904{
905 if (!range_sizek || (reg >= num_var_ranges))
906 return reg;
907
908 while (range_sizek) {
909 unsigned long max_align, align;
910 unsigned long sizek;
911
912 /* Compute the maximum size I can make a range */
913 if (range_startk)
914 max_align = ffs(range_startk) - 1;
915 else
916 max_align = 32;
917 align = fls(range_sizek) - 1;
918 if (align > max_align)
919 align = max_align;
920
921 sizek = 1 << align;
922 if (debug_print)
923 printk(KERN_DEBUG "Setting variable MTRR %d, "
924 "base: %ldMB, range: %ldMB, type %s\n",
925 reg, range_startk >> 10, sizek >> 10,
926 (type == MTRR_TYPE_UNCACHABLE)?"UC":
927 ((type == MTRR_TYPE_WRBACK)?"WB":"Other")
928 );
929 save_var_mtrr(reg++, range_startk, sizek, type);
930 range_startk += sizek;
931 range_sizek -= sizek;
932 if (reg >= num_var_ranges)
933 break;
934 }
935 return reg;
936}
937
938static unsigned __init
939range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
940 unsigned long sizek)
941{
942 unsigned long hole_basek, hole_sizek;
943 unsigned long second_basek, second_sizek;
944 unsigned long range0_basek, range0_sizek;
945 unsigned long range_basek, range_sizek;
946 unsigned long chunk_sizek;
947 unsigned long gran_sizek;
948
949 hole_basek = 0;
950 hole_sizek = 0;
951 second_basek = 0;
952 second_sizek = 0;
953 chunk_sizek = state->chunk_sizek;
954 gran_sizek = state->gran_sizek;
955
956 /* align with gran size, prevent small block used up MTRRs */
957 range_basek = ALIGN(state->range_startk, gran_sizek);
958 if ((range_basek > basek) && basek)
959 return second_sizek;
960 state->range_sizek -= (range_basek - state->range_startk);
961 range_sizek = ALIGN(state->range_sizek, gran_sizek);
962
963 while (range_sizek > state->range_sizek) {
964 range_sizek -= gran_sizek;
965 if (!range_sizek)
966 return 0;
967 }
968 state->range_sizek = range_sizek;
969
970 /* try to append some small hole */
971 range0_basek = state->range_startk;
972 range0_sizek = ALIGN(state->range_sizek, chunk_sizek);
973 if (range0_sizek == state->range_sizek) {
974 if (debug_print)
975 printk(KERN_DEBUG "rangeX: %016lx - %016lx\n",
976 range0_basek<<10,
977 (range0_basek + state->range_sizek)<<10);
978 state->reg = range_to_mtrr(state->reg, range0_basek,
979 state->range_sizek, MTRR_TYPE_WRBACK);
980 return 0;
981 }
982
983 range0_sizek -= chunk_sizek;
984 if (range0_sizek && sizek) {
985 while (range0_basek + range0_sizek > (basek + sizek)) {
986 range0_sizek -= chunk_sizek;
987 if (!range0_sizek)
988 break;
989 }
990 }
991
992 if (range0_sizek) {
993 if (debug_print)
994 printk(KERN_DEBUG "range0: %016lx - %016lx\n",
995 range0_basek<<10,
996 (range0_basek + range0_sizek)<<10);
997 state->reg = range_to_mtrr(state->reg, range0_basek,
998 range0_sizek, MTRR_TYPE_WRBACK);
999
1000 }
1001
1002 range_basek = range0_basek + range0_sizek;
1003 range_sizek = chunk_sizek;
1004
1005 if (range_basek + range_sizek > basek &&
1006 range_basek + range_sizek <= (basek + sizek)) {
1007 /* one hole */
1008 second_basek = basek;
1009 second_sizek = range_basek + range_sizek - basek;
1010 }
1011
1012 /* if last piece, only could one hole near end */
1013 if ((second_basek || !basek) &&
1014 range_sizek - (state->range_sizek - range0_sizek) - second_sizek <
1015 (chunk_sizek >> 1)) {
1016 /*
1017 * one hole in middle (second_sizek is 0) or at end
1018 * (second_sizek is 0 )
1019 */
1020 hole_sizek = range_sizek - (state->range_sizek - range0_sizek)
1021 - second_sizek;
1022 hole_basek = range_basek + range_sizek - hole_sizek
1023 - second_sizek;
1024 } else {
1025 /* fallback for big hole, or several holes */
1026 range_sizek = state->range_sizek - range0_sizek;
1027 second_basek = 0;
1028 second_sizek = 0;
1029 }
1030
1031 if (debug_print)
1032 printk(KERN_DEBUG "range: %016lx - %016lx\n", range_basek<<10,
1033 (range_basek + range_sizek)<<10);
1034 state->reg = range_to_mtrr(state->reg, range_basek, range_sizek,
1035 MTRR_TYPE_WRBACK);
1036 if (hole_sizek) {
1037 if (debug_print)
1038 printk(KERN_DEBUG "hole: %016lx - %016lx\n",
1039 hole_basek<<10, (hole_basek + hole_sizek)<<10);
1040 state->reg = range_to_mtrr(state->reg, hole_basek, hole_sizek,
1041 MTRR_TYPE_UNCACHABLE);
1042
1043 }
1044
1045 return second_sizek;
1046}
1047
1048static void __init
1049set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn,
1050 unsigned long size_pfn)
1051{
1052 unsigned long basek, sizek;
1053 unsigned long second_sizek = 0;
1054
1055 if (state->reg >= num_var_ranges)
1056 return;
1057
1058 basek = base_pfn << (PAGE_SHIFT - 10);
1059 sizek = size_pfn << (PAGE_SHIFT - 10);
1060
1061 /* See if I can merge with the last range */
1062 if ((basek <= 1024) ||
1063 (state->range_startk + state->range_sizek == basek)) {
1064 unsigned long endk = basek + sizek;
1065 state->range_sizek = endk - state->range_startk;
1066 return;
1067 }
1068 /* Write the range mtrrs */
1069 if (state->range_sizek != 0)
1070 second_sizek = range_to_mtrr_with_hole(state, basek, sizek);
1071
1072 /* Allocate an msr */
1073 state->range_startk = basek + second_sizek;
1074 state->range_sizek = sizek - second_sizek;
1075}
1076
1077/* mininum size of mtrr block that can take hole */
1078static u64 mtrr_chunk_size __initdata = (256ULL<<20);
1079
1080static int __init parse_mtrr_chunk_size_opt(char *p)
1081{
1082 if (!p)
1083 return -EINVAL;
1084 mtrr_chunk_size = memparse(p, &p);
1085 return 0;
1086}
1087early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt);
1088
1089/* granity of mtrr of block */
1090static u64 mtrr_gran_size __initdata;
1091
1092static int __init parse_mtrr_gran_size_opt(char *p)
1093{
1094 if (!p)
1095 return -EINVAL;
1096 mtrr_gran_size = memparse(p, &p);
1097 return 0;
1098}
1099early_param("mtrr_gran_size", parse_mtrr_gran_size_opt);
1100
1101static int nr_mtrr_spare_reg __initdata =
1102 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT;
1103
1104static int __init parse_mtrr_spare_reg(char *arg)
1105{
1106 if (arg)
1107 nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0);
1108 return 0;
1109}
1110
1111early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg);
1112
1113static int __init
1114x86_setup_var_mtrrs(struct res_range *range, int nr_range,
1115 u64 chunk_size, u64 gran_size)
1116{
1117 struct var_mtrr_state var_state;
1118 int i;
1119 int num_reg;
1120
1121 var_state.range_startk = 0;
1122 var_state.range_sizek = 0;
1123 var_state.reg = 0;
1124 var_state.chunk_sizek = chunk_size >> 10;
1125 var_state.gran_sizek = gran_size >> 10;
1126
1127 memset(range_state, 0, sizeof(range_state));
1128
1129 /* Write the range etc */
1130 for (i = 0; i < nr_range; i++)
1131 set_var_mtrr_range(&var_state, range[i].start,
1132 range[i].end - range[i].start + 1);
1133
1134 /* Write the last range */
1135 if (var_state.range_sizek != 0)
1136 range_to_mtrr_with_hole(&var_state, 0, 0);
1137
1138 num_reg = var_state.reg;
1139 /* Clear out the extra MTRR's */
1140 while (var_state.reg < num_var_ranges) {
1141 save_var_mtrr(var_state.reg, 0, 0, 0);
1142 var_state.reg++;
1143 }
1144
1145 return num_reg;
1146}
1147
1148struct mtrr_cleanup_result {
1149 unsigned long gran_sizek;
1150 unsigned long chunk_sizek;
1151 unsigned long lose_cover_sizek;
1152 unsigned int num_reg;
1153 int bad;
1154};
1155
1156/*
1157 * gran_size: 1M, 2M, ..., 2G
1158 * chunk size: gran_size, ..., 4G
1159 * so we need (2+13)*6
1160 */
1161#define NUM_RESULT 90
1162#define PSHIFT (PAGE_SHIFT - 10)
1163
1164static struct mtrr_cleanup_result __initdata result[NUM_RESULT];
1165static struct res_range __initdata range_new[RANGE_NUM];
1166static unsigned long __initdata min_loss_pfn[RANGE_NUM];
1167
1168static int __init mtrr_cleanup(unsigned address_bits)
1169{
1170 unsigned long extra_remove_base, extra_remove_size;
1171 unsigned long i, base, size, def, dummy;
1172 mtrr_type type;
1173 int nr_range, nr_range_new;
1174 u64 chunk_size, gran_size;
1175 unsigned long range_sums, range_sums_new;
1176 int index_good;
1177 int num_reg_good;
1178
1179 /* extra one for all 0 */
1180 int num[MTRR_NUM_TYPES + 1];
1181
1182 if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
1183 return 0;
1184 rdmsr(MTRRdefType_MSR, def, dummy);
1185 def &= 0xff;
1186 if (def != MTRR_TYPE_UNCACHABLE)
1187 return 0;
1188
1189 /* get it and store it aside */
1190 memset(range_state, 0, sizeof(range_state));
1191 for (i = 0; i < num_var_ranges; i++) {
1192 mtrr_if->get(i, &base, &size, &type);
1193 range_state[i].base_pfn = base;
1194 range_state[i].size_pfn = size;
1195 range_state[i].type = type;
1196 }
1197
1198 /* check entries number */
1199 memset(num, 0, sizeof(num));
1200 for (i = 0; i < num_var_ranges; i++) {
1201 type = range_state[i].type;
1202 size = range_state[i].size_pfn;
1203 if (type >= MTRR_NUM_TYPES)
1204 continue;
1205 if (!size)
1206 type = MTRR_NUM_TYPES;
1207 num[type]++;
1208 }
1209
1210 /* check if we got UC entries */
1211 if (!num[MTRR_TYPE_UNCACHABLE])
1212 return 0;
1213
1214 /* check if we only had WB and UC */
1215 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
1216 num_var_ranges - num[MTRR_NUM_TYPES])
1217 return 0;
1218
1219 memset(range, 0, sizeof(range));
1220 extra_remove_size = 0;
1221 if (mtrr_tom2) {
1222 extra_remove_base = 1 << (32 - PAGE_SHIFT);
1223 extra_remove_size =
1224 (mtrr_tom2 >> PAGE_SHIFT) - extra_remove_base;
1225 }
1226 nr_range = x86_get_mtrr_mem_range(range, 0, extra_remove_base,
1227 extra_remove_size);
1228 range_sums = sum_ranges(range, nr_range);
1229 printk(KERN_INFO "total RAM coverred: %ldM\n",
1230 range_sums >> (20 - PAGE_SHIFT));
1231
1232 if (mtrr_chunk_size && mtrr_gran_size) {
1233 int num_reg;
1234
1235 debug_print = 1;
1236 /* convert ranges to var ranges state */
1237 num_reg = x86_setup_var_mtrrs(range, nr_range, mtrr_chunk_size,
1238 mtrr_gran_size);
1239
1240 /* we got new setting in range_state, check it */
1241 memset(range_new, 0, sizeof(range_new));
1242 nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
1243 extra_remove_base,
1244 extra_remove_size);
1245 range_sums_new = sum_ranges(range_new, nr_range_new);
1246
1247 i = 0;
1248 result[i].chunk_sizek = mtrr_chunk_size >> 10;
1249 result[i].gran_sizek = mtrr_gran_size >> 10;
1250 result[i].num_reg = num_reg;
1251 if (range_sums < range_sums_new) {
1252 result[i].lose_cover_sizek =
1253 (range_sums_new - range_sums) << PSHIFT;
1254 result[i].bad = 1;
1255 } else
1256 result[i].lose_cover_sizek =
1257 (range_sums - range_sums_new) << PSHIFT;
1258
1259 printk(KERN_INFO "%sgran_size: %ldM \tchunk_size: %ldM \t",
1260 result[i].bad?"*BAD*":" ", result[i].gran_sizek >> 10,
1261 result[i].chunk_sizek >> 10);
1262 printk(KERN_CONT "num_reg: %d \tlose cover RAM: %s%ldM \n",
1263 result[i].num_reg, result[i].bad?"-":"",
1264 result[i].lose_cover_sizek >> 10);
1265 if (!result[i].bad) {
1266 set_var_mtrr_all(address_bits);
1267 return 1;
1268 }
1269 printk(KERN_INFO "invalid mtrr_gran_size or mtrr_chunk_size, "
1270 "will find optimal one\n");
1271 debug_print = 0;
1272 memset(result, 0, sizeof(result[0]));
1273 }
1274
1275 i = 0;
1276 memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn));
1277 memset(result, 0, sizeof(result));
1278 for (gran_size = (1ULL<<20); gran_size < (1ULL<<32); gran_size <<= 1) {
1279 for (chunk_size = gran_size; chunk_size < (1ULL<<33);
1280 chunk_size <<= 1) {
1281 int num_reg;
1282
1283 if (debug_print)
1284 printk(KERN_INFO
1285 "\ngran_size: %lldM chunk_size_size: %lldM\n",
1286 gran_size >> 20, chunk_size >> 20);
1287 if (i >= NUM_RESULT)
1288 continue;
1289
1290 /* convert ranges to var ranges state */
1291 num_reg = x86_setup_var_mtrrs(range, nr_range,
1292 chunk_size, gran_size);
1293
1294 /* we got new setting in range_state, check it */
1295 memset(range_new, 0, sizeof(range_new));
1296 nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
1297 extra_remove_base, extra_remove_size);
1298 range_sums_new = sum_ranges(range_new, nr_range_new);
1299
1300 result[i].chunk_sizek = chunk_size >> 10;
1301 result[i].gran_sizek = gran_size >> 10;
1302 result[i].num_reg = num_reg;
1303 if (range_sums < range_sums_new) {
1304 result[i].lose_cover_sizek =
1305 (range_sums_new - range_sums) << PSHIFT;
1306 result[i].bad = 1;
1307 } else
1308 result[i].lose_cover_sizek =
1309 (range_sums - range_sums_new) << PSHIFT;
1310
1311 /* double check it */
1312 if (!result[i].bad && !result[i].lose_cover_sizek) {
1313 if (nr_range_new != nr_range ||
1314 memcmp(range, range_new, sizeof(range)))
1315 result[i].bad = 1;
1316 }
1317
1318 if (!result[i].bad && (range_sums - range_sums_new <
1319 min_loss_pfn[num_reg])) {
1320 min_loss_pfn[num_reg] =
1321 range_sums - range_sums_new;
1322 }
1323 i++;
1324 }
1325 }
1326
1327 /* print out all */
1328 for (i = 0; i < NUM_RESULT; i++) {
1329 printk(KERN_INFO "%sgran_size: %ldM \tchunk_size: %ldM \t",
1330 result[i].bad?"*BAD* ":" ", result[i].gran_sizek >> 10,
1331 result[i].chunk_sizek >> 10);
1332 printk(KERN_CONT "num_reg: %d \tlose RAM: %s%ldM\n",
1333 result[i].num_reg, result[i].bad?"-":"",
1334 result[i].lose_cover_sizek >> 10);
1335 }
1336
1337 /* try to find the optimal index */
1338 if (nr_mtrr_spare_reg >= num_var_ranges)
1339 nr_mtrr_spare_reg = num_var_ranges - 1;
1340 num_reg_good = -1;
1341 for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
1342 if (!min_loss_pfn[i]) {
1343 num_reg_good = i;
1344 break;
1345 }
1346 }
1347
1348 index_good = -1;
1349 if (num_reg_good != -1) {
1350 for (i = 0; i < NUM_RESULT; i++) {
1351 if (!result[i].bad &&
1352 result[i].num_reg == num_reg_good &&
1353 !result[i].lose_cover_sizek) {
1354 index_good = i;
1355 break;
1356 }
1357 }
1358 }
1359
1360 if (index_good != -1) {
1361 printk(KERN_INFO "Found optimal setting for mtrr clean up\n");
1362 i = index_good;
1363 printk(KERN_INFO "gran_size: %ldM \tchunk_size: %ldM \t",
1364 result[i].gran_sizek >> 10,
1365 result[i].chunk_sizek >> 10);
1366 printk(KERN_CONT "num_reg: %d \tlose RAM: %ldM\n",
1367 result[i].num_reg,
1368 result[i].lose_cover_sizek >> 10);
1369 /* convert ranges to var ranges state */
1370 chunk_size = result[i].chunk_sizek;
1371 chunk_size <<= 10;
1372 gran_size = result[i].gran_sizek;
1373 gran_size <<= 10;
1374 debug_print = 1;
1375 x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
1376 set_var_mtrr_all(address_bits);
1377 return 1;
1378 }
1379
1380 printk(KERN_INFO "mtrr_cleanup: can not find optimal value\n");
1381 printk(KERN_INFO "please specify mtrr_gran_size/mtrr_chunk_size\n");
1382
1383 return 0;
1384}
1385#else
1386static int __init mtrr_cleanup(unsigned address_bits)
1387{
1388 return 0;
1389}
1390#endif
1391
1392static int __initdata changed_by_mtrr_cleanup;
1393
612static int disable_mtrr_trim; 1394static int disable_mtrr_trim;
613 1395
614static int __init disable_mtrr_trim_setup(char *str) 1396static int __init disable_mtrr_trim_setup(char *str)
@@ -648,6 +1430,19 @@ int __init amd_special_default_mtrr(void)
648 return 0; 1430 return 0;
649} 1431}
650 1432
1433static u64 __init real_trim_memory(unsigned long start_pfn,
1434 unsigned long limit_pfn)
1435{
1436 u64 trim_start, trim_size;
1437 trim_start = start_pfn;
1438 trim_start <<= PAGE_SHIFT;
1439 trim_size = limit_pfn;
1440 trim_size <<= PAGE_SHIFT;
1441 trim_size -= trim_start;
1442
1443 return e820_update_range(trim_start, trim_size, E820_RAM,
1444 E820_RESERVED);
1445}
651/** 1446/**
652 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs 1447 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs
653 * @end_pfn: ending page frame number 1448 * @end_pfn: ending page frame number
@@ -663,8 +1458,11 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
663{ 1458{
664 unsigned long i, base, size, highest_pfn = 0, def, dummy; 1459 unsigned long i, base, size, highest_pfn = 0, def, dummy;
665 mtrr_type type; 1460 mtrr_type type;
666 u64 trim_start, trim_size; 1461 int nr_range;
1462 u64 total_trim_size;
667 1463
1464 /* extra one for all 0 */
1465 int num[MTRR_NUM_TYPES + 1];
668 /* 1466 /*
669 * Make sure we only trim uncachable memory on machines that 1467 * Make sure we only trim uncachable memory on machines that
670 * support the Intel MTRR architecture: 1468 * support the Intel MTRR architecture:
@@ -676,14 +1474,22 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
676 if (def != MTRR_TYPE_UNCACHABLE) 1474 if (def != MTRR_TYPE_UNCACHABLE)
677 return 0; 1475 return 0;
678 1476
679 if (amd_special_default_mtrr()) 1477 /* get it and store it aside */
680 return 0; 1478 memset(range_state, 0, sizeof(range_state));
1479 for (i = 0; i < num_var_ranges; i++) {
1480 mtrr_if->get(i, &base, &size, &type);
1481 range_state[i].base_pfn = base;
1482 range_state[i].size_pfn = size;
1483 range_state[i].type = type;
1484 }
681 1485
682 /* Find highest cached pfn */ 1486 /* Find highest cached pfn */
683 for (i = 0; i < num_var_ranges; i++) { 1487 for (i = 0; i < num_var_ranges; i++) {
684 mtrr_if->get(i, &base, &size, &type); 1488 type = range_state[i].type;
685 if (type != MTRR_TYPE_WRBACK) 1489 if (type != MTRR_TYPE_WRBACK)
686 continue; 1490 continue;
1491 base = range_state[i].base_pfn;
1492 size = range_state[i].size_pfn;
687 if (highest_pfn < base + size) 1493 if (highest_pfn < base + size)
688 highest_pfn = base + size; 1494 highest_pfn = base + size;
689 } 1495 }
@@ -698,22 +1504,65 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
698 return 0; 1504 return 0;
699 } 1505 }
700 1506
701 if (highest_pfn < end_pfn) { 1507 /* check entries number */
1508 memset(num, 0, sizeof(num));
1509 for (i = 0; i < num_var_ranges; i++) {
1510 type = range_state[i].type;
1511 if (type >= MTRR_NUM_TYPES)
1512 continue;
1513 size = range_state[i].size_pfn;
1514 if (!size)
1515 type = MTRR_NUM_TYPES;
1516 num[type]++;
1517 }
1518
1519 /* no entry for WB? */
1520 if (!num[MTRR_TYPE_WRBACK])
1521 return 0;
1522
1523 /* check if we only had WB and UC */
1524 if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
1525 num_var_ranges - num[MTRR_NUM_TYPES])
1526 return 0;
1527
1528 memset(range, 0, sizeof(range));
1529 nr_range = 0;
1530 if (mtrr_tom2) {
1531 range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT));
1532 range[nr_range].end = (mtrr_tom2 >> PAGE_SHIFT) - 1;
1533 if (highest_pfn < range[nr_range].end + 1)
1534 highest_pfn = range[nr_range].end + 1;
1535 nr_range++;
1536 }
1537 nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0);
1538
1539 total_trim_size = 0;
1540 /* check the head */
1541 if (range[0].start)
1542 total_trim_size += real_trim_memory(0, range[0].start);
1543 /* check the holes */
1544 for (i = 0; i < nr_range - 1; i++) {
1545 if (range[i].end + 1 < range[i+1].start)
1546 total_trim_size += real_trim_memory(range[i].end + 1,
1547 range[i+1].start);
1548 }
1549 /* check the top */
1550 i = nr_range - 1;
1551 if (range[i].end + 1 < end_pfn)
1552 total_trim_size += real_trim_memory(range[i].end + 1,
1553 end_pfn);
1554
1555 if (total_trim_size) {
702 printk(KERN_WARNING "WARNING: BIOS bug: CPU MTRRs don't cover" 1556 printk(KERN_WARNING "WARNING: BIOS bug: CPU MTRRs don't cover"
703 " all of memory, losing %luMB of RAM.\n", 1557 " all of memory, losing %lluMB of RAM.\n",
704 (end_pfn - highest_pfn) >> (20 - PAGE_SHIFT)); 1558 total_trim_size >> 20);
705 1559
706 WARN_ON(1); 1560 if (!changed_by_mtrr_cleanup)
1561 WARN_ON(1);
707 1562
708 printk(KERN_INFO "update e820 for mtrr\n"); 1563 printk(KERN_INFO "update e820 for mtrr\n");
709 trim_start = highest_pfn;
710 trim_start <<= PAGE_SHIFT;
711 trim_size = end_pfn;
712 trim_size <<= PAGE_SHIFT;
713 trim_size -= trim_start;
714 update_memory_range(trim_start, trim_size, E820_RAM,
715 E820_RESERVED);
716 update_e820(); 1564 update_e820();
1565
717 return 1; 1566 return 1;
718 } 1567 }
719 1568
@@ -729,18 +1578,21 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
729 */ 1578 */
730void __init mtrr_bp_init(void) 1579void __init mtrr_bp_init(void)
731{ 1580{
1581 u32 phys_addr;
732 init_ifs(); 1582 init_ifs();
733 1583
1584 phys_addr = 32;
1585
734 if (cpu_has_mtrr) { 1586 if (cpu_has_mtrr) {
735 mtrr_if = &generic_mtrr_ops; 1587 mtrr_if = &generic_mtrr_ops;
736 size_or_mask = 0xff000000; /* 36 bits */ 1588 size_or_mask = 0xff000000; /* 36 bits */
737 size_and_mask = 0x00f00000; 1589 size_and_mask = 0x00f00000;
1590 phys_addr = 36;
738 1591
739 /* This is an AMD specific MSR, but we assume(hope?) that 1592 /* This is an AMD specific MSR, but we assume(hope?) that
740 Intel will implement it to when they extend the address 1593 Intel will implement it to when they extend the address
741 bus of the Xeon. */ 1594 bus of the Xeon. */
742 if (cpuid_eax(0x80000000) >= 0x80000008) { 1595 if (cpuid_eax(0x80000000) >= 0x80000008) {
743 u32 phys_addr;
744 phys_addr = cpuid_eax(0x80000008) & 0xff; 1596 phys_addr = cpuid_eax(0x80000008) & 0xff;
745 /* CPUID workaround for Intel 0F33/0F34 CPU */ 1597 /* CPUID workaround for Intel 0F33/0F34 CPU */
746 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 1598 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
@@ -758,6 +1610,7 @@ void __init mtrr_bp_init(void)
758 don't support PAE */ 1610 don't support PAE */
759 size_or_mask = 0xfff00000; /* 32 bits */ 1611 size_or_mask = 0xfff00000; /* 32 bits */
760 size_and_mask = 0; 1612 size_and_mask = 0;
1613 phys_addr = 32;
761 } 1614 }
762 } else { 1615 } else {
763 switch (boot_cpu_data.x86_vendor) { 1616 switch (boot_cpu_data.x86_vendor) {
@@ -791,8 +1644,15 @@ void __init mtrr_bp_init(void)
791 if (mtrr_if) { 1644 if (mtrr_if) {
792 set_num_var_ranges(); 1645 set_num_var_ranges();
793 init_table(); 1646 init_table();
794 if (use_intel()) 1647 if (use_intel()) {
795 get_mtrr_state(); 1648 get_mtrr_state();
1649
1650 if (mtrr_cleanup(phys_addr)) {
1651 changed_by_mtrr_cleanup = 1;
1652 mtrr_if->set_all();
1653 }
1654
1655 }
796 } 1656 }
797} 1657}
798 1658
@@ -822,16 +1682,17 @@ void mtrr_ap_init(void)
822 */ 1682 */
823void mtrr_save_state(void) 1683void mtrr_save_state(void)
824{ 1684{
825 smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1, 1); 1685 smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
826} 1686}
827 1687
828static int __init mtrr_init_finialize(void) 1688static int __init mtrr_init_finialize(void)
829{ 1689{
830 if (!mtrr_if) 1690 if (!mtrr_if)
831 return 0; 1691 return 0;
832 if (use_intel()) 1692 if (use_intel()) {
833 mtrr_state_warn(); 1693 if (!changed_by_mtrr_cleanup)
834 else { 1694 mtrr_state_warn();
1695 } else {
835 /* The CPUs haven't MTRR and seem to not support SMP. They have 1696 /* The CPUs haven't MTRR and seem to not support SMP. They have
836 * specific drivers, we use a tricky method to support 1697 * specific drivers, we use a tricky method to support
837 * suspend/resume for them. 1698 * suspend/resume for them.
diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.h b/arch/x86/kernel/cpu/mtrr/mtrr.h
index 2cc77eb6fea3..2dc4ec656b23 100644
--- a/arch/x86/kernel/cpu/mtrr/mtrr.h
+++ b/arch/x86/kernel/cpu/mtrr/mtrr.h
@@ -81,6 +81,8 @@ void set_mtrr_done(struct set_mtrr_context *ctxt);
81void set_mtrr_cache_disable(struct set_mtrr_context *ctxt); 81void set_mtrr_cache_disable(struct set_mtrr_context *ctxt);
82void set_mtrr_prepare_save(struct set_mtrr_context *ctxt); 82void set_mtrr_prepare_save(struct set_mtrr_context *ctxt);
83 83
84void fill_mtrr_var_range(unsigned int index,
85 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi);
84void get_mtrr_state(void); 86void get_mtrr_state(void);
85 87
86extern void set_mtrr_ops(struct mtrr_ops * ops); 88extern void set_mtrr_ops(struct mtrr_ops * ops);
@@ -92,6 +94,7 @@ extern struct mtrr_ops * mtrr_if;
92#define use_intel() (mtrr_if && mtrr_if->use_intel_if == 1) 94#define use_intel() (mtrr_if && mtrr_if->use_intel_if == 1)
93 95
94extern unsigned int num_var_ranges; 96extern unsigned int num_var_ranges;
97extern u64 mtrr_tom2;
95 98
96void mtrr_state_warn(void); 99void mtrr_state_warn(void);
97const char *mtrr_attrib_to_str(int x); 100const char *mtrr_attrib_to_str(int x);
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index f9ae93adffe5..6d4bdc02388a 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -1,11 +1,15 @@
1/* local apic based NMI watchdog for various CPUs. 1/*
2 This file also handles reservation of performance counters for coordination 2 * local apic based NMI watchdog for various CPUs.
3 with other users (like oprofile). 3 *
4 4 * This file also handles reservation of performance counters for coordination
5 Note that these events normally don't tick when the CPU idles. This means 5 * with other users (like oprofile).
6 the frequency varies with CPU load. 6 *
7 7 * Note that these events normally don't tick when the CPU idles. This means
8 Original code for K7/P6 written by Keith Owens */ 8 * the frequency varies with CPU load.
9 *
10 * Original code for K7/P6 written by Keith Owens
11 *
12 */
9 13
10#include <linux/percpu.h> 14#include <linux/percpu.h>
11#include <linux/module.h> 15#include <linux/module.h>
@@ -36,12 +40,16 @@ struct wd_ops {
36 40
37static const struct wd_ops *wd_ops; 41static const struct wd_ops *wd_ops;
38 42
39/* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's 43/*
40 * offset from MSR_P4_BSU_ESCR0. It will be the max for all platforms (for now) 44 * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
45 * offset from MSR_P4_BSU_ESCR0.
46 *
47 * It will be the max for all platforms (for now)
41 */ 48 */
42#define NMI_MAX_COUNTER_BITS 66 49#define NMI_MAX_COUNTER_BITS 66
43 50
44/* perfctr_nmi_owner tracks the ownership of the perfctr registers: 51/*
52 * perfctr_nmi_owner tracks the ownership of the perfctr registers:
45 * evtsel_nmi_owner tracks the ownership of the event selection 53 * evtsel_nmi_owner tracks the ownership of the event selection
46 * - different performance counters/ event selection may be reserved for 54 * - different performance counters/ event selection may be reserved for
47 * different subsystems this reservation system just tries to coordinate 55 * different subsystems this reservation system just tries to coordinate
@@ -73,8 +81,10 @@ static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
73 return 0; 81 return 0;
74} 82}
75 83
76/* converts an msr to an appropriate reservation bit */ 84/*
77/* returns the bit offset of the event selection register */ 85 * converts an msr to an appropriate reservation bit
86 * returns the bit offset of the event selection register
87 */
78static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr) 88static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
79{ 89{
80 /* returns the bit offset of the event selection register */ 90 /* returns the bit offset of the event selection register */
@@ -114,6 +124,7 @@ int avail_to_resrv_perfctr_nmi(unsigned int msr)
114 124
115 return (!test_bit(counter, perfctr_nmi_owner)); 125 return (!test_bit(counter, perfctr_nmi_owner));
116} 126}
127EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
117 128
118int reserve_perfctr_nmi(unsigned int msr) 129int reserve_perfctr_nmi(unsigned int msr)
119{ 130{
@@ -128,6 +139,7 @@ int reserve_perfctr_nmi(unsigned int msr)
128 return 1; 139 return 1;
129 return 0; 140 return 0;
130} 141}
142EXPORT_SYMBOL(reserve_perfctr_nmi);
131 143
132void release_perfctr_nmi(unsigned int msr) 144void release_perfctr_nmi(unsigned int msr)
133{ 145{
@@ -140,6 +152,7 @@ void release_perfctr_nmi(unsigned int msr)
140 152
141 clear_bit(counter, perfctr_nmi_owner); 153 clear_bit(counter, perfctr_nmi_owner);
142} 154}
155EXPORT_SYMBOL(release_perfctr_nmi);
143 156
144int reserve_evntsel_nmi(unsigned int msr) 157int reserve_evntsel_nmi(unsigned int msr)
145{ 158{
@@ -154,6 +167,7 @@ int reserve_evntsel_nmi(unsigned int msr)
154 return 1; 167 return 1;
155 return 0; 168 return 0;
156} 169}
170EXPORT_SYMBOL(reserve_evntsel_nmi);
157 171
158void release_evntsel_nmi(unsigned int msr) 172void release_evntsel_nmi(unsigned int msr)
159{ 173{
@@ -166,11 +180,6 @@ void release_evntsel_nmi(unsigned int msr)
166 180
167 clear_bit(counter, evntsel_nmi_owner); 181 clear_bit(counter, evntsel_nmi_owner);
168} 182}
169
170EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
171EXPORT_SYMBOL(reserve_perfctr_nmi);
172EXPORT_SYMBOL(release_perfctr_nmi);
173EXPORT_SYMBOL(reserve_evntsel_nmi);
174EXPORT_SYMBOL(release_evntsel_nmi); 183EXPORT_SYMBOL(release_evntsel_nmi);
175 184
176void disable_lapic_nmi_watchdog(void) 185void disable_lapic_nmi_watchdog(void)
@@ -180,8 +189,10 @@ void disable_lapic_nmi_watchdog(void)
180 if (atomic_read(&nmi_active) <= 0) 189 if (atomic_read(&nmi_active) <= 0)
181 return; 190 return;
182 191
183 on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1); 192 on_each_cpu(stop_apic_nmi_watchdog, NULL, 1);
184 wd_ops->unreserve(); 193
194 if (wd_ops)
195 wd_ops->unreserve();
185 196
186 BUG_ON(atomic_read(&nmi_active) != 0); 197 BUG_ON(atomic_read(&nmi_active) != 0);
187} 198}
@@ -202,7 +213,7 @@ void enable_lapic_nmi_watchdog(void)
202 return; 213 return;
203 } 214 }
204 215
205 on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1); 216 on_each_cpu(setup_apic_nmi_watchdog, NULL, 1);
206 touch_nmi_watchdog(); 217 touch_nmi_watchdog();
207} 218}
208 219
@@ -232,8 +243,8 @@ static unsigned int adjust_for_32bit_ctr(unsigned int hz)
232 return retval; 243 return retval;
233} 244}
234 245
235static void 246static void write_watchdog_counter(unsigned int perfctr_msr,
236write_watchdog_counter(unsigned int perfctr_msr, const char *descr, unsigned nmi_hz) 247 const char *descr, unsigned nmi_hz)
237{ 248{
238 u64 count = (u64)cpu_khz * 1000; 249 u64 count = (u64)cpu_khz * 1000;
239 250
@@ -244,7 +255,7 @@ write_watchdog_counter(unsigned int perfctr_msr, const char *descr, unsigned nmi
244} 255}
245 256
246static void write_watchdog_counter32(unsigned int perfctr_msr, 257static void write_watchdog_counter32(unsigned int perfctr_msr,
247 const char *descr, unsigned nmi_hz) 258 const char *descr, unsigned nmi_hz)
248{ 259{
249 u64 count = (u64)cpu_khz * 1000; 260 u64 count = (u64)cpu_khz * 1000;
250 261
@@ -254,9 +265,10 @@ static void write_watchdog_counter32(unsigned int perfctr_msr,
254 wrmsr(perfctr_msr, (u32)(-count), 0); 265 wrmsr(perfctr_msr, (u32)(-count), 0);
255} 266}
256 267
257/* AMD K7/K8/Family10h/Family11h support. AMD keeps this interface 268/*
258 nicely stable so there is not much variety */ 269 * AMD K7/K8/Family10h/Family11h support.
259 270 * AMD keeps this interface nicely stable so there is not much variety
271 */
260#define K7_EVNTSEL_ENABLE (1 << 22) 272#define K7_EVNTSEL_ENABLE (1 << 22)
261#define K7_EVNTSEL_INT (1 << 20) 273#define K7_EVNTSEL_INT (1 << 20)
262#define K7_EVNTSEL_OS (1 << 17) 274#define K7_EVNTSEL_OS (1 << 17)
@@ -289,7 +301,7 @@ static int setup_k7_watchdog(unsigned nmi_hz)
289 301
290 wd->perfctr_msr = perfctr_msr; 302 wd->perfctr_msr = perfctr_msr;
291 wd->evntsel_msr = evntsel_msr; 303 wd->evntsel_msr = evntsel_msr;
292 wd->cccr_msr = 0; //unused 304 wd->cccr_msr = 0; /* unused */
293 return 1; 305 return 1;
294} 306}
295 307
@@ -325,18 +337,19 @@ static void single_msr_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
325} 337}
326 338
327static const struct wd_ops k7_wd_ops = { 339static const struct wd_ops k7_wd_ops = {
328 .reserve = single_msr_reserve, 340 .reserve = single_msr_reserve,
329 .unreserve = single_msr_unreserve, 341 .unreserve = single_msr_unreserve,
330 .setup = setup_k7_watchdog, 342 .setup = setup_k7_watchdog,
331 .rearm = single_msr_rearm, 343 .rearm = single_msr_rearm,
332 .stop = single_msr_stop_watchdog, 344 .stop = single_msr_stop_watchdog,
333 .perfctr = MSR_K7_PERFCTR0, 345 .perfctr = MSR_K7_PERFCTR0,
334 .evntsel = MSR_K7_EVNTSEL0, 346 .evntsel = MSR_K7_EVNTSEL0,
335 .checkbit = 1ULL<<47, 347 .checkbit = 1ULL << 47,
336}; 348};
337 349
338/* Intel Model 6 (PPro+,P2,P3,P-M,Core1) */ 350/*
339 351 * Intel Model 6 (PPro+,P2,P3,P-M,Core1)
352 */
340#define P6_EVNTSEL0_ENABLE (1 << 22) 353#define P6_EVNTSEL0_ENABLE (1 << 22)
341#define P6_EVNTSEL_INT (1 << 20) 354#define P6_EVNTSEL_INT (1 << 20)
342#define P6_EVNTSEL_OS (1 << 17) 355#define P6_EVNTSEL_OS (1 << 17)
@@ -372,52 +385,58 @@ static int setup_p6_watchdog(unsigned nmi_hz)
372 385
373 wd->perfctr_msr = perfctr_msr; 386 wd->perfctr_msr = perfctr_msr;
374 wd->evntsel_msr = evntsel_msr; 387 wd->evntsel_msr = evntsel_msr;
375 wd->cccr_msr = 0; //unused 388 wd->cccr_msr = 0; /* unused */
376 return 1; 389 return 1;
377} 390}
378 391
379static void p6_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz) 392static void p6_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
380{ 393{
381 /* P6 based Pentium M need to re-unmask 394 /*
395 * P6 based Pentium M need to re-unmask
382 * the apic vector but it doesn't hurt 396 * the apic vector but it doesn't hurt
383 * other P6 variant. 397 * other P6 variant.
384 * ArchPerfom/Core Duo also needs this */ 398 * ArchPerfom/Core Duo also needs this
399 */
385 apic_write(APIC_LVTPC, APIC_DM_NMI); 400 apic_write(APIC_LVTPC, APIC_DM_NMI);
401
386 /* P6/ARCH_PERFMON has 32 bit counter write */ 402 /* P6/ARCH_PERFMON has 32 bit counter write */
387 write_watchdog_counter32(wd->perfctr_msr, NULL,nmi_hz); 403 write_watchdog_counter32(wd->perfctr_msr, NULL,nmi_hz);
388} 404}
389 405
390static const struct wd_ops p6_wd_ops = { 406static const struct wd_ops p6_wd_ops = {
391 .reserve = single_msr_reserve, 407 .reserve = single_msr_reserve,
392 .unreserve = single_msr_unreserve, 408 .unreserve = single_msr_unreserve,
393 .setup = setup_p6_watchdog, 409 .setup = setup_p6_watchdog,
394 .rearm = p6_rearm, 410 .rearm = p6_rearm,
395 .stop = single_msr_stop_watchdog, 411 .stop = single_msr_stop_watchdog,
396 .perfctr = MSR_P6_PERFCTR0, 412 .perfctr = MSR_P6_PERFCTR0,
397 .evntsel = MSR_P6_EVNTSEL0, 413 .evntsel = MSR_P6_EVNTSEL0,
398 .checkbit = 1ULL<<39, 414 .checkbit = 1ULL << 39,
399}; 415};
400 416
401/* Intel P4 performance counters. By far the most complicated of all. */ 417/*
402 418 * Intel P4 performance counters.
403#define MSR_P4_MISC_ENABLE_PERF_AVAIL (1<<7) 419 * By far the most complicated of all.
404#define P4_ESCR_EVENT_SELECT(N) ((N)<<25) 420 */
405#define P4_ESCR_OS (1<<3) 421#define MSR_P4_MISC_ENABLE_PERF_AVAIL (1 << 7)
406#define P4_ESCR_USR (1<<2) 422#define P4_ESCR_EVENT_SELECT(N) ((N) << 25)
407#define P4_CCCR_OVF_PMI0 (1<<26) 423#define P4_ESCR_OS (1 << 3)
408#define P4_CCCR_OVF_PMI1 (1<<27) 424#define P4_ESCR_USR (1 << 2)
409#define P4_CCCR_THRESHOLD(N) ((N)<<20) 425#define P4_CCCR_OVF_PMI0 (1 << 26)
410#define P4_CCCR_COMPLEMENT (1<<19) 426#define P4_CCCR_OVF_PMI1 (1 << 27)
411#define P4_CCCR_COMPARE (1<<18) 427#define P4_CCCR_THRESHOLD(N) ((N) << 20)
412#define P4_CCCR_REQUIRED (3<<16) 428#define P4_CCCR_COMPLEMENT (1 << 19)
413#define P4_CCCR_ESCR_SELECT(N) ((N)<<13) 429#define P4_CCCR_COMPARE (1 << 18)
414#define P4_CCCR_ENABLE (1<<12) 430#define P4_CCCR_REQUIRED (3 << 16)
415#define P4_CCCR_OVF (1<<31) 431#define P4_CCCR_ESCR_SELECT(N) ((N) << 13)
416 432#define P4_CCCR_ENABLE (1 << 12)
417/* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter 433#define P4_CCCR_OVF (1 << 31)
418 CRU_ESCR0 (with any non-null event selector) through a complemented
419 max threshold. [IA32-Vol3, Section 14.9.9] */
420 434
435/*
436 * Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter
437 * CRU_ESCR0 (with any non-null event selector) through a complemented
438 * max threshold. [IA32-Vol3, Section 14.9.9]
439 */
421static int setup_p4_watchdog(unsigned nmi_hz) 440static int setup_p4_watchdog(unsigned nmi_hz)
422{ 441{
423 unsigned int perfctr_msr, evntsel_msr, cccr_msr; 442 unsigned int perfctr_msr, evntsel_msr, cccr_msr;
@@ -442,7 +461,8 @@ static int setup_p4_watchdog(unsigned nmi_hz)
442#endif 461#endif
443 ht_num = 0; 462 ht_num = 0;
444 463
445 /* performance counters are shared resources 464 /*
465 * performance counters are shared resources
446 * assign each hyperthread its own set 466 * assign each hyperthread its own set
447 * (re-use the ESCR0 register, seems safe 467 * (re-use the ESCR0 register, seems safe
448 * and keeps the cccr_val the same) 468 * and keeps the cccr_val the same)
@@ -540,20 +560,21 @@ static void p4_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
540} 560}
541 561
542static const struct wd_ops p4_wd_ops = { 562static const struct wd_ops p4_wd_ops = {
543 .reserve = p4_reserve, 563 .reserve = p4_reserve,
544 .unreserve = p4_unreserve, 564 .unreserve = p4_unreserve,
545 .setup = setup_p4_watchdog, 565 .setup = setup_p4_watchdog,
546 .rearm = p4_rearm, 566 .rearm = p4_rearm,
547 .stop = stop_p4_watchdog, 567 .stop = stop_p4_watchdog,
548 /* RED-PEN this is wrong for the other sibling */ 568 /* RED-PEN this is wrong for the other sibling */
549 .perfctr = MSR_P4_BPU_PERFCTR0, 569 .perfctr = MSR_P4_BPU_PERFCTR0,
550 .evntsel = MSR_P4_BSU_ESCR0, 570 .evntsel = MSR_P4_BSU_ESCR0,
551 .checkbit = 1ULL<<39, 571 .checkbit = 1ULL << 39,
552}; 572};
553 573
554/* Watchdog using the Intel architected PerfMon. Used for Core2 and hopefully 574/*
555 all future Intel CPUs. */ 575 * Watchdog using the Intel architected PerfMon.
556 576 * Used for Core2 and hopefully all future Intel CPUs.
577 */
557#define ARCH_PERFMON_NMI_EVENT_SEL ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 578#define ARCH_PERFMON_NMI_EVENT_SEL ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
558#define ARCH_PERFMON_NMI_EVENT_UMASK ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK 579#define ARCH_PERFMON_NMI_EVENT_UMASK ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK
559 580
@@ -599,19 +620,19 @@ static int setup_intel_arch_watchdog(unsigned nmi_hz)
599 620
600 wd->perfctr_msr = perfctr_msr; 621 wd->perfctr_msr = perfctr_msr;
601 wd->evntsel_msr = evntsel_msr; 622 wd->evntsel_msr = evntsel_msr;
602 wd->cccr_msr = 0; //unused 623 wd->cccr_msr = 0; /* unused */
603 intel_arch_wd_ops.checkbit = 1ULL << (eax.split.bit_width - 1); 624 intel_arch_wd_ops.checkbit = 1ULL << (eax.split.bit_width - 1);
604 return 1; 625 return 1;
605} 626}
606 627
607static struct wd_ops intel_arch_wd_ops __read_mostly = { 628static struct wd_ops intel_arch_wd_ops __read_mostly = {
608 .reserve = single_msr_reserve, 629 .reserve = single_msr_reserve,
609 .unreserve = single_msr_unreserve, 630 .unreserve = single_msr_unreserve,
610 .setup = setup_intel_arch_watchdog, 631 .setup = setup_intel_arch_watchdog,
611 .rearm = p6_rearm, 632 .rearm = p6_rearm,
612 .stop = single_msr_stop_watchdog, 633 .stop = single_msr_stop_watchdog,
613 .perfctr = MSR_ARCH_PERFMON_PERFCTR1, 634 .perfctr = MSR_ARCH_PERFMON_PERFCTR1,
614 .evntsel = MSR_ARCH_PERFMON_EVENTSEL1, 635 .evntsel = MSR_ARCH_PERFMON_EVENTSEL1,
615}; 636};
616 637
617static void probe_nmi_watchdog(void) 638static void probe_nmi_watchdog(void)
@@ -624,8 +645,10 @@ static void probe_nmi_watchdog(void)
624 wd_ops = &k7_wd_ops; 645 wd_ops = &k7_wd_ops;
625 break; 646 break;
626 case X86_VENDOR_INTEL: 647 case X86_VENDOR_INTEL:
627 /* Work around Core Duo (Yonah) errata AE49 where perfctr1 648 /*
628 doesn't have a working enable bit. */ 649 * Work around Core Duo (Yonah) errata AE49 where perfctr1
650 * doesn't have a working enable bit.
651 */
629 if (boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 14) { 652 if (boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 14) {
630 intel_arch_wd_ops.perfctr = MSR_ARCH_PERFMON_PERFCTR0; 653 intel_arch_wd_ops.perfctr = MSR_ARCH_PERFMON_PERFCTR0;
631 intel_arch_wd_ops.evntsel = MSR_ARCH_PERFMON_EVENTSEL0; 654 intel_arch_wd_ops.evntsel = MSR_ARCH_PERFMON_EVENTSEL0;
@@ -636,7 +659,7 @@ static void probe_nmi_watchdog(void)
636 } 659 }
637 switch (boot_cpu_data.x86) { 660 switch (boot_cpu_data.x86) {
638 case 6: 661 case 6:
639 if (boot_cpu_data.x86_model > 0xd) 662 if (boot_cpu_data.x86_model > 13)
640 return; 663 return;
641 664
642 wd_ops = &p6_wd_ops; 665 wd_ops = &p6_wd_ops;
@@ -697,10 +720,11 @@ int lapic_wd_event(unsigned nmi_hz)
697{ 720{
698 struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); 721 struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
699 u64 ctr; 722 u64 ctr;
723
700 rdmsrl(wd->perfctr_msr, ctr); 724 rdmsrl(wd->perfctr_msr, ctr);
701 if (ctr & wd_ops->checkbit) { /* perfctr still running? */ 725 if (ctr & wd_ops->checkbit) /* perfctr still running? */
702 return 0; 726 return 0;
703 } 727
704 wd_ops->rearm(wd, nmi_hz); 728 wd_ops->rearm(wd, nmi_hz);
705 return 1; 729 return 1;
706} 730}
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index daff52a62248..2de5fa2bbf77 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -33,6 +33,7 @@
33#include <linux/init.h> 33#include <linux/init.h>
34#include <linux/poll.h> 34#include <linux/poll.h>
35#include <linux/smp.h> 35#include <linux/smp.h>
36#include <linux/smp_lock.h>
36#include <linux/major.h> 37#include <linux/major.h>
37#include <linux/fs.h> 38#include <linux/fs.h>
38#include <linux/smp_lock.h> 39#include <linux/smp_lock.h>
@@ -95,7 +96,7 @@ static ssize_t cpuid_read(struct file *file, char __user *buf,
95 for (; count; count -= 16) { 96 for (; count; count -= 16) {
96 cmd.eax = pos; 97 cmd.eax = pos;
97 cmd.ecx = pos >> 32; 98 cmd.ecx = pos >> 32;
98 smp_call_function_single(cpu, cpuid_smp_cpuid, &cmd, 1, 1); 99 smp_call_function_single(cpu, cpuid_smp_cpuid, &cmd, 1);
99 if (copy_to_user(tmp, &cmd, 16)) 100 if (copy_to_user(tmp, &cmd, 16))
100 return -EFAULT; 101 return -EFAULT;
101 tmp += 16; 102 tmp += 16;
@@ -107,15 +108,23 @@ static ssize_t cpuid_read(struct file *file, char __user *buf,
107 108
108static int cpuid_open(struct inode *inode, struct file *file) 109static int cpuid_open(struct inode *inode, struct file *file)
109{ 110{
110 unsigned int cpu = iminor(file->f_path.dentry->d_inode); 111 unsigned int cpu;
111 struct cpuinfo_x86 *c = &cpu_data(cpu); 112 struct cpuinfo_x86 *c;
112 113 int ret = 0;
113 if (cpu >= NR_CPUS || !cpu_online(cpu)) 114
114 return -ENXIO; /* No such CPU */ 115 lock_kernel();
116
117 cpu = iminor(file->f_path.dentry->d_inode);
118 if (cpu >= NR_CPUS || !cpu_online(cpu)) {
119 ret = -ENXIO; /* No such CPU */
120 goto out;
121 }
122 c = &cpu_data(cpu);
115 if (c->cpuid_level < 0) 123 if (c->cpuid_level < 0)
116 return -EIO; /* CPUID not supported */ 124 ret = -EIO; /* CPUID not supported */
117 125out:
118 return 0; 126 unlock_kernel();
127 return ret;
119} 128}
120 129
121/* 130/*
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
new file mode 100644
index 000000000000..28c29180b380
--- /dev/null
+++ b/arch/x86/kernel/e820.c
@@ -0,0 +1,1390 @@
1/*
2 * Handle the memory map.
3 * The functions here do the job until bootmem takes over.
4 *
5 * Getting sanitize_e820_map() in sync with i386 version by applying change:
6 * - Provisions for empty E820 memory regions (reported by certain BIOSes).
7 * Alex Achenbach <xela@slit.de>, December 2002.
8 * Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
9 *
10 */
11#include <linux/kernel.h>
12#include <linux/types.h>
13#include <linux/init.h>
14#include <linux/bootmem.h>
15#include <linux/ioport.h>
16#include <linux/string.h>
17#include <linux/kexec.h>
18#include <linux/module.h>
19#include <linux/mm.h>
20#include <linux/pfn.h>
21#include <linux/suspend.h>
22#include <linux/firmware-map.h>
23
24#include <asm/pgtable.h>
25#include <asm/page.h>
26#include <asm/e820.h>
27#include <asm/proto.h>
28#include <asm/setup.h>
29#include <asm/trampoline.h>
30
31/*
32 * The e820 map is the map that gets modified e.g. with command line parameters
33 * and that is also registered with modifications in the kernel resource tree
34 * with the iomem_resource as parent.
35 *
36 * The e820_saved is directly saved after the BIOS-provided memory map is
37 * copied. It doesn't get modified afterwards. It's registered for the
38 * /sys/firmware/memmap interface.
39 *
40 * That memory map is not modified and is used as base for kexec. The kexec'd
41 * kernel should get the same memory map as the firmware provides. Then the
42 * user can e.g. boot the original kernel with mem=1G while still booting the
43 * next kernel with full memory.
44 */
45struct e820map e820;
46struct e820map e820_saved;
47
48/* For PCI or other memory-mapped resources */
49unsigned long pci_mem_start = 0xaeedbabe;
50#ifdef CONFIG_PCI
51EXPORT_SYMBOL(pci_mem_start);
52#endif
53
54/*
55 * This function checks if any part of the range <start,end> is mapped
56 * with type.
57 */
58int
59e820_any_mapped(u64 start, u64 end, unsigned type)
60{
61 int i;
62
63 for (i = 0; i < e820.nr_map; i++) {
64 struct e820entry *ei = &e820.map[i];
65
66 if (type && ei->type != type)
67 continue;
68 if (ei->addr >= end || ei->addr + ei->size <= start)
69 continue;
70 return 1;
71 }
72 return 0;
73}
74EXPORT_SYMBOL_GPL(e820_any_mapped);
75
76/*
77 * This function checks if the entire range <start,end> is mapped with type.
78 *
79 * Note: this function only works correct if the e820 table is sorted and
80 * not-overlapping, which is the case
81 */
82int __init e820_all_mapped(u64 start, u64 end, unsigned type)
83{
84 int i;
85
86 for (i = 0; i < e820.nr_map; i++) {
87 struct e820entry *ei = &e820.map[i];
88
89 if (type && ei->type != type)
90 continue;
91 /* is the region (part) in overlap with the current region ?*/
92 if (ei->addr >= end || ei->addr + ei->size <= start)
93 continue;
94
95 /* if the region is at the beginning of <start,end> we move
96 * start to the end of the region since it's ok until there
97 */
98 if (ei->addr <= start)
99 start = ei->addr + ei->size;
100 /*
101 * if start is now at or beyond end, we're done, full
102 * coverage
103 */
104 if (start >= end)
105 return 1;
106 }
107 return 0;
108}
109
110/*
111 * Add a memory region to the kernel e820 map.
112 */
113void __init e820_add_region(u64 start, u64 size, int type)
114{
115 int x = e820.nr_map;
116
117 if (x == ARRAY_SIZE(e820.map)) {
118 printk(KERN_ERR "Ooops! Too many entries in the memory map!\n");
119 return;
120 }
121
122 e820.map[x].addr = start;
123 e820.map[x].size = size;
124 e820.map[x].type = type;
125 e820.nr_map++;
126}
127
128void __init e820_print_map(char *who)
129{
130 int i;
131
132 for (i = 0; i < e820.nr_map; i++) {
133 printk(KERN_INFO " %s: %016Lx - %016Lx ", who,
134 (unsigned long long) e820.map[i].addr,
135 (unsigned long long)
136 (e820.map[i].addr + e820.map[i].size));
137 switch (e820.map[i].type) {
138 case E820_RAM:
139 case E820_RESERVED_KERN:
140 printk(KERN_CONT "(usable)\n");
141 break;
142 case E820_RESERVED:
143 printk(KERN_CONT "(reserved)\n");
144 break;
145 case E820_ACPI:
146 printk(KERN_CONT "(ACPI data)\n");
147 break;
148 case E820_NVS:
149 printk(KERN_CONT "(ACPI NVS)\n");
150 break;
151 default:
152 printk(KERN_CONT "type %u\n", e820.map[i].type);
153 break;
154 }
155 }
156}
157
158/*
159 * Sanitize the BIOS e820 map.
160 *
161 * Some e820 responses include overlapping entries. The following
162 * replaces the original e820 map with a new one, removing overlaps,
163 * and resolving conflicting memory types in favor of highest
164 * numbered type.
165 *
166 * The input parameter biosmap points to an array of 'struct
167 * e820entry' which on entry has elements in the range [0, *pnr_map)
168 * valid, and which has space for up to max_nr_map entries.
169 * On return, the resulting sanitized e820 map entries will be in
170 * overwritten in the same location, starting at biosmap.
171 *
172 * The integer pointed to by pnr_map must be valid on entry (the
173 * current number of valid entries located at biosmap) and will
174 * be updated on return, with the new number of valid entries
175 * (something no more than max_nr_map.)
176 *
177 * The return value from sanitize_e820_map() is zero if it
178 * successfully 'sanitized' the map entries passed in, and is -1
179 * if it did nothing, which can happen if either of (1) it was
180 * only passed one map entry, or (2) any of the input map entries
181 * were invalid (start + size < start, meaning that the size was
182 * so big the described memory range wrapped around through zero.)
183 *
184 * Visually we're performing the following
185 * (1,2,3,4 = memory types)...
186 *
187 * Sample memory map (w/overlaps):
188 * ____22__________________
189 * ______________________4_
190 * ____1111________________
191 * _44_____________________
192 * 11111111________________
193 * ____________________33__
194 * ___________44___________
195 * __________33333_________
196 * ______________22________
197 * ___________________2222_
198 * _________111111111______
199 * _____________________11_
200 * _________________4______
201 *
202 * Sanitized equivalent (no overlap):
203 * 1_______________________
204 * _44_____________________
205 * ___1____________________
206 * ____22__________________
207 * ______11________________
208 * _________1______________
209 * __________3_____________
210 * ___________44___________
211 * _____________33_________
212 * _______________2________
213 * ________________1_______
214 * _________________4______
215 * ___________________2____
216 * ____________________33__
217 * ______________________4_
218 */
219
220int __init sanitize_e820_map(struct e820entry *biosmap, int max_nr_map,
221 int *pnr_map)
222{
223 struct change_member {
224 struct e820entry *pbios; /* pointer to original bios entry */
225 unsigned long long addr; /* address for this change point */
226 };
227 static struct change_member change_point_list[2*E820_X_MAX] __initdata;
228 static struct change_member *change_point[2*E820_X_MAX] __initdata;
229 static struct e820entry *overlap_list[E820_X_MAX] __initdata;
230 static struct e820entry new_bios[E820_X_MAX] __initdata;
231 struct change_member *change_tmp;
232 unsigned long current_type, last_type;
233 unsigned long long last_addr;
234 int chgidx, still_changing;
235 int overlap_entries;
236 int new_bios_entry;
237 int old_nr, new_nr, chg_nr;
238 int i;
239
240 /* if there's only one memory region, don't bother */
241 if (*pnr_map < 2)
242 return -1;
243
244 old_nr = *pnr_map;
245 BUG_ON(old_nr > max_nr_map);
246
247 /* bail out if we find any unreasonable addresses in bios map */
248 for (i = 0; i < old_nr; i++)
249 if (biosmap[i].addr + biosmap[i].size < biosmap[i].addr)
250 return -1;
251
252 /* create pointers for initial change-point information (for sorting) */
253 for (i = 0; i < 2 * old_nr; i++)
254 change_point[i] = &change_point_list[i];
255
256 /* record all known change-points (starting and ending addresses),
257 omitting those that are for empty memory regions */
258 chgidx = 0;
259 for (i = 0; i < old_nr; i++) {
260 if (biosmap[i].size != 0) {
261 change_point[chgidx]->addr = biosmap[i].addr;
262 change_point[chgidx++]->pbios = &biosmap[i];
263 change_point[chgidx]->addr = biosmap[i].addr +
264 biosmap[i].size;
265 change_point[chgidx++]->pbios = &biosmap[i];
266 }
267 }
268 chg_nr = chgidx;
269
270 /* sort change-point list by memory addresses (low -> high) */
271 still_changing = 1;
272 while (still_changing) {
273 still_changing = 0;
274 for (i = 1; i < chg_nr; i++) {
275 unsigned long long curaddr, lastaddr;
276 unsigned long long curpbaddr, lastpbaddr;
277
278 curaddr = change_point[i]->addr;
279 lastaddr = change_point[i - 1]->addr;
280 curpbaddr = change_point[i]->pbios->addr;
281 lastpbaddr = change_point[i - 1]->pbios->addr;
282
283 /*
284 * swap entries, when:
285 *
286 * curaddr > lastaddr or
287 * curaddr == lastaddr and curaddr == curpbaddr and
288 * lastaddr != lastpbaddr
289 */
290 if (curaddr < lastaddr ||
291 (curaddr == lastaddr && curaddr == curpbaddr &&
292 lastaddr != lastpbaddr)) {
293 change_tmp = change_point[i];
294 change_point[i] = change_point[i-1];
295 change_point[i-1] = change_tmp;
296 still_changing = 1;
297 }
298 }
299 }
300
301 /* create a new bios memory map, removing overlaps */
302 overlap_entries = 0; /* number of entries in the overlap table */
303 new_bios_entry = 0; /* index for creating new bios map entries */
304 last_type = 0; /* start with undefined memory type */
305 last_addr = 0; /* start with 0 as last starting address */
306
307 /* loop through change-points, determining affect on the new bios map */
308 for (chgidx = 0; chgidx < chg_nr; chgidx++) {
309 /* keep track of all overlapping bios entries */
310 if (change_point[chgidx]->addr ==
311 change_point[chgidx]->pbios->addr) {
312 /*
313 * add map entry to overlap list (> 1 entry
314 * implies an overlap)
315 */
316 overlap_list[overlap_entries++] =
317 change_point[chgidx]->pbios;
318 } else {
319 /*
320 * remove entry from list (order independent,
321 * so swap with last)
322 */
323 for (i = 0; i < overlap_entries; i++) {
324 if (overlap_list[i] ==
325 change_point[chgidx]->pbios)
326 overlap_list[i] =
327 overlap_list[overlap_entries-1];
328 }
329 overlap_entries--;
330 }
331 /*
332 * if there are overlapping entries, decide which
333 * "type" to use (larger value takes precedence --
334 * 1=usable, 2,3,4,4+=unusable)
335 */
336 current_type = 0;
337 for (i = 0; i < overlap_entries; i++)
338 if (overlap_list[i]->type > current_type)
339 current_type = overlap_list[i]->type;
340 /*
341 * continue building up new bios map based on this
342 * information
343 */
344 if (current_type != last_type) {
345 if (last_type != 0) {
346 new_bios[new_bios_entry].size =
347 change_point[chgidx]->addr - last_addr;
348 /*
349 * move forward only if the new size
350 * was non-zero
351 */
352 if (new_bios[new_bios_entry].size != 0)
353 /*
354 * no more space left for new
355 * bios entries ?
356 */
357 if (++new_bios_entry >= max_nr_map)
358 break;
359 }
360 if (current_type != 0) {
361 new_bios[new_bios_entry].addr =
362 change_point[chgidx]->addr;
363 new_bios[new_bios_entry].type = current_type;
364 last_addr = change_point[chgidx]->addr;
365 }
366 last_type = current_type;
367 }
368 }
369 /* retain count for new bios entries */
370 new_nr = new_bios_entry;
371
372 /* copy new bios mapping into original location */
373 memcpy(biosmap, new_bios, new_nr * sizeof(struct e820entry));
374 *pnr_map = new_nr;
375
376 return 0;
377}
378
379static int __init __append_e820_map(struct e820entry *biosmap, int nr_map)
380{
381 while (nr_map) {
382 u64 start = biosmap->addr;
383 u64 size = biosmap->size;
384 u64 end = start + size;
385 u32 type = biosmap->type;
386
387 /* Overflow in 64 bits? Ignore the memory map. */
388 if (start > end)
389 return -1;
390
391 e820_add_region(start, size, type);
392
393 biosmap++;
394 nr_map--;
395 }
396 return 0;
397}
398
399/*
400 * Copy the BIOS e820 map into a safe place.
401 *
402 * Sanity-check it while we're at it..
403 *
404 * If we're lucky and live on a modern system, the setup code
405 * will have given us a memory map that we can use to properly
406 * set up memory. If we aren't, we'll fake a memory map.
407 */
408static int __init append_e820_map(struct e820entry *biosmap, int nr_map)
409{
410 /* Only one memory region (or negative)? Ignore it */
411 if (nr_map < 2)
412 return -1;
413
414 return __append_e820_map(biosmap, nr_map);
415}
416
417static u64 __init e820_update_range_map(struct e820map *e820x, u64 start,
418 u64 size, unsigned old_type,
419 unsigned new_type)
420{
421 int i;
422 u64 real_updated_size = 0;
423
424 BUG_ON(old_type == new_type);
425
426 if (size > (ULLONG_MAX - start))
427 size = ULLONG_MAX - start;
428
429 for (i = 0; i < e820.nr_map; i++) {
430 struct e820entry *ei = &e820x->map[i];
431 u64 final_start, final_end;
432 if (ei->type != old_type)
433 continue;
434 /* totally covered? */
435 if (ei->addr >= start &&
436 (ei->addr + ei->size) <= (start + size)) {
437 ei->type = new_type;
438 real_updated_size += ei->size;
439 continue;
440 }
441 /* partially covered */
442 final_start = max(start, ei->addr);
443 final_end = min(start + size, ei->addr + ei->size);
444 if (final_start >= final_end)
445 continue;
446 e820_add_region(final_start, final_end - final_start,
447 new_type);
448 real_updated_size += final_end - final_start;
449
450 ei->size -= final_end - final_start;
451 if (ei->addr < final_start)
452 continue;
453 ei->addr = final_end;
454 }
455 return real_updated_size;
456}
457
458u64 __init e820_update_range(u64 start, u64 size, unsigned old_type,
459 unsigned new_type)
460{
461 return e820_update_range_map(&e820, start, size, old_type, new_type);
462}
463
464static u64 __init e820_update_range_saved(u64 start, u64 size,
465 unsigned old_type, unsigned new_type)
466{
467 return e820_update_range_map(&e820_saved, start, size, old_type,
468 new_type);
469}
470
471/* make e820 not cover the range */
472u64 __init e820_remove_range(u64 start, u64 size, unsigned old_type,
473 int checktype)
474{
475 int i;
476 u64 real_removed_size = 0;
477
478 if (size > (ULLONG_MAX - start))
479 size = ULLONG_MAX - start;
480
481 for (i = 0; i < e820.nr_map; i++) {
482 struct e820entry *ei = &e820.map[i];
483 u64 final_start, final_end;
484
485 if (checktype && ei->type != old_type)
486 continue;
487 /* totally covered? */
488 if (ei->addr >= start &&
489 (ei->addr + ei->size) <= (start + size)) {
490 real_removed_size += ei->size;
491 memset(ei, 0, sizeof(struct e820entry));
492 continue;
493 }
494 /* partially covered */
495 final_start = max(start, ei->addr);
496 final_end = min(start + size, ei->addr + ei->size);
497 if (final_start >= final_end)
498 continue;
499 real_removed_size += final_end - final_start;
500
501 ei->size -= final_end - final_start;
502 if (ei->addr < final_start)
503 continue;
504 ei->addr = final_end;
505 }
506 return real_removed_size;
507}
508
509void __init update_e820(void)
510{
511 int nr_map;
512
513 nr_map = e820.nr_map;
514 if (sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &nr_map))
515 return;
516 e820.nr_map = nr_map;
517 printk(KERN_INFO "modified physical RAM map:\n");
518 e820_print_map("modified");
519}
520static void __init update_e820_saved(void)
521{
522 int nr_map;
523
524 nr_map = e820_saved.nr_map;
525 if (sanitize_e820_map(e820_saved.map, ARRAY_SIZE(e820_saved.map), &nr_map))
526 return;
527 e820_saved.nr_map = nr_map;
528}
529#define MAX_GAP_END 0x100000000ull
530/*
531 * Search for a gap in the e820 memory space from start_addr to end_addr.
532 */
533__init int e820_search_gap(unsigned long *gapstart, unsigned long *gapsize,
534 unsigned long start_addr, unsigned long long end_addr)
535{
536 unsigned long long last;
537 int i = e820.nr_map;
538 int found = 0;
539
540 last = (end_addr && end_addr < MAX_GAP_END) ? end_addr : MAX_GAP_END;
541
542 while (--i >= 0) {
543 unsigned long long start = e820.map[i].addr;
544 unsigned long long end = start + e820.map[i].size;
545
546 if (end < start_addr)
547 continue;
548
549 /*
550 * Since "last" is at most 4GB, we know we'll
551 * fit in 32 bits if this condition is true
552 */
553 if (last > end) {
554 unsigned long gap = last - end;
555
556 if (gap >= *gapsize) {
557 *gapsize = gap;
558 *gapstart = end;
559 found = 1;
560 }
561 }
562 if (start < last)
563 last = start;
564 }
565 return found;
566}
567
568/*
569 * Search for the biggest gap in the low 32 bits of the e820
570 * memory space. We pass this space to PCI to assign MMIO resources
571 * for hotplug or unconfigured devices in.
572 * Hopefully the BIOS let enough space left.
573 */
574__init void e820_setup_gap(void)
575{
576 unsigned long gapstart, gapsize, round;
577 int found;
578
579 gapstart = 0x10000000;
580 gapsize = 0x400000;
581 found = e820_search_gap(&gapstart, &gapsize, 0, MAX_GAP_END);
582
583#ifdef CONFIG_X86_64
584 if (!found) {
585 gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024;
586 printk(KERN_ERR "PCI: Warning: Cannot find a gap in the 32bit "
587 "address range\n"
588 KERN_ERR "PCI: Unassigned devices with 32bit resource "
589 "registers may break!\n");
590 }
591#endif
592
593 /*
594 * See how much we want to round up: start off with
595 * rounding to the next 1MB area.
596 */
597 round = 0x100000;
598 while ((gapsize >> 4) > round)
599 round += round;
600 /* Fun with two's complement */
601 pci_mem_start = (gapstart + round) & -round;
602
603 printk(KERN_INFO
604 "Allocating PCI resources starting at %lx (gap: %lx:%lx)\n",
605 pci_mem_start, gapstart, gapsize);
606}
607
608/**
609 * Because of the size limitation of struct boot_params, only first
610 * 128 E820 memory entries are passed to kernel via
611 * boot_params.e820_map, others are passed via SETUP_E820_EXT node of
612 * linked list of struct setup_data, which is parsed here.
613 */
614void __init parse_e820_ext(struct setup_data *sdata, unsigned long pa_data)
615{
616 u32 map_len;
617 int entries;
618 struct e820entry *extmap;
619
620 entries = sdata->len / sizeof(struct e820entry);
621 map_len = sdata->len + sizeof(struct setup_data);
622 if (map_len > PAGE_SIZE)
623 sdata = early_ioremap(pa_data, map_len);
624 extmap = (struct e820entry *)(sdata->data);
625 __append_e820_map(extmap, entries);
626 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
627 if (map_len > PAGE_SIZE)
628 early_iounmap(sdata, map_len);
629 printk(KERN_INFO "extended physical RAM map:\n");
630 e820_print_map("extended");
631}
632
633#if defined(CONFIG_X86_64) || \
634 (defined(CONFIG_X86_32) && defined(CONFIG_HIBERNATION))
635/**
636 * Find the ranges of physical addresses that do not correspond to
637 * e820 RAM areas and mark the corresponding pages as nosave for
638 * hibernation (32 bit) or software suspend and suspend to RAM (64 bit).
639 *
640 * This function requires the e820 map to be sorted and without any
641 * overlapping entries and assumes the first e820 area to be RAM.
642 */
643void __init e820_mark_nosave_regions(unsigned long limit_pfn)
644{
645 int i;
646 unsigned long pfn;
647
648 pfn = PFN_DOWN(e820.map[0].addr + e820.map[0].size);
649 for (i = 1; i < e820.nr_map; i++) {
650 struct e820entry *ei = &e820.map[i];
651
652 if (pfn < PFN_UP(ei->addr))
653 register_nosave_region(pfn, PFN_UP(ei->addr));
654
655 pfn = PFN_DOWN(ei->addr + ei->size);
656 if (ei->type != E820_RAM && ei->type != E820_RESERVED_KERN)
657 register_nosave_region(PFN_UP(ei->addr), pfn);
658
659 if (pfn >= limit_pfn)
660 break;
661 }
662}
663#endif
664
665/*
666 * Early reserved memory areas.
667 */
668#define MAX_EARLY_RES 20
669
670struct early_res {
671 u64 start, end;
672 char name[16];
673 char overlap_ok;
674};
675static struct early_res early_res[MAX_EARLY_RES] __initdata = {
676 { 0, PAGE_SIZE, "BIOS data page" }, /* BIOS data page */
677#if defined(CONFIG_X86_64) && defined(CONFIG_X86_TRAMPOLINE)
678 { TRAMPOLINE_BASE, TRAMPOLINE_BASE + 2 * PAGE_SIZE, "TRAMPOLINE" },
679#endif
680#if defined(CONFIG_X86_32) && defined(CONFIG_SMP)
681 /*
682 * But first pinch a few for the stack/trampoline stuff
683 * FIXME: Don't need the extra page at 4K, but need to fix
684 * trampoline before removing it. (see the GDT stuff)
685 */
686 { PAGE_SIZE, PAGE_SIZE + PAGE_SIZE, "EX TRAMPOLINE" },
687 /*
688 * Has to be in very low memory so we can execute
689 * real-mode AP code.
690 */
691 { TRAMPOLINE_BASE, TRAMPOLINE_BASE + PAGE_SIZE, "TRAMPOLINE" },
692#endif
693 {}
694};
695
696static int __init find_overlapped_early(u64 start, u64 end)
697{
698 int i;
699 struct early_res *r;
700
701 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
702 r = &early_res[i];
703 if (end > r->start && start < r->end)
704 break;
705 }
706
707 return i;
708}
709
710/*
711 * Drop the i-th range from the early reservation map,
712 * by copying any higher ranges down one over it, and
713 * clearing what had been the last slot.
714 */
715static void __init drop_range(int i)
716{
717 int j;
718
719 for (j = i + 1; j < MAX_EARLY_RES && early_res[j].end; j++)
720 ;
721
722 memmove(&early_res[i], &early_res[i + 1],
723 (j - 1 - i) * sizeof(struct early_res));
724
725 early_res[j - 1].end = 0;
726}
727
728/*
729 * Split any existing ranges that:
730 * 1) are marked 'overlap_ok', and
731 * 2) overlap with the stated range [start, end)
732 * into whatever portion (if any) of the existing range is entirely
733 * below or entirely above the stated range. Drop the portion
734 * of the existing range that overlaps with the stated range,
735 * which will allow the caller of this routine to then add that
736 * stated range without conflicting with any existing range.
737 */
738static void __init drop_overlaps_that_are_ok(u64 start, u64 end)
739{
740 int i;
741 struct early_res *r;
742 u64 lower_start, lower_end;
743 u64 upper_start, upper_end;
744 char name[16];
745
746 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
747 r = &early_res[i];
748
749 /* Continue past non-overlapping ranges */
750 if (end <= r->start || start >= r->end)
751 continue;
752
753 /*
754 * Leave non-ok overlaps as is; let caller
755 * panic "Overlapping early reservations"
756 * when it hits this overlap.
757 */
758 if (!r->overlap_ok)
759 return;
760
761 /*
762 * We have an ok overlap. We will drop it from the early
763 * reservation map, and add back in any non-overlapping
764 * portions (lower or upper) as separate, overlap_ok,
765 * non-overlapping ranges.
766 */
767
768 /* 1. Note any non-overlapping (lower or upper) ranges. */
769 strncpy(name, r->name, sizeof(name) - 1);
770
771 lower_start = lower_end = 0;
772 upper_start = upper_end = 0;
773 if (r->start < start) {
774 lower_start = r->start;
775 lower_end = start;
776 }
777 if (r->end > end) {
778 upper_start = end;
779 upper_end = r->end;
780 }
781
782 /* 2. Drop the original ok overlapping range */
783 drop_range(i);
784
785 i--; /* resume for-loop on copied down entry */
786
787 /* 3. Add back in any non-overlapping ranges. */
788 if (lower_end)
789 reserve_early_overlap_ok(lower_start, lower_end, name);
790 if (upper_end)
791 reserve_early_overlap_ok(upper_start, upper_end, name);
792 }
793}
794
795static void __init __reserve_early(u64 start, u64 end, char *name,
796 int overlap_ok)
797{
798 int i;
799 struct early_res *r;
800
801 i = find_overlapped_early(start, end);
802 if (i >= MAX_EARLY_RES)
803 panic("Too many early reservations");
804 r = &early_res[i];
805 if (r->end)
806 panic("Overlapping early reservations "
807 "%llx-%llx %s to %llx-%llx %s\n",
808 start, end - 1, name?name:"", r->start,
809 r->end - 1, r->name);
810 r->start = start;
811 r->end = end;
812 r->overlap_ok = overlap_ok;
813 if (name)
814 strncpy(r->name, name, sizeof(r->name) - 1);
815}
816
817/*
818 * A few early reservtations come here.
819 *
820 * The 'overlap_ok' in the name of this routine does -not- mean it
821 * is ok for these reservations to overlap an earlier reservation.
822 * Rather it means that it is ok for subsequent reservations to
823 * overlap this one.
824 *
825 * Use this entry point to reserve early ranges when you are doing
826 * so out of "Paranoia", reserving perhaps more memory than you need,
827 * just in case, and don't mind a subsequent overlapping reservation
828 * that is known to be needed.
829 *
830 * The drop_overlaps_that_are_ok() call here isn't really needed.
831 * It would be needed if we had two colliding 'overlap_ok'
832 * reservations, so that the second such would not panic on the
833 * overlap with the first. We don't have any such as of this
834 * writing, but might as well tolerate such if it happens in
835 * the future.
836 */
837void __init reserve_early_overlap_ok(u64 start, u64 end, char *name)
838{
839 drop_overlaps_that_are_ok(start, end);
840 __reserve_early(start, end, name, 1);
841}
842
843/*
844 * Most early reservations come here.
845 *
846 * We first have drop_overlaps_that_are_ok() drop any pre-existing
847 * 'overlap_ok' ranges, so that we can then reserve this memory
848 * range without risk of panic'ing on an overlapping overlap_ok
849 * early reservation.
850 */
851void __init reserve_early(u64 start, u64 end, char *name)
852{
853 drop_overlaps_that_are_ok(start, end);
854 __reserve_early(start, end, name, 0);
855}
856
857void __init free_early(u64 start, u64 end)
858{
859 struct early_res *r;
860 int i;
861
862 i = find_overlapped_early(start, end);
863 r = &early_res[i];
864 if (i >= MAX_EARLY_RES || r->end != end || r->start != start)
865 panic("free_early on not reserved area: %llx-%llx!",
866 start, end - 1);
867
868 drop_range(i);
869}
870
871void __init early_res_to_bootmem(u64 start, u64 end)
872{
873 int i, count;
874 u64 final_start, final_end;
875
876 count = 0;
877 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++)
878 count++;
879
880 printk(KERN_INFO "(%d early reservations) ==> bootmem\n", count);
881 for (i = 0; i < count; i++) {
882 struct early_res *r = &early_res[i];
883 printk(KERN_INFO " #%d [%010llx - %010llx] %16s", i,
884 r->start, r->end, r->name);
885 final_start = max(start, r->start);
886 final_end = min(end, r->end);
887 if (final_start >= final_end) {
888 printk(KERN_CONT "\n");
889 continue;
890 }
891 printk(KERN_CONT " ==> [%010llx - %010llx]\n",
892 final_start, final_end);
893 reserve_bootmem_generic(final_start, final_end - final_start,
894 BOOTMEM_DEFAULT);
895 }
896}
897
898/* Check for already reserved areas */
899static inline int __init bad_addr(u64 *addrp, u64 size, u64 align)
900{
901 int i;
902 u64 addr = *addrp;
903 int changed = 0;
904 struct early_res *r;
905again:
906 i = find_overlapped_early(addr, addr + size);
907 r = &early_res[i];
908 if (i < MAX_EARLY_RES && r->end) {
909 *addrp = addr = round_up(r->end, align);
910 changed = 1;
911 goto again;
912 }
913 return changed;
914}
915
916/* Check for already reserved areas */
917static inline int __init bad_addr_size(u64 *addrp, u64 *sizep, u64 align)
918{
919 int i;
920 u64 addr = *addrp, last;
921 u64 size = *sizep;
922 int changed = 0;
923again:
924 last = addr + size;
925 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
926 struct early_res *r = &early_res[i];
927 if (last > r->start && addr < r->start) {
928 size = r->start - addr;
929 changed = 1;
930 goto again;
931 }
932 if (last > r->end && addr < r->end) {
933 addr = round_up(r->end, align);
934 size = last - addr;
935 changed = 1;
936 goto again;
937 }
938 if (last <= r->end && addr >= r->start) {
939 (*sizep)++;
940 return 0;
941 }
942 }
943 if (changed) {
944 *addrp = addr;
945 *sizep = size;
946 }
947 return changed;
948}
949
950/*
951 * Find a free area with specified alignment in a specific range.
952 */
953u64 __init find_e820_area(u64 start, u64 end, u64 size, u64 align)
954{
955 int i;
956
957 for (i = 0; i < e820.nr_map; i++) {
958 struct e820entry *ei = &e820.map[i];
959 u64 addr, last;
960 u64 ei_last;
961
962 if (ei->type != E820_RAM)
963 continue;
964 addr = round_up(ei->addr, align);
965 ei_last = ei->addr + ei->size;
966 if (addr < start)
967 addr = round_up(start, align);
968 if (addr >= ei_last)
969 continue;
970 while (bad_addr(&addr, size, align) && addr+size <= ei_last)
971 ;
972 last = addr + size;
973 if (last > ei_last)
974 continue;
975 if (last > end)
976 continue;
977 return addr;
978 }
979 return -1ULL;
980}
981
982/*
983 * Find next free range after *start
984 */
985u64 __init find_e820_area_size(u64 start, u64 *sizep, u64 align)
986{
987 int i;
988
989 for (i = 0; i < e820.nr_map; i++) {
990 struct e820entry *ei = &e820.map[i];
991 u64 addr, last;
992 u64 ei_last;
993
994 if (ei->type != E820_RAM)
995 continue;
996 addr = round_up(ei->addr, align);
997 ei_last = ei->addr + ei->size;
998 if (addr < start)
999 addr = round_up(start, align);
1000 if (addr >= ei_last)
1001 continue;
1002 *sizep = ei_last - addr;
1003 while (bad_addr_size(&addr, sizep, align) &&
1004 addr + *sizep <= ei_last)
1005 ;
1006 last = addr + *sizep;
1007 if (last > ei_last)
1008 continue;
1009 return addr;
1010 }
1011 return -1UL;
1012
1013}
1014
1015/*
1016 * pre allocated 4k and reserved it in e820
1017 */
1018u64 __init early_reserve_e820(u64 startt, u64 sizet, u64 align)
1019{
1020 u64 size = 0;
1021 u64 addr;
1022 u64 start;
1023
1024 start = startt;
1025 while (size < sizet)
1026 start = find_e820_area_size(start, &size, align);
1027
1028 if (size < sizet)
1029 return 0;
1030
1031 addr = round_down(start + size - sizet, align);
1032 e820_update_range(addr, sizet, E820_RAM, E820_RESERVED);
1033 e820_update_range_saved(addr, sizet, E820_RAM, E820_RESERVED);
1034 printk(KERN_INFO "update e820 for early_reserve_e820\n");
1035 update_e820();
1036 update_e820_saved();
1037
1038 return addr;
1039}
1040
1041#ifdef CONFIG_X86_32
1042# ifdef CONFIG_X86_PAE
1043# define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT))
1044# else
1045# define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT))
1046# endif
1047#else /* CONFIG_X86_32 */
1048# define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT
1049#endif
1050
1051/*
1052 * Find the highest page frame number we have available
1053 */
1054static unsigned long __init e820_end_pfn(unsigned long limit_pfn, unsigned type)
1055{
1056 int i;
1057 unsigned long last_pfn = 0;
1058 unsigned long max_arch_pfn = MAX_ARCH_PFN;
1059
1060 for (i = 0; i < e820.nr_map; i++) {
1061 struct e820entry *ei = &e820.map[i];
1062 unsigned long start_pfn;
1063 unsigned long end_pfn;
1064
1065 if (ei->type != type)
1066 continue;
1067
1068 start_pfn = ei->addr >> PAGE_SHIFT;
1069 end_pfn = (ei->addr + ei->size) >> PAGE_SHIFT;
1070
1071 if (start_pfn >= limit_pfn)
1072 continue;
1073 if (end_pfn > limit_pfn) {
1074 last_pfn = limit_pfn;
1075 break;
1076 }
1077 if (end_pfn > last_pfn)
1078 last_pfn = end_pfn;
1079 }
1080
1081 if (last_pfn > max_arch_pfn)
1082 last_pfn = max_arch_pfn;
1083
1084 printk(KERN_INFO "last_pfn = %#lx max_arch_pfn = %#lx\n",
1085 last_pfn, max_arch_pfn);
1086 return last_pfn;
1087}
1088unsigned long __init e820_end_of_ram_pfn(void)
1089{
1090 return e820_end_pfn(MAX_ARCH_PFN, E820_RAM);
1091}
1092
1093unsigned long __init e820_end_of_low_ram_pfn(void)
1094{
1095 return e820_end_pfn(1UL<<(32 - PAGE_SHIFT), E820_RAM);
1096}
1097/*
1098 * Finds an active region in the address range from start_pfn to last_pfn and
1099 * returns its range in ei_startpfn and ei_endpfn for the e820 entry.
1100 */
1101int __init e820_find_active_region(const struct e820entry *ei,
1102 unsigned long start_pfn,
1103 unsigned long last_pfn,
1104 unsigned long *ei_startpfn,
1105 unsigned long *ei_endpfn)
1106{
1107 u64 align = PAGE_SIZE;
1108
1109 *ei_startpfn = round_up(ei->addr, align) >> PAGE_SHIFT;
1110 *ei_endpfn = round_down(ei->addr + ei->size, align) >> PAGE_SHIFT;
1111
1112 /* Skip map entries smaller than a page */
1113 if (*ei_startpfn >= *ei_endpfn)
1114 return 0;
1115
1116 /* Skip if map is outside the node */
1117 if (ei->type != E820_RAM || *ei_endpfn <= start_pfn ||
1118 *ei_startpfn >= last_pfn)
1119 return 0;
1120
1121 /* Check for overlaps */
1122 if (*ei_startpfn < start_pfn)
1123 *ei_startpfn = start_pfn;
1124 if (*ei_endpfn > last_pfn)
1125 *ei_endpfn = last_pfn;
1126
1127 return 1;
1128}
1129
1130/* Walk the e820 map and register active regions within a node */
1131void __init e820_register_active_regions(int nid, unsigned long start_pfn,
1132 unsigned long last_pfn)
1133{
1134 unsigned long ei_startpfn;
1135 unsigned long ei_endpfn;
1136 int i;
1137
1138 for (i = 0; i < e820.nr_map; i++)
1139 if (e820_find_active_region(&e820.map[i],
1140 start_pfn, last_pfn,
1141 &ei_startpfn, &ei_endpfn))
1142 add_active_range(nid, ei_startpfn, ei_endpfn);
1143}
1144
1145/*
1146 * Find the hole size (in bytes) in the memory range.
1147 * @start: starting address of the memory range to scan
1148 * @end: ending address of the memory range to scan
1149 */
1150u64 __init e820_hole_size(u64 start, u64 end)
1151{
1152 unsigned long start_pfn = start >> PAGE_SHIFT;
1153 unsigned long last_pfn = end >> PAGE_SHIFT;
1154 unsigned long ei_startpfn, ei_endpfn, ram = 0;
1155 int i;
1156
1157 for (i = 0; i < e820.nr_map; i++) {
1158 if (e820_find_active_region(&e820.map[i],
1159 start_pfn, last_pfn,
1160 &ei_startpfn, &ei_endpfn))
1161 ram += ei_endpfn - ei_startpfn;
1162 }
1163 return end - start - ((u64)ram << PAGE_SHIFT);
1164}
1165
1166static void early_panic(char *msg)
1167{
1168 early_printk(msg);
1169 panic(msg);
1170}
1171
1172static int userdef __initdata;
1173
1174/* "mem=nopentium" disables the 4MB page tables. */
1175static int __init parse_memopt(char *p)
1176{
1177 u64 mem_size;
1178
1179 if (!p)
1180 return -EINVAL;
1181
1182#ifdef CONFIG_X86_32
1183 if (!strcmp(p, "nopentium")) {
1184 setup_clear_cpu_cap(X86_FEATURE_PSE);
1185 return 0;
1186 }
1187#endif
1188
1189 userdef = 1;
1190 mem_size = memparse(p, &p);
1191 e820_remove_range(mem_size, ULLONG_MAX - mem_size, E820_RAM, 1);
1192
1193 return 0;
1194}
1195early_param("mem", parse_memopt);
1196
1197static int __init parse_memmap_opt(char *p)
1198{
1199 char *oldp;
1200 u64 start_at, mem_size;
1201
1202 if (!p)
1203 return -EINVAL;
1204
1205 if (!strcmp(p, "exactmap")) {
1206#ifdef CONFIG_CRASH_DUMP
1207 /*
1208 * If we are doing a crash dump, we still need to know
1209 * the real mem size before original memory map is
1210 * reset.
1211 */
1212 saved_max_pfn = e820_end_of_ram_pfn();
1213#endif
1214 e820.nr_map = 0;
1215 userdef = 1;
1216 return 0;
1217 }
1218
1219 oldp = p;
1220 mem_size = memparse(p, &p);
1221 if (p == oldp)
1222 return -EINVAL;
1223
1224 userdef = 1;
1225 if (*p == '@') {
1226 start_at = memparse(p+1, &p);
1227 e820_add_region(start_at, mem_size, E820_RAM);
1228 } else if (*p == '#') {
1229 start_at = memparse(p+1, &p);
1230 e820_add_region(start_at, mem_size, E820_ACPI);
1231 } else if (*p == '$') {
1232 start_at = memparse(p+1, &p);
1233 e820_add_region(start_at, mem_size, E820_RESERVED);
1234 } else
1235 e820_remove_range(mem_size, ULLONG_MAX - mem_size, E820_RAM, 1);
1236
1237 return *p == '\0' ? 0 : -EINVAL;
1238}
1239early_param("memmap", parse_memmap_opt);
1240
1241void __init finish_e820_parsing(void)
1242{
1243 if (userdef) {
1244 int nr = e820.nr_map;
1245
1246 if (sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &nr) < 0)
1247 early_panic("Invalid user supplied memory map");
1248 e820.nr_map = nr;
1249
1250 printk(KERN_INFO "user-defined physical RAM map:\n");
1251 e820_print_map("user");
1252 }
1253}
1254
1255static inline const char *e820_type_to_string(int e820_type)
1256{
1257 switch (e820_type) {
1258 case E820_RESERVED_KERN:
1259 case E820_RAM: return "System RAM";
1260 case E820_ACPI: return "ACPI Tables";
1261 case E820_NVS: return "ACPI Non-volatile Storage";
1262 default: return "reserved";
1263 }
1264}
1265
1266/*
1267 * Mark e820 reserved areas as busy for the resource manager.
1268 */
1269void __init e820_reserve_resources(void)
1270{
1271 int i;
1272 struct resource *res;
1273 u64 end;
1274
1275 res = alloc_bootmem_low(sizeof(struct resource) * e820.nr_map);
1276 for (i = 0; i < e820.nr_map; i++) {
1277 end = e820.map[i].addr + e820.map[i].size - 1;
1278#ifndef CONFIG_RESOURCES_64BIT
1279 if (end > 0x100000000ULL) {
1280 res++;
1281 continue;
1282 }
1283#endif
1284 res->name = e820_type_to_string(e820.map[i].type);
1285 res->start = e820.map[i].addr;
1286 res->end = end;
1287
1288 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
1289 insert_resource(&iomem_resource, res);
1290 res++;
1291 }
1292
1293 for (i = 0; i < e820_saved.nr_map; i++) {
1294 struct e820entry *entry = &e820_saved.map[i];
1295 firmware_map_add_early(entry->addr,
1296 entry->addr + entry->size - 1,
1297 e820_type_to_string(entry->type));
1298 }
1299}
1300
1301/*
1302 * Non-standard memory setup can be specified via this quirk:
1303 */
1304char * (*arch_memory_setup_quirk)(void);
1305
1306char *__init default_machine_specific_memory_setup(void)
1307{
1308 char *who = "BIOS-e820";
1309 int new_nr;
1310 /*
1311 * Try to copy the BIOS-supplied E820-map.
1312 *
1313 * Otherwise fake a memory map; one section from 0k->640k,
1314 * the next section from 1mb->appropriate_mem_k
1315 */
1316 new_nr = boot_params.e820_entries;
1317 sanitize_e820_map(boot_params.e820_map,
1318 ARRAY_SIZE(boot_params.e820_map),
1319 &new_nr);
1320 boot_params.e820_entries = new_nr;
1321 if (append_e820_map(boot_params.e820_map, boot_params.e820_entries)
1322 < 0) {
1323 u64 mem_size;
1324
1325 /* compare results from other methods and take the greater */
1326 if (boot_params.alt_mem_k
1327 < boot_params.screen_info.ext_mem_k) {
1328 mem_size = boot_params.screen_info.ext_mem_k;
1329 who = "BIOS-88";
1330 } else {
1331 mem_size = boot_params.alt_mem_k;
1332 who = "BIOS-e801";
1333 }
1334
1335 e820.nr_map = 0;
1336 e820_add_region(0, LOWMEMSIZE(), E820_RAM);
1337 e820_add_region(HIGH_MEMORY, mem_size << 10, E820_RAM);
1338 }
1339
1340 /* In case someone cares... */
1341 return who;
1342}
1343
1344char *__init __attribute__((weak)) machine_specific_memory_setup(void)
1345{
1346 if (arch_memory_setup_quirk) {
1347 char *who = arch_memory_setup_quirk();
1348
1349 if (who)
1350 return who;
1351 }
1352 return default_machine_specific_memory_setup();
1353}
1354
1355/* Overridden in paravirt.c if CONFIG_PARAVIRT */
1356char * __init __attribute__((weak)) memory_setup(void)
1357{
1358 return machine_specific_memory_setup();
1359}
1360
1361void __init setup_memory_map(void)
1362{
1363 char *who;
1364
1365 who = memory_setup();
1366 memcpy(&e820_saved, &e820, sizeof(struct e820map));
1367 printk(KERN_INFO "BIOS-provided physical RAM map:\n");
1368 e820_print_map(who);
1369}
1370
1371#ifdef CONFIG_X86_64
1372int __init arch_get_ram_range(int slot, u64 *addr, u64 *size)
1373{
1374 int i;
1375
1376 if (slot < 0 || slot >= e820.nr_map)
1377 return -1;
1378 for (i = slot; i < e820.nr_map; i++) {
1379 if (e820.map[i].type != E820_RAM)
1380 continue;
1381 break;
1382 }
1383 if (i == e820.nr_map || e820.map[i].addr > (max_pfn << PAGE_SHIFT))
1384 return -1;
1385 *addr = e820.map[i].addr;
1386 *size = min_t(u64, e820.map[i].size + e820.map[i].addr,
1387 max_pfn << PAGE_SHIFT) - *addr;
1388 return i + 1;
1389}
1390#endif
diff --git a/arch/x86/kernel/e820_32.c b/arch/x86/kernel/e820_32.c
deleted file mode 100644
index ed733e7cf4e6..000000000000
--- a/arch/x86/kernel/e820_32.c
+++ /dev/null
@@ -1,775 +0,0 @@
1#include <linux/kernel.h>
2#include <linux/types.h>
3#include <linux/init.h>
4#include <linux/bootmem.h>
5#include <linux/ioport.h>
6#include <linux/string.h>
7#include <linux/kexec.h>
8#include <linux/module.h>
9#include <linux/mm.h>
10#include <linux/pfn.h>
11#include <linux/uaccess.h>
12#include <linux/suspend.h>
13
14#include <asm/pgtable.h>
15#include <asm/page.h>
16#include <asm/e820.h>
17#include <asm/setup.h>
18
19struct e820map e820;
20struct change_member {
21 struct e820entry *pbios; /* pointer to original bios entry */
22 unsigned long long addr; /* address for this change point */
23};
24static struct change_member change_point_list[2*E820MAX] __initdata;
25static struct change_member *change_point[2*E820MAX] __initdata;
26static struct e820entry *overlap_list[E820MAX] __initdata;
27static struct e820entry new_bios[E820MAX] __initdata;
28/* For PCI or other memory-mapped resources */
29unsigned long pci_mem_start = 0x10000000;
30#ifdef CONFIG_PCI
31EXPORT_SYMBOL(pci_mem_start);
32#endif
33extern int user_defined_memmap;
34
35static struct resource system_rom_resource = {
36 .name = "System ROM",
37 .start = 0xf0000,
38 .end = 0xfffff,
39 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
40};
41
42static struct resource extension_rom_resource = {
43 .name = "Extension ROM",
44 .start = 0xe0000,
45 .end = 0xeffff,
46 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
47};
48
49static struct resource adapter_rom_resources[] = { {
50 .name = "Adapter ROM",
51 .start = 0xc8000,
52 .end = 0,
53 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
54}, {
55 .name = "Adapter ROM",
56 .start = 0,
57 .end = 0,
58 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
59}, {
60 .name = "Adapter ROM",
61 .start = 0,
62 .end = 0,
63 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
64}, {
65 .name = "Adapter ROM",
66 .start = 0,
67 .end = 0,
68 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
69}, {
70 .name = "Adapter ROM",
71 .start = 0,
72 .end = 0,
73 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
74}, {
75 .name = "Adapter ROM",
76 .start = 0,
77 .end = 0,
78 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
79} };
80
81static struct resource video_rom_resource = {
82 .name = "Video ROM",
83 .start = 0xc0000,
84 .end = 0xc7fff,
85 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
86};
87
88#define ROMSIGNATURE 0xaa55
89
90static int __init romsignature(const unsigned char *rom)
91{
92 const unsigned short * const ptr = (const unsigned short *)rom;
93 unsigned short sig;
94
95 return probe_kernel_address(ptr, sig) == 0 && sig == ROMSIGNATURE;
96}
97
98static int __init romchecksum(const unsigned char *rom, unsigned long length)
99{
100 unsigned char sum, c;
101
102 for (sum = 0; length && probe_kernel_address(rom++, c) == 0; length--)
103 sum += c;
104 return !length && !sum;
105}
106
107static void __init probe_roms(void)
108{
109 const unsigned char *rom;
110 unsigned long start, length, upper;
111 unsigned char c;
112 int i;
113
114 /* video rom */
115 upper = adapter_rom_resources[0].start;
116 for (start = video_rom_resource.start; start < upper; start += 2048) {
117 rom = isa_bus_to_virt(start);
118 if (!romsignature(rom))
119 continue;
120
121 video_rom_resource.start = start;
122
123 if (probe_kernel_address(rom + 2, c) != 0)
124 continue;
125
126 /* 0 < length <= 0x7f * 512, historically */
127 length = c * 512;
128
129 /* if checksum okay, trust length byte */
130 if (length && romchecksum(rom, length))
131 video_rom_resource.end = start + length - 1;
132
133 request_resource(&iomem_resource, &video_rom_resource);
134 break;
135 }
136
137 start = (video_rom_resource.end + 1 + 2047) & ~2047UL;
138 if (start < upper)
139 start = upper;
140
141 /* system rom */
142 request_resource(&iomem_resource, &system_rom_resource);
143 upper = system_rom_resource.start;
144
145 /* check for extension rom (ignore length byte!) */
146 rom = isa_bus_to_virt(extension_rom_resource.start);
147 if (romsignature(rom)) {
148 length = extension_rom_resource.end - extension_rom_resource.start + 1;
149 if (romchecksum(rom, length)) {
150 request_resource(&iomem_resource, &extension_rom_resource);
151 upper = extension_rom_resource.start;
152 }
153 }
154
155 /* check for adapter roms on 2k boundaries */
156 for (i = 0; i < ARRAY_SIZE(adapter_rom_resources) && start < upper; start += 2048) {
157 rom = isa_bus_to_virt(start);
158 if (!romsignature(rom))
159 continue;
160
161 if (probe_kernel_address(rom + 2, c) != 0)
162 continue;
163
164 /* 0 < length <= 0x7f * 512, historically */
165 length = c * 512;
166
167 /* but accept any length that fits if checksum okay */
168 if (!length || start + length > upper || !romchecksum(rom, length))
169 continue;
170
171 adapter_rom_resources[i].start = start;
172 adapter_rom_resources[i].end = start + length - 1;
173 request_resource(&iomem_resource, &adapter_rom_resources[i]);
174
175 start = adapter_rom_resources[i++].end & ~2047UL;
176 }
177}
178
179/*
180 * Request address space for all standard RAM and ROM resources
181 * and also for regions reported as reserved by the e820.
182 */
183void __init init_iomem_resources(struct resource *code_resource,
184 struct resource *data_resource,
185 struct resource *bss_resource)
186{
187 int i;
188
189 probe_roms();
190 for (i = 0; i < e820.nr_map; i++) {
191 struct resource *res;
192#ifndef CONFIG_RESOURCES_64BIT
193 if (e820.map[i].addr + e820.map[i].size > 0x100000000ULL)
194 continue;
195#endif
196 res = kzalloc(sizeof(struct resource), GFP_ATOMIC);
197 switch (e820.map[i].type) {
198 case E820_RAM: res->name = "System RAM"; break;
199 case E820_ACPI: res->name = "ACPI Tables"; break;
200 case E820_NVS: res->name = "ACPI Non-volatile Storage"; break;
201 default: res->name = "reserved";
202 }
203 res->start = e820.map[i].addr;
204 res->end = res->start + e820.map[i].size - 1;
205 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
206 if (request_resource(&iomem_resource, res)) {
207 kfree(res);
208 continue;
209 }
210 if (e820.map[i].type == E820_RAM) {
211 /*
212 * We don't know which RAM region contains kernel data,
213 * so we try it repeatedly and let the resource manager
214 * test it.
215 */
216 request_resource(res, code_resource);
217 request_resource(res, data_resource);
218 request_resource(res, bss_resource);
219#ifdef CONFIG_KEXEC
220 if (crashk_res.start != crashk_res.end)
221 request_resource(res, &crashk_res);
222#endif
223 }
224 }
225}
226
227#if defined(CONFIG_PM) && defined(CONFIG_HIBERNATION)
228/**
229 * e820_mark_nosave_regions - Find the ranges of physical addresses that do not
230 * correspond to e820 RAM areas and mark the corresponding pages as nosave for
231 * hibernation.
232 *
233 * This function requires the e820 map to be sorted and without any
234 * overlapping entries and assumes the first e820 area to be RAM.
235 */
236void __init e820_mark_nosave_regions(void)
237{
238 int i;
239 unsigned long pfn;
240
241 pfn = PFN_DOWN(e820.map[0].addr + e820.map[0].size);
242 for (i = 1; i < e820.nr_map; i++) {
243 struct e820entry *ei = &e820.map[i];
244
245 if (pfn < PFN_UP(ei->addr))
246 register_nosave_region(pfn, PFN_UP(ei->addr));
247
248 pfn = PFN_DOWN(ei->addr + ei->size);
249 if (ei->type != E820_RAM)
250 register_nosave_region(PFN_UP(ei->addr), pfn);
251
252 if (pfn >= max_low_pfn)
253 break;
254 }
255}
256#endif
257
258void __init add_memory_region(unsigned long long start,
259 unsigned long long size, int type)
260{
261 int x;
262
263 x = e820.nr_map;
264
265 if (x == E820MAX) {
266 printk(KERN_ERR "Ooops! Too many entries in the memory map!\n");
267 return;
268 }
269
270 e820.map[x].addr = start;
271 e820.map[x].size = size;
272 e820.map[x].type = type;
273 e820.nr_map++;
274} /* add_memory_region */
275
276/*
277 * Sanitize the BIOS e820 map.
278 *
279 * Some e820 responses include overlapping entries. The following
280 * replaces the original e820 map with a new one, removing overlaps.
281 *
282 */
283int __init sanitize_e820_map(struct e820entry * biosmap, char * pnr_map)
284{
285 struct change_member *change_tmp;
286 unsigned long current_type, last_type;
287 unsigned long long last_addr;
288 int chgidx, still_changing;
289 int overlap_entries;
290 int new_bios_entry;
291 int old_nr, new_nr, chg_nr;
292 int i;
293
294 /*
295 Visually we're performing the following (1,2,3,4 = memory types)...
296
297 Sample memory map (w/overlaps):
298 ____22__________________
299 ______________________4_
300 ____1111________________
301 _44_____________________
302 11111111________________
303 ____________________33__
304 ___________44___________
305 __________33333_________
306 ______________22________
307 ___________________2222_
308 _________111111111______
309 _____________________11_
310 _________________4______
311
312 Sanitized equivalent (no overlap):
313 1_______________________
314 _44_____________________
315 ___1____________________
316 ____22__________________
317 ______11________________
318 _________1______________
319 __________3_____________
320 ___________44___________
321 _____________33_________
322 _______________2________
323 ________________1_______
324 _________________4______
325 ___________________2____
326 ____________________33__
327 ______________________4_
328 */
329 /* if there's only one memory region, don't bother */
330 if (*pnr_map < 2) {
331 return -1;
332 }
333
334 old_nr = *pnr_map;
335
336 /* bail out if we find any unreasonable addresses in bios map */
337 for (i=0; i<old_nr; i++)
338 if (biosmap[i].addr + biosmap[i].size < biosmap[i].addr) {
339 return -1;
340 }
341
342 /* create pointers for initial change-point information (for sorting) */
343 for (i=0; i < 2*old_nr; i++)
344 change_point[i] = &change_point_list[i];
345
346 /* record all known change-points (starting and ending addresses),
347 omitting those that are for empty memory regions */
348 chgidx = 0;
349 for (i=0; i < old_nr; i++) {
350 if (biosmap[i].size != 0) {
351 change_point[chgidx]->addr = biosmap[i].addr;
352 change_point[chgidx++]->pbios = &biosmap[i];
353 change_point[chgidx]->addr = biosmap[i].addr + biosmap[i].size;
354 change_point[chgidx++]->pbios = &biosmap[i];
355 }
356 }
357 chg_nr = chgidx; /* true number of change-points */
358
359 /* sort change-point list by memory addresses (low -> high) */
360 still_changing = 1;
361 while (still_changing) {
362 still_changing = 0;
363 for (i=1; i < chg_nr; i++) {
364 /* if <current_addr> > <last_addr>, swap */
365 /* or, if current=<start_addr> & last=<end_addr>, swap */
366 if ((change_point[i]->addr < change_point[i-1]->addr) ||
367 ((change_point[i]->addr == change_point[i-1]->addr) &&
368 (change_point[i]->addr == change_point[i]->pbios->addr) &&
369 (change_point[i-1]->addr != change_point[i-1]->pbios->addr))
370 )
371 {
372 change_tmp = change_point[i];
373 change_point[i] = change_point[i-1];
374 change_point[i-1] = change_tmp;
375 still_changing=1;
376 }
377 }
378 }
379
380 /* create a new bios memory map, removing overlaps */
381 overlap_entries=0; /* number of entries in the overlap table */
382 new_bios_entry=0; /* index for creating new bios map entries */
383 last_type = 0; /* start with undefined memory type */
384 last_addr = 0; /* start with 0 as last starting address */
385 /* loop through change-points, determining affect on the new bios map */
386 for (chgidx=0; chgidx < chg_nr; chgidx++)
387 {
388 /* keep track of all overlapping bios entries */
389 if (change_point[chgidx]->addr == change_point[chgidx]->pbios->addr)
390 {
391 /* add map entry to overlap list (> 1 entry implies an overlap) */
392 overlap_list[overlap_entries++]=change_point[chgidx]->pbios;
393 }
394 else
395 {
396 /* remove entry from list (order independent, so swap with last) */
397 for (i=0; i<overlap_entries; i++)
398 {
399 if (overlap_list[i] == change_point[chgidx]->pbios)
400 overlap_list[i] = overlap_list[overlap_entries-1];
401 }
402 overlap_entries--;
403 }
404 /* if there are overlapping entries, decide which "type" to use */
405 /* (larger value takes precedence -- 1=usable, 2,3,4,4+=unusable) */
406 current_type = 0;
407 for (i=0; i<overlap_entries; i++)
408 if (overlap_list[i]->type > current_type)
409 current_type = overlap_list[i]->type;
410 /* continue building up new bios map based on this information */
411 if (current_type != last_type) {
412 if (last_type != 0) {
413 new_bios[new_bios_entry].size =
414 change_point[chgidx]->addr - last_addr;
415 /* move forward only if the new size was non-zero */
416 if (new_bios[new_bios_entry].size != 0)
417 if (++new_bios_entry >= E820MAX)
418 break; /* no more space left for new bios entries */
419 }
420 if (current_type != 0) {
421 new_bios[new_bios_entry].addr = change_point[chgidx]->addr;
422 new_bios[new_bios_entry].type = current_type;
423 last_addr=change_point[chgidx]->addr;
424 }
425 last_type = current_type;
426 }
427 }
428 new_nr = new_bios_entry; /* retain count for new bios entries */
429
430 /* copy new bios mapping into original location */
431 memcpy(biosmap, new_bios, new_nr*sizeof(struct e820entry));
432 *pnr_map = new_nr;
433
434 return 0;
435}
436
437/*
438 * Copy the BIOS e820 map into a safe place.
439 *
440 * Sanity-check it while we're at it..
441 *
442 * If we're lucky and live on a modern system, the setup code
443 * will have given us a memory map that we can use to properly
444 * set up memory. If we aren't, we'll fake a memory map.
445 *
446 * We check to see that the memory map contains at least 2 elements
447 * before we'll use it, because the detection code in setup.S may
448 * not be perfect and most every PC known to man has two memory
449 * regions: one from 0 to 640k, and one from 1mb up. (The IBM
450 * thinkpad 560x, for example, does not cooperate with the memory
451 * detection code.)
452 */
453int __init copy_e820_map(struct e820entry *biosmap, int nr_map)
454{
455 /* Only one memory region (or negative)? Ignore it */
456 if (nr_map < 2)
457 return -1;
458
459 do {
460 u64 start = biosmap->addr;
461 u64 size = biosmap->size;
462 u64 end = start + size;
463 u32 type = biosmap->type;
464
465 /* Overflow in 64 bits? Ignore the memory map. */
466 if (start > end)
467 return -1;
468
469 add_memory_region(start, size, type);
470 } while (biosmap++, --nr_map);
471
472 return 0;
473}
474
475/*
476 * Find the highest page frame number we have available
477 */
478void __init propagate_e820_map(void)
479{
480 int i;
481
482 max_pfn = 0;
483
484 for (i = 0; i < e820.nr_map; i++) {
485 unsigned long start, end;
486 /* RAM? */
487 if (e820.map[i].type != E820_RAM)
488 continue;
489 start = PFN_UP(e820.map[i].addr);
490 end = PFN_DOWN(e820.map[i].addr + e820.map[i].size);
491 if (start >= end)
492 continue;
493 if (end > max_pfn)
494 max_pfn = end;
495 memory_present(0, start, end);
496 }
497}
498
499/*
500 * Register fully available low RAM pages with the bootmem allocator.
501 */
502void __init register_bootmem_low_pages(unsigned long max_low_pfn)
503{
504 int i;
505
506 for (i = 0; i < e820.nr_map; i++) {
507 unsigned long curr_pfn, last_pfn, size;
508 /*
509 * Reserve usable low memory
510 */
511 if (e820.map[i].type != E820_RAM)
512 continue;
513 /*
514 * We are rounding up the start address of usable memory:
515 */
516 curr_pfn = PFN_UP(e820.map[i].addr);
517 if (curr_pfn >= max_low_pfn)
518 continue;
519 /*
520 * ... and at the end of the usable range downwards:
521 */
522 last_pfn = PFN_DOWN(e820.map[i].addr + e820.map[i].size);
523
524 if (last_pfn > max_low_pfn)
525 last_pfn = max_low_pfn;
526
527 /*
528 * .. finally, did all the rounding and playing
529 * around just make the area go away?
530 */
531 if (last_pfn <= curr_pfn)
532 continue;
533
534 size = last_pfn - curr_pfn;
535 free_bootmem(PFN_PHYS(curr_pfn), PFN_PHYS(size));
536 }
537}
538
539void __init e820_register_memory(void)
540{
541 unsigned long gapstart, gapsize, round;
542 unsigned long long last;
543 int i;
544
545 /*
546 * Search for the biggest gap in the low 32 bits of the e820
547 * memory space.
548 */
549 last = 0x100000000ull;
550 gapstart = 0x10000000;
551 gapsize = 0x400000;
552 i = e820.nr_map;
553 while (--i >= 0) {
554 unsigned long long start = e820.map[i].addr;
555 unsigned long long end = start + e820.map[i].size;
556
557 /*
558 * Since "last" is at most 4GB, we know we'll
559 * fit in 32 bits if this condition is true
560 */
561 if (last > end) {
562 unsigned long gap = last - end;
563
564 if (gap > gapsize) {
565 gapsize = gap;
566 gapstart = end;
567 }
568 }
569 if (start < last)
570 last = start;
571 }
572
573 /*
574 * See how much we want to round up: start off with
575 * rounding to the next 1MB area.
576 */
577 round = 0x100000;
578 while ((gapsize >> 4) > round)
579 round += round;
580 /* Fun with two's complement */
581 pci_mem_start = (gapstart + round) & -round;
582
583 printk("Allocating PCI resources starting at %08lx (gap: %08lx:%08lx)\n",
584 pci_mem_start, gapstart, gapsize);
585}
586
587void __init print_memory_map(char *who)
588{
589 int i;
590
591 for (i = 0; i < e820.nr_map; i++) {
592 printk(" %s: %016Lx - %016Lx ", who,
593 e820.map[i].addr,
594 e820.map[i].addr + e820.map[i].size);
595 switch (e820.map[i].type) {
596 case E820_RAM: printk("(usable)\n");
597 break;
598 case E820_RESERVED:
599 printk("(reserved)\n");
600 break;
601 case E820_ACPI:
602 printk("(ACPI data)\n");
603 break;
604 case E820_NVS:
605 printk("(ACPI NVS)\n");
606 break;
607 default: printk("type %u\n", e820.map[i].type);
608 break;
609 }
610 }
611}
612
613void __init limit_regions(unsigned long long size)
614{
615 unsigned long long current_addr;
616 int i;
617
618 print_memory_map("limit_regions start");
619 for (i = 0; i < e820.nr_map; i++) {
620 current_addr = e820.map[i].addr + e820.map[i].size;
621 if (current_addr < size)
622 continue;
623
624 if (e820.map[i].type != E820_RAM)
625 continue;
626
627 if (e820.map[i].addr >= size) {
628 /*
629 * This region starts past the end of the
630 * requested size, skip it completely.
631 */
632 e820.nr_map = i;
633 } else {
634 e820.nr_map = i + 1;
635 e820.map[i].size -= current_addr - size;
636 }
637 print_memory_map("limit_regions endfor");
638 return;
639 }
640 print_memory_map("limit_regions endfunc");
641}
642
643/*
644 * This function checks if any part of the range <start,end> is mapped
645 * with type.
646 */
647int
648e820_any_mapped(u64 start, u64 end, unsigned type)
649{
650 int i;
651 for (i = 0; i < e820.nr_map; i++) {
652 const struct e820entry *ei = &e820.map[i];
653 if (type && ei->type != type)
654 continue;
655 if (ei->addr >= end || ei->addr + ei->size <= start)
656 continue;
657 return 1;
658 }
659 return 0;
660}
661EXPORT_SYMBOL_GPL(e820_any_mapped);
662
663 /*
664 * This function checks if the entire range <start,end> is mapped with type.
665 *
666 * Note: this function only works correct if the e820 table is sorted and
667 * not-overlapping, which is the case
668 */
669int __init
670e820_all_mapped(unsigned long s, unsigned long e, unsigned type)
671{
672 u64 start = s;
673 u64 end = e;
674 int i;
675 for (i = 0; i < e820.nr_map; i++) {
676 struct e820entry *ei = &e820.map[i];
677 if (type && ei->type != type)
678 continue;
679 /* is the region (part) in overlap with the current region ?*/
680 if (ei->addr >= end || ei->addr + ei->size <= start)
681 continue;
682 /* if the region is at the beginning of <start,end> we move
683 * start to the end of the region since it's ok until there
684 */
685 if (ei->addr <= start)
686 start = ei->addr + ei->size;
687 /* if start is now at or beyond end, we're done, full
688 * coverage */
689 if (start >= end)
690 return 1; /* we're done */
691 }
692 return 0;
693}
694
695static int __init parse_memmap(char *arg)
696{
697 if (!arg)
698 return -EINVAL;
699
700 if (strcmp(arg, "exactmap") == 0) {
701#ifdef CONFIG_CRASH_DUMP
702 /* If we are doing a crash dump, we
703 * still need to know the real mem
704 * size before original memory map is
705 * reset.
706 */
707 propagate_e820_map();
708 saved_max_pfn = max_pfn;
709#endif
710 e820.nr_map = 0;
711 user_defined_memmap = 1;
712 } else {
713 /* If the user specifies memory size, we
714 * limit the BIOS-provided memory map to
715 * that size. exactmap can be used to specify
716 * the exact map. mem=number can be used to
717 * trim the existing memory map.
718 */
719 unsigned long long start_at, mem_size;
720
721 mem_size = memparse(arg, &arg);
722 if (*arg == '@') {
723 start_at = memparse(arg+1, &arg);
724 add_memory_region(start_at, mem_size, E820_RAM);
725 } else if (*arg == '#') {
726 start_at = memparse(arg+1, &arg);
727 add_memory_region(start_at, mem_size, E820_ACPI);
728 } else if (*arg == '$') {
729 start_at = memparse(arg+1, &arg);
730 add_memory_region(start_at, mem_size, E820_RESERVED);
731 } else {
732 limit_regions(mem_size);
733 user_defined_memmap = 1;
734 }
735 }
736 return 0;
737}
738early_param("memmap", parse_memmap);
739void __init update_memory_range(u64 start, u64 size, unsigned old_type,
740 unsigned new_type)
741{
742 int i;
743
744 BUG_ON(old_type == new_type);
745
746 for (i = 0; i < e820.nr_map; i++) {
747 struct e820entry *ei = &e820.map[i];
748 u64 final_start, final_end;
749 if (ei->type != old_type)
750 continue;
751 /* totally covered? */
752 if (ei->addr >= start && ei->size <= size) {
753 ei->type = new_type;
754 continue;
755 }
756 /* partially covered */
757 final_start = max(start, ei->addr);
758 final_end = min(start + size, ei->addr + ei->size);
759 if (final_start >= final_end)
760 continue;
761 add_memory_region(final_start, final_end - final_start,
762 new_type);
763 }
764}
765void __init update_e820(void)
766{
767 u8 nr_map;
768
769 nr_map = e820.nr_map;
770 if (sanitize_e820_map(e820.map, &nr_map))
771 return;
772 e820.nr_map = nr_map;
773 printk(KERN_INFO "modified physical RAM map:\n");
774 print_memory_map("modified");
775}
diff --git a/arch/x86/kernel/e820_64.c b/arch/x86/kernel/e820_64.c
deleted file mode 100644
index 124480c0008d..000000000000
--- a/arch/x86/kernel/e820_64.c
+++ /dev/null
@@ -1,952 +0,0 @@
1/*
2 * Handle the memory map.
3 * The functions here do the job until bootmem takes over.
4 *
5 * Getting sanitize_e820_map() in sync with i386 version by applying change:
6 * - Provisions for empty E820 memory regions (reported by certain BIOSes).
7 * Alex Achenbach <xela@slit.de>, December 2002.
8 * Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
9 *
10 */
11#include <linux/kernel.h>
12#include <linux/types.h>
13#include <linux/init.h>
14#include <linux/bootmem.h>
15#include <linux/ioport.h>
16#include <linux/string.h>
17#include <linux/kexec.h>
18#include <linux/module.h>
19#include <linux/mm.h>
20#include <linux/suspend.h>
21#include <linux/pfn.h>
22
23#include <asm/pgtable.h>
24#include <asm/page.h>
25#include <asm/e820.h>
26#include <asm/proto.h>
27#include <asm/setup.h>
28#include <asm/sections.h>
29#include <asm/kdebug.h>
30#include <asm/trampoline.h>
31
32struct e820map e820;
33
34/*
35 * PFN of last memory page.
36 */
37unsigned long end_pfn;
38
39/*
40 * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
41 * The direct mapping extends to max_pfn_mapped, so that we can directly access
42 * apertures, ACPI and other tables without having to play with fixmaps.
43 */
44unsigned long max_pfn_mapped;
45
46/*
47 * Last pfn which the user wants to use.
48 */
49static unsigned long __initdata end_user_pfn = MAXMEM>>PAGE_SHIFT;
50
51/*
52 * Early reserved memory areas.
53 */
54#define MAX_EARLY_RES 20
55
56struct early_res {
57 unsigned long start, end;
58 char name[16];
59};
60static struct early_res early_res[MAX_EARLY_RES] __initdata = {
61 { 0, PAGE_SIZE, "BIOS data page" }, /* BIOS data page */
62#ifdef CONFIG_X86_TRAMPOLINE
63 { TRAMPOLINE_BASE, TRAMPOLINE_BASE + 2 * PAGE_SIZE, "TRAMPOLINE" },
64#endif
65 {}
66};
67
68void __init reserve_early(unsigned long start, unsigned long end, char *name)
69{
70 int i;
71 struct early_res *r;
72 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
73 r = &early_res[i];
74 if (end > r->start && start < r->end)
75 panic("Overlapping early reservations %lx-%lx %s to %lx-%lx %s\n",
76 start, end - 1, name?name:"", r->start, r->end - 1, r->name);
77 }
78 if (i >= MAX_EARLY_RES)
79 panic("Too many early reservations");
80 r = &early_res[i];
81 r->start = start;
82 r->end = end;
83 if (name)
84 strncpy(r->name, name, sizeof(r->name) - 1);
85}
86
87void __init free_early(unsigned long start, unsigned long end)
88{
89 struct early_res *r;
90 int i, j;
91
92 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
93 r = &early_res[i];
94 if (start == r->start && end == r->end)
95 break;
96 }
97 if (i >= MAX_EARLY_RES || !early_res[i].end)
98 panic("free_early on not reserved area: %lx-%lx!", start, end);
99
100 for (j = i + 1; j < MAX_EARLY_RES && early_res[j].end; j++)
101 ;
102
103 memmove(&early_res[i], &early_res[i + 1],
104 (j - 1 - i) * sizeof(struct early_res));
105
106 early_res[j - 1].end = 0;
107}
108
109void __init early_res_to_bootmem(unsigned long start, unsigned long end)
110{
111 int i;
112 unsigned long final_start, final_end;
113 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
114 struct early_res *r = &early_res[i];
115 final_start = max(start, r->start);
116 final_end = min(end, r->end);
117 if (final_start >= final_end)
118 continue;
119 printk(KERN_INFO " early res: %d [%lx-%lx] %s\n", i,
120 final_start, final_end - 1, r->name);
121 reserve_bootmem_generic(final_start, final_end - final_start);
122 }
123}
124
125/* Check for already reserved areas */
126static inline int __init
127bad_addr(unsigned long *addrp, unsigned long size, unsigned long align)
128{
129 int i;
130 unsigned long addr = *addrp, last;
131 int changed = 0;
132again:
133 last = addr + size;
134 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
135 struct early_res *r = &early_res[i];
136 if (last >= r->start && addr < r->end) {
137 *addrp = addr = round_up(r->end, align);
138 changed = 1;
139 goto again;
140 }
141 }
142 return changed;
143}
144
145/* Check for already reserved areas */
146static inline int __init
147bad_addr_size(unsigned long *addrp, unsigned long *sizep, unsigned long align)
148{
149 int i;
150 unsigned long addr = *addrp, last;
151 unsigned long size = *sizep;
152 int changed = 0;
153again:
154 last = addr + size;
155 for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
156 struct early_res *r = &early_res[i];
157 if (last > r->start && addr < r->start) {
158 size = r->start - addr;
159 changed = 1;
160 goto again;
161 }
162 if (last > r->end && addr < r->end) {
163 addr = round_up(r->end, align);
164 size = last - addr;
165 changed = 1;
166 goto again;
167 }
168 if (last <= r->end && addr >= r->start) {
169 (*sizep)++;
170 return 0;
171 }
172 }
173 if (changed) {
174 *addrp = addr;
175 *sizep = size;
176 }
177 return changed;
178}
179/*
180 * This function checks if any part of the range <start,end> is mapped
181 * with type.
182 */
183int
184e820_any_mapped(unsigned long start, unsigned long end, unsigned type)
185{
186 int i;
187
188 for (i = 0; i < e820.nr_map; i++) {
189 struct e820entry *ei = &e820.map[i];
190
191 if (type && ei->type != type)
192 continue;
193 if (ei->addr >= end || ei->addr + ei->size <= start)
194 continue;
195 return 1;
196 }
197 return 0;
198}
199EXPORT_SYMBOL_GPL(e820_any_mapped);
200
201/*
202 * This function checks if the entire range <start,end> is mapped with type.
203 *
204 * Note: this function only works correct if the e820 table is sorted and
205 * not-overlapping, which is the case
206 */
207int __init e820_all_mapped(unsigned long start, unsigned long end,
208 unsigned type)
209{
210 int i;
211
212 for (i = 0; i < e820.nr_map; i++) {
213 struct e820entry *ei = &e820.map[i];
214
215 if (type && ei->type != type)
216 continue;
217 /* is the region (part) in overlap with the current region ?*/
218 if (ei->addr >= end || ei->addr + ei->size <= start)
219 continue;
220
221 /* if the region is at the beginning of <start,end> we move
222 * start to the end of the region since it's ok until there
223 */
224 if (ei->addr <= start)
225 start = ei->addr + ei->size;
226 /*
227 * if start is now at or beyond end, we're done, full
228 * coverage
229 */
230 if (start >= end)
231 return 1;
232 }
233 return 0;
234}
235
236/*
237 * Find a free area with specified alignment in a specific range.
238 */
239unsigned long __init find_e820_area(unsigned long start, unsigned long end,
240 unsigned long size, unsigned long align)
241{
242 int i;
243
244 for (i = 0; i < e820.nr_map; i++) {
245 struct e820entry *ei = &e820.map[i];
246 unsigned long addr, last;
247 unsigned long ei_last;
248
249 if (ei->type != E820_RAM)
250 continue;
251 addr = round_up(ei->addr, align);
252 ei_last = ei->addr + ei->size;
253 if (addr < start)
254 addr = round_up(start, align);
255 if (addr >= ei_last)
256 continue;
257 while (bad_addr(&addr, size, align) && addr+size <= ei_last)
258 ;
259 last = addr + size;
260 if (last > ei_last)
261 continue;
262 if (last > end)
263 continue;
264 return addr;
265 }
266 return -1UL;
267}
268
269/*
270 * Find next free range after *start
271 */
272unsigned long __init find_e820_area_size(unsigned long start,
273 unsigned long *sizep,
274 unsigned long align)
275{
276 int i;
277
278 for (i = 0; i < e820.nr_map; i++) {
279 struct e820entry *ei = &e820.map[i];
280 unsigned long addr, last;
281 unsigned long ei_last;
282
283 if (ei->type != E820_RAM)
284 continue;
285 addr = round_up(ei->addr, align);
286 ei_last = ei->addr + ei->size;
287 if (addr < start)
288 addr = round_up(start, align);
289 if (addr >= ei_last)
290 continue;
291 *sizep = ei_last - addr;
292 while (bad_addr_size(&addr, sizep, align) &&
293 addr + *sizep <= ei_last)
294 ;
295 last = addr + *sizep;
296 if (last > ei_last)
297 continue;
298 return addr;
299 }
300 return -1UL;
301
302}
303/*
304 * Find the highest page frame number we have available
305 */
306unsigned long __init e820_end_of_ram(void)
307{
308 unsigned long end_pfn;
309
310 end_pfn = find_max_pfn_with_active_regions();
311
312 if (end_pfn > max_pfn_mapped)
313 max_pfn_mapped = end_pfn;
314 if (max_pfn_mapped > MAXMEM>>PAGE_SHIFT)
315 max_pfn_mapped = MAXMEM>>PAGE_SHIFT;
316 if (end_pfn > end_user_pfn)
317 end_pfn = end_user_pfn;
318 if (end_pfn > max_pfn_mapped)
319 end_pfn = max_pfn_mapped;
320
321 printk(KERN_INFO "max_pfn_mapped = %lu\n", max_pfn_mapped);
322 return end_pfn;
323}
324
325/*
326 * Mark e820 reserved areas as busy for the resource manager.
327 */
328void __init e820_reserve_resources(void)
329{
330 int i;
331 struct resource *res;
332
333 res = alloc_bootmem_low(sizeof(struct resource) * e820.nr_map);
334 for (i = 0; i < e820.nr_map; i++) {
335 switch (e820.map[i].type) {
336 case E820_RAM: res->name = "System RAM"; break;
337 case E820_ACPI: res->name = "ACPI Tables"; break;
338 case E820_NVS: res->name = "ACPI Non-volatile Storage"; break;
339 default: res->name = "reserved";
340 }
341 res->start = e820.map[i].addr;
342 res->end = res->start + e820.map[i].size - 1;
343 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
344 insert_resource(&iomem_resource, res);
345 res++;
346 }
347}
348
349/*
350 * Find the ranges of physical addresses that do not correspond to
351 * e820 RAM areas and mark the corresponding pages as nosave for software
352 * suspend and suspend to RAM.
353 *
354 * This function requires the e820 map to be sorted and without any
355 * overlapping entries and assumes the first e820 area to be RAM.
356 */
357void __init e820_mark_nosave_regions(void)
358{
359 int i;
360 unsigned long paddr;
361
362 paddr = round_down(e820.map[0].addr + e820.map[0].size, PAGE_SIZE);
363 for (i = 1; i < e820.nr_map; i++) {
364 struct e820entry *ei = &e820.map[i];
365
366 if (paddr < ei->addr)
367 register_nosave_region(PFN_DOWN(paddr),
368 PFN_UP(ei->addr));
369
370 paddr = round_down(ei->addr + ei->size, PAGE_SIZE);
371 if (ei->type != E820_RAM)
372 register_nosave_region(PFN_UP(ei->addr),
373 PFN_DOWN(paddr));
374
375 if (paddr >= (end_pfn << PAGE_SHIFT))
376 break;
377 }
378}
379
380/*
381 * Finds an active region in the address range from start_pfn to end_pfn and
382 * returns its range in ei_startpfn and ei_endpfn for the e820 entry.
383 */
384static int __init e820_find_active_region(const struct e820entry *ei,
385 unsigned long start_pfn,
386 unsigned long end_pfn,
387 unsigned long *ei_startpfn,
388 unsigned long *ei_endpfn)
389{
390 *ei_startpfn = round_up(ei->addr, PAGE_SIZE) >> PAGE_SHIFT;
391 *ei_endpfn = round_down(ei->addr + ei->size, PAGE_SIZE) >> PAGE_SHIFT;
392
393 /* Skip map entries smaller than a page */
394 if (*ei_startpfn >= *ei_endpfn)
395 return 0;
396
397 /* Check if max_pfn_mapped should be updated */
398 if (ei->type != E820_RAM && *ei_endpfn > max_pfn_mapped)
399 max_pfn_mapped = *ei_endpfn;
400
401 /* Skip if map is outside the node */
402 if (ei->type != E820_RAM || *ei_endpfn <= start_pfn ||
403 *ei_startpfn >= end_pfn)
404 return 0;
405
406 /* Check for overlaps */
407 if (*ei_startpfn < start_pfn)
408 *ei_startpfn = start_pfn;
409 if (*ei_endpfn > end_pfn)
410 *ei_endpfn = end_pfn;
411
412 /* Obey end_user_pfn to save on memmap */
413 if (*ei_startpfn >= end_user_pfn)
414 return 0;
415 if (*ei_endpfn > end_user_pfn)
416 *ei_endpfn = end_user_pfn;
417
418 return 1;
419}
420
421/* Walk the e820 map and register active regions within a node */
422void __init
423e820_register_active_regions(int nid, unsigned long start_pfn,
424 unsigned long end_pfn)
425{
426 unsigned long ei_startpfn;
427 unsigned long ei_endpfn;
428 int i;
429
430 for (i = 0; i < e820.nr_map; i++)
431 if (e820_find_active_region(&e820.map[i],
432 start_pfn, end_pfn,
433 &ei_startpfn, &ei_endpfn))
434 add_active_range(nid, ei_startpfn, ei_endpfn);
435}
436
437/*
438 * Add a memory region to the kernel e820 map.
439 */
440void __init add_memory_region(unsigned long start, unsigned long size, int type)
441{
442 int x = e820.nr_map;
443
444 if (x == E820MAX) {
445 printk(KERN_ERR "Ooops! Too many entries in the memory map!\n");
446 return;
447 }
448
449 e820.map[x].addr = start;
450 e820.map[x].size = size;
451 e820.map[x].type = type;
452 e820.nr_map++;
453}
454
455/*
456 * Find the hole size (in bytes) in the memory range.
457 * @start: starting address of the memory range to scan
458 * @end: ending address of the memory range to scan
459 */
460unsigned long __init e820_hole_size(unsigned long start, unsigned long end)
461{
462 unsigned long start_pfn = start >> PAGE_SHIFT;
463 unsigned long end_pfn = end >> PAGE_SHIFT;
464 unsigned long ei_startpfn, ei_endpfn, ram = 0;
465 int i;
466
467 for (i = 0; i < e820.nr_map; i++) {
468 if (e820_find_active_region(&e820.map[i],
469 start_pfn, end_pfn,
470 &ei_startpfn, &ei_endpfn))
471 ram += ei_endpfn - ei_startpfn;
472 }
473 return end - start - (ram << PAGE_SHIFT);
474}
475
476static void __init e820_print_map(char *who)
477{
478 int i;
479
480 for (i = 0; i < e820.nr_map; i++) {
481 printk(KERN_INFO " %s: %016Lx - %016Lx ", who,
482 (unsigned long long) e820.map[i].addr,
483 (unsigned long long)
484 (e820.map[i].addr + e820.map[i].size));
485 switch (e820.map[i].type) {
486 case E820_RAM:
487 printk(KERN_CONT "(usable)\n");
488 break;
489 case E820_RESERVED:
490 printk(KERN_CONT "(reserved)\n");
491 break;
492 case E820_ACPI:
493 printk(KERN_CONT "(ACPI data)\n");
494 break;
495 case E820_NVS:
496 printk(KERN_CONT "(ACPI NVS)\n");
497 break;
498 default:
499 printk(KERN_CONT "type %u\n", e820.map[i].type);
500 break;
501 }
502 }
503}
504
505/*
506 * Sanitize the BIOS e820 map.
507 *
508 * Some e820 responses include overlapping entries. The following
509 * replaces the original e820 map with a new one, removing overlaps.
510 *
511 */
512static int __init sanitize_e820_map(struct e820entry *biosmap, char *pnr_map)
513{
514 struct change_member {
515 struct e820entry *pbios; /* pointer to original bios entry */
516 unsigned long long addr; /* address for this change point */
517 };
518 static struct change_member change_point_list[2*E820MAX] __initdata;
519 static struct change_member *change_point[2*E820MAX] __initdata;
520 static struct e820entry *overlap_list[E820MAX] __initdata;
521 static struct e820entry new_bios[E820MAX] __initdata;
522 struct change_member *change_tmp;
523 unsigned long current_type, last_type;
524 unsigned long long last_addr;
525 int chgidx, still_changing;
526 int overlap_entries;
527 int new_bios_entry;
528 int old_nr, new_nr, chg_nr;
529 int i;
530
531 /*
532 Visually we're performing the following
533 (1,2,3,4 = memory types)...
534
535 Sample memory map (w/overlaps):
536 ____22__________________
537 ______________________4_
538 ____1111________________
539 _44_____________________
540 11111111________________
541 ____________________33__
542 ___________44___________
543 __________33333_________
544 ______________22________
545 ___________________2222_
546 _________111111111______
547 _____________________11_
548 _________________4______
549
550 Sanitized equivalent (no overlap):
551 1_______________________
552 _44_____________________
553 ___1____________________
554 ____22__________________
555 ______11________________
556 _________1______________
557 __________3_____________
558 ___________44___________
559 _____________33_________
560 _______________2________
561 ________________1_______
562 _________________4______
563 ___________________2____
564 ____________________33__
565 ______________________4_
566 */
567
568 /* if there's only one memory region, don't bother */
569 if (*pnr_map < 2)
570 return -1;
571
572 old_nr = *pnr_map;
573
574 /* bail out if we find any unreasonable addresses in bios map */
575 for (i = 0; i < old_nr; i++)
576 if (biosmap[i].addr + biosmap[i].size < biosmap[i].addr)
577 return -1;
578
579 /* create pointers for initial change-point information (for sorting) */
580 for (i = 0; i < 2 * old_nr; i++)
581 change_point[i] = &change_point_list[i];
582
583 /* record all known change-points (starting and ending addresses),
584 omitting those that are for empty memory regions */
585 chgidx = 0;
586 for (i = 0; i < old_nr; i++) {
587 if (biosmap[i].size != 0) {
588 change_point[chgidx]->addr = biosmap[i].addr;
589 change_point[chgidx++]->pbios = &biosmap[i];
590 change_point[chgidx]->addr = biosmap[i].addr +
591 biosmap[i].size;
592 change_point[chgidx++]->pbios = &biosmap[i];
593 }
594 }
595 chg_nr = chgidx;
596
597 /* sort change-point list by memory addresses (low -> high) */
598 still_changing = 1;
599 while (still_changing) {
600 still_changing = 0;
601 for (i = 1; i < chg_nr; i++) {
602 unsigned long long curaddr, lastaddr;
603 unsigned long long curpbaddr, lastpbaddr;
604
605 curaddr = change_point[i]->addr;
606 lastaddr = change_point[i - 1]->addr;
607 curpbaddr = change_point[i]->pbios->addr;
608 lastpbaddr = change_point[i - 1]->pbios->addr;
609
610 /*
611 * swap entries, when:
612 *
613 * curaddr > lastaddr or
614 * curaddr == lastaddr and curaddr == curpbaddr and
615 * lastaddr != lastpbaddr
616 */
617 if (curaddr < lastaddr ||
618 (curaddr == lastaddr && curaddr == curpbaddr &&
619 lastaddr != lastpbaddr)) {
620 change_tmp = change_point[i];
621 change_point[i] = change_point[i-1];
622 change_point[i-1] = change_tmp;
623 still_changing = 1;
624 }
625 }
626 }
627
628 /* create a new bios memory map, removing overlaps */
629 overlap_entries = 0; /* number of entries in the overlap table */
630 new_bios_entry = 0; /* index for creating new bios map entries */
631 last_type = 0; /* start with undefined memory type */
632 last_addr = 0; /* start with 0 as last starting address */
633
634 /* loop through change-points, determining affect on the new bios map */
635 for (chgidx = 0; chgidx < chg_nr; chgidx++) {
636 /* keep track of all overlapping bios entries */
637 if (change_point[chgidx]->addr ==
638 change_point[chgidx]->pbios->addr) {
639 /*
640 * add map entry to overlap list (> 1 entry
641 * implies an overlap)
642 */
643 overlap_list[overlap_entries++] =
644 change_point[chgidx]->pbios;
645 } else {
646 /*
647 * remove entry from list (order independent,
648 * so swap with last)
649 */
650 for (i = 0; i < overlap_entries; i++) {
651 if (overlap_list[i] ==
652 change_point[chgidx]->pbios)
653 overlap_list[i] =
654 overlap_list[overlap_entries-1];
655 }
656 overlap_entries--;
657 }
658 /*
659 * if there are overlapping entries, decide which
660 * "type" to use (larger value takes precedence --
661 * 1=usable, 2,3,4,4+=unusable)
662 */
663 current_type = 0;
664 for (i = 0; i < overlap_entries; i++)
665 if (overlap_list[i]->type > current_type)
666 current_type = overlap_list[i]->type;
667 /*
668 * continue building up new bios map based on this
669 * information
670 */
671 if (current_type != last_type) {
672 if (last_type != 0) {
673 new_bios[new_bios_entry].size =
674 change_point[chgidx]->addr - last_addr;
675 /*
676 * move forward only if the new size
677 * was non-zero
678 */
679 if (new_bios[new_bios_entry].size != 0)
680 /*
681 * no more space left for new
682 * bios entries ?
683 */
684 if (++new_bios_entry >= E820MAX)
685 break;
686 }
687 if (current_type != 0) {
688 new_bios[new_bios_entry].addr =
689 change_point[chgidx]->addr;
690 new_bios[new_bios_entry].type = current_type;
691 last_addr = change_point[chgidx]->addr;
692 }
693 last_type = current_type;
694 }
695 }
696 /* retain count for new bios entries */
697 new_nr = new_bios_entry;
698
699 /* copy new bios mapping into original location */
700 memcpy(biosmap, new_bios, new_nr * sizeof(struct e820entry));
701 *pnr_map = new_nr;
702
703 return 0;
704}
705
706/*
707 * Copy the BIOS e820 map into a safe place.
708 *
709 * Sanity-check it while we're at it..
710 *
711 * If we're lucky and live on a modern system, the setup code
712 * will have given us a memory map that we can use to properly
713 * set up memory. If we aren't, we'll fake a memory map.
714 */
715static int __init copy_e820_map(struct e820entry *biosmap, int nr_map)
716{
717 /* Only one memory region (or negative)? Ignore it */
718 if (nr_map < 2)
719 return -1;
720
721 do {
722 u64 start = biosmap->addr;
723 u64 size = biosmap->size;
724 u64 end = start + size;
725 u32 type = biosmap->type;
726
727 /* Overflow in 64 bits? Ignore the memory map. */
728 if (start > end)
729 return -1;
730
731 add_memory_region(start, size, type);
732 } while (biosmap++, --nr_map);
733 return 0;
734}
735
736static void early_panic(char *msg)
737{
738 early_printk(msg);
739 panic(msg);
740}
741
742/* We're not void only for x86 32-bit compat */
743char * __init machine_specific_memory_setup(void)
744{
745 char *who = "BIOS-e820";
746 /*
747 * Try to copy the BIOS-supplied E820-map.
748 *
749 * Otherwise fake a memory map; one section from 0k->640k,
750 * the next section from 1mb->appropriate_mem_k
751 */
752 sanitize_e820_map(boot_params.e820_map, &boot_params.e820_entries);
753 if (copy_e820_map(boot_params.e820_map, boot_params.e820_entries) < 0)
754 early_panic("Cannot find a valid memory map");
755 printk(KERN_INFO "BIOS-provided physical RAM map:\n");
756 e820_print_map(who);
757
758 /* In case someone cares... */
759 return who;
760}
761
762static int __init parse_memopt(char *p)
763{
764 if (!p)
765 return -EINVAL;
766 end_user_pfn = memparse(p, &p);
767 end_user_pfn >>= PAGE_SHIFT;
768 return 0;
769}
770early_param("mem", parse_memopt);
771
772static int userdef __initdata;
773
774static int __init parse_memmap_opt(char *p)
775{
776 char *oldp;
777 unsigned long long start_at, mem_size;
778
779 if (!strcmp(p, "exactmap")) {
780#ifdef CONFIG_CRASH_DUMP
781 /*
782 * If we are doing a crash dump, we still need to know
783 * the real mem size before original memory map is
784 * reset.
785 */
786 e820_register_active_regions(0, 0, -1UL);
787 saved_max_pfn = e820_end_of_ram();
788 remove_all_active_ranges();
789#endif
790 max_pfn_mapped = 0;
791 e820.nr_map = 0;
792 userdef = 1;
793 return 0;
794 }
795
796 oldp = p;
797 mem_size = memparse(p, &p);
798 if (p == oldp)
799 return -EINVAL;
800
801 userdef = 1;
802 if (*p == '@') {
803 start_at = memparse(p+1, &p);
804 add_memory_region(start_at, mem_size, E820_RAM);
805 } else if (*p == '#') {
806 start_at = memparse(p+1, &p);
807 add_memory_region(start_at, mem_size, E820_ACPI);
808 } else if (*p == '$') {
809 start_at = memparse(p+1, &p);
810 add_memory_region(start_at, mem_size, E820_RESERVED);
811 } else {
812 end_user_pfn = (mem_size >> PAGE_SHIFT);
813 }
814 return *p == '\0' ? 0 : -EINVAL;
815}
816early_param("memmap", parse_memmap_opt);
817
818void __init finish_e820_parsing(void)
819{
820 if (userdef) {
821 char nr = e820.nr_map;
822
823 if (sanitize_e820_map(e820.map, &nr) < 0)
824 early_panic("Invalid user supplied memory map");
825 e820.nr_map = nr;
826
827 printk(KERN_INFO "user-defined physical RAM map:\n");
828 e820_print_map("user");
829 }
830}
831
832void __init update_memory_range(u64 start, u64 size, unsigned old_type,
833 unsigned new_type)
834{
835 int i;
836
837 BUG_ON(old_type == new_type);
838
839 for (i = 0; i < e820.nr_map; i++) {
840 struct e820entry *ei = &e820.map[i];
841 u64 final_start, final_end;
842 if (ei->type != old_type)
843 continue;
844 /* totally covered? */
845 if (ei->addr >= start && ei->size <= size) {
846 ei->type = new_type;
847 continue;
848 }
849 /* partially covered */
850 final_start = max(start, ei->addr);
851 final_end = min(start + size, ei->addr + ei->size);
852 if (final_start >= final_end)
853 continue;
854 add_memory_region(final_start, final_end - final_start,
855 new_type);
856 }
857}
858
859void __init update_e820(void)
860{
861 u8 nr_map;
862
863 nr_map = e820.nr_map;
864 if (sanitize_e820_map(e820.map, &nr_map))
865 return;
866 e820.nr_map = nr_map;
867 printk(KERN_INFO "modified physical RAM map:\n");
868 e820_print_map("modified");
869}
870
871unsigned long pci_mem_start = 0xaeedbabe;
872EXPORT_SYMBOL(pci_mem_start);
873
874/*
875 * Search for the biggest gap in the low 32 bits of the e820
876 * memory space. We pass this space to PCI to assign MMIO resources
877 * for hotplug or unconfigured devices in.
878 * Hopefully the BIOS let enough space left.
879 */
880__init void e820_setup_gap(void)
881{
882 unsigned long gapstart, gapsize, round;
883 unsigned long last;
884 int i;
885 int found = 0;
886
887 last = 0x100000000ull;
888 gapstart = 0x10000000;
889 gapsize = 0x400000;
890 i = e820.nr_map;
891 while (--i >= 0) {
892 unsigned long long start = e820.map[i].addr;
893 unsigned long long end = start + e820.map[i].size;
894
895 /*
896 * Since "last" is at most 4GB, we know we'll
897 * fit in 32 bits if this condition is true
898 */
899 if (last > end) {
900 unsigned long gap = last - end;
901
902 if (gap > gapsize) {
903 gapsize = gap;
904 gapstart = end;
905 found = 1;
906 }
907 }
908 if (start < last)
909 last = start;
910 }
911
912 if (!found) {
913 gapstart = (end_pfn << PAGE_SHIFT) + 1024*1024;
914 printk(KERN_ERR "PCI: Warning: Cannot find a gap in the 32bit "
915 "address range\n"
916 KERN_ERR "PCI: Unassigned devices with 32bit resource "
917 "registers may break!\n");
918 }
919
920 /*
921 * See how much we want to round up: start off with
922 * rounding to the next 1MB area.
923 */
924 round = 0x100000;
925 while ((gapsize >> 4) > round)
926 round += round;
927 /* Fun with two's complement */
928 pci_mem_start = (gapstart + round) & -round;
929
930 printk(KERN_INFO
931 "Allocating PCI resources starting at %lx (gap: %lx:%lx)\n",
932 pci_mem_start, gapstart, gapsize);
933}
934
935int __init arch_get_ram_range(int slot, u64 *addr, u64 *size)
936{
937 int i;
938
939 if (slot < 0 || slot >= e820.nr_map)
940 return -1;
941 for (i = slot; i < e820.nr_map; i++) {
942 if (e820.map[i].type != E820_RAM)
943 continue;
944 break;
945 }
946 if (i == e820.nr_map || e820.map[i].addr > (max_pfn << PAGE_SHIFT))
947 return -1;
948 *addr = e820.map[i].addr;
949 *size = min_t(u64, e820.map[i].size + e820.map[i].addr,
950 max_pfn << PAGE_SHIFT) - *addr;
951 return i + 1;
952}
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 9f51e1ea9e82..a0e11c0cc872 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -50,7 +50,7 @@ static void __init fix_hypertransport_config(int num, int slot, int func)
50static void __init via_bugs(int num, int slot, int func) 50static void __init via_bugs(int num, int slot, int func)
51{ 51{
52#ifdef CONFIG_GART_IOMMU 52#ifdef CONFIG_GART_IOMMU
53 if ((end_pfn > MAX_DMA32_PFN || force_iommu) && 53 if ((max_pfn > MAX_DMA32_PFN || force_iommu) &&
54 !gart_iommu_aperture_allowed) { 54 !gart_iommu_aperture_allowed) {
55 printk(KERN_INFO 55 printk(KERN_INFO
56 "Looks like a VIA chipset. Disabling IOMMU." 56 "Looks like a VIA chipset. Disabling IOMMU."
@@ -98,17 +98,6 @@ static void __init nvidia_bugs(int num, int slot, int func)
98 98
99} 99}
100 100
101static void __init ati_bugs(int num, int slot, int func)
102{
103#ifdef CONFIG_X86_IO_APIC
104 if (timer_over_8254 == 1) {
105 timer_over_8254 = 0;
106 printk(KERN_INFO
107 "ATI board detected. Disabling timer routing over 8254.\n");
108 }
109#endif
110}
111
112#define QFLAG_APPLY_ONCE 0x1 101#define QFLAG_APPLY_ONCE 0x1
113#define QFLAG_APPLIED 0x2 102#define QFLAG_APPLIED 0x2
114#define QFLAG_DONE (QFLAG_APPLY_ONCE|QFLAG_APPLIED) 103#define QFLAG_DONE (QFLAG_APPLY_ONCE|QFLAG_APPLIED)
@@ -126,14 +115,23 @@ static struct chipset early_qrk[] __initdata = {
126 PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, nvidia_bugs }, 115 PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, nvidia_bugs },
127 { PCI_VENDOR_ID_VIA, PCI_ANY_ID, 116 { PCI_VENDOR_ID_VIA, PCI_ANY_ID,
128 PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, via_bugs }, 117 PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, via_bugs },
129 { PCI_VENDOR_ID_ATI, PCI_ANY_ID,
130 PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, ati_bugs },
131 { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB, 118 { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB,
132 PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, fix_hypertransport_config }, 119 PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, fix_hypertransport_config },
133 {} 120 {}
134}; 121};
135 122
136static void __init check_dev_quirk(int num, int slot, int func) 123/**
124 * check_dev_quirk - apply early quirks to a given PCI device
125 * @num: bus number
126 * @slot: slot number
127 * @func: PCI function
128 *
129 * Check the vendor & device ID against the early quirks table.
130 *
131 * If the device is single function, let early_quirks() know so we don't
132 * poke at this device again.
133 */
134static int __init check_dev_quirk(int num, int slot, int func)
137{ 135{
138 u16 class; 136 u16 class;
139 u16 vendor; 137 u16 vendor;
@@ -144,7 +142,7 @@ static void __init check_dev_quirk(int num, int slot, int func)
144 class = read_pci_config_16(num, slot, func, PCI_CLASS_DEVICE); 142 class = read_pci_config_16(num, slot, func, PCI_CLASS_DEVICE);
145 143
146 if (class == 0xffff) 144 if (class == 0xffff)
147 return; 145 return -1; /* no class, treat as single function */
148 146
149 vendor = read_pci_config_16(num, slot, func, PCI_VENDOR_ID); 147 vendor = read_pci_config_16(num, slot, func, PCI_VENDOR_ID);
150 148
@@ -167,7 +165,9 @@ static void __init check_dev_quirk(int num, int slot, int func)
167 type = read_pci_config_byte(num, slot, func, 165 type = read_pci_config_byte(num, slot, func,
168 PCI_HEADER_TYPE); 166 PCI_HEADER_TYPE);
169 if (!(type & 0x80)) 167 if (!(type & 0x80))
170 return; 168 return -1;
169
170 return 0;
171} 171}
172 172
173void __init early_quirks(void) 173void __init early_quirks(void)
@@ -180,6 +180,9 @@ void __init early_quirks(void)
180 /* Poor man's PCI discovery */ 180 /* Poor man's PCI discovery */
181 for (num = 0; num < 32; num++) 181 for (num = 0; num < 32; num++)
182 for (slot = 0; slot < 32; slot++) 182 for (slot = 0; slot < 32; slot++)
183 for (func = 0; func < 8; func++) 183 for (func = 0; func < 8; func++) {
184 check_dev_quirk(num, slot, func); 184 /* Only probe function 0 on single fn devices */
185 if (check_dev_quirk(num, slot, func))
186 break;
187 }
185} 188}
diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c
index 643fd861b724..ff9e7350da54 100644
--- a/arch/x86/kernel/early_printk.c
+++ b/arch/x86/kernel/early_printk.c
@@ -196,7 +196,7 @@ static struct console simnow_console = {
196static struct console *early_console = &early_vga_console; 196static struct console *early_console = &early_vga_console;
197static int early_console_initialized; 197static int early_console_initialized;
198 198
199void early_printk(const char *fmt, ...) 199asmlinkage void early_printk(const char *fmt, ...)
200{ 200{
201 char buf[512]; 201 char buf[512];
202 int n; 202 int n;
diff --git a/arch/x86/kernel/efi.c b/arch/x86/kernel/efi.c
index 77d424cf68b3..06cc8d4254b1 100644
--- a/arch/x86/kernel/efi.c
+++ b/arch/x86/kernel/efi.c
@@ -64,6 +64,17 @@ static int __init setup_noefi(char *arg)
64} 64}
65early_param("noefi", setup_noefi); 65early_param("noefi", setup_noefi);
66 66
67int add_efi_memmap;
68EXPORT_SYMBOL(add_efi_memmap);
69
70static int __init setup_add_efi_memmap(char *arg)
71{
72 add_efi_memmap = 1;
73 return 0;
74}
75early_param("add_efi_memmap", setup_add_efi_memmap);
76
77
67static efi_status_t virt_efi_get_time(efi_time_t *tm, efi_time_cap_t *tc) 78static efi_status_t virt_efi_get_time(efi_time_t *tm, efi_time_cap_t *tc)
68{ 79{
69 return efi_call_virt2(get_time, tm, tc); 80 return efi_call_virt2(get_time, tm, tc);
@@ -213,6 +224,50 @@ unsigned long efi_get_time(void)
213 eft.minute, eft.second); 224 eft.minute, eft.second);
214} 225}
215 226
227/*
228 * Tell the kernel about the EFI memory map. This might include
229 * more than the max 128 entries that can fit in the e820 legacy
230 * (zeropage) memory map.
231 */
232
233static void __init do_add_efi_memmap(void)
234{
235 void *p;
236
237 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
238 efi_memory_desc_t *md = p;
239 unsigned long long start = md->phys_addr;
240 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
241 int e820_type;
242
243 if (md->attribute & EFI_MEMORY_WB)
244 e820_type = E820_RAM;
245 else
246 e820_type = E820_RESERVED;
247 e820_add_region(start, size, e820_type);
248 }
249 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
250}
251
252void __init efi_reserve_early(void)
253{
254 unsigned long pmap;
255
256#ifdef CONFIG_X86_32
257 pmap = boot_params.efi_info.efi_memmap;
258#else
259 pmap = (boot_params.efi_info.efi_memmap |
260 ((__u64)boot_params.efi_info.efi_memmap_hi<<32));
261#endif
262 memmap.phys_map = (void *)pmap;
263 memmap.nr_map = boot_params.efi_info.efi_memmap_size /
264 boot_params.efi_info.efi_memdesc_size;
265 memmap.desc_version = boot_params.efi_info.efi_memdesc_version;
266 memmap.desc_size = boot_params.efi_info.efi_memdesc_size;
267 reserve_early(pmap, pmap + memmap.nr_map * memmap.desc_size,
268 "EFI memmap");
269}
270
216#if EFI_DEBUG 271#if EFI_DEBUG
217static void __init print_efi_memmap(void) 272static void __init print_efi_memmap(void)
218{ 273{
@@ -244,19 +299,11 @@ void __init efi_init(void)
244 299
245#ifdef CONFIG_X86_32 300#ifdef CONFIG_X86_32
246 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab; 301 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
247 memmap.phys_map = (void *)boot_params.efi_info.efi_memmap;
248#else 302#else
249 efi_phys.systab = (efi_system_table_t *) 303 efi_phys.systab = (efi_system_table_t *)
250 (boot_params.efi_info.efi_systab | 304 (boot_params.efi_info.efi_systab |
251 ((__u64)boot_params.efi_info.efi_systab_hi<<32)); 305 ((__u64)boot_params.efi_info.efi_systab_hi<<32));
252 memmap.phys_map = (void *)
253 (boot_params.efi_info.efi_memmap |
254 ((__u64)boot_params.efi_info.efi_memmap_hi<<32));
255#endif 306#endif
256 memmap.nr_map = boot_params.efi_info.efi_memmap_size /
257 boot_params.efi_info.efi_memdesc_size;
258 memmap.desc_version = boot_params.efi_info.efi_memdesc_version;
259 memmap.desc_size = boot_params.efi_info.efi_memdesc_size;
260 307
261 efi.systab = early_ioremap((unsigned long)efi_phys.systab, 308 efi.systab = early_ioremap((unsigned long)efi_phys.systab,
262 sizeof(efi_system_table_t)); 309 sizeof(efi_system_table_t));
@@ -370,6 +417,8 @@ void __init efi_init(void)
370 if (memmap.desc_size != sizeof(efi_memory_desc_t)) 417 if (memmap.desc_size != sizeof(efi_memory_desc_t))
371 printk(KERN_WARNING "Kernel-defined memdesc" 418 printk(KERN_WARNING "Kernel-defined memdesc"
372 "doesn't match the one from EFI!\n"); 419 "doesn't match the one from EFI!\n");
420 if (add_efi_memmap)
421 do_add_efi_memmap();
373 422
374 /* Setup for EFI runtime service */ 423 /* Setup for EFI runtime service */
375 reboot_type = BOOT_EFI; 424 reboot_type = BOOT_EFI;
@@ -424,7 +473,7 @@ void __init efi_enter_virtual_mode(void)
424 size = md->num_pages << EFI_PAGE_SHIFT; 473 size = md->num_pages << EFI_PAGE_SHIFT;
425 end = md->phys_addr + size; 474 end = md->phys_addr + size;
426 475
427 if (PFN_UP(end) <= max_pfn_mapped) 476 if (PFN_UP(end) <= max_low_pfn_mapped)
428 va = __va(md->phys_addr); 477 va = __va(md->phys_addr);
429 else 478 else
430 va = efi_ioremap(md->phys_addr, size); 479 va = efi_ioremap(md->phys_addr, size);
diff --git a/arch/x86/kernel/efi_32.c b/arch/x86/kernel/efi_32.c
index 5d23d85624d4..4b63c8e1f13b 100644
--- a/arch/x86/kernel/efi_32.c
+++ b/arch/x86/kernel/efi_32.c
@@ -49,13 +49,13 @@ void efi_call_phys_prelog(void)
49 local_irq_save(efi_rt_eflags); 49 local_irq_save(efi_rt_eflags);
50 50
51 /* 51 /*
52 * If I don't have PSE, I should just duplicate two entries in page 52 * If I don't have PAE, I should just duplicate two entries in page
53 * directory. If I have PSE, I just need to duplicate one entry in 53 * directory. If I have PAE, I just need to duplicate one entry in
54 * page directory. 54 * page directory.
55 */ 55 */
56 cr4 = read_cr4(); 56 cr4 = read_cr4();
57 57
58 if (cr4 & X86_CR4_PSE) { 58 if (cr4 & X86_CR4_PAE) {
59 efi_bak_pg_dir_pointer[0].pgd = 59 efi_bak_pg_dir_pointer[0].pgd =
60 swapper_pg_dir[pgd_index(0)].pgd; 60 swapper_pg_dir[pgd_index(0)].pgd;
61 swapper_pg_dir[0].pgd = 61 swapper_pg_dir[0].pgd =
@@ -93,7 +93,7 @@ void efi_call_phys_epilog(void)
93 93
94 cr4 = read_cr4(); 94 cr4 = read_cr4();
95 95
96 if (cr4 & X86_CR4_PSE) { 96 if (cr4 & X86_CR4_PAE) {
97 swapper_pg_dir[pgd_index(0)].pgd = 97 swapper_pg_dir[pgd_index(0)].pgd =
98 efi_bak_pg_dir_pointer[0].pgd; 98 efi_bak_pg_dir_pointer[0].pgd;
99 } else { 99 } else {
diff --git a/arch/x86/kernel/efi_64.c b/arch/x86/kernel/efi_64.c
index d0060fdcccac..652c5287215f 100644
--- a/arch/x86/kernel/efi_64.c
+++ b/arch/x86/kernel/efi_64.c
@@ -97,13 +97,7 @@ void __init efi_call_phys_epilog(void)
97 early_runtime_code_mapping_set_exec(0); 97 early_runtime_code_mapping_set_exec(0);
98} 98}
99 99
100void __init efi_reserve_bootmem(void) 100void __iomem *__init efi_ioremap(unsigned long phys_addr, unsigned long size)
101{
102 reserve_bootmem_generic((unsigned long)memmap.phys_map,
103 memmap.nr_map * memmap.desc_size);
104}
105
106void __iomem * __init efi_ioremap(unsigned long phys_addr, unsigned long size)
107{ 101{
108 static unsigned pages_mapped __initdata; 102 static unsigned pages_mapped __initdata;
109 unsigned i, pages; 103 unsigned i, pages;
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index c778e4fa55a2..6bc07f0f1202 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -51,14 +51,15 @@
51#include <asm/percpu.h> 51#include <asm/percpu.h>
52#include <asm/dwarf2.h> 52#include <asm/dwarf2.h>
53#include <asm/processor-flags.h> 53#include <asm/processor-flags.h>
54#include "irq_vectors.h" 54#include <asm/ftrace.h>
55#include <asm/irq_vectors.h>
55 56
56/* 57/*
57 * We use macros for low-level operations which need to be overridden 58 * We use macros for low-level operations which need to be overridden
58 * for paravirtualization. The following will never clobber any registers: 59 * for paravirtualization. The following will never clobber any registers:
59 * INTERRUPT_RETURN (aka. "iret") 60 * INTERRUPT_RETURN (aka. "iret")
60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 61 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61 * ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit"). 62 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
62 * 63 *
63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 64 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 65 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
@@ -349,7 +350,7 @@ sysenter_past_esp:
349 xorl %ebp,%ebp 350 xorl %ebp,%ebp
350 TRACE_IRQS_ON 351 TRACE_IRQS_ON
3511: mov PT_FS(%esp), %fs 3521: mov PT_FS(%esp), %fs
352 ENABLE_INTERRUPTS_SYSCALL_RET 353 ENABLE_INTERRUPTS_SYSEXIT
353 CFI_ENDPROC 354 CFI_ENDPROC
354.pushsection .fixup,"ax" 355.pushsection .fixup,"ax"
3552: movl $0,PT_FS(%esp) 3562: movl $0,PT_FS(%esp)
@@ -874,10 +875,10 @@ ENTRY(native_iret)
874.previous 875.previous
875END(native_iret) 876END(native_iret)
876 877
877ENTRY(native_irq_enable_syscall_ret) 878ENTRY(native_irq_enable_sysexit)
878 sti 879 sti
879 sysexit 880 sysexit
880END(native_irq_enable_syscall_ret) 881END(native_irq_enable_sysexit)
881#endif 882#endif
882 883
883KPROBE_ENTRY(int3) 884KPROBE_ENTRY(int3)
@@ -1024,6 +1025,7 @@ ENTRY(xen_sysenter_target)
1024 RING0_INT_FRAME 1025 RING0_INT_FRAME
1025 addl $5*4, %esp /* remove xen-provided frame */ 1026 addl $5*4, %esp /* remove xen-provided frame */
1026 jmp sysenter_past_esp 1027 jmp sysenter_past_esp
1028 CFI_ENDPROC
1027 1029
1028ENTRY(xen_hypervisor_callback) 1030ENTRY(xen_hypervisor_callback)
1029 CFI_STARTPROC 1031 CFI_STARTPROC
@@ -1110,6 +1112,77 @@ ENDPROC(xen_failsafe_callback)
1110 1112
1111#endif /* CONFIG_XEN */ 1113#endif /* CONFIG_XEN */
1112 1114
1115#ifdef CONFIG_FTRACE
1116#ifdef CONFIG_DYNAMIC_FTRACE
1117
1118ENTRY(mcount)
1119 pushl %eax
1120 pushl %ecx
1121 pushl %edx
1122 movl 0xc(%esp), %eax
1123 subl $MCOUNT_INSN_SIZE, %eax
1124
1125.globl mcount_call
1126mcount_call:
1127 call ftrace_stub
1128
1129 popl %edx
1130 popl %ecx
1131 popl %eax
1132
1133 ret
1134END(mcount)
1135
1136ENTRY(ftrace_caller)
1137 pushl %eax
1138 pushl %ecx
1139 pushl %edx
1140 movl 0xc(%esp), %eax
1141 movl 0x4(%ebp), %edx
1142 subl $MCOUNT_INSN_SIZE, %eax
1143
1144.globl ftrace_call
1145ftrace_call:
1146 call ftrace_stub
1147
1148 popl %edx
1149 popl %ecx
1150 popl %eax
1151
1152.globl ftrace_stub
1153ftrace_stub:
1154 ret
1155END(ftrace_caller)
1156
1157#else /* ! CONFIG_DYNAMIC_FTRACE */
1158
1159ENTRY(mcount)
1160 cmpl $ftrace_stub, ftrace_trace_function
1161 jnz trace
1162.globl ftrace_stub
1163ftrace_stub:
1164 ret
1165
1166 /* taken from glibc */
1167trace:
1168 pushl %eax
1169 pushl %ecx
1170 pushl %edx
1171 movl 0xc(%esp), %eax
1172 movl 0x4(%ebp), %edx
1173 subl $MCOUNT_INSN_SIZE, %eax
1174
1175 call *ftrace_trace_function
1176
1177 popl %edx
1178 popl %ecx
1179 popl %eax
1180
1181 jmp ftrace_stub
1182END(mcount)
1183#endif /* CONFIG_DYNAMIC_FTRACE */
1184#endif /* CONFIG_FTRACE */
1185
1113.section .rodata,"a" 1186.section .rodata,"a"
1114#include "syscall_table_32.S" 1187#include "syscall_table_32.S"
1115 1188
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 556a8df522a7..ae63e584c340 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -51,16 +51,121 @@
51#include <asm/page.h> 51#include <asm/page.h>
52#include <asm/irqflags.h> 52#include <asm/irqflags.h>
53#include <asm/paravirt.h> 53#include <asm/paravirt.h>
54#include <asm/ftrace.h>
54 55
55 .code64 56 .code64
56 57
58#ifdef CONFIG_FTRACE
59#ifdef CONFIG_DYNAMIC_FTRACE
60ENTRY(mcount)
61
62 subq $0x38, %rsp
63 movq %rax, (%rsp)
64 movq %rcx, 8(%rsp)
65 movq %rdx, 16(%rsp)
66 movq %rsi, 24(%rsp)
67 movq %rdi, 32(%rsp)
68 movq %r8, 40(%rsp)
69 movq %r9, 48(%rsp)
70
71 movq 0x38(%rsp), %rdi
72 subq $MCOUNT_INSN_SIZE, %rdi
73
74.globl mcount_call
75mcount_call:
76 call ftrace_stub
77
78 movq 48(%rsp), %r9
79 movq 40(%rsp), %r8
80 movq 32(%rsp), %rdi
81 movq 24(%rsp), %rsi
82 movq 16(%rsp), %rdx
83 movq 8(%rsp), %rcx
84 movq (%rsp), %rax
85 addq $0x38, %rsp
86
87 retq
88END(mcount)
89
90ENTRY(ftrace_caller)
91
92 /* taken from glibc */
93 subq $0x38, %rsp
94 movq %rax, (%rsp)
95 movq %rcx, 8(%rsp)
96 movq %rdx, 16(%rsp)
97 movq %rsi, 24(%rsp)
98 movq %rdi, 32(%rsp)
99 movq %r8, 40(%rsp)
100 movq %r9, 48(%rsp)
101
102 movq 0x38(%rsp), %rdi
103 movq 8(%rbp), %rsi
104 subq $MCOUNT_INSN_SIZE, %rdi
105
106.globl ftrace_call
107ftrace_call:
108 call ftrace_stub
109
110 movq 48(%rsp), %r9
111 movq 40(%rsp), %r8
112 movq 32(%rsp), %rdi
113 movq 24(%rsp), %rsi
114 movq 16(%rsp), %rdx
115 movq 8(%rsp), %rcx
116 movq (%rsp), %rax
117 addq $0x38, %rsp
118
119.globl ftrace_stub
120ftrace_stub:
121 retq
122END(ftrace_caller)
123
124#else /* ! CONFIG_DYNAMIC_FTRACE */
125ENTRY(mcount)
126 cmpq $ftrace_stub, ftrace_trace_function
127 jnz trace
128.globl ftrace_stub
129ftrace_stub:
130 retq
131
132trace:
133 /* taken from glibc */
134 subq $0x38, %rsp
135 movq %rax, (%rsp)
136 movq %rcx, 8(%rsp)
137 movq %rdx, 16(%rsp)
138 movq %rsi, 24(%rsp)
139 movq %rdi, 32(%rsp)
140 movq %r8, 40(%rsp)
141 movq %r9, 48(%rsp)
142
143 movq 0x38(%rsp), %rdi
144 movq 8(%rbp), %rsi
145 subq $MCOUNT_INSN_SIZE, %rdi
146
147 call *ftrace_trace_function
148
149 movq 48(%rsp), %r9
150 movq 40(%rsp), %r8
151 movq 32(%rsp), %rdi
152 movq 24(%rsp), %rsi
153 movq 16(%rsp), %rdx
154 movq 8(%rsp), %rcx
155 movq (%rsp), %rax
156 addq $0x38, %rsp
157
158 jmp ftrace_stub
159END(mcount)
160#endif /* CONFIG_DYNAMIC_FTRACE */
161#endif /* CONFIG_FTRACE */
162
57#ifndef CONFIG_PREEMPT 163#ifndef CONFIG_PREEMPT
58#define retint_kernel retint_restore_args 164#define retint_kernel retint_restore_args
59#endif 165#endif
60 166
61#ifdef CONFIG_PARAVIRT 167#ifdef CONFIG_PARAVIRT
62ENTRY(native_irq_enable_syscall_ret) 168ENTRY(native_usergs_sysret64)
63 movq %gs:pda_oldrsp,%rsp
64 swapgs 169 swapgs
65 sysretq 170 sysretq
66#endif /* CONFIG_PARAVIRT */ 171#endif /* CONFIG_PARAVIRT */
@@ -104,7 +209,7 @@ ENTRY(native_irq_enable_syscall_ret)
104 .macro FAKE_STACK_FRAME child_rip 209 .macro FAKE_STACK_FRAME child_rip
105 /* push in order ss, rsp, eflags, cs, rip */ 210 /* push in order ss, rsp, eflags, cs, rip */
106 xorl %eax, %eax 211 xorl %eax, %eax
107 pushq %rax /* ss */ 212 pushq $__KERNEL_DS /* ss */
108 CFI_ADJUST_CFA_OFFSET 8 213 CFI_ADJUST_CFA_OFFSET 8
109 /*CFI_REL_OFFSET ss,0*/ 214 /*CFI_REL_OFFSET ss,0*/
110 pushq %rax /* rsp */ 215 pushq %rax /* rsp */
@@ -169,13 +274,13 @@ ENTRY(ret_from_fork)
169 CFI_ADJUST_CFA_OFFSET -4 274 CFI_ADJUST_CFA_OFFSET -4
170 call schedule_tail 275 call schedule_tail
171 GET_THREAD_INFO(%rcx) 276 GET_THREAD_INFO(%rcx)
172 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx) 277 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
173 jnz rff_trace 278 jnz rff_trace
174rff_action: 279rff_action:
175 RESTORE_REST 280 RESTORE_REST
176 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread? 281 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
177 je int_ret_from_sys_call 282 je int_ret_from_sys_call
178 testl $_TIF_IA32,threadinfo_flags(%rcx) 283 testl $_TIF_IA32,TI_flags(%rcx)
179 jnz int_ret_from_sys_call 284 jnz int_ret_from_sys_call
180 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET 285 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
181 jmp ret_from_sys_call 286 jmp ret_from_sys_call
@@ -244,7 +349,8 @@ ENTRY(system_call_after_swapgs)
244 movq %rcx,RIP-ARGOFFSET(%rsp) 349 movq %rcx,RIP-ARGOFFSET(%rsp)
245 CFI_REL_OFFSET rip,RIP-ARGOFFSET 350 CFI_REL_OFFSET rip,RIP-ARGOFFSET
246 GET_THREAD_INFO(%rcx) 351 GET_THREAD_INFO(%rcx)
247 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) 352 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \
353 TI_flags(%rcx)
248 jnz tracesys 354 jnz tracesys
249 cmpq $__NR_syscall_max,%rax 355 cmpq $__NR_syscall_max,%rax
250 ja badsys 356 ja badsys
@@ -263,7 +369,7 @@ sysret_check:
263 GET_THREAD_INFO(%rcx) 369 GET_THREAD_INFO(%rcx)
264 DISABLE_INTERRUPTS(CLBR_NONE) 370 DISABLE_INTERRUPTS(CLBR_NONE)
265 TRACE_IRQS_OFF 371 TRACE_IRQS_OFF
266 movl threadinfo_flags(%rcx),%edx 372 movl TI_flags(%rcx),%edx
267 andl %edi,%edx 373 andl %edi,%edx
268 jnz sysret_careful 374 jnz sysret_careful
269 CFI_REMEMBER_STATE 375 CFI_REMEMBER_STATE
@@ -275,7 +381,8 @@ sysret_check:
275 CFI_REGISTER rip,rcx 381 CFI_REGISTER rip,rcx
276 RESTORE_ARGS 0,-ARG_SKIP,1 382 RESTORE_ARGS 0,-ARG_SKIP,1
277 /*CFI_REGISTER rflags,r11*/ 383 /*CFI_REGISTER rflags,r11*/
278 ENABLE_INTERRUPTS_SYSCALL_RET 384 movq %gs:pda_oldrsp, %rsp
385 USERGS_SYSRET64
279 386
280 CFI_RESTORE_STATE 387 CFI_RESTORE_STATE
281 /* Handle reschedules */ 388 /* Handle reschedules */
@@ -305,7 +412,7 @@ sysret_signal:
305 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1 412 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
306 xorl %esi,%esi # oldset -> arg2 413 xorl %esi,%esi # oldset -> arg2
307 call ptregscall_common 414 call ptregscall_common
3081: movl $_TIF_NEED_RESCHED,%edi 4151: movl $_TIF_WORK_MASK,%edi
309 /* Use IRET because user could have changed frame. This 416 /* Use IRET because user could have changed frame. This
310 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */ 417 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
311 DISABLE_INTERRUPTS(CLBR_NONE) 418 DISABLE_INTERRUPTS(CLBR_NONE)
@@ -347,10 +454,10 @@ int_ret_from_sys_call:
347int_with_check: 454int_with_check:
348 LOCKDEP_SYS_EXIT_IRQ 455 LOCKDEP_SYS_EXIT_IRQ
349 GET_THREAD_INFO(%rcx) 456 GET_THREAD_INFO(%rcx)
350 movl threadinfo_flags(%rcx),%edx 457 movl TI_flags(%rcx),%edx
351 andl %edi,%edx 458 andl %edi,%edx
352 jnz int_careful 459 jnz int_careful
353 andl $~TS_COMPAT,threadinfo_status(%rcx) 460 andl $~TS_COMPAT,TI_status(%rcx)
354 jmp retint_swapgs 461 jmp retint_swapgs
355 462
356 /* Either reschedule or signal or syscall exit tracking needed. */ 463 /* Either reschedule or signal or syscall exit tracking needed. */
@@ -393,7 +500,7 @@ int_signal:
393 movq %rsp,%rdi # &ptregs -> arg1 500 movq %rsp,%rdi # &ptregs -> arg1
394 xorl %esi,%esi # oldset -> arg2 501 xorl %esi,%esi # oldset -> arg2
395 call do_notify_resume 502 call do_notify_resume
3961: movl $_TIF_NEED_RESCHED,%edi 5031: movl $_TIF_WORK_MASK,%edi
397int_restore_rest: 504int_restore_rest:
398 RESTORE_REST 505 RESTORE_REST
399 DISABLE_INTERRUPTS(CLBR_NONE) 506 DISABLE_INTERRUPTS(CLBR_NONE)
@@ -420,7 +527,6 @@ END(\label)
420 PTREGSCALL stub_clone, sys_clone, %r8 527 PTREGSCALL stub_clone, sys_clone, %r8
421 PTREGSCALL stub_fork, sys_fork, %rdi 528 PTREGSCALL stub_fork, sys_fork, %rdi
422 PTREGSCALL stub_vfork, sys_vfork, %rdi 529 PTREGSCALL stub_vfork, sys_vfork, %rdi
423 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
424 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx 530 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
425 PTREGSCALL stub_iopl, sys_iopl, %rsi 531 PTREGSCALL stub_iopl, sys_iopl, %rsi
426 532
@@ -559,7 +665,7 @@ retint_with_reschedule:
559 movl $_TIF_WORK_MASK,%edi 665 movl $_TIF_WORK_MASK,%edi
560retint_check: 666retint_check:
561 LOCKDEP_SYS_EXIT_IRQ 667 LOCKDEP_SYS_EXIT_IRQ
562 movl threadinfo_flags(%rcx),%edx 668 movl TI_flags(%rcx),%edx
563 andl %edi,%edx 669 andl %edi,%edx
564 CFI_REMEMBER_STATE 670 CFI_REMEMBER_STATE
565 jnz retint_careful 671 jnz retint_careful
@@ -647,17 +753,16 @@ retint_signal:
647 RESTORE_REST 753 RESTORE_REST
648 DISABLE_INTERRUPTS(CLBR_NONE) 754 DISABLE_INTERRUPTS(CLBR_NONE)
649 TRACE_IRQS_OFF 755 TRACE_IRQS_OFF
650 movl $_TIF_NEED_RESCHED,%edi
651 GET_THREAD_INFO(%rcx) 756 GET_THREAD_INFO(%rcx)
652 jmp retint_check 757 jmp retint_with_reschedule
653 758
654#ifdef CONFIG_PREEMPT 759#ifdef CONFIG_PREEMPT
655 /* Returning to kernel space. Check if we need preemption */ 760 /* Returning to kernel space. Check if we need preemption */
656 /* rcx: threadinfo. interrupts off. */ 761 /* rcx: threadinfo. interrupts off. */
657ENTRY(retint_kernel) 762ENTRY(retint_kernel)
658 cmpl $0,threadinfo_preempt_count(%rcx) 763 cmpl $0,TI_preempt_count(%rcx)
659 jnz retint_restore_args 764 jnz retint_restore_args
660 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx) 765 bt $TIF_NEED_RESCHED,TI_flags(%rcx)
661 jnc retint_restore_args 766 jnc retint_restore_args
662 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ 767 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
663 jnc retint_restore_args 768 jnc retint_restore_args
@@ -711,6 +816,9 @@ END(invalidate_interrupt\num)
711ENTRY(call_function_interrupt) 816ENTRY(call_function_interrupt)
712 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt 817 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
713END(call_function_interrupt) 818END(call_function_interrupt)
819ENTRY(call_function_single_interrupt)
820 apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
821END(call_function_single_interrupt)
714ENTRY(irq_move_cleanup_interrupt) 822ENTRY(irq_move_cleanup_interrupt)
715 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt 823 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
716END(irq_move_cleanup_interrupt) 824END(irq_move_cleanup_interrupt)
@@ -720,6 +828,10 @@ ENTRY(apic_timer_interrupt)
720 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt 828 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
721END(apic_timer_interrupt) 829END(apic_timer_interrupt)
722 830
831ENTRY(uv_bau_message_intr1)
832 apicinterrupt 220,uv_bau_message_interrupt
833END(uv_bau_message_intr1)
834
723ENTRY(error_interrupt) 835ENTRY(error_interrupt)
724 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt 836 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
725END(error_interrupt) 837END(error_interrupt)
@@ -733,6 +845,7 @@ END(spurious_interrupt)
733 */ 845 */
734 .macro zeroentry sym 846 .macro zeroentry sym
735 INTR_FRAME 847 INTR_FRAME
848 PARAVIRT_ADJUST_EXCEPTION_FRAME
736 pushq $0 /* push error code/oldrax */ 849 pushq $0 /* push error code/oldrax */
737 CFI_ADJUST_CFA_OFFSET 8 850 CFI_ADJUST_CFA_OFFSET 8
738 pushq %rax /* push real oldrax to the rdi slot */ 851 pushq %rax /* push real oldrax to the rdi slot */
@@ -745,6 +858,7 @@ END(spurious_interrupt)
745 858
746 .macro errorentry sym 859 .macro errorentry sym
747 XCPT_FRAME 860 XCPT_FRAME
861 PARAVIRT_ADJUST_EXCEPTION_FRAME
748 pushq %rax 862 pushq %rax
749 CFI_ADJUST_CFA_OFFSET 8 863 CFI_ADJUST_CFA_OFFSET 8
750 CFI_REL_OFFSET rax,0 864 CFI_REL_OFFSET rax,0
@@ -814,7 +928,7 @@ paranoid_restore\trace:
814 jmp irq_return 928 jmp irq_return
815paranoid_userspace\trace: 929paranoid_userspace\trace:
816 GET_THREAD_INFO(%rcx) 930 GET_THREAD_INFO(%rcx)
817 movl threadinfo_flags(%rcx),%ebx 931 movl TI_flags(%rcx),%ebx
818 andl $_TIF_WORK_MASK,%ebx 932 andl $_TIF_WORK_MASK,%ebx
819 jz paranoid_swapgs\trace 933 jz paranoid_swapgs\trace
820 movq %rsp,%rdi /* &pt_regs */ 934 movq %rsp,%rdi /* &pt_regs */
@@ -912,7 +1026,7 @@ error_exit:
912 testl %eax,%eax 1026 testl %eax,%eax
913 jne retint_kernel 1027 jne retint_kernel
914 LOCKDEP_SYS_EXIT_IRQ 1028 LOCKDEP_SYS_EXIT_IRQ
915 movl threadinfo_flags(%rcx),%edx 1029 movl TI_flags(%rcx),%edx
916 movl $_TIF_WORK_MASK,%edi 1030 movl $_TIF_WORK_MASK,%edi
917 andl %edi,%edx 1031 andl %edi,%edx
918 jnz retint_careful 1032 jnz retint_careful
@@ -926,11 +1040,11 @@ error_kernelspace:
926 iret run with kernel gs again, so don't set the user space flag. 1040 iret run with kernel gs again, so don't set the user space flag.
927 B stepping K8s sometimes report an truncated RIP for IRET 1041 B stepping K8s sometimes report an truncated RIP for IRET
928 exceptions returning to compat mode. Check for these here too. */ 1042 exceptions returning to compat mode. Check for these here too. */
929 leaq irq_return(%rip),%rbp 1043 leaq irq_return(%rip),%rcx
930 cmpq %rbp,RIP(%rsp) 1044 cmpq %rcx,RIP(%rsp)
931 je error_swapgs 1045 je error_swapgs
932 movl %ebp,%ebp /* zero extend */ 1046 movl %ecx,%ecx /* zero extend */
933 cmpq %rbp,RIP(%rsp) 1047 cmpq %rcx,RIP(%rsp)
934 je error_swapgs 1048 je error_swapgs
935 cmpq $gs_change,RIP(%rsp) 1049 cmpq $gs_change,RIP(%rsp)
936 je error_swapgs 1050 je error_swapgs
@@ -939,7 +1053,7 @@ KPROBE_END(error_entry)
939 1053
940 /* Reload gs selector with exception handling */ 1054 /* Reload gs selector with exception handling */
941 /* edi: new selector */ 1055 /* edi: new selector */
942ENTRY(load_gs_index) 1056ENTRY(native_load_gs_index)
943 CFI_STARTPROC 1057 CFI_STARTPROC
944 pushf 1058 pushf
945 CFI_ADJUST_CFA_OFFSET 8 1059 CFI_ADJUST_CFA_OFFSET 8
@@ -953,7 +1067,7 @@ gs_change:
953 CFI_ADJUST_CFA_OFFSET -8 1067 CFI_ADJUST_CFA_OFFSET -8
954 ret 1068 ret
955 CFI_ENDPROC 1069 CFI_ENDPROC
956ENDPROC(load_gs_index) 1070ENDPROC(native_load_gs_index)
957 1071
958 .section __ex_table,"a" 1072 .section __ex_table,"a"
959 .align 8 1073 .align 8
@@ -1120,10 +1234,6 @@ ENTRY(coprocessor_segment_overrun)
1120 zeroentry do_coprocessor_segment_overrun 1234 zeroentry do_coprocessor_segment_overrun
1121END(coprocessor_segment_overrun) 1235END(coprocessor_segment_overrun)
1122 1236
1123ENTRY(reserved)
1124 zeroentry do_reserved
1125END(reserved)
1126
1127 /* runs on exception stack */ 1237 /* runs on exception stack */
1128ENTRY(double_fault) 1238ENTRY(double_fault)
1129 XCPT_FRAME 1239 XCPT_FRAME
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
new file mode 100644
index 000000000000..ab115cd15fdf
--- /dev/null
+++ b/arch/x86/kernel/ftrace.c
@@ -0,0 +1,141 @@
1/*
2 * Code for replacing ftrace calls with jumps.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 *
6 * Thanks goes to Ingo Molnar, for suggesting the idea.
7 * Mathieu Desnoyers, for suggesting postponing the modifications.
8 * Arjan van de Ven, for keeping me straight, and explaining to me
9 * the dangers of modifying code on the run.
10 */
11
12#include <linux/spinlock.h>
13#include <linux/hardirq.h>
14#include <linux/ftrace.h>
15#include <linux/percpu.h>
16#include <linux/init.h>
17#include <linux/list.h>
18
19#include <asm/alternative.h>
20#include <asm/ftrace.h>
21
22
23/* Long is fine, even if it is only 4 bytes ;-) */
24static long *ftrace_nop;
25
26union ftrace_code_union {
27 char code[MCOUNT_INSN_SIZE];
28 struct {
29 char e8;
30 int offset;
31 } __attribute__((packed));
32};
33
34
35static int notrace ftrace_calc_offset(long ip, long addr)
36{
37 return (int)(addr - ip);
38}
39
40notrace unsigned char *ftrace_nop_replace(void)
41{
42 return (char *)ftrace_nop;
43}
44
45notrace unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
46{
47 static union ftrace_code_union calc;
48
49 calc.e8 = 0xe8;
50 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
51
52 /*
53 * No locking needed, this must be called via kstop_machine
54 * which in essence is like running on a uniprocessor machine.
55 */
56 return calc.code;
57}
58
59notrace int
60ftrace_modify_code(unsigned long ip, unsigned char *old_code,
61 unsigned char *new_code)
62{
63 unsigned replaced;
64 unsigned old = *(unsigned *)old_code; /* 4 bytes */
65 unsigned new = *(unsigned *)new_code; /* 4 bytes */
66 unsigned char newch = new_code[4];
67 int faulted = 0;
68
69 /*
70 * Note: Due to modules and __init, code can
71 * disappear and change, we need to protect against faulting
72 * as well as code changing.
73 *
74 * No real locking needed, this code is run through
75 * kstop_machine.
76 */
77 asm volatile (
78 "1: lock\n"
79 " cmpxchg %3, (%2)\n"
80 " jnz 2f\n"
81 " movb %b4, 4(%2)\n"
82 "2:\n"
83 ".section .fixup, \"ax\"\n"
84 "3: movl $1, %0\n"
85 " jmp 2b\n"
86 ".previous\n"
87 _ASM_EXTABLE(1b, 3b)
88 : "=r"(faulted), "=a"(replaced)
89 : "r"(ip), "r"(new), "c"(newch),
90 "0"(faulted), "a"(old)
91 : "memory");
92 sync_core();
93
94 if (replaced != old && replaced != new)
95 faulted = 2;
96
97 return faulted;
98}
99
100notrace int ftrace_update_ftrace_func(ftrace_func_t func)
101{
102 unsigned long ip = (unsigned long)(&ftrace_call);
103 unsigned char old[MCOUNT_INSN_SIZE], *new;
104 int ret;
105
106 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
107 new = ftrace_call_replace(ip, (unsigned long)func);
108 ret = ftrace_modify_code(ip, old, new);
109
110 return ret;
111}
112
113notrace int ftrace_mcount_set(unsigned long *data)
114{
115 unsigned long ip = (long)(&mcount_call);
116 unsigned long *addr = data;
117 unsigned char old[MCOUNT_INSN_SIZE], *new;
118
119 /*
120 * Replace the mcount stub with a pointer to the
121 * ip recorder function.
122 */
123 memcpy(old, &mcount_call, MCOUNT_INSN_SIZE);
124 new = ftrace_call_replace(ip, *addr);
125 *addr = ftrace_modify_code(ip, old, new);
126
127 return 0;
128}
129
130int __init ftrace_dyn_arch_init(void *data)
131{
132 const unsigned char *const *noptable = find_nop_table();
133
134 /* This is running in kstop_machine */
135
136 ftrace_mcount_set(data);
137
138 ftrace_nop = (unsigned long *)noptable[MCOUNT_INSN_SIZE];
139
140 return 0;
141}
diff --git a/arch/x86/kernel/genapic_64.c b/arch/x86/kernel/genapic_64.c
index cbaaf69bedb2..1fa8be5bd217 100644
--- a/arch/x86/kernel/genapic_64.c
+++ b/arch/x86/kernel/genapic_64.c
@@ -51,7 +51,7 @@ void __init setup_apic_routing(void)
51 else 51 else
52#endif 52#endif
53 53
54 if (num_possible_cpus() <= 8) 54 if (max_physical_apicid < 8)
55 genapic = &apic_flat; 55 genapic = &apic_flat;
56 else 56 else
57 genapic = &apic_physflat; 57 genapic = &apic_physflat;
diff --git a/arch/x86/kernel/genx2apic_uv_x.c b/arch/x86/kernel/genx2apic_uv_x.c
index ebf13908a743..711f11c30b06 100644
--- a/arch/x86/kernel/genx2apic_uv_x.c
+++ b/arch/x86/kernel/genx2apic_uv_x.c
@@ -5,9 +5,10 @@
5 * 5 *
6 * SGI UV APIC functions (note: not an Intel compatible APIC) 6 * SGI UV APIC functions (note: not an Intel compatible APIC)
7 * 7 *
8 * Copyright (C) 2007 Silicon Graphics, Inc. All rights reserved. 8 * Copyright (C) 2007-2008 Silicon Graphics, Inc. All rights reserved.
9 */ 9 */
10 10
11#include <linux/kernel.h>
11#include <linux/threads.h> 12#include <linux/threads.h>
12#include <linux/cpumask.h> 13#include <linux/cpumask.h>
13#include <linux/string.h> 14#include <linux/string.h>
@@ -20,6 +21,7 @@
20#include <asm/smp.h> 21#include <asm/smp.h>
21#include <asm/ipi.h> 22#include <asm/ipi.h>
22#include <asm/genapic.h> 23#include <asm/genapic.h>
24#include <asm/pgtable.h>
23#include <asm/uv/uv_mmrs.h> 25#include <asm/uv/uv_mmrs.h>
24#include <asm/uv/uv_hub.h> 26#include <asm/uv/uv_hub.h>
25 27
@@ -55,37 +57,37 @@ static cpumask_t uv_vector_allocation_domain(int cpu)
55int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip) 57int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip)
56{ 58{
57 unsigned long val; 59 unsigned long val;
58 int nasid; 60 int pnode;
59 61
60 nasid = uv_apicid_to_nasid(phys_apicid); 62 pnode = uv_apicid_to_pnode(phys_apicid);
61 val = (1UL << UVH_IPI_INT_SEND_SHFT) | 63 val = (1UL << UVH_IPI_INT_SEND_SHFT) |
62 (phys_apicid << UVH_IPI_INT_APIC_ID_SHFT) | 64 (phys_apicid << UVH_IPI_INT_APIC_ID_SHFT) |
63 (((long)start_rip << UVH_IPI_INT_VECTOR_SHFT) >> 12) | 65 (((long)start_rip << UVH_IPI_INT_VECTOR_SHFT) >> 12) |
64 APIC_DM_INIT; 66 APIC_DM_INIT;
65 uv_write_global_mmr64(nasid, UVH_IPI_INT, val); 67 uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
66 mdelay(10); 68 mdelay(10);
67 69
68 val = (1UL << UVH_IPI_INT_SEND_SHFT) | 70 val = (1UL << UVH_IPI_INT_SEND_SHFT) |
69 (phys_apicid << UVH_IPI_INT_APIC_ID_SHFT) | 71 (phys_apicid << UVH_IPI_INT_APIC_ID_SHFT) |
70 (((long)start_rip << UVH_IPI_INT_VECTOR_SHFT) >> 12) | 72 (((long)start_rip << UVH_IPI_INT_VECTOR_SHFT) >> 12) |
71 APIC_DM_STARTUP; 73 APIC_DM_STARTUP;
72 uv_write_global_mmr64(nasid, UVH_IPI_INT, val); 74 uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
73 return 0; 75 return 0;
74} 76}
75 77
76static void uv_send_IPI_one(int cpu, int vector) 78static void uv_send_IPI_one(int cpu, int vector)
77{ 79{
78 unsigned long val, apicid, lapicid; 80 unsigned long val, apicid, lapicid;
79 int nasid; 81 int pnode;
80 82
81 apicid = per_cpu(x86_cpu_to_apicid, cpu); /* ZZZ - cache node-local ? */ 83 apicid = per_cpu(x86_cpu_to_apicid, cpu); /* ZZZ - cache node-local ? */
82 lapicid = apicid & 0x3f; /* ZZZ macro needed */ 84 lapicid = apicid & 0x3f; /* ZZZ macro needed */
83 nasid = uv_apicid_to_nasid(apicid); 85 pnode = uv_apicid_to_pnode(apicid);
84 val = 86 val =
85 (1UL << UVH_IPI_INT_SEND_SHFT) | (lapicid << 87 (1UL << UVH_IPI_INT_SEND_SHFT) | (lapicid <<
86 UVH_IPI_INT_APIC_ID_SHFT) | 88 UVH_IPI_INT_APIC_ID_SHFT) |
87 (vector << UVH_IPI_INT_VECTOR_SHFT); 89 (vector << UVH_IPI_INT_VECTOR_SHFT);
88 uv_write_global_mmr64(nasid, UVH_IPI_INT, val); 90 uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
89} 91}
90 92
91static void uv_send_IPI_mask(cpumask_t mask, int vector) 93static void uv_send_IPI_mask(cpumask_t mask, int vector)
@@ -159,39 +161,146 @@ struct genapic apic_x2apic_uv_x = {
159 .phys_pkg_id = phys_pkg_id, /* Fixme ZZZ */ 161 .phys_pkg_id = phys_pkg_id, /* Fixme ZZZ */
160}; 162};
161 163
162static __cpuinit void set_x2apic_extra_bits(int nasid) 164static __cpuinit void set_x2apic_extra_bits(int pnode)
163{ 165{
164 __get_cpu_var(x2apic_extra_bits) = ((nasid >> 1) << 6); 166 __get_cpu_var(x2apic_extra_bits) = (pnode << 6);
165} 167}
166 168
167/* 169/*
168 * Called on boot cpu. 170 * Called on boot cpu.
169 */ 171 */
172static __init int boot_pnode_to_blade(int pnode)
173{
174 int blade;
175
176 for (blade = 0; blade < uv_num_possible_blades(); blade++)
177 if (pnode == uv_blade_info[blade].pnode)
178 return blade;
179 BUG();
180}
181
182struct redir_addr {
183 unsigned long redirect;
184 unsigned long alias;
185};
186
187#define DEST_SHIFT UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT
188
189static __initdata struct redir_addr redir_addrs[] = {
190 {UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR, UVH_SI_ALIAS0_OVERLAY_CONFIG},
191 {UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR, UVH_SI_ALIAS1_OVERLAY_CONFIG},
192 {UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR, UVH_SI_ALIAS2_OVERLAY_CONFIG},
193};
194
195static __init void get_lowmem_redirect(unsigned long *base, unsigned long *size)
196{
197 union uvh_si_alias0_overlay_config_u alias;
198 union uvh_rh_gam_alias210_redirect_config_2_mmr_u redirect;
199 int i;
200
201 for (i = 0; i < ARRAY_SIZE(redir_addrs); i++) {
202 alias.v = uv_read_local_mmr(redir_addrs[i].alias);
203 if (alias.s.base == 0) {
204 *size = (1UL << alias.s.m_alias);
205 redirect.v = uv_read_local_mmr(redir_addrs[i].redirect);
206 *base = (unsigned long)redirect.s.dest_base << DEST_SHIFT;
207 return;
208 }
209 }
210 BUG();
211}
212
213static __init void map_low_mmrs(void)
214{
215 init_extra_mapping_uc(UV_GLOBAL_MMR32_BASE, UV_GLOBAL_MMR32_SIZE);
216 init_extra_mapping_uc(UV_LOCAL_MMR_BASE, UV_LOCAL_MMR_SIZE);
217}
218
219enum map_type {map_wb, map_uc};
220
221static void map_high(char *id, unsigned long base, int shift, enum map_type map_type)
222{
223 unsigned long bytes, paddr;
224
225 paddr = base << shift;
226 bytes = (1UL << shift);
227 printk(KERN_INFO "UV: Map %s_HI 0x%lx - 0x%lx\n", id, paddr,
228 paddr + bytes);
229 if (map_type == map_uc)
230 init_extra_mapping_uc(paddr, bytes);
231 else
232 init_extra_mapping_wb(paddr, bytes);
233
234}
235static __init void map_gru_high(int max_pnode)
236{
237 union uvh_rh_gam_gru_overlay_config_mmr_u gru;
238 int shift = UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT;
239
240 gru.v = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR);
241 if (gru.s.enable)
242 map_high("GRU", gru.s.base, shift, map_wb);
243}
244
245static __init void map_config_high(int max_pnode)
246{
247 union uvh_rh_gam_cfg_overlay_config_mmr_u cfg;
248 int shift = UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR_BASE_SHFT;
249
250 cfg.v = uv_read_local_mmr(UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR);
251 if (cfg.s.enable)
252 map_high("CONFIG", cfg.s.base, shift, map_uc);
253}
254
255static __init void map_mmr_high(int max_pnode)
256{
257 union uvh_rh_gam_mmr_overlay_config_mmr_u mmr;
258 int shift = UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT;
259
260 mmr.v = uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR);
261 if (mmr.s.enable)
262 map_high("MMR", mmr.s.base, shift, map_uc);
263}
264
265static __init void map_mmioh_high(int max_pnode)
266{
267 union uvh_rh_gam_mmioh_overlay_config_mmr_u mmioh;
268 int shift = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT;
269
270 mmioh.v = uv_read_local_mmr(UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR);
271 if (mmioh.s.enable)
272 map_high("MMIOH", mmioh.s.base, shift, map_uc);
273}
274
170static __init void uv_system_init(void) 275static __init void uv_system_init(void)
171{ 276{
172 union uvh_si_addr_map_config_u m_n_config; 277 union uvh_si_addr_map_config_u m_n_config;
173 int bytes, nid, cpu, lcpu, nasid, last_nasid, blade; 278 union uvh_node_id_u node_id;
174 unsigned long mmr_base; 279 unsigned long gnode_upper, lowmem_redir_base, lowmem_redir_size;
280 int bytes, nid, cpu, lcpu, pnode, blade, i, j, m_val, n_val;
281 int max_pnode = 0;
282 unsigned long mmr_base, present;
283
284 map_low_mmrs();
175 285
176 m_n_config.v = uv_read_local_mmr(UVH_SI_ADDR_MAP_CONFIG); 286 m_n_config.v = uv_read_local_mmr(UVH_SI_ADDR_MAP_CONFIG);
287 m_val = m_n_config.s.m_skt;
288 n_val = m_n_config.s.n_skt;
177 mmr_base = 289 mmr_base =
178 uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR) & 290 uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR) &
179 ~UV_MMR_ENABLE; 291 ~UV_MMR_ENABLE;
180 printk(KERN_DEBUG "UV: global MMR base 0x%lx\n", mmr_base); 292 printk(KERN_DEBUG "UV: global MMR base 0x%lx\n", mmr_base);
181 293
182 last_nasid = -1; 294 for(i = 0; i < UVH_NODE_PRESENT_TABLE_DEPTH; i++)
183 for_each_possible_cpu(cpu) { 295 uv_possible_blades +=
184 nid = cpu_to_node(cpu); 296 hweight64(uv_read_local_mmr( UVH_NODE_PRESENT_TABLE + i * 8));
185 nasid = uv_apicid_to_nasid(per_cpu(x86_cpu_to_apicid, cpu));
186 if (nasid != last_nasid)
187 uv_possible_blades++;
188 last_nasid = nasid;
189 }
190 printk(KERN_DEBUG "UV: Found %d blades\n", uv_num_possible_blades()); 297 printk(KERN_DEBUG "UV: Found %d blades\n", uv_num_possible_blades());
191 298
192 bytes = sizeof(struct uv_blade_info) * uv_num_possible_blades(); 299 bytes = sizeof(struct uv_blade_info) * uv_num_possible_blades();
193 uv_blade_info = alloc_bootmem_pages(bytes); 300 uv_blade_info = alloc_bootmem_pages(bytes);
194 301
302 get_lowmem_redirect(&lowmem_redir_base, &lowmem_redir_size);
303
195 bytes = sizeof(uv_node_to_blade[0]) * num_possible_nodes(); 304 bytes = sizeof(uv_node_to_blade[0]) * num_possible_nodes();
196 uv_node_to_blade = alloc_bootmem_pages(bytes); 305 uv_node_to_blade = alloc_bootmem_pages(bytes);
197 memset(uv_node_to_blade, 255, bytes); 306 memset(uv_node_to_blade, 255, bytes);
@@ -200,43 +309,62 @@ static __init void uv_system_init(void)
200 uv_cpu_to_blade = alloc_bootmem_pages(bytes); 309 uv_cpu_to_blade = alloc_bootmem_pages(bytes);
201 memset(uv_cpu_to_blade, 255, bytes); 310 memset(uv_cpu_to_blade, 255, bytes);
202 311
203 last_nasid = -1; 312 blade = 0;
204 blade = -1; 313 for (i = 0; i < UVH_NODE_PRESENT_TABLE_DEPTH; i++) {
205 lcpu = -1; 314 present = uv_read_local_mmr(UVH_NODE_PRESENT_TABLE + i * 8);
206 for_each_possible_cpu(cpu) { 315 for (j = 0; j < 64; j++) {
207 nid = cpu_to_node(cpu); 316 if (!test_bit(j, &present))
208 nasid = uv_apicid_to_nasid(per_cpu(x86_cpu_to_apicid, cpu)); 317 continue;
209 if (nasid != last_nasid) { 318 uv_blade_info[blade].pnode = (i * 64 + j);
210 blade++; 319 uv_blade_info[blade].nr_possible_cpus = 0;
211 lcpu = -1;
212 uv_blade_info[blade].nr_posible_cpus = 0;
213 uv_blade_info[blade].nr_online_cpus = 0; 320 uv_blade_info[blade].nr_online_cpus = 0;
321 blade++;
214 } 322 }
215 last_nasid = nasid; 323 }
216 lcpu++;
217 324
218 uv_cpu_hub_info(cpu)->m_val = m_n_config.s.m_skt; 325 node_id.v = uv_read_local_mmr(UVH_NODE_ID);
219 uv_cpu_hub_info(cpu)->n_val = m_n_config.s.n_skt; 326 gnode_upper = (((unsigned long)node_id.s.node_id) &
327 ~((1 << n_val) - 1)) << m_val;
328
329 for_each_present_cpu(cpu) {
330 nid = cpu_to_node(cpu);
331 pnode = uv_apicid_to_pnode(per_cpu(x86_cpu_to_apicid, cpu));
332 blade = boot_pnode_to_blade(pnode);
333 lcpu = uv_blade_info[blade].nr_possible_cpus;
334 uv_blade_info[blade].nr_possible_cpus++;
335
336 uv_cpu_hub_info(cpu)->lowmem_remap_base = lowmem_redir_base;
337 uv_cpu_hub_info(cpu)->lowmem_remap_top =
338 lowmem_redir_base + lowmem_redir_size;
339 uv_cpu_hub_info(cpu)->m_val = m_val;
340 uv_cpu_hub_info(cpu)->n_val = m_val;
220 uv_cpu_hub_info(cpu)->numa_blade_id = blade; 341 uv_cpu_hub_info(cpu)->numa_blade_id = blade;
221 uv_cpu_hub_info(cpu)->blade_processor_id = lcpu; 342 uv_cpu_hub_info(cpu)->blade_processor_id = lcpu;
222 uv_cpu_hub_info(cpu)->local_nasid = nasid; 343 uv_cpu_hub_info(cpu)->pnode = pnode;
223 uv_cpu_hub_info(cpu)->gnode_upper = 344 uv_cpu_hub_info(cpu)->pnode_mask = (1 << n_val) - 1;
224 nasid & ~((1 << uv_hub_info->n_val) - 1); 345 uv_cpu_hub_info(cpu)->gpa_mask = (1 << (m_val + n_val)) - 1;
346 uv_cpu_hub_info(cpu)->gnode_upper = gnode_upper;
225 uv_cpu_hub_info(cpu)->global_mmr_base = mmr_base; 347 uv_cpu_hub_info(cpu)->global_mmr_base = mmr_base;
226 uv_cpu_hub_info(cpu)->coherency_domain_number = 0;/* ZZZ */ 348 uv_cpu_hub_info(cpu)->coherency_domain_number = 0;/* ZZZ */
227 uv_blade_info[blade].nasid = nasid;
228 uv_blade_info[blade].nr_posible_cpus++;
229 uv_node_to_blade[nid] = blade; 349 uv_node_to_blade[nid] = blade;
230 uv_cpu_to_blade[cpu] = blade; 350 uv_cpu_to_blade[cpu] = blade;
351 max_pnode = max(pnode, max_pnode);
231 352
232 printk(KERN_DEBUG "UV cpu %d, apicid 0x%x, nasid %d, nid %d\n", 353 printk(KERN_DEBUG "UV: cpu %d, apicid 0x%x, pnode %d, nid %d, "
233 cpu, per_cpu(x86_cpu_to_apicid, cpu), nasid, nid); 354 "lcpu %d, blade %d\n",
234 printk(KERN_DEBUG "UV lcpu %d, blade %d\n", lcpu, blade); 355 cpu, per_cpu(x86_cpu_to_apicid, cpu), pnode, nid,
356 lcpu, blade);
235 } 357 }
358
359 map_gru_high(max_pnode);
360 map_mmr_high(max_pnode);
361 map_config_high(max_pnode);
362 map_mmioh_high(max_pnode);
236} 363}
237 364
238/* 365/*
239 * Called on each cpu to initialize the per_cpu UV data area. 366 * Called on each cpu to initialize the per_cpu UV data area.
367 * ZZZ hotplug not supported yet
240 */ 368 */
241void __cpuinit uv_cpu_init(void) 369void __cpuinit uv_cpu_init(void)
242{ 370{
@@ -246,5 +374,5 @@ void __cpuinit uv_cpu_init(void)
246 uv_blade_info[uv_numa_blade_id()].nr_online_cpus++; 374 uv_blade_info[uv_numa_blade_id()].nr_online_cpus++;
247 375
248 if (get_uv_system_type() == UV_NON_UNIQUE_APIC) 376 if (get_uv_system_type() == UV_NON_UNIQUE_APIC)
249 set_x2apic_extra_bits(uv_hub_info->local_nasid); 377 set_x2apic_extra_bits(uv_hub_info->pnode);
250} 378}
diff --git a/arch/x86/kernel/head.c b/arch/x86/kernel/head.c
new file mode 100644
index 000000000000..3e66bd364a9d
--- /dev/null
+++ b/arch/x86/kernel/head.c
@@ -0,0 +1,55 @@
1#include <linux/kernel.h>
2#include <linux/init.h>
3
4#include <asm/setup.h>
5#include <asm/bios_ebda.h>
6
7#define BIOS_LOWMEM_KILOBYTES 0x413
8
9/*
10 * The BIOS places the EBDA/XBDA at the top of conventional
11 * memory, and usually decreases the reported amount of
12 * conventional memory (int 0x12) too. This also contains a
13 * workaround for Dell systems that neglect to reserve EBDA.
14 * The same workaround also avoids a problem with the AMD768MPX
15 * chipset: reserve a page before VGA to prevent PCI prefetch
16 * into it (errata #56). Usually the page is reserved anyways,
17 * unless you have no PS/2 mouse plugged in.
18 */
19void __init reserve_ebda_region(void)
20{
21 unsigned int lowmem, ebda_addr;
22
23 /* To determine the position of the EBDA and the */
24 /* end of conventional memory, we need to look at */
25 /* the BIOS data area. In a paravirtual environment */
26 /* that area is absent. We'll just have to assume */
27 /* that the paravirt case can handle memory setup */
28 /* correctly, without our help. */
29 if (paravirt_enabled())
30 return;
31
32 /* end of low (conventional) memory */
33 lowmem = *(unsigned short *)__va(BIOS_LOWMEM_KILOBYTES);
34 lowmem <<= 10;
35
36 /* start of EBDA area */
37 ebda_addr = get_bios_ebda();
38
39 /* Fixup: bios puts an EBDA in the top 64K segment */
40 /* of conventional memory, but does not adjust lowmem. */
41 if ((lowmem - ebda_addr) <= 0x10000)
42 lowmem = ebda_addr;
43
44 /* Fixup: bios does not report an EBDA at all. */
45 /* Some old Dells seem to need 4k anyhow (bugzilla 2990) */
46 if ((ebda_addr == 0) && (lowmem >= 0x9f000))
47 lowmem = 0x9f000;
48
49 /* Paranoia: should never happen, but... */
50 if ((lowmem == 0) || (lowmem >= 0x100000))
51 lowmem = 0x9f000;
52
53 /* reserve all memory between lowmem and the 1MB mark */
54 reserve_early_overlap_ok(lowmem, 0x100000, "BIOS reserved");
55}
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index 3db059058927..fa1d25dd83e3 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -8,7 +8,34 @@
8#include <linux/init.h> 8#include <linux/init.h>
9#include <linux/start_kernel.h> 9#include <linux/start_kernel.h>
10 10
11#include <asm/setup.h>
12#include <asm/sections.h>
13#include <asm/e820.h>
14#include <asm/bios_ebda.h>
15
11void __init i386_start_kernel(void) 16void __init i386_start_kernel(void)
12{ 17{
18 reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
19
20#ifdef CONFIG_BLK_DEV_INITRD
21 /* Reserve INITRD */
22 if (boot_params.hdr.type_of_loader && boot_params.hdr.ramdisk_image) {
23 u64 ramdisk_image = boot_params.hdr.ramdisk_image;
24 u64 ramdisk_size = boot_params.hdr.ramdisk_size;
25 u64 ramdisk_end = ramdisk_image + ramdisk_size;
26 reserve_early(ramdisk_image, ramdisk_end, "RAMDISK");
27 }
28#endif
29 reserve_early(init_pg_tables_start, init_pg_tables_end,
30 "INIT_PG_TABLE");
31
32 reserve_ebda_region();
33
34 /*
35 * At this point everything still needed from the boot loader
36 * or BIOS or kernel text should be early reserved or marked not
37 * RAM in e820. All other memory is free game.
38 */
39
13 start_kernel(); 40 start_kernel();
14} 41}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index e25c57b8aa84..c97819829146 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -25,6 +25,20 @@
25#include <asm/e820.h> 25#include <asm/e820.h>
26#include <asm/bios_ebda.h> 26#include <asm/bios_ebda.h>
27 27
28/* boot cpu pda */
29static struct x8664_pda _boot_cpu_pda __read_mostly;
30
31#ifdef CONFIG_SMP
32/*
33 * We install an empty cpu_pda pointer table to indicate to early users
34 * (numa_set_node) that the cpu_pda pointer table for cpus other than
35 * the boot cpu is not yet setup.
36 */
37static struct x8664_pda *__cpu_pda[NR_CPUS] __initdata;
38#else
39static struct x8664_pda *__cpu_pda[NR_CPUS] __read_mostly;
40#endif
41
28static void __init zap_identity_mappings(void) 42static void __init zap_identity_mappings(void)
29{ 43{
30 pgd_t *pgd = pgd_offset_k(0UL); 44 pgd_t *pgd = pgd_offset_k(0UL);
@@ -51,74 +65,6 @@ static void __init copy_bootdata(char *real_mode_data)
51 } 65 }
52} 66}
53 67
54#define BIOS_LOWMEM_KILOBYTES 0x413
55
56/*
57 * The BIOS places the EBDA/XBDA at the top of conventional
58 * memory, and usually decreases the reported amount of
59 * conventional memory (int 0x12) too. This also contains a
60 * workaround for Dell systems that neglect to reserve EBDA.
61 * The same workaround also avoids a problem with the AMD768MPX
62 * chipset: reserve a page before VGA to prevent PCI prefetch
63 * into it (errata #56). Usually the page is reserved anyways,
64 * unless you have no PS/2 mouse plugged in.
65 */
66static void __init reserve_ebda_region(void)
67{
68 unsigned int lowmem, ebda_addr;
69
70 /* To determine the position of the EBDA and the */
71 /* end of conventional memory, we need to look at */
72 /* the BIOS data area. In a paravirtual environment */
73 /* that area is absent. We'll just have to assume */
74 /* that the paravirt case can handle memory setup */
75 /* correctly, without our help. */
76 if (paravirt_enabled())
77 return;
78
79 /* end of low (conventional) memory */
80 lowmem = *(unsigned short *)__va(BIOS_LOWMEM_KILOBYTES);
81 lowmem <<= 10;
82
83 /* start of EBDA area */
84 ebda_addr = get_bios_ebda();
85
86 /* Fixup: bios puts an EBDA in the top 64K segment */
87 /* of conventional memory, but does not adjust lowmem. */
88 if ((lowmem - ebda_addr) <= 0x10000)
89 lowmem = ebda_addr;
90
91 /* Fixup: bios does not report an EBDA at all. */
92 /* Some old Dells seem to need 4k anyhow (bugzilla 2990) */
93 if ((ebda_addr == 0) && (lowmem >= 0x9f000))
94 lowmem = 0x9f000;
95
96 /* Paranoia: should never happen, but... */
97 if ((lowmem == 0) || (lowmem >= 0x100000))
98 lowmem = 0x9f000;
99
100 /* reserve all memory between lowmem and the 1MB mark */
101 reserve_early(lowmem, 0x100000, "BIOS reserved");
102}
103
104static void __init reserve_setup_data(void)
105{
106 struct setup_data *data;
107 unsigned long pa_data;
108 char buf[32];
109
110 if (boot_params.hdr.version < 0x0209)
111 return;
112 pa_data = boot_params.hdr.setup_data;
113 while (pa_data) {
114 data = early_ioremap(pa_data, sizeof(*data));
115 sprintf(buf, "setup data %x", data->type);
116 reserve_early(pa_data, pa_data+sizeof(*data)+data->len, buf);
117 pa_data = data->next;
118 early_iounmap(data, sizeof(*data));
119 }
120}
121
122void __init x86_64_start_kernel(char * real_mode_data) 68void __init x86_64_start_kernel(char * real_mode_data)
123{ 69{
124 int i; 70 int i;
@@ -156,10 +102,17 @@ void __init x86_64_start_kernel(char * real_mode_data)
156 102
157 early_printk("Kernel alive\n"); 103 early_printk("Kernel alive\n");
158 104
159 for (i = 0; i < NR_CPUS; i++) 105 _cpu_pda = __cpu_pda;
160 cpu_pda(i) = &boot_cpu_pda[i]; 106 cpu_pda(0) = &_boot_cpu_pda;
161
162 pda_init(0); 107 pda_init(0);
108
109 early_printk("Kernel really alive\n");
110
111 x86_64_start_reservations(real_mode_data);
112}
113
114void __init x86_64_start_reservations(char *real_mode_data)
115{
163 copy_bootdata(__va(real_mode_data)); 116 copy_bootdata(__va(real_mode_data));
164 117
165 reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS"); 118 reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
@@ -175,7 +128,6 @@ void __init x86_64_start_kernel(char * real_mode_data)
175#endif 128#endif
176 129
177 reserve_ebda_region(); 130 reserve_ebda_region();
178 reserve_setup_data();
179 131
180 /* 132 /*
181 * At this point everything still needed from the boot loader 133 * At this point everything still needed from the boot loader
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index f7357cc0162c..f67e93441caf 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -194,6 +194,7 @@ default_entry:
194 xorl %ebx,%ebx /* %ebx is kept at zero */ 194 xorl %ebx,%ebx /* %ebx is kept at zero */
195 195
196 movl $pa(pg0), %edi 196 movl $pa(pg0), %edi
197 movl %edi, pa(init_pg_tables_start)
197 movl $pa(swapper_pg_pmd), %edx 198 movl $pa(swapper_pg_pmd), %edx
198 movl $PTE_ATTR, %eax 199 movl $PTE_ATTR, %eax
19910: 20010:
@@ -219,6 +220,8 @@ default_entry:
219 jb 10b 220 jb 10b
2201: 2211:
221 movl %edi,pa(init_pg_tables_end) 222 movl %edi,pa(init_pg_tables_end)
223 shrl $12, %eax
224 movl %eax, pa(max_pfn_mapped)
222 225
223 /* Do early initialization of the fixmap area */ 226 /* Do early initialization of the fixmap area */
224 movl $pa(swapper_pg_fixmap)+PDE_ATTR,%eax 227 movl $pa(swapper_pg_fixmap)+PDE_ATTR,%eax
@@ -228,6 +231,7 @@ default_entry:
228page_pde_offset = (__PAGE_OFFSET >> 20); 231page_pde_offset = (__PAGE_OFFSET >> 20);
229 232
230 movl $pa(pg0), %edi 233 movl $pa(pg0), %edi
234 movl %edi, pa(init_pg_tables_start)
231 movl $pa(swapper_pg_dir), %edx 235 movl $pa(swapper_pg_dir), %edx
232 movl $PTE_ATTR, %eax 236 movl $PTE_ATTR, %eax
23310: 23710:
@@ -249,6 +253,8 @@ page_pde_offset = (__PAGE_OFFSET >> 20);
249 cmpl %ebp,%eax 253 cmpl %ebp,%eax
250 jb 10b 254 jb 10b
251 movl %edi,pa(init_pg_tables_end) 255 movl %edi,pa(init_pg_tables_end)
256 shrl $12, %eax
257 movl %eax, pa(max_pfn_mapped)
252 258
253 /* Do early initialization of the fixmap area */ 259 /* Do early initialization of the fixmap area */
254 movl $pa(swapper_pg_fixmap)+PDE_ATTR,%eax 260 movl $pa(swapper_pg_fixmap)+PDE_ATTR,%eax
@@ -446,10 +452,13 @@ is386: movl $2,%ecx # set MP
446 je 1f 452 je 1f
447 movl $(__KERNEL_PERCPU), %eax 453 movl $(__KERNEL_PERCPU), %eax
448 movl %eax,%fs # set this cpu's percpu 454 movl %eax,%fs # set this cpu's percpu
449 jmp initialize_secondary # all other CPUs call initialize_secondary 455 movl (stack_start), %esp
4501: 4561:
451#endif /* CONFIG_SMP */ 457#endif /* CONFIG_SMP */
452 jmp i386_start_kernel 458 jmp *(initial_code)
459.align 4
460ENTRY(initial_code)
461 .long i386_start_kernel
453 462
454/* 463/*
455 * We depend on ET to be correct. This checks for 287/387. 464 * We depend on ET to be correct. This checks for 287/387.
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 10a1955bb1d1..b07ac7b217cb 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -18,6 +18,7 @@
18#include <asm/page.h> 18#include <asm/page.h>
19#include <asm/msr.h> 19#include <asm/msr.h>
20#include <asm/cache.h> 20#include <asm/cache.h>
21#include <asm/processor-flags.h>
21 22
22#ifdef CONFIG_PARAVIRT 23#ifdef CONFIG_PARAVIRT
23#include <asm/asm-offsets.h> 24#include <asm/asm-offsets.h>
@@ -31,6 +32,13 @@
31 * 32 *
32 */ 33 */
33 34
35#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
36
37L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET)
38L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET)
39L4_START_KERNEL = pgd_index(__START_KERNEL_map)
40L3_START_KERNEL = pud_index(__START_KERNEL_map)
41
34 .text 42 .text
35 .section .text.head 43 .section .text.head
36 .code64 44 .code64
@@ -76,8 +84,8 @@ startup_64:
76 /* Fixup the physical addresses in the page table 84 /* Fixup the physical addresses in the page table
77 */ 85 */
78 addq %rbp, init_level4_pgt + 0(%rip) 86 addq %rbp, init_level4_pgt + 0(%rip)
79 addq %rbp, init_level4_pgt + (258*8)(%rip) 87 addq %rbp, init_level4_pgt + (L4_PAGE_OFFSET*8)(%rip)
80 addq %rbp, init_level4_pgt + (511*8)(%rip) 88 addq %rbp, init_level4_pgt + (L4_START_KERNEL*8)(%rip)
81 89
82 addq %rbp, level3_ident_pgt + 0(%rip) 90 addq %rbp, level3_ident_pgt + 0(%rip)
83 91
@@ -128,7 +136,7 @@ ident_complete:
128 /* Fixup phys_base */ 136 /* Fixup phys_base */
129 addq %rbp, phys_base(%rip) 137 addq %rbp, phys_base(%rip)
130 138
131#ifdef CONFIG_SMP 139#ifdef CONFIG_X86_TRAMPOLINE
132 addq %rbp, trampoline_level4_pgt + 0(%rip) 140 addq %rbp, trampoline_level4_pgt + 0(%rip)
133 addq %rbp, trampoline_level4_pgt + (511*8)(%rip) 141 addq %rbp, trampoline_level4_pgt + (511*8)(%rip)
134#endif 142#endif
@@ -154,9 +162,7 @@ ENTRY(secondary_startup_64)
154 */ 162 */
155 163
156 /* Enable PAE mode and PGE */ 164 /* Enable PAE mode and PGE */
157 xorq %rax, %rax 165 movl $(X86_CR4_PAE | X86_CR4_PGE), %eax
158 btsq $5, %rax
159 btsq $7, %rax
160 movq %rax, %cr4 166 movq %rax, %cr4
161 167
162 /* Setup early boot stage 4 level pagetables. */ 168 /* Setup early boot stage 4 level pagetables. */
@@ -184,19 +190,15 @@ ENTRY(secondary_startup_64)
1841: wrmsr /* Make changes effective */ 1901: wrmsr /* Make changes effective */
185 191
186 /* Setup cr0 */ 192 /* Setup cr0 */
187#define CR0_PM 1 /* protected mode */ 193#define CR0_STATE (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \
188#define CR0_MP (1<<1) 194 X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \
189#define CR0_ET (1<<4) 195 X86_CR0_PG)
190#define CR0_NE (1<<5) 196 movl $CR0_STATE, %eax
191#define CR0_WP (1<<16)
192#define CR0_AM (1<<18)
193#define CR0_PAGING (1<<31)
194 movl $CR0_PM|CR0_MP|CR0_ET|CR0_NE|CR0_WP|CR0_AM|CR0_PAGING,%eax
195 /* Make changes effective */ 197 /* Make changes effective */
196 movq %rax, %cr0 198 movq %rax, %cr0
197 199
198 /* Setup a boot time stack */ 200 /* Setup a boot time stack */
199 movq init_rsp(%rip),%rsp 201 movq stack_start(%rip),%rsp
200 202
201 /* zero EFLAGS after setting rsp */ 203 /* zero EFLAGS after setting rsp */
202 pushq $0 204 pushq $0
@@ -208,7 +210,7 @@ ENTRY(secondary_startup_64)
208 * addresses where we're currently running on. We have to do that here 210 * addresses where we're currently running on. We have to do that here
209 * because in 32bit we couldn't load a 64bit linear address. 211 * because in 32bit we couldn't load a 64bit linear address.
210 */ 212 */
211 lgdt cpu_gdt_descr(%rip) 213 lgdt early_gdt_descr(%rip)
212 214
213 /* set up data segments. actually 0 would do too */ 215 /* set up data segments. actually 0 would do too */
214 movl $__KERNEL_DS,%eax 216 movl $__KERNEL_DS,%eax
@@ -257,8 +259,9 @@ ENTRY(secondary_startup_64)
257 .quad x86_64_start_kernel 259 .quad x86_64_start_kernel
258 __FINITDATA 260 __FINITDATA
259 261
260 ENTRY(init_rsp) 262 ENTRY(stack_start)
261 .quad init_thread_union+THREAD_SIZE-8 263 .quad init_thread_union+THREAD_SIZE-8
264 .word 0
262 265
263bad_address: 266bad_address:
264 jmp bad_address 267 jmp bad_address
@@ -327,11 +330,11 @@ early_idt_ripmsg:
327ENTRY(name) 330ENTRY(name)
328 331
329/* Automate the creation of 1 to 1 mapping pmd entries */ 332/* Automate the creation of 1 to 1 mapping pmd entries */
330#define PMDS(START, PERM, COUNT) \ 333#define PMDS(START, PERM, COUNT) \
331 i = 0 ; \ 334 i = 0 ; \
332 .rept (COUNT) ; \ 335 .rept (COUNT) ; \
333 .quad (START) + (i << 21) + (PERM) ; \ 336 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
334 i = i + 1 ; \ 337 i = i + 1 ; \
335 .endr 338 .endr
336 339
337 /* 340 /*
@@ -342,9 +345,9 @@ ENTRY(name)
342 */ 345 */
343NEXT_PAGE(init_level4_pgt) 346NEXT_PAGE(init_level4_pgt)
344 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE 347 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
345 .fill 257,8,0 348 .org init_level4_pgt + L4_PAGE_OFFSET*8, 0
346 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE 349 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
347 .fill 252,8,0 350 .org init_level4_pgt + L4_START_KERNEL*8, 0
348 /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */ 351 /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
349 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE 352 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
350 353
@@ -353,7 +356,7 @@ NEXT_PAGE(level3_ident_pgt)
353 .fill 511,8,0 356 .fill 511,8,0
354 357
355NEXT_PAGE(level3_kernel_pgt) 358NEXT_PAGE(level3_kernel_pgt)
356 .fill 510,8,0 359 .fill L3_START_KERNEL,8,0
357 /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */ 360 /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
358 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE 361 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE
359 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE 362 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
@@ -384,7 +387,7 @@ NEXT_PAGE(level2_kernel_pgt)
384 * If you want to increase this then increase MODULES_VADDR 387 * If you want to increase this then increase MODULES_VADDR
385 * too.) 388 * too.)
386 */ 389 */
387 PMDS(0, __PAGE_KERNEL_LARGE_EXEC|_PAGE_GLOBAL, 390 PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
388 KERNEL_IMAGE_SIZE/PMD_SIZE) 391 KERNEL_IMAGE_SIZE/PMD_SIZE)
389 392
390NEXT_PAGE(level2_spare_pgt) 393NEXT_PAGE(level2_spare_pgt)
@@ -395,54 +398,16 @@ NEXT_PAGE(level2_spare_pgt)
395 398
396 .data 399 .data
397 .align 16 400 .align 16
398 .globl cpu_gdt_descr 401 .globl early_gdt_descr
399cpu_gdt_descr: 402early_gdt_descr:
400 .word gdt_end-cpu_gdt_table-1 403 .word GDT_ENTRIES*8-1
401gdt: 404 .quad per_cpu__gdt_page
402 .quad cpu_gdt_table
403#ifdef CONFIG_SMP
404 .rept NR_CPUS-1
405 .word 0
406 .quad 0
407 .endr
408#endif
409 405
410ENTRY(phys_base) 406ENTRY(phys_base)
411 /* This must match the first entry in level2_kernel_pgt */ 407 /* This must match the first entry in level2_kernel_pgt */
412 .quad 0x0000000000000000 408 .quad 0x0000000000000000
413 409
414/* We need valid kernel segments for data and code in long mode too
415 * IRET will check the segment types kkeil 2000/10/28
416 * Also sysret mandates a special GDT layout
417 */
418
419 .section .data.page_aligned, "aw"
420 .align PAGE_SIZE
421
422/* The TLS descriptors are currently at a different place compared to i386.
423 Hopefully nobody expects them at a fixed place (Wine?) */
424 410
425ENTRY(cpu_gdt_table)
426 .quad 0x0000000000000000 /* NULL descriptor */
427 .quad 0x00cf9b000000ffff /* __KERNEL32_CS */
428 .quad 0x00af9b000000ffff /* __KERNEL_CS */
429 .quad 0x00cf93000000ffff /* __KERNEL_DS */
430 .quad 0x00cffb000000ffff /* __USER32_CS */
431 .quad 0x00cff3000000ffff /* __USER_DS, __USER32_DS */
432 .quad 0x00affb000000ffff /* __USER_CS */
433 .quad 0x0 /* unused */
434 .quad 0,0 /* TSS */
435 .quad 0,0 /* LDT */
436 .quad 0,0,0 /* three TLS descriptors */
437 .quad 0x0000f40000000000 /* node/CPU stored in limit */
438gdt_end:
439 /* asm/segment.h:GDT_ENTRIES must match this */
440 /* This should be a multiple of the cache line size */
441 /* GDTs of other CPUs are now dynamically allocated */
442
443 /* zero the remaining page */
444 .fill PAGE_SIZE / 8 - GDT_ENTRIES,8,0
445
446 .section .bss, "aw", @nobits 411 .section .bss, "aw", @nobits
447 .align L1_CACHE_BYTES 412 .align L1_CACHE_BYTES
448ENTRY(idt_table) 413ENTRY(idt_table)
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index 9b5cfcdfc426..0ea6a19bfdfe 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -17,7 +17,7 @@
17 17
18/* FSEC = 10^-15 18/* FSEC = 10^-15
19 NSEC = 10^-9 */ 19 NSEC = 10^-9 */
20#define FSEC_PER_NSEC 1000000 20#define FSEC_PER_NSEC 1000000L
21 21
22/* 22/*
23 * HPET address is set in acpi/boot.c, when an ACPI entry exists 23 * HPET address is set in acpi/boot.c, when an ACPI entry exists
@@ -36,26 +36,15 @@ static inline void hpet_writel(unsigned long d, unsigned long a)
36} 36}
37 37
38#ifdef CONFIG_X86_64 38#ifdef CONFIG_X86_64
39
40#include <asm/pgtable.h> 39#include <asm/pgtable.h>
41 40#endif
42static inline void hpet_set_mapping(void)
43{
44 set_fixmap_nocache(FIX_HPET_BASE, hpet_address);
45 __set_fixmap(VSYSCALL_HPET, hpet_address, PAGE_KERNEL_VSYSCALL_NOCACHE);
46 hpet_virt_address = (void __iomem *)fix_to_virt(FIX_HPET_BASE);
47}
48
49static inline void hpet_clear_mapping(void)
50{
51 hpet_virt_address = NULL;
52}
53
54#else
55 41
56static inline void hpet_set_mapping(void) 42static inline void hpet_set_mapping(void)
57{ 43{
58 hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE); 44 hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE);
45#ifdef CONFIG_X86_64
46 __set_fixmap(VSYSCALL_HPET, hpet_address, PAGE_KERNEL_VSYSCALL_NOCACHE);
47#endif
59} 48}
60 49
61static inline void hpet_clear_mapping(void) 50static inline void hpet_clear_mapping(void)
@@ -63,7 +52,6 @@ static inline void hpet_clear_mapping(void)
63 iounmap(hpet_virt_address); 52 iounmap(hpet_virt_address);
64 hpet_virt_address = NULL; 53 hpet_virt_address = NULL;
65} 54}
66#endif
67 55
68/* 56/*
69 * HPET command line enable / disable 57 * HPET command line enable / disable
@@ -206,20 +194,19 @@ static void hpet_enable_legacy_int(void)
206 194
207static void hpet_legacy_clockevent_register(void) 195static void hpet_legacy_clockevent_register(void)
208{ 196{
209 uint64_t hpet_freq;
210
211 /* Start HPET legacy interrupts */ 197 /* Start HPET legacy interrupts */
212 hpet_enable_legacy_int(); 198 hpet_enable_legacy_int();
213 199
214 /* 200 /*
215 * The period is a femto seconds value. We need to calculate the 201 * The mult factor is defined as (include/linux/clockchips.h)
216 * scaled math multiplication factor for nanosecond to hpet tick 202 * mult/2^shift = cyc/ns (in contrast to ns/cyc in clocksource.h)
217 * conversion. 203 * hpet_period is in units of femtoseconds (per cycle), so
204 * mult/2^shift = cyc/ns = 10^6/hpet_period
205 * mult = (10^6 * 2^shift)/hpet_period
206 * mult = (FSEC_PER_NSEC << hpet_clockevent.shift)/hpet_period
218 */ 207 */
219 hpet_freq = 1000000000000000ULL; 208 hpet_clockevent.mult = div_sc((unsigned long) FSEC_PER_NSEC,
220 do_div(hpet_freq, hpet_period); 209 hpet_period, hpet_clockevent.shift);
221 hpet_clockevent.mult = div_sc((unsigned long) hpet_freq,
222 NSEC_PER_SEC, hpet_clockevent.shift);
223 /* Calculate the min / max delta */ 210 /* Calculate the min / max delta */
224 hpet_clockevent.max_delta_ns = clockevent_delta2ns(0x7FFFFFFF, 211 hpet_clockevent.max_delta_ns = clockevent_delta2ns(0x7FFFFFFF,
225 &hpet_clockevent); 212 &hpet_clockevent);
@@ -324,7 +311,7 @@ static struct clocksource clocksource_hpet = {
324 311
325static int hpet_clocksource_register(void) 312static int hpet_clocksource_register(void)
326{ 313{
327 u64 tmp, start, now; 314 u64 start, now;
328 cycle_t t1; 315 cycle_t t1;
329 316
330 /* Start the counter */ 317 /* Start the counter */
@@ -351,21 +338,15 @@ static int hpet_clocksource_register(void)
351 return -ENODEV; 338 return -ENODEV;
352 } 339 }
353 340
354 /* Initialize and register HPET clocksource 341 /*
355 * 342 * The definition of mult is (include/linux/clocksource.h)
356 * hpet period is in femto seconds per cycle 343 * mult/2^shift = ns/cyc and hpet_period is in units of fsec/cyc
357 * so we need to convert this to ns/cyc units 344 * so we first need to convert hpet_period to ns/cyc units:
358 * approximated by mult/2^shift 345 * mult/2^shift = ns/cyc = hpet_period/10^6
359 * 346 * mult = (hpet_period * 2^shift)/10^6
360 * fsec/cyc * 1nsec/1000000fsec = nsec/cyc = mult/2^shift 347 * mult = (hpet_period << shift)/FSEC_PER_NSEC
361 * fsec/cyc * 1ns/1000000fsec * 2^shift = mult
362 * fsec/cyc * 2^shift * 1nsec/1000000fsec = mult
363 * (fsec/cyc << shift)/1000000 = mult
364 * (hpet_period << shift)/FSEC_PER_NSEC = mult
365 */ 348 */
366 tmp = (u64)hpet_period << HPET_SHIFT; 349 clocksource_hpet.mult = div_sc(hpet_period, FSEC_PER_NSEC, HPET_SHIFT);
367 do_div(tmp, FSEC_PER_NSEC);
368 clocksource_hpet.mult = (u32)tmp;
369 350
370 clocksource_register(&clocksource_hpet); 351 clocksource_register(&clocksource_hpet);
371 352
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index deb43785e923..dd7ebee446af 100644
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -1,7 +1,14 @@
1#include <linux/module.h> 1#include <linux/module.h>
2
2#include <asm/checksum.h> 3#include <asm/checksum.h>
3#include <asm/desc.h>
4#include <asm/pgtable.h> 4#include <asm/pgtable.h>
5#include <asm/desc.h>
6#include <asm/ftrace.h>
7
8#ifdef CONFIG_FTRACE
9/* mcount is defined in assembly */
10EXPORT_SYMBOL(mcount);
11#endif
5 12
6/* Networking helper routines. */ 13/* Networking helper routines. */
7EXPORT_SYMBOL(csum_partial_copy_generic); 14EXPORT_SYMBOL(csum_partial_copy_generic);
diff --git a/arch/x86/kernel/i8259_32.c b/arch/x86/kernel/i8259.c
index fe631967d625..dc92b49d9204 100644
--- a/arch/x86/kernel/i8259_32.c
+++ b/arch/x86/kernel/i8259.c
@@ -1,8 +1,10 @@
1#include <linux/linkage.h>
1#include <linux/errno.h> 2#include <linux/errno.h>
2#include <linux/signal.h> 3#include <linux/signal.h>
3#include <linux/sched.h> 4#include <linux/sched.h>
4#include <linux/ioport.h> 5#include <linux/ioport.h>
5#include <linux/interrupt.h> 6#include <linux/interrupt.h>
7#include <linux/timex.h>
6#include <linux/slab.h> 8#include <linux/slab.h>
7#include <linux/random.h> 9#include <linux/random.h>
8#include <linux/init.h> 10#include <linux/init.h>
@@ -10,10 +12,12 @@
10#include <linux/sysdev.h> 12#include <linux/sysdev.h>
11#include <linux/bitops.h> 13#include <linux/bitops.h>
12 14
15#include <asm/acpi.h>
13#include <asm/atomic.h> 16#include <asm/atomic.h>
14#include <asm/system.h> 17#include <asm/system.h>
15#include <asm/io.h> 18#include <asm/io.h>
16#include <asm/timer.h> 19#include <asm/timer.h>
20#include <asm/hw_irq.h>
17#include <asm/pgtable.h> 21#include <asm/pgtable.h>
18#include <asm/delay.h> 22#include <asm/delay.h>
19#include <asm/desc.h> 23#include <asm/desc.h>
@@ -32,7 +36,7 @@ static int i8259A_auto_eoi;
32DEFINE_SPINLOCK(i8259A_lock); 36DEFINE_SPINLOCK(i8259A_lock);
33static void mask_and_ack_8259A(unsigned int); 37static void mask_and_ack_8259A(unsigned int);
34 38
35static struct irq_chip i8259A_chip = { 39struct irq_chip i8259A_chip = {
36 .name = "XT-PIC", 40 .name = "XT-PIC",
37 .mask = disable_8259A_irq, 41 .mask = disable_8259A_irq,
38 .disable = disable_8259A_irq, 42 .disable = disable_8259A_irq,
@@ -125,14 +129,14 @@ static inline int i8259A_irq_real(unsigned int irq)
125 int irqmask = 1<<irq; 129 int irqmask = 1<<irq;
126 130
127 if (irq < 8) { 131 if (irq < 8) {
128 outb(0x0B,PIC_MASTER_CMD); /* ISR register */ 132 outb(0x0B, PIC_MASTER_CMD); /* ISR register */
129 value = inb(PIC_MASTER_CMD) & irqmask; 133 value = inb(PIC_MASTER_CMD) & irqmask;
130 outb(0x0A,PIC_MASTER_CMD); /* back to the IRR register */ 134 outb(0x0A, PIC_MASTER_CMD); /* back to the IRR register */
131 return value; 135 return value;
132 } 136 }
133 outb(0x0B,PIC_SLAVE_CMD); /* ISR register */ 137 outb(0x0B, PIC_SLAVE_CMD); /* ISR register */
134 value = inb(PIC_SLAVE_CMD) & (irqmask >> 8); 138 value = inb(PIC_SLAVE_CMD) & (irqmask >> 8);
135 outb(0x0A,PIC_SLAVE_CMD); /* back to the IRR register */ 139 outb(0x0A, PIC_SLAVE_CMD); /* back to the IRR register */
136 return value; 140 return value;
137} 141}
138 142
@@ -171,12 +175,14 @@ handle_real_irq:
171 if (irq & 8) { 175 if (irq & 8) {
172 inb(PIC_SLAVE_IMR); /* DUMMY - (do we need this?) */ 176 inb(PIC_SLAVE_IMR); /* DUMMY - (do we need this?) */
173 outb(cached_slave_mask, PIC_SLAVE_IMR); 177 outb(cached_slave_mask, PIC_SLAVE_IMR);
174 outb(0x60+(irq&7),PIC_SLAVE_CMD);/* 'Specific EOI' to slave */ 178 /* 'Specific EOI' to slave */
175 outb(0x60+PIC_CASCADE_IR,PIC_MASTER_CMD); /* 'Specific EOI' to master-IRQ2 */ 179 outb(0x60+(irq&7), PIC_SLAVE_CMD);
180 /* 'Specific EOI' to master-IRQ2 */
181 outb(0x60+PIC_CASCADE_IR, PIC_MASTER_CMD);
176 } else { 182 } else {
177 inb(PIC_MASTER_IMR); /* DUMMY - (do we need this?) */ 183 inb(PIC_MASTER_IMR); /* DUMMY - (do we need this?) */
178 outb(cached_master_mask, PIC_MASTER_IMR); 184 outb(cached_master_mask, PIC_MASTER_IMR);
179 outb(0x60+irq,PIC_MASTER_CMD); /* 'Specific EOI to master */ 185 outb(0x60+irq, PIC_MASTER_CMD); /* 'Specific EOI to master */
180 } 186 }
181 spin_unlock_irqrestore(&i8259A_lock, flags); 187 spin_unlock_irqrestore(&i8259A_lock, flags);
182 return; 188 return;
@@ -199,7 +205,8 @@ spurious_8259A_irq:
199 * lets ACK and report it. [once per IRQ] 205 * lets ACK and report it. [once per IRQ]
200 */ 206 */
201 if (!(spurious_irq_mask & irqmask)) { 207 if (!(spurious_irq_mask & irqmask)) {
202 printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq); 208 printk(KERN_DEBUG
209 "spurious 8259A interrupt: IRQ%d.\n", irq);
203 spurious_irq_mask |= irqmask; 210 spurious_irq_mask |= irqmask;
204 } 211 }
205 atomic_inc(&irq_err_count); 212 atomic_inc(&irq_err_count);
@@ -290,17 +297,28 @@ void init_8259A(int auto_eoi)
290 * outb_pic - this has to work on a wide range of PC hardware. 297 * outb_pic - this has to work on a wide range of PC hardware.
291 */ 298 */
292 outb_pic(0x11, PIC_MASTER_CMD); /* ICW1: select 8259A-1 init */ 299 outb_pic(0x11, PIC_MASTER_CMD); /* ICW1: select 8259A-1 init */
293 outb_pic(0x20 + 0, PIC_MASTER_IMR); /* ICW2: 8259A-1 IR0-7 mapped to 0x20-0x27 */ 300
294 outb_pic(1U << PIC_CASCADE_IR, PIC_MASTER_IMR); /* 8259A-1 (the master) has a slave on IR2 */ 301 /* ICW2: 8259A-1 IR0-7 mapped to 0x30-0x37 on x86-64,
302 to 0x20-0x27 on i386 */
303 outb_pic(IRQ0_VECTOR, PIC_MASTER_IMR);
304
305 /* 8259A-1 (the master) has a slave on IR2 */
306 outb_pic(1U << PIC_CASCADE_IR, PIC_MASTER_IMR);
307
295 if (auto_eoi) /* master does Auto EOI */ 308 if (auto_eoi) /* master does Auto EOI */
296 outb_pic(MASTER_ICW4_DEFAULT | PIC_ICW4_AEOI, PIC_MASTER_IMR); 309 outb_pic(MASTER_ICW4_DEFAULT | PIC_ICW4_AEOI, PIC_MASTER_IMR);
297 else /* master expects normal EOI */ 310 else /* master expects normal EOI */
298 outb_pic(MASTER_ICW4_DEFAULT, PIC_MASTER_IMR); 311 outb_pic(MASTER_ICW4_DEFAULT, PIC_MASTER_IMR);
299 312
300 outb_pic(0x11, PIC_SLAVE_CMD); /* ICW1: select 8259A-2 init */ 313 outb_pic(0x11, PIC_SLAVE_CMD); /* ICW1: select 8259A-2 init */
301 outb_pic(0x20 + 8, PIC_SLAVE_IMR); /* ICW2: 8259A-2 IR0-7 mapped to 0x28-0x2f */ 314
302 outb_pic(PIC_CASCADE_IR, PIC_SLAVE_IMR); /* 8259A-2 is a slave on master's IR2 */ 315 /* ICW2: 8259A-2 IR0-7 mapped to IRQ8_VECTOR */
303 outb_pic(SLAVE_ICW4_DEFAULT, PIC_SLAVE_IMR); /* (slave's support for AEOI in flat mode is to be investigated) */ 316 outb_pic(IRQ8_VECTOR, PIC_SLAVE_IMR);
317 /* 8259A-2 is a slave on master's IR2 */
318 outb_pic(PIC_CASCADE_IR, PIC_SLAVE_IMR);
319 /* (slave's support for AEOI in flat mode is to be investigated) */
320 outb_pic(SLAVE_ICW4_DEFAULT, PIC_SLAVE_IMR);
321
304 if (auto_eoi) 322 if (auto_eoi)
305 /* 323 /*
306 * In AEOI mode we just have to mask the interrupt 324 * In AEOI mode we just have to mask the interrupt
@@ -317,93 +335,3 @@ void init_8259A(int auto_eoi)
317 335
318 spin_unlock_irqrestore(&i8259A_lock, flags); 336 spin_unlock_irqrestore(&i8259A_lock, flags);
319} 337}
320
321/*
322 * Note that on a 486, we don't want to do a SIGFPE on an irq13
323 * as the irq is unreliable, and exception 16 works correctly
324 * (ie as explained in the intel literature). On a 386, you
325 * can't use exception 16 due to bad IBM design, so we have to
326 * rely on the less exact irq13.
327 *
328 * Careful.. Not only is IRQ13 unreliable, but it is also
329 * leads to races. IBM designers who came up with it should
330 * be shot.
331 */
332
333
334static irqreturn_t math_error_irq(int cpl, void *dev_id)
335{
336 extern void math_error(void __user *);
337 outb(0,0xF0);
338 if (ignore_fpu_irq || !boot_cpu_data.hard_math)
339 return IRQ_NONE;
340 math_error((void __user *)get_irq_regs()->ip);
341 return IRQ_HANDLED;
342}
343
344/*
345 * New motherboards sometimes make IRQ 13 be a PCI interrupt,
346 * so allow interrupt sharing.
347 */
348static struct irqaction fpu_irq = {
349 .handler = math_error_irq,
350 .mask = CPU_MASK_NONE,
351 .name = "fpu",
352};
353
354void __init init_ISA_irqs (void)
355{
356 int i;
357
358#ifdef CONFIG_X86_LOCAL_APIC
359 init_bsp_APIC();
360#endif
361 init_8259A(0);
362
363 /*
364 * 16 old-style INTA-cycle interrupts:
365 */
366 for (i = 0; i < 16; i++) {
367 set_irq_chip_and_handler_name(i, &i8259A_chip,
368 handle_level_irq, "XT");
369 }
370}
371
372/* Overridden in paravirt.c */
373void init_IRQ(void) __attribute__((weak, alias("native_init_IRQ")));
374
375void __init native_init_IRQ(void)
376{
377 int i;
378
379 /* all the set up before the call gates are initialised */
380 pre_intr_init_hook();
381
382 /*
383 * Cover the whole vector space, no vector can escape
384 * us. (some of these will be overridden and become
385 * 'special' SMP interrupts)
386 */
387 for (i = 0; i < (NR_VECTORS - FIRST_EXTERNAL_VECTOR); i++) {
388 int vector = FIRST_EXTERNAL_VECTOR + i;
389 if (i >= NR_IRQS)
390 break;
391 /* SYSCALL_VECTOR was reserved in trap_init. */
392 if (!test_bit(vector, used_vectors))
393 set_intr_gate(vector, interrupt[i]);
394 }
395
396 /* setup after call gates are initialised (usually add in
397 * the architecture specific gates)
398 */
399 intr_init_hook();
400
401 /*
402 * External FPU? Set up irq13 if so, for
403 * original braindamaged IBM FERR coupling.
404 */
405 if (boot_cpu_data.hard_math && !cpu_has_fpu)
406 setup_irq(FPU_IRQ, &fpu_irq);
407
408 irq_ctx_init(smp_processor_id());
409}
diff --git a/arch/x86/kernel/i8259_64.c b/arch/x86/kernel/i8259_64.c
deleted file mode 100644
index fa57a1568508..000000000000
--- a/arch/x86/kernel/i8259_64.c
+++ /dev/null
@@ -1,512 +0,0 @@
1#include <linux/linkage.h>
2#include <linux/errno.h>
3#include <linux/signal.h>
4#include <linux/sched.h>
5#include <linux/ioport.h>
6#include <linux/interrupt.h>
7#include <linux/timex.h>
8#include <linux/slab.h>
9#include <linux/random.h>
10#include <linux/init.h>
11#include <linux/kernel_stat.h>
12#include <linux/sysdev.h>
13#include <linux/bitops.h>
14
15#include <asm/acpi.h>
16#include <asm/atomic.h>
17#include <asm/system.h>
18#include <asm/io.h>
19#include <asm/hw_irq.h>
20#include <asm/pgtable.h>
21#include <asm/delay.h>
22#include <asm/desc.h>
23#include <asm/apic.h>
24#include <asm/i8259.h>
25
26/*
27 * Common place to define all x86 IRQ vectors
28 *
29 * This builds up the IRQ handler stubs using some ugly macros in irq.h
30 *
31 * These macros create the low-level assembly IRQ routines that save
32 * register context and call do_IRQ(). do_IRQ() then does all the
33 * operations that are needed to keep the AT (or SMP IOAPIC)
34 * interrupt-controller happy.
35 */
36
37#define BI(x,y) \
38 BUILD_IRQ(x##y)
39
40#define BUILD_16_IRQS(x) \
41 BI(x,0) BI(x,1) BI(x,2) BI(x,3) \
42 BI(x,4) BI(x,5) BI(x,6) BI(x,7) \
43 BI(x,8) BI(x,9) BI(x,a) BI(x,b) \
44 BI(x,c) BI(x,d) BI(x,e) BI(x,f)
45
46/*
47 * ISA PIC or low IO-APIC triggered (INTA-cycle or APIC) interrupts:
48 * (these are usually mapped to vectors 0x30-0x3f)
49 */
50
51/*
52 * The IO-APIC gives us many more interrupt sources. Most of these
53 * are unused but an SMP system is supposed to have enough memory ...
54 * sometimes (mostly wrt. hw bugs) we get corrupted vectors all
55 * across the spectrum, so we really want to be prepared to get all
56 * of these. Plus, more powerful systems might have more than 64
57 * IO-APIC registers.
58 *
59 * (these are usually mapped into the 0x30-0xff vector range)
60 */
61 BUILD_16_IRQS(0x2) BUILD_16_IRQS(0x3)
62BUILD_16_IRQS(0x4) BUILD_16_IRQS(0x5) BUILD_16_IRQS(0x6) BUILD_16_IRQS(0x7)
63BUILD_16_IRQS(0x8) BUILD_16_IRQS(0x9) BUILD_16_IRQS(0xa) BUILD_16_IRQS(0xb)
64BUILD_16_IRQS(0xc) BUILD_16_IRQS(0xd) BUILD_16_IRQS(0xe) BUILD_16_IRQS(0xf)
65
66#undef BUILD_16_IRQS
67#undef BI
68
69
70#define IRQ(x,y) \
71 IRQ##x##y##_interrupt
72
73#define IRQLIST_16(x) \
74 IRQ(x,0), IRQ(x,1), IRQ(x,2), IRQ(x,3), \
75 IRQ(x,4), IRQ(x,5), IRQ(x,6), IRQ(x,7), \
76 IRQ(x,8), IRQ(x,9), IRQ(x,a), IRQ(x,b), \
77 IRQ(x,c), IRQ(x,d), IRQ(x,e), IRQ(x,f)
78
79/* for the irq vectors */
80static void (*__initdata interrupt[NR_VECTORS - FIRST_EXTERNAL_VECTOR])(void) = {
81 IRQLIST_16(0x2), IRQLIST_16(0x3),
82 IRQLIST_16(0x4), IRQLIST_16(0x5), IRQLIST_16(0x6), IRQLIST_16(0x7),
83 IRQLIST_16(0x8), IRQLIST_16(0x9), IRQLIST_16(0xa), IRQLIST_16(0xb),
84 IRQLIST_16(0xc), IRQLIST_16(0xd), IRQLIST_16(0xe), IRQLIST_16(0xf)
85};
86
87#undef IRQ
88#undef IRQLIST_16
89
90/*
91 * This is the 'legacy' 8259A Programmable Interrupt Controller,
92 * present in the majority of PC/AT boxes.
93 * plus some generic x86 specific things if generic specifics makes
94 * any sense at all.
95 * this file should become arch/i386/kernel/irq.c when the old irq.c
96 * moves to arch independent land
97 */
98
99static int i8259A_auto_eoi;
100DEFINE_SPINLOCK(i8259A_lock);
101static void mask_and_ack_8259A(unsigned int);
102
103static struct irq_chip i8259A_chip = {
104 .name = "XT-PIC",
105 .mask = disable_8259A_irq,
106 .disable = disable_8259A_irq,
107 .unmask = enable_8259A_irq,
108 .mask_ack = mask_and_ack_8259A,
109};
110
111/*
112 * 8259A PIC functions to handle ISA devices:
113 */
114
115/*
116 * This contains the irq mask for both 8259A irq controllers,
117 */
118unsigned int cached_irq_mask = 0xffff;
119
120/*
121 * Not all IRQs can be routed through the IO-APIC, eg. on certain (older)
122 * boards the timer interrupt is not really connected to any IO-APIC pin,
123 * it's fed to the master 8259A's IR0 line only.
124 *
125 * Any '1' bit in this mask means the IRQ is routed through the IO-APIC.
126 * this 'mixed mode' IRQ handling costs nothing because it's only used
127 * at IRQ setup time.
128 */
129unsigned long io_apic_irqs;
130
131void disable_8259A_irq(unsigned int irq)
132{
133 unsigned int mask = 1 << irq;
134 unsigned long flags;
135
136 spin_lock_irqsave(&i8259A_lock, flags);
137 cached_irq_mask |= mask;
138 if (irq & 8)
139 outb(cached_slave_mask, PIC_SLAVE_IMR);
140 else
141 outb(cached_master_mask, PIC_MASTER_IMR);
142 spin_unlock_irqrestore(&i8259A_lock, flags);
143}
144
145void enable_8259A_irq(unsigned int irq)
146{
147 unsigned int mask = ~(1 << irq);
148 unsigned long flags;
149
150 spin_lock_irqsave(&i8259A_lock, flags);
151 cached_irq_mask &= mask;
152 if (irq & 8)
153 outb(cached_slave_mask, PIC_SLAVE_IMR);
154 else
155 outb(cached_master_mask, PIC_MASTER_IMR);
156 spin_unlock_irqrestore(&i8259A_lock, flags);
157}
158
159int i8259A_irq_pending(unsigned int irq)
160{
161 unsigned int mask = 1<<irq;
162 unsigned long flags;
163 int ret;
164
165 spin_lock_irqsave(&i8259A_lock, flags);
166 if (irq < 8)
167 ret = inb(PIC_MASTER_CMD) & mask;
168 else
169 ret = inb(PIC_SLAVE_CMD) & (mask >> 8);
170 spin_unlock_irqrestore(&i8259A_lock, flags);
171
172 return ret;
173}
174
175void make_8259A_irq(unsigned int irq)
176{
177 disable_irq_nosync(irq);
178 io_apic_irqs &= ~(1<<irq);
179 set_irq_chip_and_handler_name(irq, &i8259A_chip, handle_level_irq,
180 "XT");
181 enable_irq(irq);
182}
183
184/*
185 * This function assumes to be called rarely. Switching between
186 * 8259A registers is slow.
187 * This has to be protected by the irq controller spinlock
188 * before being called.
189 */
190static inline int i8259A_irq_real(unsigned int irq)
191{
192 int value;
193 int irqmask = 1<<irq;
194
195 if (irq < 8) {
196 outb(0x0B,PIC_MASTER_CMD); /* ISR register */
197 value = inb(PIC_MASTER_CMD) & irqmask;
198 outb(0x0A,PIC_MASTER_CMD); /* back to the IRR register */
199 return value;
200 }
201 outb(0x0B,PIC_SLAVE_CMD); /* ISR register */
202 value = inb(PIC_SLAVE_CMD) & (irqmask >> 8);
203 outb(0x0A,PIC_SLAVE_CMD); /* back to the IRR register */
204 return value;
205}
206
207/*
208 * Careful! The 8259A is a fragile beast, it pretty
209 * much _has_ to be done exactly like this (mask it
210 * first, _then_ send the EOI, and the order of EOI
211 * to the two 8259s is important!
212 */
213static void mask_and_ack_8259A(unsigned int irq)
214{
215 unsigned int irqmask = 1 << irq;
216 unsigned long flags;
217
218 spin_lock_irqsave(&i8259A_lock, flags);
219 /*
220 * Lightweight spurious IRQ detection. We do not want
221 * to overdo spurious IRQ handling - it's usually a sign
222 * of hardware problems, so we only do the checks we can
223 * do without slowing down good hardware unnecessarily.
224 *
225 * Note that IRQ7 and IRQ15 (the two spurious IRQs
226 * usually resulting from the 8259A-1|2 PICs) occur
227 * even if the IRQ is masked in the 8259A. Thus we
228 * can check spurious 8259A IRQs without doing the
229 * quite slow i8259A_irq_real() call for every IRQ.
230 * This does not cover 100% of spurious interrupts,
231 * but should be enough to warn the user that there
232 * is something bad going on ...
233 */
234 if (cached_irq_mask & irqmask)
235 goto spurious_8259A_irq;
236 cached_irq_mask |= irqmask;
237
238handle_real_irq:
239 if (irq & 8) {
240 inb(PIC_SLAVE_IMR); /* DUMMY - (do we need this?) */
241 outb(cached_slave_mask, PIC_SLAVE_IMR);
242 /* 'Specific EOI' to slave */
243 outb(0x60+(irq&7),PIC_SLAVE_CMD);
244 /* 'Specific EOI' to master-IRQ2 */
245 outb(0x60+PIC_CASCADE_IR,PIC_MASTER_CMD);
246 } else {
247 inb(PIC_MASTER_IMR); /* DUMMY - (do we need this?) */
248 outb(cached_master_mask, PIC_MASTER_IMR);
249 /* 'Specific EOI' to master */
250 outb(0x60+irq,PIC_MASTER_CMD);
251 }
252 spin_unlock_irqrestore(&i8259A_lock, flags);
253 return;
254
255spurious_8259A_irq:
256 /*
257 * this is the slow path - should happen rarely.
258 */
259 if (i8259A_irq_real(irq))
260 /*
261 * oops, the IRQ _is_ in service according to the
262 * 8259A - not spurious, go handle it.
263 */
264 goto handle_real_irq;
265
266 {
267 static int spurious_irq_mask;
268 /*
269 * At this point we can be sure the IRQ is spurious,
270 * lets ACK and report it. [once per IRQ]
271 */
272 if (!(spurious_irq_mask & irqmask)) {
273 printk(KERN_DEBUG
274 "spurious 8259A interrupt: IRQ%d.\n", irq);
275 spurious_irq_mask |= irqmask;
276 }
277 atomic_inc(&irq_err_count);
278 /*
279 * Theoretically we do not have to handle this IRQ,
280 * but in Linux this does not cause problems and is
281 * simpler for us.
282 */
283 goto handle_real_irq;
284 }
285}
286
287static char irq_trigger[2];
288/**
289 * ELCR registers (0x4d0, 0x4d1) control edge/level of IRQ
290 */
291static void restore_ELCR(char *trigger)
292{
293 outb(trigger[0], 0x4d0);
294 outb(trigger[1], 0x4d1);
295}
296
297static void save_ELCR(char *trigger)
298{
299 /* IRQ 0,1,2,8,13 are marked as reserved */
300 trigger[0] = inb(0x4d0) & 0xF8;
301 trigger[1] = inb(0x4d1) & 0xDE;
302}
303
304static int i8259A_resume(struct sys_device *dev)
305{
306 init_8259A(i8259A_auto_eoi);
307 restore_ELCR(irq_trigger);
308 return 0;
309}
310
311static int i8259A_suspend(struct sys_device *dev, pm_message_t state)
312{
313 save_ELCR(irq_trigger);
314 return 0;
315}
316
317static int i8259A_shutdown(struct sys_device *dev)
318{
319 /* Put the i8259A into a quiescent state that
320 * the kernel initialization code can get it
321 * out of.
322 */
323 outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */
324 outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-1 */
325 return 0;
326}
327
328static struct sysdev_class i8259_sysdev_class = {
329 .name = "i8259",
330 .suspend = i8259A_suspend,
331 .resume = i8259A_resume,
332 .shutdown = i8259A_shutdown,
333};
334
335static struct sys_device device_i8259A = {
336 .id = 0,
337 .cls = &i8259_sysdev_class,
338};
339
340static int __init i8259A_init_sysfs(void)
341{
342 int error = sysdev_class_register(&i8259_sysdev_class);
343 if (!error)
344 error = sysdev_register(&device_i8259A);
345 return error;
346}
347
348device_initcall(i8259A_init_sysfs);
349
350void init_8259A(int auto_eoi)
351{
352 unsigned long flags;
353
354 i8259A_auto_eoi = auto_eoi;
355
356 spin_lock_irqsave(&i8259A_lock, flags);
357
358 outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */
359 outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */
360
361 /*
362 * outb_pic - this has to work on a wide range of PC hardware.
363 */
364 outb_pic(0x11, PIC_MASTER_CMD); /* ICW1: select 8259A-1 init */
365 /* ICW2: 8259A-1 IR0-7 mapped to 0x30-0x37 */
366 outb_pic(IRQ0_VECTOR, PIC_MASTER_IMR);
367 /* 8259A-1 (the master) has a slave on IR2 */
368 outb_pic(0x04, PIC_MASTER_IMR);
369 if (auto_eoi) /* master does Auto EOI */
370 outb_pic(MASTER_ICW4_DEFAULT | PIC_ICW4_AEOI, PIC_MASTER_IMR);
371 else /* master expects normal EOI */
372 outb_pic(MASTER_ICW4_DEFAULT, PIC_MASTER_IMR);
373
374 outb_pic(0x11, PIC_SLAVE_CMD); /* ICW1: select 8259A-2 init */
375 /* ICW2: 8259A-2 IR0-7 mapped to 0x38-0x3f */
376 outb_pic(IRQ8_VECTOR, PIC_SLAVE_IMR);
377 /* 8259A-2 is a slave on master's IR2 */
378 outb_pic(PIC_CASCADE_IR, PIC_SLAVE_IMR);
379 /* (slave's support for AEOI in flat mode is to be investigated) */
380 outb_pic(SLAVE_ICW4_DEFAULT, PIC_SLAVE_IMR);
381
382 if (auto_eoi)
383 /*
384 * In AEOI mode we just have to mask the interrupt
385 * when acking.
386 */
387 i8259A_chip.mask_ack = disable_8259A_irq;
388 else
389 i8259A_chip.mask_ack = mask_and_ack_8259A;
390
391 udelay(100); /* wait for 8259A to initialize */
392
393 outb(cached_master_mask, PIC_MASTER_IMR); /* restore master IRQ mask */
394 outb(cached_slave_mask, PIC_SLAVE_IMR); /* restore slave IRQ mask */
395
396 spin_unlock_irqrestore(&i8259A_lock, flags);
397}
398
399
400
401
402/*
403 * IRQ2 is cascade interrupt to second interrupt controller
404 */
405
406static struct irqaction irq2 = {
407 .handler = no_action,
408 .mask = CPU_MASK_NONE,
409 .name = "cascade",
410};
411DEFINE_PER_CPU(vector_irq_t, vector_irq) = {
412 [0 ... IRQ0_VECTOR - 1] = -1,
413 [IRQ0_VECTOR] = 0,
414 [IRQ1_VECTOR] = 1,
415 [IRQ2_VECTOR] = 2,
416 [IRQ3_VECTOR] = 3,
417 [IRQ4_VECTOR] = 4,
418 [IRQ5_VECTOR] = 5,
419 [IRQ6_VECTOR] = 6,
420 [IRQ7_VECTOR] = 7,
421 [IRQ8_VECTOR] = 8,
422 [IRQ9_VECTOR] = 9,
423 [IRQ10_VECTOR] = 10,
424 [IRQ11_VECTOR] = 11,
425 [IRQ12_VECTOR] = 12,
426 [IRQ13_VECTOR] = 13,
427 [IRQ14_VECTOR] = 14,
428 [IRQ15_VECTOR] = 15,
429 [IRQ15_VECTOR + 1 ... NR_VECTORS - 1] = -1
430};
431
432void __init init_ISA_irqs (void)
433{
434 int i;
435
436 init_bsp_APIC();
437 init_8259A(0);
438
439 for (i = 0; i < NR_IRQS; i++) {
440 irq_desc[i].status = IRQ_DISABLED;
441 irq_desc[i].action = NULL;
442 irq_desc[i].depth = 1;
443
444 if (i < 16) {
445 /*
446 * 16 old-style INTA-cycle interrupts:
447 */
448 set_irq_chip_and_handler_name(i, &i8259A_chip,
449 handle_level_irq, "XT");
450 } else {
451 /*
452 * 'high' PCI IRQs filled in on demand
453 */
454 irq_desc[i].chip = &no_irq_chip;
455 }
456 }
457}
458
459void init_IRQ(void) __attribute__((weak, alias("native_init_IRQ")));
460
461void __init native_init_IRQ(void)
462{
463 int i;
464
465 init_ISA_irqs();
466 /*
467 * Cover the whole vector space, no vector can escape
468 * us. (some of these will be overridden and become
469 * 'special' SMP interrupts)
470 */
471 for (i = 0; i < (NR_VECTORS - FIRST_EXTERNAL_VECTOR); i++) {
472 int vector = FIRST_EXTERNAL_VECTOR + i;
473 if (vector != IA32_SYSCALL_VECTOR)
474 set_intr_gate(vector, interrupt[i]);
475 }
476
477#ifdef CONFIG_SMP
478 /*
479 * The reschedule interrupt is a CPU-to-CPU reschedule-helper
480 * IPI, driven by wakeup.
481 */
482 set_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
483
484 /* IPIs for invalidation */
485 set_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
486 set_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
487 set_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
488 set_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
489 set_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
490 set_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
491 set_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
492 set_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
493
494 /* IPI for generic function call */
495 set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
496
497 /* Low priority IPI to cleanup after moving an irq */
498 set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
499#endif
500 set_intr_gate(THERMAL_APIC_VECTOR, thermal_interrupt);
501 set_intr_gate(THRESHOLD_APIC_VECTOR, threshold_interrupt);
502
503 /* self generated IPI for local APIC timer */
504 set_intr_gate(LOCAL_TIMER_VECTOR, apic_timer_interrupt);
505
506 /* IPI vectors for APIC spurious and error interrupts */
507 set_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
508 set_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
509
510 if (!acpi_ioapic)
511 setup_irq(2, &irq2);
512}
diff --git a/arch/x86/kernel/io_apic_32.c b/arch/x86/kernel/io_apic_32.c
index 4dc8600d9d20..558abf4c796a 100644
--- a/arch/x86/kernel/io_apic_32.c
+++ b/arch/x86/kernel/io_apic_32.c
@@ -25,6 +25,7 @@
25#include <linux/init.h> 25#include <linux/init.h>
26#include <linux/delay.h> 26#include <linux/delay.h>
27#include <linux/sched.h> 27#include <linux/sched.h>
28#include <linux/bootmem.h>
28#include <linux/mc146818rtc.h> 29#include <linux/mc146818rtc.h>
29#include <linux/compiler.h> 30#include <linux/compiler.h>
30#include <linux/acpi.h> 31#include <linux/acpi.h>
@@ -58,7 +59,7 @@ static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
58static DEFINE_SPINLOCK(ioapic_lock); 59static DEFINE_SPINLOCK(ioapic_lock);
59static DEFINE_SPINLOCK(vector_lock); 60static DEFINE_SPINLOCK(vector_lock);
60 61
61int timer_over_8254 __initdata = 1; 62int timer_through_8259 __initdata;
62 63
63/* 64/*
64 * Is the SiS APIC rmw bug present ? 65 * Is the SiS APIC rmw bug present ?
@@ -72,15 +73,21 @@ int sis_apic_bug = -1;
72int nr_ioapic_registers[MAX_IO_APICS]; 73int nr_ioapic_registers[MAX_IO_APICS];
73 74
74/* I/O APIC entries */ 75/* I/O APIC entries */
75struct mpc_config_ioapic mp_ioapics[MAX_IO_APICS]; 76struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
76int nr_ioapics; 77int nr_ioapics;
77 78
78/* MP IRQ source entries */ 79/* MP IRQ source entries */
79struct mpc_config_intsrc mp_irqs[MAX_IRQ_SOURCES]; 80struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
80 81
81/* # of MP IRQ source entries */ 82/* # of MP IRQ source entries */
82int mp_irq_entries; 83int mp_irq_entries;
83 84
85#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
86int mp_bus_id_to_type[MAX_MP_BUSSES];
87#endif
88
89DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
90
84static int disable_timer_pin_1 __initdata; 91static int disable_timer_pin_1 __initdata;
85 92
86/* 93/*
@@ -110,7 +117,7 @@ struct io_apic {
110static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) 117static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
111{ 118{
112 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx) 119 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
113 + (mp_ioapics[idx].mpc_apicaddr & ~PAGE_MASK); 120 + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
114} 121}
115 122
116static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg) 123static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
@@ -239,7 +246,7 @@ static void __init replace_pin_at_irq(unsigned int irq,
239 } 246 }
240} 247}
241 248
242static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable) 249static void __modify_IO_APIC_irq(unsigned int irq, unsigned long enable, unsigned long disable)
243{ 250{
244 struct irq_pin_list *entry = irq_2_pin + irq; 251 struct irq_pin_list *entry = irq_2_pin + irq;
245 unsigned int pin, reg; 252 unsigned int pin, reg;
@@ -259,30 +266,32 @@ static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsign
259} 266}
260 267
261/* mask = 1 */ 268/* mask = 1 */
262static void __mask_IO_APIC_irq (unsigned int irq) 269static void __mask_IO_APIC_irq(unsigned int irq)
263{ 270{
264 __modify_IO_APIC_irq(irq, 0x00010000, 0); 271 __modify_IO_APIC_irq(irq, IO_APIC_REDIR_MASKED, 0);
265} 272}
266 273
267/* mask = 0 */ 274/* mask = 0 */
268static void __unmask_IO_APIC_irq (unsigned int irq) 275static void __unmask_IO_APIC_irq(unsigned int irq)
269{ 276{
270 __modify_IO_APIC_irq(irq, 0, 0x00010000); 277 __modify_IO_APIC_irq(irq, 0, IO_APIC_REDIR_MASKED);
271} 278}
272 279
273/* mask = 1, trigger = 0 */ 280/* mask = 1, trigger = 0 */
274static void __mask_and_edge_IO_APIC_irq (unsigned int irq) 281static void __mask_and_edge_IO_APIC_irq(unsigned int irq)
275{ 282{
276 __modify_IO_APIC_irq(irq, 0x00010000, 0x00008000); 283 __modify_IO_APIC_irq(irq, IO_APIC_REDIR_MASKED,
284 IO_APIC_REDIR_LEVEL_TRIGGER);
277} 285}
278 286
279/* mask = 0, trigger = 1 */ 287/* mask = 0, trigger = 1 */
280static void __unmask_and_level_IO_APIC_irq (unsigned int irq) 288static void __unmask_and_level_IO_APIC_irq(unsigned int irq)
281{ 289{
282 __modify_IO_APIC_irq(irq, 0x00008000, 0x00010000); 290 __modify_IO_APIC_irq(irq, IO_APIC_REDIR_LEVEL_TRIGGER,
291 IO_APIC_REDIR_MASKED);
283} 292}
284 293
285static void mask_IO_APIC_irq (unsigned int irq) 294static void mask_IO_APIC_irq(unsigned int irq)
286{ 295{
287 unsigned long flags; 296 unsigned long flags;
288 297
@@ -291,7 +300,7 @@ static void mask_IO_APIC_irq (unsigned int irq)
291 spin_unlock_irqrestore(&ioapic_lock, flags); 300 spin_unlock_irqrestore(&ioapic_lock, flags);
292} 301}
293 302
294static void unmask_IO_APIC_irq (unsigned int irq) 303static void unmask_IO_APIC_irq(unsigned int irq)
295{ 304{
296 unsigned long flags; 305 unsigned long flags;
297 306
@@ -303,7 +312,7 @@ static void unmask_IO_APIC_irq (unsigned int irq)
303static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) 312static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
304{ 313{
305 struct IO_APIC_route_entry entry; 314 struct IO_APIC_route_entry entry;
306 315
307 /* Check delivery_mode to be sure we're not clearing an SMI pin */ 316 /* Check delivery_mode to be sure we're not clearing an SMI pin */
308 entry = ioapic_read_entry(apic, pin); 317 entry = ioapic_read_entry(apic, pin);
309 if (entry.delivery_mode == dest_SMI) 318 if (entry.delivery_mode == dest_SMI)
@@ -315,7 +324,7 @@ static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
315 ioapic_mask_entry(apic, pin); 324 ioapic_mask_entry(apic, pin);
316} 325}
317 326
318static void clear_IO_APIC (void) 327static void clear_IO_APIC(void)
319{ 328{
320 int apic, pin; 329 int apic, pin;
321 330
@@ -332,7 +341,7 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t cpumask)
332 struct irq_pin_list *entry = irq_2_pin + irq; 341 struct irq_pin_list *entry = irq_2_pin + irq;
333 unsigned int apicid_value; 342 unsigned int apicid_value;
334 cpumask_t tmp; 343 cpumask_t tmp;
335 344
336 cpus_and(tmp, cpumask, cpu_online_map); 345 cpus_and(tmp, cpumask, cpu_online_map);
337 if (cpus_empty(tmp)) 346 if (cpus_empty(tmp))
338 tmp = TARGET_CPUS; 347 tmp = TARGET_CPUS;
@@ -361,7 +370,7 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t cpumask)
361# include <linux/kernel_stat.h> /* kstat */ 370# include <linux/kernel_stat.h> /* kstat */
362# include <linux/slab.h> /* kmalloc() */ 371# include <linux/slab.h> /* kmalloc() */
363# include <linux/timer.h> 372# include <linux/timer.h>
364 373
365#define IRQBALANCE_CHECK_ARCH -999 374#define IRQBALANCE_CHECK_ARCH -999
366#define MAX_BALANCED_IRQ_INTERVAL (5*HZ) 375#define MAX_BALANCED_IRQ_INTERVAL (5*HZ)
367#define MIN_BALANCED_IRQ_INTERVAL (HZ/2) 376#define MIN_BALANCED_IRQ_INTERVAL (HZ/2)
@@ -373,14 +382,14 @@ static int physical_balance __read_mostly;
373static long balanced_irq_interval __read_mostly = MAX_BALANCED_IRQ_INTERVAL; 382static long balanced_irq_interval __read_mostly = MAX_BALANCED_IRQ_INTERVAL;
374 383
375static struct irq_cpu_info { 384static struct irq_cpu_info {
376 unsigned long * last_irq; 385 unsigned long *last_irq;
377 unsigned long * irq_delta; 386 unsigned long *irq_delta;
378 unsigned long irq; 387 unsigned long irq;
379} irq_cpu_data[NR_CPUS]; 388} irq_cpu_data[NR_CPUS];
380 389
381#define CPU_IRQ(cpu) (irq_cpu_data[cpu].irq) 390#define CPU_IRQ(cpu) (irq_cpu_data[cpu].irq)
382#define LAST_CPU_IRQ(cpu,irq) (irq_cpu_data[cpu].last_irq[irq]) 391#define LAST_CPU_IRQ(cpu, irq) (irq_cpu_data[cpu].last_irq[irq])
383#define IRQ_DELTA(cpu,irq) (irq_cpu_data[cpu].irq_delta[irq]) 392#define IRQ_DELTA(cpu, irq) (irq_cpu_data[cpu].irq_delta[irq])
384 393
385#define IDLE_ENOUGH(cpu,now) \ 394#define IDLE_ENOUGH(cpu,now) \
386 (idle_cpu(cpu) && ((now) - per_cpu(irq_stat, (cpu)).idle_timestamp > 1)) 395 (idle_cpu(cpu) && ((now) - per_cpu(irq_stat, (cpu)).idle_timestamp > 1))
@@ -419,8 +428,8 @@ inside:
419 if (cpu == -1) 428 if (cpu == -1)
420 cpu = NR_CPUS-1; 429 cpu = NR_CPUS-1;
421 } 430 }
422 } while (!cpu_online(cpu) || !IRQ_ALLOWED(cpu,allowed_mask) || 431 } while (!cpu_online(cpu) || !IRQ_ALLOWED(cpu, allowed_mask) ||
423 (search_idle && !IDLE_ENOUGH(cpu,now))); 432 (search_idle && !IDLE_ENOUGH(cpu, now)));
424 433
425 return cpu; 434 return cpu;
426} 435}
@@ -430,15 +439,14 @@ static inline void balance_irq(int cpu, int irq)
430 unsigned long now = jiffies; 439 unsigned long now = jiffies;
431 cpumask_t allowed_mask; 440 cpumask_t allowed_mask;
432 unsigned int new_cpu; 441 unsigned int new_cpu;
433 442
434 if (irqbalance_disabled) 443 if (irqbalance_disabled)
435 return; 444 return;
436 445
437 cpus_and(allowed_mask, cpu_online_map, balance_irq_affinity[irq]); 446 cpus_and(allowed_mask, cpu_online_map, balance_irq_affinity[irq]);
438 new_cpu = move(cpu, allowed_mask, now, 1); 447 new_cpu = move(cpu, allowed_mask, now, 1);
439 if (cpu != new_cpu) { 448 if (cpu != new_cpu)
440 set_pending_irq(irq, cpumask_of_cpu(new_cpu)); 449 set_pending_irq(irq, cpumask_of_cpu(new_cpu));
441 }
442} 450}
443 451
444static inline void rotate_irqs_among_cpus(unsigned long useful_load_threshold) 452static inline void rotate_irqs_among_cpus(unsigned long useful_load_threshold)
@@ -450,14 +458,14 @@ static inline void rotate_irqs_among_cpus(unsigned long useful_load_threshold)
450 if (!irq_desc[j].action) 458 if (!irq_desc[j].action)
451 continue; 459 continue;
452 /* Is it a significant load ? */ 460 /* Is it a significant load ? */
453 if (IRQ_DELTA(CPU_TO_PACKAGEINDEX(i),j) < 461 if (IRQ_DELTA(CPU_TO_PACKAGEINDEX(i), j) <
454 useful_load_threshold) 462 useful_load_threshold)
455 continue; 463 continue;
456 balance_irq(i, j); 464 balance_irq(i, j);
457 } 465 }
458 } 466 }
459 balanced_irq_interval = max((long)MIN_BALANCED_IRQ_INTERVAL, 467 balanced_irq_interval = max((long)MIN_BALANCED_IRQ_INTERVAL,
460 balanced_irq_interval - BALANCED_IRQ_LESS_DELTA); 468 balanced_irq_interval - BALANCED_IRQ_LESS_DELTA);
461 return; 469 return;
462} 470}
463 471
@@ -486,22 +494,22 @@ static void do_irq_balance(void)
486 /* Is this an active IRQ or balancing disabled ? */ 494 /* Is this an active IRQ or balancing disabled ? */
487 if (!irq_desc[j].action || irq_balancing_disabled(j)) 495 if (!irq_desc[j].action || irq_balancing_disabled(j))
488 continue; 496 continue;
489 if ( package_index == i ) 497 if (package_index == i)
490 IRQ_DELTA(package_index,j) = 0; 498 IRQ_DELTA(package_index, j) = 0;
491 /* Determine the total count per processor per IRQ */ 499 /* Determine the total count per processor per IRQ */
492 value_now = (unsigned long) kstat_cpu(i).irqs[j]; 500 value_now = (unsigned long) kstat_cpu(i).irqs[j];
493 501
494 /* Determine the activity per processor per IRQ */ 502 /* Determine the activity per processor per IRQ */
495 delta = value_now - LAST_CPU_IRQ(i,j); 503 delta = value_now - LAST_CPU_IRQ(i, j);
496 504
497 /* Update last_cpu_irq[][] for the next time */ 505 /* Update last_cpu_irq[][] for the next time */
498 LAST_CPU_IRQ(i,j) = value_now; 506 LAST_CPU_IRQ(i, j) = value_now;
499 507
500 /* Ignore IRQs whose rate is less than the clock */ 508 /* Ignore IRQs whose rate is less than the clock */
501 if (delta < useful_load_threshold) 509 if (delta < useful_load_threshold)
502 continue; 510 continue;
503 /* update the load for the processor or package total */ 511 /* update the load for the processor or package total */
504 IRQ_DELTA(package_index,j) += delta; 512 IRQ_DELTA(package_index, j) += delta;
505 513
506 /* Keep track of the higher numbered sibling as well */ 514 /* Keep track of the higher numbered sibling as well */
507 if (i != package_index) 515 if (i != package_index)
@@ -527,7 +535,8 @@ static void do_irq_balance(void)
527 max_cpu_irq = ULONG_MAX; 535 max_cpu_irq = ULONG_MAX;
528 536
529tryanothercpu: 537tryanothercpu:
530 /* Look for heaviest loaded processor. 538 /*
539 * Look for heaviest loaded processor.
531 * We may come back to get the next heaviest loaded processor. 540 * We may come back to get the next heaviest loaded processor.
532 * Skip processors with trivial loads. 541 * Skip processors with trivial loads.
533 */ 542 */
@@ -536,7 +545,7 @@ tryanothercpu:
536 for_each_online_cpu(i) { 545 for_each_online_cpu(i) {
537 if (i != CPU_TO_PACKAGEINDEX(i)) 546 if (i != CPU_TO_PACKAGEINDEX(i))
538 continue; 547 continue;
539 if (max_cpu_irq <= CPU_IRQ(i)) 548 if (max_cpu_irq <= CPU_IRQ(i))
540 continue; 549 continue;
541 if (tmp_cpu_irq < CPU_IRQ(i)) { 550 if (tmp_cpu_irq < CPU_IRQ(i)) {
542 tmp_cpu_irq = CPU_IRQ(i); 551 tmp_cpu_irq = CPU_IRQ(i);
@@ -545,8 +554,9 @@ tryanothercpu:
545 } 554 }
546 555
547 if (tmp_loaded == -1) { 556 if (tmp_loaded == -1) {
548 /* In the case of small number of heavy interrupt sources, 557 /*
549 * loading some of the cpus too much. We use Ingo's original 558 * In the case of small number of heavy interrupt sources,
559 * loading some of the cpus too much. We use Ingo's original
550 * approach to rotate them around. 560 * approach to rotate them around.
551 */ 561 */
552 if (!first_attempt && imbalance >= useful_load_threshold) { 562 if (!first_attempt && imbalance >= useful_load_threshold) {
@@ -555,13 +565,14 @@ tryanothercpu:
555 } 565 }
556 goto not_worth_the_effort; 566 goto not_worth_the_effort;
557 } 567 }
558 568
559 first_attempt = 0; /* heaviest search */ 569 first_attempt = 0; /* heaviest search */
560 max_cpu_irq = tmp_cpu_irq; /* load */ 570 max_cpu_irq = tmp_cpu_irq; /* load */
561 max_loaded = tmp_loaded; /* processor */ 571 max_loaded = tmp_loaded; /* processor */
562 imbalance = (max_cpu_irq - min_cpu_irq) / 2; 572 imbalance = (max_cpu_irq - min_cpu_irq) / 2;
563 573
564 /* if imbalance is less than approx 10% of max load, then 574 /*
575 * if imbalance is less than approx 10% of max load, then
565 * observe diminishing returns action. - quit 576 * observe diminishing returns action. - quit
566 */ 577 */
567 if (imbalance < (max_cpu_irq >> 3)) 578 if (imbalance < (max_cpu_irq >> 3))
@@ -577,26 +588,25 @@ tryanotherirq:
577 /* Is this an active IRQ? */ 588 /* Is this an active IRQ? */
578 if (!irq_desc[j].action) 589 if (!irq_desc[j].action)
579 continue; 590 continue;
580 if (imbalance <= IRQ_DELTA(max_loaded,j)) 591 if (imbalance <= IRQ_DELTA(max_loaded, j))
581 continue; 592 continue;
582 /* Try to find the IRQ that is closest to the imbalance 593 /* Try to find the IRQ that is closest to the imbalance
583 * without going over. 594 * without going over.
584 */ 595 */
585 if (move_this_load < IRQ_DELTA(max_loaded,j)) { 596 if (move_this_load < IRQ_DELTA(max_loaded, j)) {
586 move_this_load = IRQ_DELTA(max_loaded,j); 597 move_this_load = IRQ_DELTA(max_loaded, j);
587 selected_irq = j; 598 selected_irq = j;
588 } 599 }
589 } 600 }
590 if (selected_irq == -1) { 601 if (selected_irq == -1)
591 goto tryanothercpu; 602 goto tryanothercpu;
592 }
593 603
594 imbalance = move_this_load; 604 imbalance = move_this_load;
595 605
596 /* For physical_balance case, we accumulated both load 606 /* For physical_balance case, we accumulated both load
597 * values in the one of the siblings cpu_irq[], 607 * values in the one of the siblings cpu_irq[],
598 * to use the same code for physical and logical processors 608 * to use the same code for physical and logical processors
599 * as much as possible. 609 * as much as possible.
600 * 610 *
601 * NOTE: the cpu_irq[] array holds the sum of the load for 611 * NOTE: the cpu_irq[] array holds the sum of the load for
602 * sibling A and sibling B in the slot for the lowest numbered 612 * sibling A and sibling B in the slot for the lowest numbered
@@ -625,11 +635,11 @@ tryanotherirq:
625 /* mark for change destination */ 635 /* mark for change destination */
626 set_pending_irq(selected_irq, cpumask_of_cpu(min_loaded)); 636 set_pending_irq(selected_irq, cpumask_of_cpu(min_loaded));
627 637
628 /* Since we made a change, come back sooner to 638 /* Since we made a change, come back sooner to
629 * check for more variation. 639 * check for more variation.
630 */ 640 */
631 balanced_irq_interval = max((long)MIN_BALANCED_IRQ_INTERVAL, 641 balanced_irq_interval = max((long)MIN_BALANCED_IRQ_INTERVAL,
632 balanced_irq_interval - BALANCED_IRQ_LESS_DELTA); 642 balanced_irq_interval - BALANCED_IRQ_LESS_DELTA);
633 return; 643 return;
634 } 644 }
635 goto tryanotherirq; 645 goto tryanotherirq;
@@ -640,7 +650,7 @@ not_worth_the_effort:
640 * upward 650 * upward
641 */ 651 */
642 balanced_irq_interval = min((long)MAX_BALANCED_IRQ_INTERVAL, 652 balanced_irq_interval = min((long)MAX_BALANCED_IRQ_INTERVAL,
643 balanced_irq_interval + BALANCED_IRQ_MORE_DELTA); 653 balanced_irq_interval + BALANCED_IRQ_MORE_DELTA);
644 return; 654 return;
645} 655}
646 656
@@ -679,13 +689,13 @@ static int __init balanced_irq_init(void)
679 cpumask_t tmp; 689 cpumask_t tmp;
680 690
681 cpus_shift_right(tmp, cpu_online_map, 2); 691 cpus_shift_right(tmp, cpu_online_map, 2);
682 c = &boot_cpu_data; 692 c = &boot_cpu_data;
683 /* When not overwritten by the command line ask subarchitecture. */ 693 /* When not overwritten by the command line ask subarchitecture. */
684 if (irqbalance_disabled == IRQBALANCE_CHECK_ARCH) 694 if (irqbalance_disabled == IRQBALANCE_CHECK_ARCH)
685 irqbalance_disabled = NO_BALANCE_IRQ; 695 irqbalance_disabled = NO_BALANCE_IRQ;
686 if (irqbalance_disabled) 696 if (irqbalance_disabled)
687 return 0; 697 return 0;
688 698
689 /* disable irqbalance completely if there is only one processor online */ 699 /* disable irqbalance completely if there is only one processor online */
690 if (num_online_cpus() < 2) { 700 if (num_online_cpus() < 2) {
691 irqbalance_disabled = 1; 701 irqbalance_disabled = 1;
@@ -699,16 +709,14 @@ static int __init balanced_irq_init(void)
699 physical_balance = 1; 709 physical_balance = 1;
700 710
701 for_each_online_cpu(i) { 711 for_each_online_cpu(i) {
702 irq_cpu_data[i].irq_delta = kmalloc(sizeof(unsigned long) * NR_IRQS, GFP_KERNEL); 712 irq_cpu_data[i].irq_delta = kzalloc(sizeof(unsigned long) * NR_IRQS, GFP_KERNEL);
703 irq_cpu_data[i].last_irq = kmalloc(sizeof(unsigned long) * NR_IRQS, GFP_KERNEL); 713 irq_cpu_data[i].last_irq = kzalloc(sizeof(unsigned long) * NR_IRQS, GFP_KERNEL);
704 if (irq_cpu_data[i].irq_delta == NULL || irq_cpu_data[i].last_irq == NULL) { 714 if (irq_cpu_data[i].irq_delta == NULL || irq_cpu_data[i].last_irq == NULL) {
705 printk(KERN_ERR "balanced_irq_init: out of memory"); 715 printk(KERN_ERR "balanced_irq_init: out of memory");
706 goto failed; 716 goto failed;
707 } 717 }
708 memset(irq_cpu_data[i].irq_delta,0,sizeof(unsigned long) * NR_IRQS);
709 memset(irq_cpu_data[i].last_irq,0,sizeof(unsigned long) * NR_IRQS);
710 } 718 }
711 719
712 printk(KERN_INFO "Starting balanced_irq\n"); 720 printk(KERN_INFO "Starting balanced_irq\n");
713 if (!IS_ERR(kthread_run(balanced_irq, NULL, "kirqd"))) 721 if (!IS_ERR(kthread_run(balanced_irq, NULL, "kirqd")))
714 return 0; 722 return 0;
@@ -801,10 +809,10 @@ static int find_irq_entry(int apic, int pin, int type)
801 int i; 809 int i;
802 810
803 for (i = 0; i < mp_irq_entries; i++) 811 for (i = 0; i < mp_irq_entries; i++)
804 if (mp_irqs[i].mpc_irqtype == type && 812 if (mp_irqs[i].mp_irqtype == type &&
805 (mp_irqs[i].mpc_dstapic == mp_ioapics[apic].mpc_apicid || 813 (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
806 mp_irqs[i].mpc_dstapic == MP_APIC_ALL) && 814 mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
807 mp_irqs[i].mpc_dstirq == pin) 815 mp_irqs[i].mp_dstirq == pin)
808 return i; 816 return i;
809 817
810 return -1; 818 return -1;
@@ -818,13 +826,13 @@ static int __init find_isa_irq_pin(int irq, int type)
818 int i; 826 int i;
819 827
820 for (i = 0; i < mp_irq_entries; i++) { 828 for (i = 0; i < mp_irq_entries; i++) {
821 int lbus = mp_irqs[i].mpc_srcbus; 829 int lbus = mp_irqs[i].mp_srcbus;
822 830
823 if (test_bit(lbus, mp_bus_not_pci) && 831 if (test_bit(lbus, mp_bus_not_pci) &&
824 (mp_irqs[i].mpc_irqtype == type) && 832 (mp_irqs[i].mp_irqtype == type) &&
825 (mp_irqs[i].mpc_srcbusirq == irq)) 833 (mp_irqs[i].mp_srcbusirq == irq))
826 834
827 return mp_irqs[i].mpc_dstirq; 835 return mp_irqs[i].mp_dstirq;
828 } 836 }
829 return -1; 837 return -1;
830} 838}
@@ -834,17 +842,17 @@ static int __init find_isa_irq_apic(int irq, int type)
834 int i; 842 int i;
835 843
836 for (i = 0; i < mp_irq_entries; i++) { 844 for (i = 0; i < mp_irq_entries; i++) {
837 int lbus = mp_irqs[i].mpc_srcbus; 845 int lbus = mp_irqs[i].mp_srcbus;
838 846
839 if (test_bit(lbus, mp_bus_not_pci) && 847 if (test_bit(lbus, mp_bus_not_pci) &&
840 (mp_irqs[i].mpc_irqtype == type) && 848 (mp_irqs[i].mp_irqtype == type) &&
841 (mp_irqs[i].mpc_srcbusirq == irq)) 849 (mp_irqs[i].mp_srcbusirq == irq))
842 break; 850 break;
843 } 851 }
844 if (i < mp_irq_entries) { 852 if (i < mp_irq_entries) {
845 int apic; 853 int apic;
846 for(apic = 0; apic < nr_ioapics; apic++) { 854 for (apic = 0; apic < nr_ioapics; apic++) {
847 if (mp_ioapics[apic].mpc_apicid == mp_irqs[i].mpc_dstapic) 855 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
848 return apic; 856 return apic;
849 } 857 }
850 } 858 }
@@ -864,28 +872,28 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
864 872
865 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, " 873 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, "
866 "slot:%d, pin:%d.\n", bus, slot, pin); 874 "slot:%d, pin:%d.\n", bus, slot, pin);
867 if (mp_bus_id_to_pci_bus[bus] == -1) { 875 if (test_bit(bus, mp_bus_not_pci)) {
868 printk(KERN_WARNING "PCI BIOS passed nonexistent PCI bus %d!\n", bus); 876 printk(KERN_WARNING "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
869 return -1; 877 return -1;
870 } 878 }
871 for (i = 0; i < mp_irq_entries; i++) { 879 for (i = 0; i < mp_irq_entries; i++) {
872 int lbus = mp_irqs[i].mpc_srcbus; 880 int lbus = mp_irqs[i].mp_srcbus;
873 881
874 for (apic = 0; apic < nr_ioapics; apic++) 882 for (apic = 0; apic < nr_ioapics; apic++)
875 if (mp_ioapics[apic].mpc_apicid == mp_irqs[i].mpc_dstapic || 883 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
876 mp_irqs[i].mpc_dstapic == MP_APIC_ALL) 884 mp_irqs[i].mp_dstapic == MP_APIC_ALL)
877 break; 885 break;
878 886
879 if (!test_bit(lbus, mp_bus_not_pci) && 887 if (!test_bit(lbus, mp_bus_not_pci) &&
880 !mp_irqs[i].mpc_irqtype && 888 !mp_irqs[i].mp_irqtype &&
881 (bus == lbus) && 889 (bus == lbus) &&
882 (slot == ((mp_irqs[i].mpc_srcbusirq >> 2) & 0x1f))) { 890 (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
883 int irq = pin_2_irq(i,apic,mp_irqs[i].mpc_dstirq); 891 int irq = pin_2_irq(i, apic, mp_irqs[i].mp_dstirq);
884 892
885 if (!(apic || IO_APIC_IRQ(irq))) 893 if (!(apic || IO_APIC_IRQ(irq)))
886 continue; 894 continue;
887 895
888 if (pin == (mp_irqs[i].mpc_srcbusirq & 3)) 896 if (pin == (mp_irqs[i].mp_srcbusirq & 3))
889 return irq; 897 return irq;
890 /* 898 /*
891 * Use the first all-but-pin matching entry as a 899 * Use the first all-but-pin matching entry as a
@@ -900,7 +908,7 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
900EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector); 908EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
901 909
902/* 910/*
903 * This function currently is only a helper for the i386 smp boot process where 911 * This function currently is only a helper for the i386 smp boot process where
904 * we need to reprogram the ioredtbls to cater for the cpus which have come online 912 * we need to reprogram the ioredtbls to cater for the cpus which have come online
905 * so mask in all cases should simply be TARGET_CPUS 913 * so mask in all cases should simply be TARGET_CPUS
906 */ 914 */
@@ -952,7 +960,7 @@ static int EISA_ELCR(unsigned int irq)
952 * EISA conforming in the MP table, that means its trigger type must 960 * EISA conforming in the MP table, that means its trigger type must
953 * be read in from the ELCR */ 961 * be read in from the ELCR */
954 962
955#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].mpc_srcbusirq)) 963#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
956#define default_EISA_polarity(idx) default_ISA_polarity(idx) 964#define default_EISA_polarity(idx) default_ISA_polarity(idx)
957 965
958/* PCI interrupts are always polarity one level triggered, 966/* PCI interrupts are always polarity one level triggered,
@@ -969,118 +977,115 @@ static int EISA_ELCR(unsigned int irq)
969 977
970static int MPBIOS_polarity(int idx) 978static int MPBIOS_polarity(int idx)
971{ 979{
972 int bus = mp_irqs[idx].mpc_srcbus; 980 int bus = mp_irqs[idx].mp_srcbus;
973 int polarity; 981 int polarity;
974 982
975 /* 983 /*
976 * Determine IRQ line polarity (high active or low active): 984 * Determine IRQ line polarity (high active or low active):
977 */ 985 */
978 switch (mp_irqs[idx].mpc_irqflag & 3) 986 switch (mp_irqs[idx].mp_irqflag & 3) {
987 case 0: /* conforms, ie. bus-type dependent polarity */
979 { 988 {
980 case 0: /* conforms, ie. bus-type dependent polarity */ 989 polarity = test_bit(bus, mp_bus_not_pci)?
981 { 990 default_ISA_polarity(idx):
982 polarity = test_bit(bus, mp_bus_not_pci)? 991 default_PCI_polarity(idx);
983 default_ISA_polarity(idx): 992 break;
984 default_PCI_polarity(idx); 993 }
985 break; 994 case 1: /* high active */
986 } 995 {
987 case 1: /* high active */ 996 polarity = 0;
988 { 997 break;
989 polarity = 0; 998 }
990 break; 999 case 2: /* reserved */
991 } 1000 {
992 case 2: /* reserved */ 1001 printk(KERN_WARNING "broken BIOS!!\n");
993 { 1002 polarity = 1;
994 printk(KERN_WARNING "broken BIOS!!\n"); 1003 break;
995 polarity = 1; 1004 }
996 break; 1005 case 3: /* low active */
997 } 1006 {
998 case 3: /* low active */ 1007 polarity = 1;
999 { 1008 break;
1000 polarity = 1; 1009 }
1001 break; 1010 default: /* invalid */
1002 } 1011 {
1003 default: /* invalid */ 1012 printk(KERN_WARNING "broken BIOS!!\n");
1004 { 1013 polarity = 1;
1005 printk(KERN_WARNING "broken BIOS!!\n"); 1014 break;
1006 polarity = 1; 1015 }
1007 break;
1008 }
1009 } 1016 }
1010 return polarity; 1017 return polarity;
1011} 1018}
1012 1019
1013static int MPBIOS_trigger(int idx) 1020static int MPBIOS_trigger(int idx)
1014{ 1021{
1015 int bus = mp_irqs[idx].mpc_srcbus; 1022 int bus = mp_irqs[idx].mp_srcbus;
1016 int trigger; 1023 int trigger;
1017 1024
1018 /* 1025 /*
1019 * Determine IRQ trigger mode (edge or level sensitive): 1026 * Determine IRQ trigger mode (edge or level sensitive):
1020 */ 1027 */
1021 switch ((mp_irqs[idx].mpc_irqflag>>2) & 3) 1028 switch ((mp_irqs[idx].mp_irqflag>>2) & 3) {
1029 case 0: /* conforms, ie. bus-type dependent */
1022 { 1030 {
1023 case 0: /* conforms, ie. bus-type dependent */ 1031 trigger = test_bit(bus, mp_bus_not_pci)?
1024 { 1032 default_ISA_trigger(idx):
1025 trigger = test_bit(bus, mp_bus_not_pci)? 1033 default_PCI_trigger(idx);
1026 default_ISA_trigger(idx):
1027 default_PCI_trigger(idx);
1028#if defined(CONFIG_EISA) || defined(CONFIG_MCA) 1034#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
1029 switch (mp_bus_id_to_type[bus]) 1035 switch (mp_bus_id_to_type[bus]) {
1030 { 1036 case MP_BUS_ISA: /* ISA pin */
1031 case MP_BUS_ISA: /* ISA pin */ 1037 {
1032 { 1038 /* set before the switch */
1033 /* set before the switch */
1034 break;
1035 }
1036 case MP_BUS_EISA: /* EISA pin */
1037 {
1038 trigger = default_EISA_trigger(idx);
1039 break;
1040 }
1041 case MP_BUS_PCI: /* PCI pin */
1042 {
1043 /* set before the switch */
1044 break;
1045 }
1046 case MP_BUS_MCA: /* MCA pin */
1047 {
1048 trigger = default_MCA_trigger(idx);
1049 break;
1050 }
1051 default:
1052 {
1053 printk(KERN_WARNING "broken BIOS!!\n");
1054 trigger = 1;
1055 break;
1056 }
1057 }
1058#endif
1059 break; 1039 break;
1060 } 1040 }
1061 case 1: /* edge */ 1041 case MP_BUS_EISA: /* EISA pin */
1062 { 1042 {
1063 trigger = 0; 1043 trigger = default_EISA_trigger(idx);
1064 break; 1044 break;
1065 } 1045 }
1066 case 2: /* reserved */ 1046 case MP_BUS_PCI: /* PCI pin */
1067 { 1047 {
1068 printk(KERN_WARNING "broken BIOS!!\n"); 1048 /* set before the switch */
1069 trigger = 1;
1070 break; 1049 break;
1071 } 1050 }
1072 case 3: /* level */ 1051 case MP_BUS_MCA: /* MCA pin */
1073 { 1052 {
1074 trigger = 1; 1053 trigger = default_MCA_trigger(idx);
1075 break; 1054 break;
1076 } 1055 }
1077 default: /* invalid */ 1056 default:
1078 { 1057 {
1079 printk(KERN_WARNING "broken BIOS!!\n"); 1058 printk(KERN_WARNING "broken BIOS!!\n");
1080 trigger = 0; 1059 trigger = 1;
1081 break; 1060 break;
1082 } 1061 }
1083 } 1062 }
1063#endif
1064 break;
1065 }
1066 case 1: /* edge */
1067 {
1068 trigger = 0;
1069 break;
1070 }
1071 case 2: /* reserved */
1072 {
1073 printk(KERN_WARNING "broken BIOS!!\n");
1074 trigger = 1;
1075 break;
1076 }
1077 case 3: /* level */
1078 {
1079 trigger = 1;
1080 break;
1081 }
1082 default: /* invalid */
1083 {
1084 printk(KERN_WARNING "broken BIOS!!\n");
1085 trigger = 0;
1086 break;
1087 }
1088 }
1084 return trigger; 1089 return trigger;
1085} 1090}
1086 1091
@@ -1097,16 +1102,16 @@ static inline int irq_trigger(int idx)
1097static int pin_2_irq(int idx, int apic, int pin) 1102static int pin_2_irq(int idx, int apic, int pin)
1098{ 1103{
1099 int irq, i; 1104 int irq, i;
1100 int bus = mp_irqs[idx].mpc_srcbus; 1105 int bus = mp_irqs[idx].mp_srcbus;
1101 1106
1102 /* 1107 /*
1103 * Debugging check, we are in big trouble if this message pops up! 1108 * Debugging check, we are in big trouble if this message pops up!
1104 */ 1109 */
1105 if (mp_irqs[idx].mpc_dstirq != pin) 1110 if (mp_irqs[idx].mp_dstirq != pin)
1106 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n"); 1111 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
1107 1112
1108 if (test_bit(bus, mp_bus_not_pci)) 1113 if (test_bit(bus, mp_bus_not_pci))
1109 irq = mp_irqs[idx].mpc_srcbusirq; 1114 irq = mp_irqs[idx].mp_srcbusirq;
1110 else { 1115 else {
1111 /* 1116 /*
1112 * PCI IRQs are mapped in order 1117 * PCI IRQs are mapped in order
@@ -1148,8 +1153,8 @@ static inline int IO_APIC_irq_trigger(int irq)
1148 1153
1149 for (apic = 0; apic < nr_ioapics; apic++) { 1154 for (apic = 0; apic < nr_ioapics; apic++) {
1150 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { 1155 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1151 idx = find_irq_entry(apic,pin,mp_INT); 1156 idx = find_irq_entry(apic, pin, mp_INT);
1152 if ((idx != -1) && (irq == pin_2_irq(idx,apic,pin))) 1157 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1153 return irq_trigger(idx); 1158 return irq_trigger(idx);
1154 } 1159 }
1155 } 1160 }
@@ -1164,7 +1169,7 @@ static u8 irq_vector[NR_IRQ_VECTORS] __read_mostly = { FIRST_DEVICE_VECTOR , 0 }
1164 1169
1165static int __assign_irq_vector(int irq) 1170static int __assign_irq_vector(int irq)
1166{ 1171{
1167 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0; 1172 static int current_vector = FIRST_DEVICE_VECTOR, current_offset;
1168 int vector, offset; 1173 int vector, offset;
1169 1174
1170 BUG_ON((unsigned)irq >= NR_IRQ_VECTORS); 1175 BUG_ON((unsigned)irq >= NR_IRQ_VECTORS);
@@ -1176,7 +1181,7 @@ static int __assign_irq_vector(int irq)
1176 offset = current_offset; 1181 offset = current_offset;
1177next: 1182next:
1178 vector += 8; 1183 vector += 8;
1179 if (vector >= FIRST_SYSTEM_VECTOR) { 1184 if (vector >= first_system_vector) {
1180 offset = (offset + 1) % 8; 1185 offset = (offset + 1) % 8;
1181 vector = FIRST_DEVICE_VECTOR + offset; 1186 vector = FIRST_DEVICE_VECTOR + offset;
1182 } 1187 }
@@ -1203,6 +1208,11 @@ static int assign_irq_vector(int irq)
1203 1208
1204 return vector; 1209 return vector;
1205} 1210}
1211
1212void setup_vector_irq(int cpu)
1213{
1214}
1215
1206static struct irq_chip ioapic_chip; 1216static struct irq_chip ioapic_chip;
1207 1217
1208#define IOAPIC_AUTO -1 1218#define IOAPIC_AUTO -1
@@ -1237,25 +1247,25 @@ static void __init setup_IO_APIC_irqs(void)
1237 /* 1247 /*
1238 * add it to the IO-APIC irq-routing table: 1248 * add it to the IO-APIC irq-routing table:
1239 */ 1249 */
1240 memset(&entry,0,sizeof(entry)); 1250 memset(&entry, 0, sizeof(entry));
1241 1251
1242 entry.delivery_mode = INT_DELIVERY_MODE; 1252 entry.delivery_mode = INT_DELIVERY_MODE;
1243 entry.dest_mode = INT_DEST_MODE; 1253 entry.dest_mode = INT_DEST_MODE;
1244 entry.mask = 0; /* enable IRQ */ 1254 entry.mask = 0; /* enable IRQ */
1245 entry.dest.logical.logical_dest = 1255 entry.dest.logical.logical_dest =
1246 cpu_mask_to_apicid(TARGET_CPUS); 1256 cpu_mask_to_apicid(TARGET_CPUS);
1247 1257
1248 idx = find_irq_entry(apic,pin,mp_INT); 1258 idx = find_irq_entry(apic, pin, mp_INT);
1249 if (idx == -1) { 1259 if (idx == -1) {
1250 if (first_notcon) { 1260 if (first_notcon) {
1251 apic_printk(APIC_VERBOSE, KERN_DEBUG 1261 apic_printk(APIC_VERBOSE, KERN_DEBUG
1252 " IO-APIC (apicid-pin) %d-%d", 1262 " IO-APIC (apicid-pin) %d-%d",
1253 mp_ioapics[apic].mpc_apicid, 1263 mp_ioapics[apic].mp_apicid,
1254 pin); 1264 pin);
1255 first_notcon = 0; 1265 first_notcon = 0;
1256 } else 1266 } else
1257 apic_printk(APIC_VERBOSE, ", %d-%d", 1267 apic_printk(APIC_VERBOSE, ", %d-%d",
1258 mp_ioapics[apic].mpc_apicid, pin); 1268 mp_ioapics[apic].mp_apicid, pin);
1259 continue; 1269 continue;
1260 } 1270 }
1261 1271
@@ -1289,7 +1299,7 @@ static void __init setup_IO_APIC_irqs(void)
1289 vector = assign_irq_vector(irq); 1299 vector = assign_irq_vector(irq);
1290 entry.vector = vector; 1300 entry.vector = vector;
1291 ioapic_register_intr(irq, vector, IOAPIC_AUTO); 1301 ioapic_register_intr(irq, vector, IOAPIC_AUTO);
1292 1302
1293 if (!apic && (irq < 16)) 1303 if (!apic && (irq < 16))
1294 disable_8259A_irq(irq); 1304 disable_8259A_irq(irq);
1295 } 1305 }
@@ -1302,25 +1312,21 @@ static void __init setup_IO_APIC_irqs(void)
1302} 1312}
1303 1313
1304/* 1314/*
1305 * Set up the 8259A-master output pin: 1315 * Set up the timer pin, possibly with the 8259A-master behind.
1306 */ 1316 */
1307static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, int vector) 1317static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
1318 int vector)
1308{ 1319{
1309 struct IO_APIC_route_entry entry; 1320 struct IO_APIC_route_entry entry;
1310 1321
1311 memset(&entry,0,sizeof(entry)); 1322 memset(&entry, 0, sizeof(entry));
1312
1313 disable_8259A_irq(0);
1314
1315 /* mask LVT0 */
1316 apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
1317 1323
1318 /* 1324 /*
1319 * We use logical delivery to get the timer IRQ 1325 * We use logical delivery to get the timer IRQ
1320 * to the first CPU. 1326 * to the first CPU.
1321 */ 1327 */
1322 entry.dest_mode = INT_DEST_MODE; 1328 entry.dest_mode = INT_DEST_MODE;
1323 entry.mask = 0; /* unmask IRQ now */ 1329 entry.mask = 1; /* mask IRQ now */
1324 entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS); 1330 entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
1325 entry.delivery_mode = INT_DELIVERY_MODE; 1331 entry.delivery_mode = INT_DELIVERY_MODE;
1326 entry.polarity = 0; 1332 entry.polarity = 0;
@@ -1329,17 +1335,14 @@ static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, in
1329 1335
1330 /* 1336 /*
1331 * The timer IRQ doesn't have to know that behind the 1337 * The timer IRQ doesn't have to know that behind the
1332 * scene we have a 8259A-master in AEOI mode ... 1338 * scene we may have a 8259A-master in AEOI mode ...
1333 */ 1339 */
1334 irq_desc[0].chip = &ioapic_chip; 1340 ioapic_register_intr(0, vector, IOAPIC_EDGE);
1335 set_irq_handler(0, handle_edge_irq);
1336 1341
1337 /* 1342 /*
1338 * Add it to the IO-APIC irq-routing table: 1343 * Add it to the IO-APIC irq-routing table:
1339 */ 1344 */
1340 ioapic_write_entry(apic, pin, entry); 1345 ioapic_write_entry(apic, pin, entry);
1341
1342 enable_8259A_irq(0);
1343} 1346}
1344 1347
1345void __init print_IO_APIC(void) 1348void __init print_IO_APIC(void)
@@ -1354,10 +1357,10 @@ void __init print_IO_APIC(void)
1354 if (apic_verbosity == APIC_QUIET) 1357 if (apic_verbosity == APIC_QUIET)
1355 return; 1358 return;
1356 1359
1357 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries); 1360 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1358 for (i = 0; i < nr_ioapics; i++) 1361 for (i = 0; i < nr_ioapics; i++)
1359 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n", 1362 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1360 mp_ioapics[i].mpc_apicid, nr_ioapic_registers[i]); 1363 mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
1361 1364
1362 /* 1365 /*
1363 * We are a bit conservative about what we expect. We have to 1366 * We are a bit conservative about what we expect. We have to
@@ -1376,7 +1379,7 @@ void __init print_IO_APIC(void)
1376 reg_03.raw = io_apic_read(apic, 3); 1379 reg_03.raw = io_apic_read(apic, 3);
1377 spin_unlock_irqrestore(&ioapic_lock, flags); 1380 spin_unlock_irqrestore(&ioapic_lock, flags);
1378 1381
1379 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mpc_apicid); 1382 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
1380 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw); 1383 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1381 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID); 1384 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1382 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type); 1385 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
@@ -1459,7 +1462,7 @@ void __init print_IO_APIC(void)
1459 1462
1460#if 0 1463#if 0
1461 1464
1462static void print_APIC_bitfield (int base) 1465static void print_APIC_bitfield(int base)
1463{ 1466{
1464 unsigned int v; 1467 unsigned int v;
1465 int i, j; 1468 int i, j;
@@ -1480,7 +1483,7 @@ static void print_APIC_bitfield (int base)
1480 } 1483 }
1481} 1484}
1482 1485
1483void /*__init*/ print_local_APIC(void * dummy) 1486void /*__init*/ print_local_APIC(void *dummy)
1484{ 1487{
1485 unsigned int v, ver, maxlvt; 1488 unsigned int v, ver, maxlvt;
1486 1489
@@ -1489,6 +1492,7 @@ void /*__init*/ print_local_APIC(void * dummy)
1489 1492
1490 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n", 1493 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1491 smp_processor_id(), hard_smp_processor_id()); 1494 smp_processor_id(), hard_smp_processor_id());
1495 v = apic_read(APIC_ID);
1492 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, 1496 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v,
1493 GET_APIC_ID(read_apic_id())); 1497 GET_APIC_ID(read_apic_id()));
1494 v = apic_read(APIC_LVR); 1498 v = apic_read(APIC_LVR);
@@ -1563,9 +1567,9 @@ void /*__init*/ print_local_APIC(void * dummy)
1563 printk("\n"); 1567 printk("\n");
1564} 1568}
1565 1569
1566void print_all_local_APICs (void) 1570void print_all_local_APICs(void)
1567{ 1571{
1568 on_each_cpu(print_local_APIC, NULL, 1, 1); 1572 on_each_cpu(print_local_APIC, NULL, 1);
1569} 1573}
1570 1574
1571void /*__init*/ print_PIC(void) 1575void /*__init*/ print_PIC(void)
@@ -1586,11 +1590,11 @@ void /*__init*/ print_PIC(void)
1586 v = inb(0xa0) << 8 | inb(0x20); 1590 v = inb(0xa0) << 8 | inb(0x20);
1587 printk(KERN_DEBUG "... PIC IRR: %04x\n", v); 1591 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1588 1592
1589 outb(0x0b,0xa0); 1593 outb(0x0b, 0xa0);
1590 outb(0x0b,0x20); 1594 outb(0x0b, 0x20);
1591 v = inb(0xa0) << 8 | inb(0x20); 1595 v = inb(0xa0) << 8 | inb(0x20);
1592 outb(0x0a,0xa0); 1596 outb(0x0a, 0xa0);
1593 outb(0x0a,0x20); 1597 outb(0x0a, 0x20);
1594 1598
1595 spin_unlock_irqrestore(&i8259A_lock, flags); 1599 spin_unlock_irqrestore(&i8259A_lock, flags);
1596 1600
@@ -1626,7 +1630,7 @@ static void __init enable_IO_APIC(void)
1626 spin_unlock_irqrestore(&ioapic_lock, flags); 1630 spin_unlock_irqrestore(&ioapic_lock, flags);
1627 nr_ioapic_registers[apic] = reg_01.bits.entries+1; 1631 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
1628 } 1632 }
1629 for(apic = 0; apic < nr_ioapics; apic++) { 1633 for (apic = 0; apic < nr_ioapics; apic++) {
1630 int pin; 1634 int pin;
1631 /* See if any of the pins is in ExtINT mode */ 1635 /* See if any of the pins is in ExtINT mode */
1632 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { 1636 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
@@ -1716,7 +1720,6 @@ void disable_IO_APIC(void)
1716 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999 1720 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
1717 */ 1721 */
1718 1722
1719#ifndef CONFIG_X86_NUMAQ
1720static void __init setup_ioapic_ids_from_mpc(void) 1723static void __init setup_ioapic_ids_from_mpc(void)
1721{ 1724{
1722 union IO_APIC_reg_00 reg_00; 1725 union IO_APIC_reg_00 reg_00;
@@ -1726,6 +1729,11 @@ static void __init setup_ioapic_ids_from_mpc(void)
1726 unsigned char old_id; 1729 unsigned char old_id;
1727 unsigned long flags; 1730 unsigned long flags;
1728 1731
1732#ifdef CONFIG_X86_NUMAQ
1733 if (found_numaq)
1734 return;
1735#endif
1736
1729 /* 1737 /*
1730 * Don't check I/O APIC IDs for xAPIC systems. They have 1738 * Don't check I/O APIC IDs for xAPIC systems. They have
1731 * no meaning without the serial APIC bus. 1739 * no meaning without the serial APIC bus.
@@ -1748,15 +1756,15 @@ static void __init setup_ioapic_ids_from_mpc(void)
1748 spin_lock_irqsave(&ioapic_lock, flags); 1756 spin_lock_irqsave(&ioapic_lock, flags);
1749 reg_00.raw = io_apic_read(apic, 0); 1757 reg_00.raw = io_apic_read(apic, 0);
1750 spin_unlock_irqrestore(&ioapic_lock, flags); 1758 spin_unlock_irqrestore(&ioapic_lock, flags);
1751
1752 old_id = mp_ioapics[apic].mpc_apicid;
1753 1759
1754 if (mp_ioapics[apic].mpc_apicid >= get_physical_broadcast()) { 1760 old_id = mp_ioapics[apic].mp_apicid;
1761
1762 if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
1755 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n", 1763 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1756 apic, mp_ioapics[apic].mpc_apicid); 1764 apic, mp_ioapics[apic].mp_apicid);
1757 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1765 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1758 reg_00.bits.ID); 1766 reg_00.bits.ID);
1759 mp_ioapics[apic].mpc_apicid = reg_00.bits.ID; 1767 mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
1760 } 1768 }
1761 1769
1762 /* 1770 /*
@@ -1765,9 +1773,9 @@ static void __init setup_ioapic_ids_from_mpc(void)
1765 * 'stuck on smp_invalidate_needed IPI wait' messages. 1773 * 'stuck on smp_invalidate_needed IPI wait' messages.
1766 */ 1774 */
1767 if (check_apicid_used(phys_id_present_map, 1775 if (check_apicid_used(phys_id_present_map,
1768 mp_ioapics[apic].mpc_apicid)) { 1776 mp_ioapics[apic].mp_apicid)) {
1769 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n", 1777 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1770 apic, mp_ioapics[apic].mpc_apicid); 1778 apic, mp_ioapics[apic].mp_apicid);
1771 for (i = 0; i < get_physical_broadcast(); i++) 1779 for (i = 0; i < get_physical_broadcast(); i++)
1772 if (!physid_isset(i, phys_id_present_map)) 1780 if (!physid_isset(i, phys_id_present_map))
1773 break; 1781 break;
@@ -1776,13 +1784,13 @@ static void __init setup_ioapic_ids_from_mpc(void)
1776 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1784 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1777 i); 1785 i);
1778 physid_set(i, phys_id_present_map); 1786 physid_set(i, phys_id_present_map);
1779 mp_ioapics[apic].mpc_apicid = i; 1787 mp_ioapics[apic].mp_apicid = i;
1780 } else { 1788 } else {
1781 physid_mask_t tmp; 1789 physid_mask_t tmp;
1782 tmp = apicid_to_cpu_present(mp_ioapics[apic].mpc_apicid); 1790 tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
1783 apic_printk(APIC_VERBOSE, "Setting %d in the " 1791 apic_printk(APIC_VERBOSE, "Setting %d in the "
1784 "phys_id_present_map\n", 1792 "phys_id_present_map\n",
1785 mp_ioapics[apic].mpc_apicid); 1793 mp_ioapics[apic].mp_apicid);
1786 physids_or(phys_id_present_map, phys_id_present_map, tmp); 1794 physids_or(phys_id_present_map, phys_id_present_map, tmp);
1787 } 1795 }
1788 1796
@@ -1791,21 +1799,21 @@ static void __init setup_ioapic_ids_from_mpc(void)
1791 * We need to adjust the IRQ routing table 1799 * We need to adjust the IRQ routing table
1792 * if the ID changed. 1800 * if the ID changed.
1793 */ 1801 */
1794 if (old_id != mp_ioapics[apic].mpc_apicid) 1802 if (old_id != mp_ioapics[apic].mp_apicid)
1795 for (i = 0; i < mp_irq_entries; i++) 1803 for (i = 0; i < mp_irq_entries; i++)
1796 if (mp_irqs[i].mpc_dstapic == old_id) 1804 if (mp_irqs[i].mp_dstapic == old_id)
1797 mp_irqs[i].mpc_dstapic 1805 mp_irqs[i].mp_dstapic
1798 = mp_ioapics[apic].mpc_apicid; 1806 = mp_ioapics[apic].mp_apicid;
1799 1807
1800 /* 1808 /*
1801 * Read the right value from the MPC table and 1809 * Read the right value from the MPC table and
1802 * write it into the ID register. 1810 * write it into the ID register.
1803 */ 1811 */
1804 apic_printk(APIC_VERBOSE, KERN_INFO 1812 apic_printk(APIC_VERBOSE, KERN_INFO
1805 "...changing IO-APIC physical APIC ID to %d ...", 1813 "...changing IO-APIC physical APIC ID to %d ...",
1806 mp_ioapics[apic].mpc_apicid); 1814 mp_ioapics[apic].mp_apicid);
1807 1815
1808 reg_00.bits.ID = mp_ioapics[apic].mpc_apicid; 1816 reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
1809 spin_lock_irqsave(&ioapic_lock, flags); 1817 spin_lock_irqsave(&ioapic_lock, flags);
1810 io_apic_write(apic, 0, reg_00.raw); 1818 io_apic_write(apic, 0, reg_00.raw);
1811 spin_unlock_irqrestore(&ioapic_lock, flags); 1819 spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -1816,15 +1824,12 @@ static void __init setup_ioapic_ids_from_mpc(void)
1816 spin_lock_irqsave(&ioapic_lock, flags); 1824 spin_lock_irqsave(&ioapic_lock, flags);
1817 reg_00.raw = io_apic_read(apic, 0); 1825 reg_00.raw = io_apic_read(apic, 0);
1818 spin_unlock_irqrestore(&ioapic_lock, flags); 1826 spin_unlock_irqrestore(&ioapic_lock, flags);
1819 if (reg_00.bits.ID != mp_ioapics[apic].mpc_apicid) 1827 if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
1820 printk("could not set ID!\n"); 1828 printk("could not set ID!\n");
1821 else 1829 else
1822 apic_printk(APIC_VERBOSE, " ok.\n"); 1830 apic_printk(APIC_VERBOSE, " ok.\n");
1823 } 1831 }
1824} 1832}
1825#else
1826static void __init setup_ioapic_ids_from_mpc(void) { }
1827#endif
1828 1833
1829int no_timer_check __initdata; 1834int no_timer_check __initdata;
1830 1835
@@ -2015,12 +2020,12 @@ static inline void init_IO_APIC_traps(void)
2015 * The local APIC irq-chip implementation: 2020 * The local APIC irq-chip implementation:
2016 */ 2021 */
2017 2022
2018static void ack_apic(unsigned int irq) 2023static void ack_lapic_irq(unsigned int irq)
2019{ 2024{
2020 ack_APIC_irq(); 2025 ack_APIC_irq();
2021} 2026}
2022 2027
2023static void mask_lapic_irq (unsigned int irq) 2028static void mask_lapic_irq(unsigned int irq)
2024{ 2029{
2025 unsigned long v; 2030 unsigned long v;
2026 2031
@@ -2028,7 +2033,7 @@ static void mask_lapic_irq (unsigned int irq)
2028 apic_write_around(APIC_LVT0, v | APIC_LVT_MASKED); 2033 apic_write_around(APIC_LVT0, v | APIC_LVT_MASKED);
2029} 2034}
2030 2035
2031static void unmask_lapic_irq (unsigned int irq) 2036static void unmask_lapic_irq(unsigned int irq)
2032{ 2037{
2033 unsigned long v; 2038 unsigned long v;
2034 2039
@@ -2037,23 +2042,31 @@ static void unmask_lapic_irq (unsigned int irq)
2037} 2042}
2038 2043
2039static struct irq_chip lapic_chip __read_mostly = { 2044static struct irq_chip lapic_chip __read_mostly = {
2040 .name = "local-APIC-edge", 2045 .name = "local-APIC",
2041 .mask = mask_lapic_irq, 2046 .mask = mask_lapic_irq,
2042 .unmask = unmask_lapic_irq, 2047 .unmask = unmask_lapic_irq,
2043 .eoi = ack_apic, 2048 .ack = ack_lapic_irq,
2044}; 2049};
2045 2050
2051static void lapic_register_intr(int irq, int vector)
2052{
2053 irq_desc[irq].status &= ~IRQ_LEVEL;
2054 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2055 "edge");
2056 set_intr_gate(vector, interrupt[irq]);
2057}
2058
2046static void __init setup_nmi(void) 2059static void __init setup_nmi(void)
2047{ 2060{
2048 /* 2061 /*
2049 * Dirty trick to enable the NMI watchdog ... 2062 * Dirty trick to enable the NMI watchdog ...
2050 * We put the 8259A master into AEOI mode and 2063 * We put the 8259A master into AEOI mode and
2051 * unmask on all local APICs LVT0 as NMI. 2064 * unmask on all local APICs LVT0 as NMI.
2052 * 2065 *
2053 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire') 2066 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
2054 * is from Maciej W. Rozycki - so we do not have to EOI from 2067 * is from Maciej W. Rozycki - so we do not have to EOI from
2055 * the NMI handler or the timer interrupt. 2068 * the NMI handler or the timer interrupt.
2056 */ 2069 */
2057 apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ..."); 2070 apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");
2058 2071
2059 enable_NMI_through_LVT0(); 2072 enable_NMI_through_LVT0();
@@ -2129,11 +2142,16 @@ static inline void __init unlock_ExtINT_logic(void)
2129static inline void __init check_timer(void) 2142static inline void __init check_timer(void)
2130{ 2143{
2131 int apic1, pin1, apic2, pin2; 2144 int apic1, pin1, apic2, pin2;
2145 int no_pin1 = 0;
2132 int vector; 2146 int vector;
2147 unsigned int ver;
2133 unsigned long flags; 2148 unsigned long flags;
2134 2149
2135 local_irq_save(flags); 2150 local_irq_save(flags);
2136 2151
2152 ver = apic_read(APIC_LVR);
2153 ver = GET_APIC_VERSION(ver);
2154
2137 /* 2155 /*
2138 * get/set the timer IRQ vector: 2156 * get/set the timer IRQ vector:
2139 */ 2157 */
@@ -2142,17 +2160,17 @@ static inline void __init check_timer(void)
2142 set_intr_gate(vector, interrupt[0]); 2160 set_intr_gate(vector, interrupt[0]);
2143 2161
2144 /* 2162 /*
2145 * Subtle, code in do_timer_interrupt() expects an AEOI 2163 * As IRQ0 is to be enabled in the 8259A, the virtual
2146 * mode for the 8259A whenever interrupts are routed 2164 * wire has to be disabled in the local APIC. Also
2147 * through I/O APICs. Also IRQ0 has to be enabled in 2165 * timer interrupts need to be acknowledged manually in
2148 * the 8259A which implies the virtual wire has to be 2166 * the 8259A for the i82489DX when using the NMI
2149 * disabled in the local APIC. 2167 * watchdog as that APIC treats NMIs as level-triggered.
2168 * The AEOI mode will finish them in the 8259A
2169 * automatically.
2150 */ 2170 */
2151 apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT); 2171 apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2152 init_8259A(1); 2172 init_8259A(1);
2153 timer_ack = 1; 2173 timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
2154 if (timer_over_8254 > 0)
2155 enable_8259A_irq(0);
2156 2174
2157 pin1 = find_isa_irq_pin(0, mp_INT); 2175 pin1 = find_isa_irq_pin(0, mp_INT);
2158 apic1 = find_isa_irq_apic(0, mp_INT); 2176 apic1 = find_isa_irq_apic(0, mp_INT);
@@ -2162,14 +2180,33 @@ static inline void __init check_timer(void)
2162 printk(KERN_INFO "..TIMER: vector=0x%02X apic1=%d pin1=%d apic2=%d pin2=%d\n", 2180 printk(KERN_INFO "..TIMER: vector=0x%02X apic1=%d pin1=%d apic2=%d pin2=%d\n",
2163 vector, apic1, pin1, apic2, pin2); 2181 vector, apic1, pin1, apic2, pin2);
2164 2182
2183 /*
2184 * Some BIOS writers are clueless and report the ExtINTA
2185 * I/O APIC input from the cascaded 8259A as the timer
2186 * interrupt input. So just in case, if only one pin
2187 * was found above, try it both directly and through the
2188 * 8259A.
2189 */
2190 if (pin1 == -1) {
2191 pin1 = pin2;
2192 apic1 = apic2;
2193 no_pin1 = 1;
2194 } else if (pin2 == -1) {
2195 pin2 = pin1;
2196 apic2 = apic1;
2197 }
2198
2165 if (pin1 != -1) { 2199 if (pin1 != -1) {
2166 /* 2200 /*
2167 * Ok, does IRQ0 through the IOAPIC work? 2201 * Ok, does IRQ0 through the IOAPIC work?
2168 */ 2202 */
2203 if (no_pin1) {
2204 add_pin_to_irq(0, apic1, pin1);
2205 setup_timer_IRQ0_pin(apic1, pin1, vector);
2206 }
2169 unmask_IO_APIC_irq(0); 2207 unmask_IO_APIC_irq(0);
2170 if (timer_irq_works()) { 2208 if (timer_irq_works()) {
2171 if (nmi_watchdog == NMI_IO_APIC) { 2209 if (nmi_watchdog == NMI_IO_APIC) {
2172 disable_8259A_irq(0);
2173 setup_nmi(); 2210 setup_nmi();
2174 enable_8259A_irq(0); 2211 enable_8259A_irq(0);
2175 } 2212 }
@@ -2178,45 +2215,47 @@ static inline void __init check_timer(void)
2178 goto out; 2215 goto out;
2179 } 2216 }
2180 clear_IO_APIC_pin(apic1, pin1); 2217 clear_IO_APIC_pin(apic1, pin1);
2181 printk(KERN_ERR "..MP-BIOS bug: 8254 timer not connected to " 2218 if (!no_pin1)
2182 "IO-APIC\n"); 2219 printk(KERN_ERR "..MP-BIOS bug: "
2183 } 2220 "8254 timer not connected to IO-APIC\n");
2184 2221
2185 printk(KERN_INFO "...trying to set up timer (IRQ0) through the 8259A ... "); 2222 printk(KERN_INFO "...trying to set up timer (IRQ0) "
2186 if (pin2 != -1) { 2223 "through the 8259A ... ");
2187 printk("\n..... (found pin %d) ...", pin2); 2224 printk("\n..... (found pin %d) ...", pin2);
2188 /* 2225 /*
2189 * legacy devices should be connected to IO APIC #0 2226 * legacy devices should be connected to IO APIC #0
2190 */ 2227 */
2191 setup_ExtINT_IRQ0_pin(apic2, pin2, vector); 2228 replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
2229 setup_timer_IRQ0_pin(apic2, pin2, vector);
2230 unmask_IO_APIC_irq(0);
2231 enable_8259A_irq(0);
2192 if (timer_irq_works()) { 2232 if (timer_irq_works()) {
2193 printk("works.\n"); 2233 printk("works.\n");
2194 if (pin1 != -1) 2234 timer_through_8259 = 1;
2195 replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
2196 else
2197 add_pin_to_irq(0, apic2, pin2);
2198 if (nmi_watchdog == NMI_IO_APIC) { 2235 if (nmi_watchdog == NMI_IO_APIC) {
2236 disable_8259A_irq(0);
2199 setup_nmi(); 2237 setup_nmi();
2238 enable_8259A_irq(0);
2200 } 2239 }
2201 goto out; 2240 goto out;
2202 } 2241 }
2203 /* 2242 /*
2204 * Cleanup, just in case ... 2243 * Cleanup, just in case ...
2205 */ 2244 */
2245 disable_8259A_irq(0);
2206 clear_IO_APIC_pin(apic2, pin2); 2246 clear_IO_APIC_pin(apic2, pin2);
2247 printk(" failed.\n");
2207 } 2248 }
2208 printk(" failed.\n");
2209 2249
2210 if (nmi_watchdog == NMI_IO_APIC) { 2250 if (nmi_watchdog == NMI_IO_APIC) {
2211 printk(KERN_WARNING "timer doesn't work through the IO-APIC - disabling NMI Watchdog!\n"); 2251 printk(KERN_WARNING "timer doesn't work through the IO-APIC - disabling NMI Watchdog!\n");
2212 nmi_watchdog = 0; 2252 nmi_watchdog = NMI_NONE;
2213 } 2253 }
2254 timer_ack = 0;
2214 2255
2215 printk(KERN_INFO "...trying to set up timer as Virtual Wire IRQ..."); 2256 printk(KERN_INFO "...trying to set up timer as Virtual Wire IRQ...");
2216 2257
2217 disable_8259A_irq(0); 2258 lapic_register_intr(0, vector);
2218 set_irq_chip_and_handler_name(0, &lapic_chip, handle_fasteoi_irq,
2219 "fasteoi");
2220 apic_write_around(APIC_LVT0, APIC_DM_FIXED | vector); /* Fixed mode */ 2259 apic_write_around(APIC_LVT0, APIC_DM_FIXED | vector); /* Fixed mode */
2221 enable_8259A_irq(0); 2260 enable_8259A_irq(0);
2222 2261
@@ -2224,12 +2263,12 @@ static inline void __init check_timer(void)
2224 printk(" works.\n"); 2263 printk(" works.\n");
2225 goto out; 2264 goto out;
2226 } 2265 }
2266 disable_8259A_irq(0);
2227 apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | vector); 2267 apic_write_around(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | vector);
2228 printk(" failed.\n"); 2268 printk(" failed.\n");
2229 2269
2230 printk(KERN_INFO "...trying to set up timer as ExtINT IRQ..."); 2270 printk(KERN_INFO "...trying to set up timer as ExtINT IRQ...");
2231 2271
2232 timer_ack = 0;
2233 init_8259A(0); 2272 init_8259A(0);
2234 make_8259A_irq(0); 2273 make_8259A_irq(0);
2235 apic_write_around(APIC_LVT0, APIC_DM_EXTINT); 2274 apic_write_around(APIC_LVT0, APIC_DM_EXTINT);
@@ -2248,11 +2287,21 @@ out:
2248} 2287}
2249 2288
2250/* 2289/*
2251 * 2290 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2252 * IRQ's that are handled by the PIC in the MPS IOAPIC case. 2291 * to devices. However there may be an I/O APIC pin available for
2253 * - IRQ2 is the cascade IRQ, and cannot be a io-apic IRQ. 2292 * this interrupt regardless. The pin may be left unconnected, but
2254 * Linux doesn't really care, as it's not actually used 2293 * typically it will be reused as an ExtINT cascade interrupt for
2255 * for any interrupt handling anyway. 2294 * the master 8259A. In the MPS case such a pin will normally be
2295 * reported as an ExtINT interrupt in the MP table. With ACPI
2296 * there is no provision for ExtINT interrupts, and in the absence
2297 * of an override it would be treated as an ordinary ISA I/O APIC
2298 * interrupt, that is edge-triggered and unmasked by default. We
2299 * used to do this, but it caused problems on some systems because
2300 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2301 * the same ExtINT cascade interrupt to drive the local APIC of the
2302 * bootstrap processor. Therefore we refrain from routing IRQ2 to
2303 * the I/O APIC in all cases now. No actual device should request
2304 * it anyway. --macro
2256 */ 2305 */
2257#define PIC_IRQS (1 << PIC_CASCADE_IR) 2306#define PIC_IRQS (1 << PIC_CASCADE_IR)
2258 2307
@@ -2261,15 +2310,12 @@ void __init setup_IO_APIC(void)
2261 int i; 2310 int i;
2262 2311
2263 /* Reserve all the system vectors. */ 2312 /* Reserve all the system vectors. */
2264 for (i = FIRST_SYSTEM_VECTOR; i < NR_VECTORS; i++) 2313 for (i = first_system_vector; i < NR_VECTORS; i++)
2265 set_bit(i, used_vectors); 2314 set_bit(i, used_vectors);
2266 2315
2267 enable_IO_APIC(); 2316 enable_IO_APIC();
2268 2317
2269 if (acpi_ioapic) 2318 io_apic_irqs = ~PIC_IRQS;
2270 io_apic_irqs = ~0; /* all IRQs go through IOAPIC */
2271 else
2272 io_apic_irqs = ~PIC_IRQS;
2273 2319
2274 printk("ENABLING IO-APIC IRQs\n"); 2320 printk("ENABLING IO-APIC IRQs\n");
2275 2321
@@ -2286,28 +2332,14 @@ void __init setup_IO_APIC(void)
2286 print_IO_APIC(); 2332 print_IO_APIC();
2287} 2333}
2288 2334
2289static int __init setup_disable_8254_timer(char *s)
2290{
2291 timer_over_8254 = -1;
2292 return 1;
2293}
2294static int __init setup_enable_8254_timer(char *s)
2295{
2296 timer_over_8254 = 2;
2297 return 1;
2298}
2299
2300__setup("disable_8254_timer", setup_disable_8254_timer);
2301__setup("enable_8254_timer", setup_enable_8254_timer);
2302
2303/* 2335/*
2304 * Called after all the initialization is done. If we didnt find any 2336 * Called after all the initialization is done. If we didnt find any
2305 * APIC bugs then we can allow the modify fast path 2337 * APIC bugs then we can allow the modify fast path
2306 */ 2338 */
2307 2339
2308static int __init io_apic_bug_finalize(void) 2340static int __init io_apic_bug_finalize(void)
2309{ 2341{
2310 if(sis_apic_bug == -1) 2342 if (sis_apic_bug == -1)
2311 sis_apic_bug = 0; 2343 sis_apic_bug = 0;
2312 return 0; 2344 return 0;
2313} 2345}
@@ -2318,17 +2350,17 @@ struct sysfs_ioapic_data {
2318 struct sys_device dev; 2350 struct sys_device dev;
2319 struct IO_APIC_route_entry entry[0]; 2351 struct IO_APIC_route_entry entry[0];
2320}; 2352};
2321static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS]; 2353static struct sysfs_ioapic_data *mp_ioapic_data[MAX_IO_APICS];
2322 2354
2323static int ioapic_suspend(struct sys_device *dev, pm_message_t state) 2355static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
2324{ 2356{
2325 struct IO_APIC_route_entry *entry; 2357 struct IO_APIC_route_entry *entry;
2326 struct sysfs_ioapic_data *data; 2358 struct sysfs_ioapic_data *data;
2327 int i; 2359 int i;
2328 2360
2329 data = container_of(dev, struct sysfs_ioapic_data, dev); 2361 data = container_of(dev, struct sysfs_ioapic_data, dev);
2330 entry = data->entry; 2362 entry = data->entry;
2331 for (i = 0; i < nr_ioapic_registers[dev->id]; i ++) 2363 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
2332 entry[i] = ioapic_read_entry(dev->id, i); 2364 entry[i] = ioapic_read_entry(dev->id, i);
2333 2365
2334 return 0; 2366 return 0;
@@ -2341,18 +2373,18 @@ static int ioapic_resume(struct sys_device *dev)
2341 unsigned long flags; 2373 unsigned long flags;
2342 union IO_APIC_reg_00 reg_00; 2374 union IO_APIC_reg_00 reg_00;
2343 int i; 2375 int i;
2344 2376
2345 data = container_of(dev, struct sysfs_ioapic_data, dev); 2377 data = container_of(dev, struct sysfs_ioapic_data, dev);
2346 entry = data->entry; 2378 entry = data->entry;
2347 2379
2348 spin_lock_irqsave(&ioapic_lock, flags); 2380 spin_lock_irqsave(&ioapic_lock, flags);
2349 reg_00.raw = io_apic_read(dev->id, 0); 2381 reg_00.raw = io_apic_read(dev->id, 0);
2350 if (reg_00.bits.ID != mp_ioapics[dev->id].mpc_apicid) { 2382 if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
2351 reg_00.bits.ID = mp_ioapics[dev->id].mpc_apicid; 2383 reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
2352 io_apic_write(dev->id, 0, reg_00.raw); 2384 io_apic_write(dev->id, 0, reg_00.raw);
2353 } 2385 }
2354 spin_unlock_irqrestore(&ioapic_lock, flags); 2386 spin_unlock_irqrestore(&ioapic_lock, flags);
2355 for (i = 0; i < nr_ioapic_registers[dev->id]; i ++) 2387 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
2356 ioapic_write_entry(dev->id, i, entry[i]); 2388 ioapic_write_entry(dev->id, i, entry[i]);
2357 2389
2358 return 0; 2390 return 0;
@@ -2366,24 +2398,23 @@ static struct sysdev_class ioapic_sysdev_class = {
2366 2398
2367static int __init ioapic_init_sysfs(void) 2399static int __init ioapic_init_sysfs(void)
2368{ 2400{
2369 struct sys_device * dev; 2401 struct sys_device *dev;
2370 int i, size, error = 0; 2402 int i, size, error = 0;
2371 2403
2372 error = sysdev_class_register(&ioapic_sysdev_class); 2404 error = sysdev_class_register(&ioapic_sysdev_class);
2373 if (error) 2405 if (error)
2374 return error; 2406 return error;
2375 2407
2376 for (i = 0; i < nr_ioapics; i++ ) { 2408 for (i = 0; i < nr_ioapics; i++) {
2377 size = sizeof(struct sys_device) + nr_ioapic_registers[i] 2409 size = sizeof(struct sys_device) + nr_ioapic_registers[i]
2378 * sizeof(struct IO_APIC_route_entry); 2410 * sizeof(struct IO_APIC_route_entry);
2379 mp_ioapic_data[i] = kmalloc(size, GFP_KERNEL); 2411 mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
2380 if (!mp_ioapic_data[i]) { 2412 if (!mp_ioapic_data[i]) {
2381 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i); 2413 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
2382 continue; 2414 continue;
2383 } 2415 }
2384 memset(mp_ioapic_data[i], 0, size);
2385 dev = &mp_ioapic_data[i]->dev; 2416 dev = &mp_ioapic_data[i]->dev;
2386 dev->id = i; 2417 dev->id = i;
2387 dev->cls = &ioapic_sysdev_class; 2418 dev->cls = &ioapic_sysdev_class;
2388 error = sysdev_register(dev); 2419 error = sysdev_register(dev);
2389 if (error) { 2420 if (error) {
@@ -2458,7 +2489,7 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
2458 msg->address_lo = 2489 msg->address_lo =
2459 MSI_ADDR_BASE_LO | 2490 MSI_ADDR_BASE_LO |
2460 ((INT_DEST_MODE == 0) ? 2491 ((INT_DEST_MODE == 0) ?
2461 MSI_ADDR_DEST_MODE_PHYSICAL: 2492MSI_ADDR_DEST_MODE_PHYSICAL:
2462 MSI_ADDR_DEST_MODE_LOGICAL) | 2493 MSI_ADDR_DEST_MODE_LOGICAL) |
2463 ((INT_DELIVERY_MODE != dest_LowestPrio) ? 2494 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
2464 MSI_ADDR_REDIRECTION_CPU: 2495 MSI_ADDR_REDIRECTION_CPU:
@@ -2469,7 +2500,7 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
2469 MSI_DATA_TRIGGER_EDGE | 2500 MSI_DATA_TRIGGER_EDGE |
2470 MSI_DATA_LEVEL_ASSERT | 2501 MSI_DATA_LEVEL_ASSERT |
2471 ((INT_DELIVERY_MODE != dest_LowestPrio) ? 2502 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
2472 MSI_DATA_DELIVERY_FIXED: 2503MSI_DATA_DELIVERY_FIXED:
2473 MSI_DATA_DELIVERY_LOWPRI) | 2504 MSI_DATA_DELIVERY_LOWPRI) |
2474 MSI_DATA_VECTOR(vector); 2505 MSI_DATA_VECTOR(vector);
2475 } 2506 }
@@ -2640,12 +2671,12 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
2640#endif /* CONFIG_HT_IRQ */ 2671#endif /* CONFIG_HT_IRQ */
2641 2672
2642/* -------------------------------------------------------------------------- 2673/* --------------------------------------------------------------------------
2643 ACPI-based IOAPIC Configuration 2674 ACPI-based IOAPIC Configuration
2644 -------------------------------------------------------------------------- */ 2675 -------------------------------------------------------------------------- */
2645 2676
2646#ifdef CONFIG_ACPI 2677#ifdef CONFIG_ACPI
2647 2678
2648int __init io_apic_get_unique_id (int ioapic, int apic_id) 2679int __init io_apic_get_unique_id(int ioapic, int apic_id)
2649{ 2680{
2650 union IO_APIC_reg_00 reg_00; 2681 union IO_APIC_reg_00 reg_00;
2651 static physid_mask_t apic_id_map = PHYSID_MASK_NONE; 2682 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
@@ -2654,10 +2685,10 @@ int __init io_apic_get_unique_id (int ioapic, int apic_id)
2654 int i = 0; 2685 int i = 0;
2655 2686
2656 /* 2687 /*
2657 * The P4 platform supports up to 256 APIC IDs on two separate APIC 2688 * The P4 platform supports up to 256 APIC IDs on two separate APIC
2658 * buses (one for LAPICs, one for IOAPICs), where predecessors only 2689 * buses (one for LAPICs, one for IOAPICs), where predecessors only
2659 * supports up to 16 on one shared APIC bus. 2690 * supports up to 16 on one shared APIC bus.
2660 * 2691 *
2661 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full 2692 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
2662 * advantage of new APIC bus architecture. 2693 * advantage of new APIC bus architecture.
2663 */ 2694 */
@@ -2676,7 +2707,7 @@ int __init io_apic_get_unique_id (int ioapic, int apic_id)
2676 } 2707 }
2677 2708
2678 /* 2709 /*
2679 * Every APIC in a system must have a unique ID or we get lots of nice 2710 * Every APIC in a system must have a unique ID or we get lots of nice
2680 * 'stuck on smp_invalidate_needed IPI wait' messages. 2711 * 'stuck on smp_invalidate_needed IPI wait' messages.
2681 */ 2712 */
2682 if (check_apicid_used(apic_id_map, apic_id)) { 2713 if (check_apicid_used(apic_id_map, apic_id)) {
@@ -2693,7 +2724,7 @@ int __init io_apic_get_unique_id (int ioapic, int apic_id)
2693 "trying %d\n", ioapic, apic_id, i); 2724 "trying %d\n", ioapic, apic_id, i);
2694 2725
2695 apic_id = i; 2726 apic_id = i;
2696 } 2727 }
2697 2728
2698 tmp = apicid_to_cpu_present(apic_id); 2729 tmp = apicid_to_cpu_present(apic_id);
2699 physids_or(apic_id_map, apic_id_map, tmp); 2730 physids_or(apic_id_map, apic_id_map, tmp);
@@ -2720,7 +2751,7 @@ int __init io_apic_get_unique_id (int ioapic, int apic_id)
2720} 2751}
2721 2752
2722 2753
2723int __init io_apic_get_version (int ioapic) 2754int __init io_apic_get_version(int ioapic)
2724{ 2755{
2725 union IO_APIC_reg_01 reg_01; 2756 union IO_APIC_reg_01 reg_01;
2726 unsigned long flags; 2757 unsigned long flags;
@@ -2733,7 +2764,7 @@ int __init io_apic_get_version (int ioapic)
2733} 2764}
2734 2765
2735 2766
2736int __init io_apic_get_redir_entries (int ioapic) 2767int __init io_apic_get_redir_entries(int ioapic)
2737{ 2768{
2738 union IO_APIC_reg_01 reg_01; 2769 union IO_APIC_reg_01 reg_01;
2739 unsigned long flags; 2770 unsigned long flags;
@@ -2746,7 +2777,7 @@ int __init io_apic_get_redir_entries (int ioapic)
2746} 2777}
2747 2778
2748 2779
2749int io_apic_set_pci_routing (int ioapic, int pin, int irq, int edge_level, int active_high_low) 2780int io_apic_set_pci_routing(int ioapic, int pin, int irq, int edge_level, int active_high_low)
2750{ 2781{
2751 struct IO_APIC_route_entry entry; 2782 struct IO_APIC_route_entry entry;
2752 2783
@@ -2762,7 +2793,7 @@ int io_apic_set_pci_routing (int ioapic, int pin, int irq, int edge_level, int a
2762 * corresponding device driver registers for this IRQ. 2793 * corresponding device driver registers for this IRQ.
2763 */ 2794 */
2764 2795
2765 memset(&entry,0,sizeof(entry)); 2796 memset(&entry, 0, sizeof(entry));
2766 2797
2767 entry.delivery_mode = INT_DELIVERY_MODE; 2798 entry.delivery_mode = INT_DELIVERY_MODE;
2768 entry.dest_mode = INT_DEST_MODE; 2799 entry.dest_mode = INT_DEST_MODE;
@@ -2781,7 +2812,7 @@ int io_apic_set_pci_routing (int ioapic, int pin, int irq, int edge_level, int a
2781 2812
2782 apic_printk(APIC_DEBUG, KERN_DEBUG "IOAPIC[%d]: Set PCI routing entry " 2813 apic_printk(APIC_DEBUG, KERN_DEBUG "IOAPIC[%d]: Set PCI routing entry "
2783 "(%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i)\n", ioapic, 2814 "(%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i)\n", ioapic,
2784 mp_ioapics[ioapic].mpc_apicid, pin, entry.vector, irq, 2815 mp_ioapics[ioapic].mp_apicid, pin, entry.vector, irq,
2785 edge_level, active_high_low); 2816 edge_level, active_high_low);
2786 2817
2787 ioapic_register_intr(irq, entry.vector, edge_level); 2818 ioapic_register_intr(irq, entry.vector, edge_level);
@@ -2802,8 +2833,8 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
2802 return -1; 2833 return -1;
2803 2834
2804 for (i = 0; i < mp_irq_entries; i++) 2835 for (i = 0; i < mp_irq_entries; i++)
2805 if (mp_irqs[i].mpc_irqtype == mp_INT && 2836 if (mp_irqs[i].mp_irqtype == mp_INT &&
2806 mp_irqs[i].mpc_srcbusirq == bus_irq) 2837 mp_irqs[i].mp_srcbusirq == bus_irq)
2807 break; 2838 break;
2808 if (i >= mp_irq_entries) 2839 if (i >= mp_irq_entries)
2809 return -1; 2840 return -1;
@@ -2836,3 +2867,34 @@ static int __init parse_noapic(char *arg)
2836 return 0; 2867 return 0;
2837} 2868}
2838early_param("noapic", parse_noapic); 2869early_param("noapic", parse_noapic);
2870
2871void __init ioapic_init_mappings(void)
2872{
2873 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
2874 int i;
2875
2876 for (i = 0; i < nr_ioapics; i++) {
2877 if (smp_found_config) {
2878 ioapic_phys = mp_ioapics[i].mp_apicaddr;
2879 if (!ioapic_phys) {
2880 printk(KERN_ERR
2881 "WARNING: bogus zero IO-APIC "
2882 "address found in MPTABLE, "
2883 "disabling IO/APIC support!\n");
2884 smp_found_config = 0;
2885 skip_ioapic_setup = 1;
2886 goto fake_ioapic_page;
2887 }
2888 } else {
2889fake_ioapic_page:
2890 ioapic_phys = (unsigned long)
2891 alloc_bootmem_pages(PAGE_SIZE);
2892 ioapic_phys = __pa(ioapic_phys);
2893 }
2894 set_fixmap_nocache(idx, ioapic_phys);
2895 printk(KERN_DEBUG "mapped IOAPIC to %08lx (%08lx)\n",
2896 __fix_to_virt(idx), ioapic_phys);
2897 idx++;
2898 }
2899}
2900
diff --git a/arch/x86/kernel/io_apic_64.c b/arch/x86/kernel/io_apic_64.c
index ef1a8dfcc529..6510cde36b35 100644
--- a/arch/x86/kernel/io_apic_64.c
+++ b/arch/x86/kernel/io_apic_64.c
@@ -61,7 +61,7 @@ struct irq_cfg {
61}; 61};
62 62
63/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ 63/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
64struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = { 64static struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = {
65 [0] = { .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, }, 65 [0] = { .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, },
66 [1] = { .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, }, 66 [1] = { .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, },
67 [2] = { .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, }, 67 [2] = { .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, },
@@ -82,6 +82,10 @@ struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = {
82 82
83static int assign_irq_vector(int irq, cpumask_t mask); 83static int assign_irq_vector(int irq, cpumask_t mask);
84 84
85int first_system_vector = 0xfe;
86
87char system_vectors[NR_VECTORS] = { [0 ... NR_VECTORS-1] = SYS_VECTOR_FREE};
88
85#define __apicdebuginit __init 89#define __apicdebuginit __init
86 90
87int sis_apic_bug; /* not actually supported, dummy for compile */ 91int sis_apic_bug; /* not actually supported, dummy for compile */
@@ -90,7 +94,7 @@ static int no_timer_check;
90 94
91static int disable_timer_pin_1 __initdata; 95static int disable_timer_pin_1 __initdata;
92 96
93int timer_over_8254 __initdata = 1; 97int timer_through_8259 __initdata;
94 98
95/* Where if anywhere is the i8259 connect in external int mode */ 99/* Where if anywhere is the i8259 connect in external int mode */
96static struct { int pin, apic; } ioapic_i8259 = { -1, -1 }; 100static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
@@ -104,15 +108,17 @@ DEFINE_SPINLOCK(vector_lock);
104int nr_ioapic_registers[MAX_IO_APICS]; 108int nr_ioapic_registers[MAX_IO_APICS];
105 109
106/* I/O APIC entries */ 110/* I/O APIC entries */
107struct mpc_config_ioapic mp_ioapics[MAX_IO_APICS]; 111struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
108int nr_ioapics; 112int nr_ioapics;
109 113
110/* MP IRQ source entries */ 114/* MP IRQ source entries */
111struct mpc_config_intsrc mp_irqs[MAX_IRQ_SOURCES]; 115struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
112 116
113/* # of MP IRQ source entries */ 117/* # of MP IRQ source entries */
114int mp_irq_entries; 118int mp_irq_entries;
115 119
120DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
121
116/* 122/*
117 * Rough estimation of how many shared IRQs there are, can 123 * Rough estimation of how many shared IRQs there are, can
118 * be changed anytime. 124 * be changed anytime.
@@ -140,7 +146,7 @@ struct io_apic {
140static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) 146static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
141{ 147{
142 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx) 148 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
143 + (mp_ioapics[idx].mpc_apicaddr & ~PAGE_MASK); 149 + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
144} 150}
145 151
146static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg) 152static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
@@ -183,7 +189,7 @@ static bool io_apic_level_ack_pending(unsigned int irq)
183 break; 189 break;
184 reg = io_apic_read(entry->apic, 0x10 + pin*2); 190 reg = io_apic_read(entry->apic, 0x10 + pin*2);
185 /* Is the remote IRR bit set? */ 191 /* Is the remote IRR bit set? */
186 if ((reg >> 14) & 1) { 192 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
187 spin_unlock_irqrestore(&ioapic_lock, flags); 193 spin_unlock_irqrestore(&ioapic_lock, flags);
188 return true; 194 return true;
189 } 195 }
@@ -298,7 +304,7 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
298 break; 304 break;
299 io_apic_write(apic, 0x11 + pin*2, dest); 305 io_apic_write(apic, 0x11 + pin*2, dest);
300 reg = io_apic_read(apic, 0x10 + pin*2); 306 reg = io_apic_read(apic, 0x10 + pin*2);
301 reg &= ~0x000000ff; 307 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
302 reg |= vector; 308 reg |= vector;
303 io_apic_modify(apic, reg); 309 io_apic_modify(apic, reg);
304 if (!entry->next) 310 if (!entry->next)
@@ -360,16 +366,37 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin)
360 entry->pin = pin; 366 entry->pin = pin;
361} 367}
362 368
369/*
370 * Reroute an IRQ to a different pin.
371 */
372static void __init replace_pin_at_irq(unsigned int irq,
373 int oldapic, int oldpin,
374 int newapic, int newpin)
375{
376 struct irq_pin_list *entry = irq_2_pin + irq;
377
378 while (1) {
379 if (entry->apic == oldapic && entry->pin == oldpin) {
380 entry->apic = newapic;
381 entry->pin = newpin;
382 }
383 if (!entry->next)
384 break;
385 entry = irq_2_pin + entry->next;
386 }
387}
388
363 389
364#define DO_ACTION(name,R,ACTION, FINAL) \ 390#define DO_ACTION(name,R,ACTION, FINAL) \
365 \ 391 \
366 static void name##_IO_APIC_irq (unsigned int irq) \ 392 static void name##_IO_APIC_irq (unsigned int irq) \
367 __DO_ACTION(R, ACTION, FINAL) 393 __DO_ACTION(R, ACTION, FINAL)
368 394
369DO_ACTION( __mask, 0, |= 0x00010000, io_apic_sync(entry->apic) ) 395/* mask = 1 */
370 /* mask = 1 */ 396DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, io_apic_sync(entry->apic))
371DO_ACTION( __unmask, 0, &= 0xfffeffff, ) 397
372 /* mask = 0 */ 398/* mask = 0 */
399DO_ACTION(__unmask, 0, &= ~IO_APIC_REDIR_MASKED, )
373 400
374static void mask_IO_APIC_irq (unsigned int irq) 401static void mask_IO_APIC_irq (unsigned int irq)
375{ 402{
@@ -430,20 +457,6 @@ static int __init disable_timer_pin_setup(char *arg)
430} 457}
431__setup("disable_timer_pin_1", disable_timer_pin_setup); 458__setup("disable_timer_pin_1", disable_timer_pin_setup);
432 459
433static int __init setup_disable_8254_timer(char *s)
434{
435 timer_over_8254 = -1;
436 return 1;
437}
438static int __init setup_enable_8254_timer(char *s)
439{
440 timer_over_8254 = 2;
441 return 1;
442}
443
444__setup("disable_8254_timer", setup_disable_8254_timer);
445__setup("enable_8254_timer", setup_enable_8254_timer);
446
447 460
448/* 461/*
449 * Find the IRQ entry number of a certain pin. 462 * Find the IRQ entry number of a certain pin.
@@ -453,10 +466,10 @@ static int find_irq_entry(int apic, int pin, int type)
453 int i; 466 int i;
454 467
455 for (i = 0; i < mp_irq_entries; i++) 468 for (i = 0; i < mp_irq_entries; i++)
456 if (mp_irqs[i].mpc_irqtype == type && 469 if (mp_irqs[i].mp_irqtype == type &&
457 (mp_irqs[i].mpc_dstapic == mp_ioapics[apic].mpc_apicid || 470 (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
458 mp_irqs[i].mpc_dstapic == MP_APIC_ALL) && 471 mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
459 mp_irqs[i].mpc_dstirq == pin) 472 mp_irqs[i].mp_dstirq == pin)
460 return i; 473 return i;
461 474
462 return -1; 475 return -1;
@@ -470,13 +483,13 @@ static int __init find_isa_irq_pin(int irq, int type)
470 int i; 483 int i;
471 484
472 for (i = 0; i < mp_irq_entries; i++) { 485 for (i = 0; i < mp_irq_entries; i++) {
473 int lbus = mp_irqs[i].mpc_srcbus; 486 int lbus = mp_irqs[i].mp_srcbus;
474 487
475 if (test_bit(lbus, mp_bus_not_pci) && 488 if (test_bit(lbus, mp_bus_not_pci) &&
476 (mp_irqs[i].mpc_irqtype == type) && 489 (mp_irqs[i].mp_irqtype == type) &&
477 (mp_irqs[i].mpc_srcbusirq == irq)) 490 (mp_irqs[i].mp_srcbusirq == irq))
478 491
479 return mp_irqs[i].mpc_dstirq; 492 return mp_irqs[i].mp_dstirq;
480 } 493 }
481 return -1; 494 return -1;
482} 495}
@@ -486,17 +499,17 @@ static int __init find_isa_irq_apic(int irq, int type)
486 int i; 499 int i;
487 500
488 for (i = 0; i < mp_irq_entries; i++) { 501 for (i = 0; i < mp_irq_entries; i++) {
489 int lbus = mp_irqs[i].mpc_srcbus; 502 int lbus = mp_irqs[i].mp_srcbus;
490 503
491 if (test_bit(lbus, mp_bus_not_pci) && 504 if (test_bit(lbus, mp_bus_not_pci) &&
492 (mp_irqs[i].mpc_irqtype == type) && 505 (mp_irqs[i].mp_irqtype == type) &&
493 (mp_irqs[i].mpc_srcbusirq == irq)) 506 (mp_irqs[i].mp_srcbusirq == irq))
494 break; 507 break;
495 } 508 }
496 if (i < mp_irq_entries) { 509 if (i < mp_irq_entries) {
497 int apic; 510 int apic;
498 for(apic = 0; apic < nr_ioapics; apic++) { 511 for(apic = 0; apic < nr_ioapics; apic++) {
499 if (mp_ioapics[apic].mpc_apicid == mp_irqs[i].mpc_dstapic) 512 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
500 return apic; 513 return apic;
501 } 514 }
502 } 515 }
@@ -516,28 +529,28 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
516 529
517 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n", 530 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
518 bus, slot, pin); 531 bus, slot, pin);
519 if (mp_bus_id_to_pci_bus[bus] == -1) { 532 if (test_bit(bus, mp_bus_not_pci)) {
520 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus); 533 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
521 return -1; 534 return -1;
522 } 535 }
523 for (i = 0; i < mp_irq_entries; i++) { 536 for (i = 0; i < mp_irq_entries; i++) {
524 int lbus = mp_irqs[i].mpc_srcbus; 537 int lbus = mp_irqs[i].mp_srcbus;
525 538
526 for (apic = 0; apic < nr_ioapics; apic++) 539 for (apic = 0; apic < nr_ioapics; apic++)
527 if (mp_ioapics[apic].mpc_apicid == mp_irqs[i].mpc_dstapic || 540 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
528 mp_irqs[i].mpc_dstapic == MP_APIC_ALL) 541 mp_irqs[i].mp_dstapic == MP_APIC_ALL)
529 break; 542 break;
530 543
531 if (!test_bit(lbus, mp_bus_not_pci) && 544 if (!test_bit(lbus, mp_bus_not_pci) &&
532 !mp_irqs[i].mpc_irqtype && 545 !mp_irqs[i].mp_irqtype &&
533 (bus == lbus) && 546 (bus == lbus) &&
534 (slot == ((mp_irqs[i].mpc_srcbusirq >> 2) & 0x1f))) { 547 (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
535 int irq = pin_2_irq(i,apic,mp_irqs[i].mpc_dstirq); 548 int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
536 549
537 if (!(apic || IO_APIC_IRQ(irq))) 550 if (!(apic || IO_APIC_IRQ(irq)))
538 continue; 551 continue;
539 552
540 if (pin == (mp_irqs[i].mpc_srcbusirq & 3)) 553 if (pin == (mp_irqs[i].mp_srcbusirq & 3))
541 return irq; 554 return irq;
542 /* 555 /*
543 * Use the first all-but-pin matching entry as a 556 * Use the first all-but-pin matching entry as a
@@ -565,13 +578,13 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
565 578
566static int MPBIOS_polarity(int idx) 579static int MPBIOS_polarity(int idx)
567{ 580{
568 int bus = mp_irqs[idx].mpc_srcbus; 581 int bus = mp_irqs[idx].mp_srcbus;
569 int polarity; 582 int polarity;
570 583
571 /* 584 /*
572 * Determine IRQ line polarity (high active or low active): 585 * Determine IRQ line polarity (high active or low active):
573 */ 586 */
574 switch (mp_irqs[idx].mpc_irqflag & 3) 587 switch (mp_irqs[idx].mp_irqflag & 3)
575 { 588 {
576 case 0: /* conforms, ie. bus-type dependent polarity */ 589 case 0: /* conforms, ie. bus-type dependent polarity */
577 if (test_bit(bus, mp_bus_not_pci)) 590 if (test_bit(bus, mp_bus_not_pci))
@@ -607,13 +620,13 @@ static int MPBIOS_polarity(int idx)
607 620
608static int MPBIOS_trigger(int idx) 621static int MPBIOS_trigger(int idx)
609{ 622{
610 int bus = mp_irqs[idx].mpc_srcbus; 623 int bus = mp_irqs[idx].mp_srcbus;
611 int trigger; 624 int trigger;
612 625
613 /* 626 /*
614 * Determine IRQ trigger mode (edge or level sensitive): 627 * Determine IRQ trigger mode (edge or level sensitive):
615 */ 628 */
616 switch ((mp_irqs[idx].mpc_irqflag>>2) & 3) 629 switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
617 { 630 {
618 case 0: /* conforms, ie. bus-type dependent */ 631 case 0: /* conforms, ie. bus-type dependent */
619 if (test_bit(bus, mp_bus_not_pci)) 632 if (test_bit(bus, mp_bus_not_pci))
@@ -660,16 +673,16 @@ static inline int irq_trigger(int idx)
660static int pin_2_irq(int idx, int apic, int pin) 673static int pin_2_irq(int idx, int apic, int pin)
661{ 674{
662 int irq, i; 675 int irq, i;
663 int bus = mp_irqs[idx].mpc_srcbus; 676 int bus = mp_irqs[idx].mp_srcbus;
664 677
665 /* 678 /*
666 * Debugging check, we are in big trouble if this message pops up! 679 * Debugging check, we are in big trouble if this message pops up!
667 */ 680 */
668 if (mp_irqs[idx].mpc_dstirq != pin) 681 if (mp_irqs[idx].mp_dstirq != pin)
669 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n"); 682 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
670 683
671 if (test_bit(bus, mp_bus_not_pci)) { 684 if (test_bit(bus, mp_bus_not_pci)) {
672 irq = mp_irqs[idx].mpc_srcbusirq; 685 irq = mp_irqs[idx].mp_srcbusirq;
673 } else { 686 } else {
674 /* 687 /*
675 * PCI IRQs are mapped in order 688 * PCI IRQs are mapped in order
@@ -730,7 +743,7 @@ static int __assign_irq_vector(int irq, cpumask_t mask)
730 offset = current_offset; 743 offset = current_offset;
731next: 744next:
732 vector += 8; 745 vector += 8;
733 if (vector >= FIRST_SYSTEM_VECTOR) { 746 if (vector >= first_system_vector) {
734 /* If we run out of vectors on large boxen, must share them. */ 747 /* If we run out of vectors on large boxen, must share them. */
735 offset = (offset + 1) % 8; 748 offset = (offset + 1) % 8;
736 vector = FIRST_DEVICE_VECTOR + offset; 749 vector = FIRST_DEVICE_VECTOR + offset;
@@ -788,7 +801,7 @@ static void __clear_irq_vector(int irq)
788 cpus_clear(cfg->domain); 801 cpus_clear(cfg->domain);
789} 802}
790 803
791void __setup_vector_irq(int cpu) 804static void __setup_vector_irq(int cpu)
792{ 805{
793 /* Initialize vector_irq on a new cpu */ 806 /* Initialize vector_irq on a new cpu */
794 /* This function must be called with vector_lock held */ 807 /* This function must be called with vector_lock held */
@@ -811,6 +824,13 @@ void __setup_vector_irq(int cpu)
811 } 824 }
812} 825}
813 826
827void setup_vector_irq(int cpu)
828{
829 spin_lock(&vector_lock);
830 __setup_vector_irq(smp_processor_id());
831 spin_unlock(&vector_lock);
832}
833
814 834
815static struct irq_chip ioapic_chip; 835static struct irq_chip ioapic_chip;
816 836
@@ -846,7 +866,7 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
846 apic_printk(APIC_VERBOSE,KERN_DEBUG 866 apic_printk(APIC_VERBOSE,KERN_DEBUG
847 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> " 867 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
848 "IRQ %d Mode:%i Active:%i)\n", 868 "IRQ %d Mode:%i Active:%i)\n",
849 apic, mp_ioapics[apic].mpc_apicid, pin, cfg->vector, 869 apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
850 irq, trigger, polarity); 870 irq, trigger, polarity);
851 871
852 /* 872 /*
@@ -887,10 +907,10 @@ static void __init setup_IO_APIC_irqs(void)
887 idx = find_irq_entry(apic,pin,mp_INT); 907 idx = find_irq_entry(apic,pin,mp_INT);
888 if (idx == -1) { 908 if (idx == -1) {
889 if (first_notcon) { 909 if (first_notcon) {
890 apic_printk(APIC_VERBOSE, KERN_DEBUG " IO-APIC (apicid-pin) %d-%d", mp_ioapics[apic].mpc_apicid, pin); 910 apic_printk(APIC_VERBOSE, KERN_DEBUG " IO-APIC (apicid-pin) %d-%d", mp_ioapics[apic].mp_apicid, pin);
891 first_notcon = 0; 911 first_notcon = 0;
892 } else 912 } else
893 apic_printk(APIC_VERBOSE, ", %d-%d", mp_ioapics[apic].mpc_apicid, pin); 913 apic_printk(APIC_VERBOSE, ", %d-%d", mp_ioapics[apic].mp_apicid, pin);
894 continue; 914 continue;
895 } 915 }
896 if (!first_notcon) { 916 if (!first_notcon) {
@@ -911,26 +931,21 @@ static void __init setup_IO_APIC_irqs(void)
911} 931}
912 932
913/* 933/*
914 * Set up the 8259A-master output pin as broadcast to all 934 * Set up the timer pin, possibly with the 8259A-master behind.
915 * CPUs.
916 */ 935 */
917static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, int vector) 936static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
937 int vector)
918{ 938{
919 struct IO_APIC_route_entry entry; 939 struct IO_APIC_route_entry entry;
920 940
921 memset(&entry, 0, sizeof(entry)); 941 memset(&entry, 0, sizeof(entry));
922 942
923 disable_8259A_irq(0);
924
925 /* mask LVT0 */
926 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
927
928 /* 943 /*
929 * We use logical delivery to get the timer IRQ 944 * We use logical delivery to get the timer IRQ
930 * to the first CPU. 945 * to the first CPU.
931 */ 946 */
932 entry.dest_mode = INT_DEST_MODE; 947 entry.dest_mode = INT_DEST_MODE;
933 entry.mask = 0; /* unmask IRQ now */ 948 entry.mask = 1; /* mask IRQ now */
934 entry.dest = cpu_mask_to_apicid(TARGET_CPUS); 949 entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
935 entry.delivery_mode = INT_DELIVERY_MODE; 950 entry.delivery_mode = INT_DELIVERY_MODE;
936 entry.polarity = 0; 951 entry.polarity = 0;
@@ -939,7 +954,7 @@ static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, in
939 954
940 /* 955 /*
941 * The timer IRQ doesn't have to know that behind the 956 * The timer IRQ doesn't have to know that behind the
942 * scene we have a 8259A-master in AEOI mode ... 957 * scene we may have a 8259A-master in AEOI mode ...
943 */ 958 */
944 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge"); 959 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
945 960
@@ -947,8 +962,6 @@ static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, in
947 * Add it to the IO-APIC irq-routing table: 962 * Add it to the IO-APIC irq-routing table:
948 */ 963 */
949 ioapic_write_entry(apic, pin, entry); 964 ioapic_write_entry(apic, pin, entry);
950
951 enable_8259A_irq(0);
952} 965}
953 966
954void __apicdebuginit print_IO_APIC(void) 967void __apicdebuginit print_IO_APIC(void)
@@ -965,7 +978,7 @@ void __apicdebuginit print_IO_APIC(void)
965 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries); 978 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
966 for (i = 0; i < nr_ioapics; i++) 979 for (i = 0; i < nr_ioapics; i++)
967 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n", 980 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
968 mp_ioapics[i].mpc_apicid, nr_ioapic_registers[i]); 981 mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
969 982
970 /* 983 /*
971 * We are a bit conservative about what we expect. We have to 984 * We are a bit conservative about what we expect. We have to
@@ -983,7 +996,7 @@ void __apicdebuginit print_IO_APIC(void)
983 spin_unlock_irqrestore(&ioapic_lock, flags); 996 spin_unlock_irqrestore(&ioapic_lock, flags);
984 997
985 printk("\n"); 998 printk("\n");
986 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mpc_apicid); 999 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
987 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw); 1000 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
988 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID); 1001 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
989 1002
@@ -1077,6 +1090,7 @@ void __apicdebuginit print_local_APIC(void * dummy)
1077 1090
1078 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n", 1091 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1079 smp_processor_id(), hard_smp_processor_id()); 1092 smp_processor_id(), hard_smp_processor_id());
1093 v = apic_read(APIC_ID);
1080 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, GET_APIC_ID(read_apic_id())); 1094 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, GET_APIC_ID(read_apic_id()));
1081 v = apic_read(APIC_LVR); 1095 v = apic_read(APIC_LVR);
1082 printk(KERN_INFO "... APIC VERSION: %08x\n", v); 1096 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
@@ -1146,7 +1160,7 @@ void __apicdebuginit print_local_APIC(void * dummy)
1146 1160
1147void print_all_local_APICs (void) 1161void print_all_local_APICs (void)
1148{ 1162{
1149 on_each_cpu(print_local_APIC, NULL, 1, 1); 1163 on_each_cpu(print_local_APIC, NULL, 1);
1150} 1164}
1151 1165
1152void __apicdebuginit print_PIC(void) 1166void __apicdebuginit print_PIC(void)
@@ -1540,7 +1554,7 @@ static inline void init_IO_APIC_traps(void)
1540 } 1554 }
1541} 1555}
1542 1556
1543static void enable_lapic_irq (unsigned int irq) 1557static void unmask_lapic_irq(unsigned int irq)
1544{ 1558{
1545 unsigned long v; 1559 unsigned long v;
1546 1560
@@ -1548,7 +1562,7 @@ static void enable_lapic_irq (unsigned int irq)
1548 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED); 1562 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
1549} 1563}
1550 1564
1551static void disable_lapic_irq (unsigned int irq) 1565static void mask_lapic_irq(unsigned int irq)
1552{ 1566{
1553 unsigned long v; 1567 unsigned long v;
1554 1568
@@ -1561,19 +1575,20 @@ static void ack_lapic_irq (unsigned int irq)
1561 ack_APIC_irq(); 1575 ack_APIC_irq();
1562} 1576}
1563 1577
1564static void end_lapic_irq (unsigned int i) { /* nothing */ } 1578static struct irq_chip lapic_chip __read_mostly = {
1565 1579 .name = "local-APIC",
1566static struct hw_interrupt_type lapic_irq_type __read_mostly = { 1580 .mask = mask_lapic_irq,
1567 .name = "local-APIC", 1581 .unmask = unmask_lapic_irq,
1568 .typename = "local-APIC-edge", 1582 .ack = ack_lapic_irq,
1569 .startup = NULL, /* startup_irq() not used for IRQ0 */
1570 .shutdown = NULL, /* shutdown_irq() not used for IRQ0 */
1571 .enable = enable_lapic_irq,
1572 .disable = disable_lapic_irq,
1573 .ack = ack_lapic_irq,
1574 .end = end_lapic_irq,
1575}; 1583};
1576 1584
1585static void lapic_register_intr(int irq)
1586{
1587 irq_desc[irq].status &= ~IRQ_LEVEL;
1588 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
1589 "edge");
1590}
1591
1577static void __init setup_nmi(void) 1592static void __init setup_nmi(void)
1578{ 1593{
1579 /* 1594 /*
@@ -1659,6 +1674,7 @@ static inline void __init check_timer(void)
1659 struct irq_cfg *cfg = irq_cfg + 0; 1674 struct irq_cfg *cfg = irq_cfg + 0;
1660 int apic1, pin1, apic2, pin2; 1675 int apic1, pin1, apic2, pin2;
1661 unsigned long flags; 1676 unsigned long flags;
1677 int no_pin1 = 0;
1662 1678
1663 local_irq_save(flags); 1679 local_irq_save(flags);
1664 1680
@@ -1669,16 +1685,11 @@ static inline void __init check_timer(void)
1669 assign_irq_vector(0, TARGET_CPUS); 1685 assign_irq_vector(0, TARGET_CPUS);
1670 1686
1671 /* 1687 /*
1672 * Subtle, code in do_timer_interrupt() expects an AEOI 1688 * As IRQ0 is to be enabled in the 8259A, the virtual
1673 * mode for the 8259A whenever interrupts are routed 1689 * wire has to be disabled in the local APIC.
1674 * through I/O APICs. Also IRQ0 has to be enabled in
1675 * the 8259A which implies the virtual wire has to be
1676 * disabled in the local APIC.
1677 */ 1690 */
1678 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT); 1691 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
1679 init_8259A(1); 1692 init_8259A(1);
1680 if (timer_over_8254 > 0)
1681 enable_8259A_irq(0);
1682 1693
1683 pin1 = find_isa_irq_pin(0, mp_INT); 1694 pin1 = find_isa_irq_pin(0, mp_INT);
1684 apic1 = find_isa_irq_apic(0, mp_INT); 1695 apic1 = find_isa_irq_apic(0, mp_INT);
@@ -1688,15 +1699,33 @@ static inline void __init check_timer(void)
1688 apic_printk(APIC_VERBOSE,KERN_INFO "..TIMER: vector=0x%02X apic1=%d pin1=%d apic2=%d pin2=%d\n", 1699 apic_printk(APIC_VERBOSE,KERN_INFO "..TIMER: vector=0x%02X apic1=%d pin1=%d apic2=%d pin2=%d\n",
1689 cfg->vector, apic1, pin1, apic2, pin2); 1700 cfg->vector, apic1, pin1, apic2, pin2);
1690 1701
1702 /*
1703 * Some BIOS writers are clueless and report the ExtINTA
1704 * I/O APIC input from the cascaded 8259A as the timer
1705 * interrupt input. So just in case, if only one pin
1706 * was found above, try it both directly and through the
1707 * 8259A.
1708 */
1709 if (pin1 == -1) {
1710 pin1 = pin2;
1711 apic1 = apic2;
1712 no_pin1 = 1;
1713 } else if (pin2 == -1) {
1714 pin2 = pin1;
1715 apic2 = apic1;
1716 }
1717
1691 if (pin1 != -1) { 1718 if (pin1 != -1) {
1692 /* 1719 /*
1693 * Ok, does IRQ0 through the IOAPIC work? 1720 * Ok, does IRQ0 through the IOAPIC work?
1694 */ 1721 */
1722 if (no_pin1) {
1723 add_pin_to_irq(0, apic1, pin1);
1724 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
1725 }
1695 unmask_IO_APIC_irq(0); 1726 unmask_IO_APIC_irq(0);
1696 if (!no_timer_check && timer_irq_works()) { 1727 if (!no_timer_check && timer_irq_works()) {
1697 nmi_watchdog_default();
1698 if (nmi_watchdog == NMI_IO_APIC) { 1728 if (nmi_watchdog == NMI_IO_APIC) {
1699 disable_8259A_irq(0);
1700 setup_nmi(); 1729 setup_nmi();
1701 enable_8259A_irq(0); 1730 enable_8259A_irq(0);
1702 } 1731 }
@@ -1705,43 +1734,48 @@ static inline void __init check_timer(void)
1705 goto out; 1734 goto out;
1706 } 1735 }
1707 clear_IO_APIC_pin(apic1, pin1); 1736 clear_IO_APIC_pin(apic1, pin1);
1708 apic_printk(APIC_QUIET,KERN_ERR "..MP-BIOS bug: 8254 timer not " 1737 if (!no_pin1)
1709 "connected to IO-APIC\n"); 1738 apic_printk(APIC_QUIET,KERN_ERR "..MP-BIOS bug: "
1710 } 1739 "8254 timer not connected to IO-APIC\n");
1711 1740
1712 apic_printk(APIC_VERBOSE,KERN_INFO "...trying to set up timer (IRQ0) " 1741 apic_printk(APIC_VERBOSE,KERN_INFO
1713 "through the 8259A ... "); 1742 "...trying to set up timer (IRQ0) "
1714 if (pin2 != -1) { 1743 "through the 8259A ... ");
1715 apic_printk(APIC_VERBOSE,"\n..... (found apic %d pin %d) ...", 1744 apic_printk(APIC_VERBOSE,"\n..... (found apic %d pin %d) ...",
1716 apic2, pin2); 1745 apic2, pin2);
1717 /* 1746 /*
1718 * legacy devices should be connected to IO APIC #0 1747 * legacy devices should be connected to IO APIC #0
1719 */ 1748 */
1720 setup_ExtINT_IRQ0_pin(apic2, pin2, cfg->vector); 1749 replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
1750 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
1751 unmask_IO_APIC_irq(0);
1752 enable_8259A_irq(0);
1721 if (timer_irq_works()) { 1753 if (timer_irq_works()) {
1722 apic_printk(APIC_VERBOSE," works.\n"); 1754 apic_printk(APIC_VERBOSE," works.\n");
1723 nmi_watchdog_default(); 1755 timer_through_8259 = 1;
1724 if (nmi_watchdog == NMI_IO_APIC) { 1756 if (nmi_watchdog == NMI_IO_APIC) {
1757 disable_8259A_irq(0);
1725 setup_nmi(); 1758 setup_nmi();
1759 enable_8259A_irq(0);
1726 } 1760 }
1727 goto out; 1761 goto out;
1728 } 1762 }
1729 /* 1763 /*
1730 * Cleanup, just in case ... 1764 * Cleanup, just in case ...
1731 */ 1765 */
1766 disable_8259A_irq(0);
1732 clear_IO_APIC_pin(apic2, pin2); 1767 clear_IO_APIC_pin(apic2, pin2);
1768 apic_printk(APIC_VERBOSE," failed.\n");
1733 } 1769 }
1734 apic_printk(APIC_VERBOSE," failed.\n");
1735 1770
1736 if (nmi_watchdog == NMI_IO_APIC) { 1771 if (nmi_watchdog == NMI_IO_APIC) {
1737 printk(KERN_WARNING "timer doesn't work through the IO-APIC - disabling NMI Watchdog!\n"); 1772 printk(KERN_WARNING "timer doesn't work through the IO-APIC - disabling NMI Watchdog!\n");
1738 nmi_watchdog = 0; 1773 nmi_watchdog = NMI_NONE;
1739 } 1774 }
1740 1775
1741 apic_printk(APIC_VERBOSE, KERN_INFO "...trying to set up timer as Virtual Wire IRQ..."); 1776 apic_printk(APIC_VERBOSE, KERN_INFO "...trying to set up timer as Virtual Wire IRQ...");
1742 1777
1743 disable_8259A_irq(0); 1778 lapic_register_intr(0);
1744 irq_desc[0].chip = &lapic_irq_type;
1745 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ 1779 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
1746 enable_8259A_irq(0); 1780 enable_8259A_irq(0);
1747 1781
@@ -1749,6 +1783,7 @@ static inline void __init check_timer(void)
1749 apic_printk(APIC_VERBOSE," works.\n"); 1783 apic_printk(APIC_VERBOSE," works.\n");
1750 goto out; 1784 goto out;
1751 } 1785 }
1786 disable_8259A_irq(0);
1752 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector); 1787 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
1753 apic_printk(APIC_VERBOSE," failed.\n"); 1788 apic_printk(APIC_VERBOSE," failed.\n");
1754 1789
@@ -1778,11 +1813,21 @@ static int __init notimercheck(char *s)
1778__setup("no_timer_check", notimercheck); 1813__setup("no_timer_check", notimercheck);
1779 1814
1780/* 1815/*
1781 * 1816 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
1782 * IRQs that are handled by the PIC in the MPS IOAPIC case. 1817 * to devices. However there may be an I/O APIC pin available for
1783 * - IRQ2 is the cascade IRQ, and cannot be a io-apic IRQ. 1818 * this interrupt regardless. The pin may be left unconnected, but
1784 * Linux doesn't really care, as it's not actually used 1819 * typically it will be reused as an ExtINT cascade interrupt for
1785 * for any interrupt handling anyway. 1820 * the master 8259A. In the MPS case such a pin will normally be
1821 * reported as an ExtINT interrupt in the MP table. With ACPI
1822 * there is no provision for ExtINT interrupts, and in the absence
1823 * of an override it would be treated as an ordinary ISA I/O APIC
1824 * interrupt, that is edge-triggered and unmasked by default. We
1825 * used to do this, but it caused problems on some systems because
1826 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
1827 * the same ExtINT cascade interrupt to drive the local APIC of the
1828 * bootstrap processor. Therefore we refrain from routing IRQ2 to
1829 * the I/O APIC in all cases now. No actual device should request
1830 * it anyway. --macro
1786 */ 1831 */
1787#define PIC_IRQS (1<<2) 1832#define PIC_IRQS (1<<2)
1788 1833
@@ -1793,10 +1838,7 @@ void __init setup_IO_APIC(void)
1793 * calling enable_IO_APIC() is moved to setup_local_APIC for BP 1838 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
1794 */ 1839 */
1795 1840
1796 if (acpi_ioapic) 1841 io_apic_irqs = ~PIC_IRQS;
1797 io_apic_irqs = ~0; /* all IRQs go through IOAPIC */
1798 else
1799 io_apic_irqs = ~PIC_IRQS;
1800 1842
1801 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n"); 1843 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
1802 1844
@@ -1841,8 +1883,8 @@ static int ioapic_resume(struct sys_device *dev)
1841 1883
1842 spin_lock_irqsave(&ioapic_lock, flags); 1884 spin_lock_irqsave(&ioapic_lock, flags);
1843 reg_00.raw = io_apic_read(dev->id, 0); 1885 reg_00.raw = io_apic_read(dev->id, 0);
1844 if (reg_00.bits.ID != mp_ioapics[dev->id].mpc_apicid) { 1886 if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
1845 reg_00.bits.ID = mp_ioapics[dev->id].mpc_apicid; 1887 reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
1846 io_apic_write(dev->id, 0, reg_00.raw); 1888 io_apic_write(dev->id, 0, reg_00.raw);
1847 } 1889 }
1848 spin_unlock_irqrestore(&ioapic_lock, flags); 1890 spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -2242,8 +2284,8 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
2242 return -1; 2284 return -1;
2243 2285
2244 for (i = 0; i < mp_irq_entries; i++) 2286 for (i = 0; i < mp_irq_entries; i++)
2245 if (mp_irqs[i].mpc_irqtype == mp_INT && 2287 if (mp_irqs[i].mp_irqtype == mp_INT &&
2246 mp_irqs[i].mpc_srcbusirq == bus_irq) 2288 mp_irqs[i].mp_srcbusirq == bus_irq)
2247 break; 2289 break;
2248 if (i >= mp_irq_entries) 2290 if (i >= mp_irq_entries)
2249 return -1; 2291 return -1;
@@ -2336,7 +2378,7 @@ void __init ioapic_init_mappings(void)
2336 ioapic_res = ioapic_setup_resources(); 2378 ioapic_res = ioapic_setup_resources();
2337 for (i = 0; i < nr_ioapics; i++) { 2379 for (i = 0; i < nr_ioapics; i++) {
2338 if (smp_found_config) { 2380 if (smp_found_config) {
2339 ioapic_phys = mp_ioapics[i].mpc_apicaddr; 2381 ioapic_phys = mp_ioapics[i].mp_apicaddr;
2340 } else { 2382 } else {
2341 ioapic_phys = (unsigned long) 2383 ioapic_phys = (unsigned long)
2342 alloc_bootmem_pages(PAGE_SIZE); 2384 alloc_bootmem_pages(PAGE_SIZE);
diff --git a/arch/x86/kernel/ipi.c b/arch/x86/kernel/ipi.c
index c0df7b89ca23..9d98cda39ad9 100644
--- a/arch/x86/kernel/ipi.c
+++ b/arch/x86/kernel/ipi.c
@@ -8,7 +8,6 @@
8#include <linux/kernel_stat.h> 8#include <linux/kernel_stat.h>
9#include <linux/mc146818rtc.h> 9#include <linux/mc146818rtc.h>
10#include <linux/cache.h> 10#include <linux/cache.h>
11#include <linux/interrupt.h>
12#include <linux/cpu.h> 11#include <linux/cpu.h>
13#include <linux/module.h> 12#include <linux/module.h>
14 13
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 147352df28b9..47a6f6f12478 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -48,6 +48,29 @@ void ack_bad_irq(unsigned int irq)
48#endif 48#endif
49} 49}
50 50
51#ifdef CONFIG_DEBUG_STACKOVERFLOW
52/* Debugging check for stack overflow: is there less than 1KB free? */
53static int check_stack_overflow(void)
54{
55 long sp;
56
57 __asm__ __volatile__("andl %%esp,%0" :
58 "=r" (sp) : "0" (THREAD_SIZE - 1));
59
60 return sp < (sizeof(struct thread_info) + STACK_WARN);
61}
62
63static void print_stack_overflow(void)
64{
65 printk(KERN_WARNING "low stack detected by irq handler\n");
66 dump_stack();
67}
68
69#else
70static inline int check_stack_overflow(void) { return 0; }
71static inline void print_stack_overflow(void) { }
72#endif
73
51#ifdef CONFIG_4KSTACKS 74#ifdef CONFIG_4KSTACKS
52/* 75/*
53 * per-CPU IRQ handling contexts (thread information and stack) 76 * per-CPU IRQ handling contexts (thread information and stack)
@@ -59,48 +82,29 @@ union irq_ctx {
59 82
60static union irq_ctx *hardirq_ctx[NR_CPUS] __read_mostly; 83static union irq_ctx *hardirq_ctx[NR_CPUS] __read_mostly;
61static union irq_ctx *softirq_ctx[NR_CPUS] __read_mostly; 84static union irq_ctx *softirq_ctx[NR_CPUS] __read_mostly;
62#endif
63 85
64/* 86static char softirq_stack[NR_CPUS * THREAD_SIZE]
65 * do_IRQ handles all normal device IRQ's (the special 87 __attribute__((__section__(".bss.page_aligned")));
66 * SMP cross-CPU interrupts have their own specific
67 * handlers).
68 */
69unsigned int do_IRQ(struct pt_regs *regs)
70{
71 struct pt_regs *old_regs;
72 /* high bit used in ret_from_ code */
73 int irq = ~regs->orig_ax;
74 struct irq_desc *desc = irq_desc + irq;
75#ifdef CONFIG_4KSTACKS
76 union irq_ctx *curctx, *irqctx;
77 u32 *isp;
78#endif
79 88
80 if (unlikely((unsigned)irq >= NR_IRQS)) { 89static char hardirq_stack[NR_CPUS * THREAD_SIZE]
81 printk(KERN_EMERG "%s: cannot handle IRQ %d\n", 90 __attribute__((__section__(".bss.page_aligned")));
82 __func__, irq);
83 BUG();
84 }
85 91
86 old_regs = set_irq_regs(regs); 92static void call_on_stack(void *func, void *stack)
87 irq_enter(); 93{
88#ifdef CONFIG_DEBUG_STACKOVERFLOW 94 asm volatile("xchgl %%ebx,%%esp \n"
89 /* Debugging check for stack overflow: is there less than 1KB free? */ 95 "call *%%edi \n"
90 { 96 "movl %%ebx,%%esp \n"
91 long sp; 97 : "=b" (stack)
92 98 : "0" (stack),
93 __asm__ __volatile__("andl %%esp,%0" : 99 "D"(func)
94 "=r" (sp) : "0" (THREAD_SIZE - 1)); 100 : "memory", "cc", "edx", "ecx", "eax");
95 if (unlikely(sp < (sizeof(struct thread_info) + STACK_WARN))) { 101}
96 printk("do_IRQ: stack overflow: %ld\n",
97 sp - sizeof(struct thread_info));
98 dump_stack();
99 }
100 }
101#endif
102 102
103#ifdef CONFIG_4KSTACKS 103static inline int
104execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
105{
106 union irq_ctx *curctx, *irqctx;
107 u32 *isp, arg1, arg2;
104 108
105 curctx = (union irq_ctx *) current_thread_info(); 109 curctx = (union irq_ctx *) current_thread_info();
106 irqctx = hardirq_ctx[smp_processor_id()]; 110 irqctx = hardirq_ctx[smp_processor_id()];
@@ -111,52 +115,39 @@ unsigned int do_IRQ(struct pt_regs *regs)
111 * handler) we can't do that and just have to keep using the 115 * handler) we can't do that and just have to keep using the
112 * current stack (which is the irq stack already after all) 116 * current stack (which is the irq stack already after all)
113 */ 117 */
114 if (curctx != irqctx) { 118 if (unlikely(curctx == irqctx))
115 int arg1, arg2, bx; 119 return 0;
116
117 /* build the stack frame on the IRQ stack */
118 isp = (u32*) ((char*)irqctx + sizeof(*irqctx));
119 irqctx->tinfo.task = curctx->tinfo.task;
120 irqctx->tinfo.previous_esp = current_stack_pointer;
121 120
122 /* 121 /* build the stack frame on the IRQ stack */
123 * Copy the softirq bits in preempt_count so that the 122 isp = (u32 *) ((char*)irqctx + sizeof(*irqctx));
124 * softirq checks work in the hardirq context. 123 irqctx->tinfo.task = curctx->tinfo.task;
125 */ 124 irqctx->tinfo.previous_esp = current_stack_pointer;
126 irqctx->tinfo.preempt_count =
127 (irqctx->tinfo.preempt_count & ~SOFTIRQ_MASK) |
128 (curctx->tinfo.preempt_count & SOFTIRQ_MASK);
129
130 asm volatile(
131 " xchgl %%ebx,%%esp \n"
132 " call *%%edi \n"
133 " movl %%ebx,%%esp \n"
134 : "=a" (arg1), "=d" (arg2), "=b" (bx)
135 : "0" (irq), "1" (desc), "2" (isp),
136 "D" (desc->handle_irq)
137 : "memory", "cc", "ecx"
138 );
139 } else
140#endif
141 desc->handle_irq(irq, desc);
142 125
143 irq_exit(); 126 /*
144 set_irq_regs(old_regs); 127 * Copy the softirq bits in preempt_count so that the
128 * softirq checks work in the hardirq context.
129 */
130 irqctx->tinfo.preempt_count =
131 (irqctx->tinfo.preempt_count & ~SOFTIRQ_MASK) |
132 (curctx->tinfo.preempt_count & SOFTIRQ_MASK);
133
134 if (unlikely(overflow))
135 call_on_stack(print_stack_overflow, isp);
136
137 asm volatile("xchgl %%ebx,%%esp \n"
138 "call *%%edi \n"
139 "movl %%ebx,%%esp \n"
140 : "=a" (arg1), "=d" (arg2), "=b" (isp)
141 : "0" (irq), "1" (desc), "2" (isp),
142 "D" (desc->handle_irq)
143 : "memory", "cc", "ecx");
145 return 1; 144 return 1;
146} 145}
147 146
148#ifdef CONFIG_4KSTACKS
149
150static char softirq_stack[NR_CPUS * THREAD_SIZE]
151 __attribute__((__section__(".bss.page_aligned")));
152
153static char hardirq_stack[NR_CPUS * THREAD_SIZE]
154 __attribute__((__section__(".bss.page_aligned")));
155
156/* 147/*
157 * allocate per-cpu stacks for hardirq and for softirq processing 148 * allocate per-cpu stacks for hardirq and for softirq processing
158 */ 149 */
159void irq_ctx_init(int cpu) 150void __cpuinit irq_ctx_init(int cpu)
160{ 151{
161 union irq_ctx *irqctx; 152 union irq_ctx *irqctx;
162 153
@@ -164,25 +155,25 @@ void irq_ctx_init(int cpu)
164 return; 155 return;
165 156
166 irqctx = (union irq_ctx*) &hardirq_stack[cpu*THREAD_SIZE]; 157 irqctx = (union irq_ctx*) &hardirq_stack[cpu*THREAD_SIZE];
167 irqctx->tinfo.task = NULL; 158 irqctx->tinfo.task = NULL;
168 irqctx->tinfo.exec_domain = NULL; 159 irqctx->tinfo.exec_domain = NULL;
169 irqctx->tinfo.cpu = cpu; 160 irqctx->tinfo.cpu = cpu;
170 irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; 161 irqctx->tinfo.preempt_count = HARDIRQ_OFFSET;
171 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); 162 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0);
172 163
173 hardirq_ctx[cpu] = irqctx; 164 hardirq_ctx[cpu] = irqctx;
174 165
175 irqctx = (union irq_ctx*) &softirq_stack[cpu*THREAD_SIZE]; 166 irqctx = (union irq_ctx*) &softirq_stack[cpu*THREAD_SIZE];
176 irqctx->tinfo.task = NULL; 167 irqctx->tinfo.task = NULL;
177 irqctx->tinfo.exec_domain = NULL; 168 irqctx->tinfo.exec_domain = NULL;
178 irqctx->tinfo.cpu = cpu; 169 irqctx->tinfo.cpu = cpu;
179 irqctx->tinfo.preempt_count = 0; 170 irqctx->tinfo.preempt_count = 0;
180 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); 171 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0);
181 172
182 softirq_ctx[cpu] = irqctx; 173 softirq_ctx[cpu] = irqctx;
183 174
184 printk("CPU %u irqstacks, hard=%p soft=%p\n", 175 printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n",
185 cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); 176 cpu,hardirq_ctx[cpu],softirq_ctx[cpu]);
186} 177}
187 178
188void irq_ctx_exit(int cpu) 179void irq_ctx_exit(int cpu)
@@ -211,25 +202,56 @@ asmlinkage void do_softirq(void)
211 /* build the stack frame on the softirq stack */ 202 /* build the stack frame on the softirq stack */
212 isp = (u32*) ((char*)irqctx + sizeof(*irqctx)); 203 isp = (u32*) ((char*)irqctx + sizeof(*irqctx));
213 204
214 asm volatile( 205 call_on_stack(__do_softirq, isp);
215 " xchgl %%ebx,%%esp \n"
216 " call __do_softirq \n"
217 " movl %%ebx,%%esp \n"
218 : "=b"(isp)
219 : "0"(isp)
220 : "memory", "cc", "edx", "ecx", "eax"
221 );
222 /* 206 /*
223 * Shouldnt happen, we returned above if in_interrupt(): 207 * Shouldnt happen, we returned above if in_interrupt():
224 */ 208 */
225 WARN_ON_ONCE(softirq_count()); 209 WARN_ON_ONCE(softirq_count());
226 } 210 }
227 211
228 local_irq_restore(flags); 212 local_irq_restore(flags);
229} 213}
214
215#else
216static inline int
217execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) { return 0; }
230#endif 218#endif
231 219
232/* 220/*
221 * do_IRQ handles all normal device IRQ's (the special
222 * SMP cross-CPU interrupts have their own specific
223 * handlers).
224 */
225unsigned int do_IRQ(struct pt_regs *regs)
226{
227 struct pt_regs *old_regs;
228 /* high bit used in ret_from_ code */
229 int overflow, irq = ~regs->orig_ax;
230 struct irq_desc *desc = irq_desc + irq;
231
232 if (unlikely((unsigned)irq >= NR_IRQS)) {
233 printk(KERN_EMERG "%s: cannot handle IRQ %d\n",
234 __func__, irq);
235 BUG();
236 }
237
238 old_regs = set_irq_regs(regs);
239 irq_enter();
240
241 overflow = check_stack_overflow();
242
243 if (!execute_on_irq_stack(overflow, desc, irq)) {
244 if (unlikely(overflow))
245 print_stack_overflow();
246 desc->handle_irq(irq, desc);
247 }
248
249 irq_exit();
250 set_irq_regs(old_regs);
251 return 1;
252}
253
254/*
233 * Interrupt statistics: 255 * Interrupt statistics:
234 */ 256 */
235 257
@@ -313,16 +335,20 @@ skip:
313 per_cpu(irq_stat,j).irq_tlb_count); 335 per_cpu(irq_stat,j).irq_tlb_count);
314 seq_printf(p, " TLB shootdowns\n"); 336 seq_printf(p, " TLB shootdowns\n");
315#endif 337#endif
338#ifdef CONFIG_X86_MCE
316 seq_printf(p, "TRM: "); 339 seq_printf(p, "TRM: ");
317 for_each_online_cpu(j) 340 for_each_online_cpu(j)
318 seq_printf(p, "%10u ", 341 seq_printf(p, "%10u ",
319 per_cpu(irq_stat,j).irq_thermal_count); 342 per_cpu(irq_stat,j).irq_thermal_count);
320 seq_printf(p, " Thermal event interrupts\n"); 343 seq_printf(p, " Thermal event interrupts\n");
344#endif
345#ifdef CONFIG_X86_LOCAL_APIC
321 seq_printf(p, "SPU: "); 346 seq_printf(p, "SPU: ");
322 for_each_online_cpu(j) 347 for_each_online_cpu(j)
323 seq_printf(p, "%10u ", 348 seq_printf(p, "%10u ",
324 per_cpu(irq_stat,j).irq_spurious_count); 349 per_cpu(irq_stat,j).irq_spurious_count);
325 seq_printf(p, " Spurious interrupts\n"); 350 seq_printf(p, " Spurious interrupts\n");
351#endif
326 seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); 352 seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
327#if defined(CONFIG_X86_IO_APIC) 353#if defined(CONFIG_X86_IO_APIC)
328 seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count)); 354 seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
@@ -331,6 +357,40 @@ skip:
331 return 0; 357 return 0;
332} 358}
333 359
360/*
361 * /proc/stat helpers
362 */
363u64 arch_irq_stat_cpu(unsigned int cpu)
364{
365 u64 sum = nmi_count(cpu);
366
367#ifdef CONFIG_X86_LOCAL_APIC
368 sum += per_cpu(irq_stat, cpu).apic_timer_irqs;
369#endif
370#ifdef CONFIG_SMP
371 sum += per_cpu(irq_stat, cpu).irq_resched_count;
372 sum += per_cpu(irq_stat, cpu).irq_call_count;
373 sum += per_cpu(irq_stat, cpu).irq_tlb_count;
374#endif
375#ifdef CONFIG_X86_MCE
376 sum += per_cpu(irq_stat, cpu).irq_thermal_count;
377#endif
378#ifdef CONFIG_X86_LOCAL_APIC
379 sum += per_cpu(irq_stat, cpu).irq_spurious_count;
380#endif
381 return sum;
382}
383
384u64 arch_irq_stat(void)
385{
386 u64 sum = atomic_read(&irq_err_count);
387
388#ifdef CONFIG_X86_IO_APIC
389 sum += atomic_read(&irq_mis_count);
390#endif
391 return sum;
392}
393
334#ifdef CONFIG_HOTPLUG_CPU 394#ifdef CONFIG_HOTPLUG_CPU
335#include <mach_apic.h> 395#include <mach_apic.h>
336 396
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 3aac15466a91..1f78b238d8d2 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -135,6 +135,7 @@ skip:
135 seq_printf(p, "%10u ", cpu_pda(j)->irq_tlb_count); 135 seq_printf(p, "%10u ", cpu_pda(j)->irq_tlb_count);
136 seq_printf(p, " TLB shootdowns\n"); 136 seq_printf(p, " TLB shootdowns\n");
137#endif 137#endif
138#ifdef CONFIG_X86_MCE
138 seq_printf(p, "TRM: "); 139 seq_printf(p, "TRM: ");
139 for_each_online_cpu(j) 140 for_each_online_cpu(j)
140 seq_printf(p, "%10u ", cpu_pda(j)->irq_thermal_count); 141 seq_printf(p, "%10u ", cpu_pda(j)->irq_thermal_count);
@@ -143,6 +144,7 @@ skip:
143 for_each_online_cpu(j) 144 for_each_online_cpu(j)
144 seq_printf(p, "%10u ", cpu_pda(j)->irq_threshold_count); 145 seq_printf(p, "%10u ", cpu_pda(j)->irq_threshold_count);
145 seq_printf(p, " Threshold APIC interrupts\n"); 146 seq_printf(p, " Threshold APIC interrupts\n");
147#endif
146 seq_printf(p, "SPU: "); 148 seq_printf(p, "SPU: ");
147 for_each_online_cpu(j) 149 for_each_online_cpu(j)
148 seq_printf(p, "%10u ", cpu_pda(j)->irq_spurious_count); 150 seq_printf(p, "%10u ", cpu_pda(j)->irq_spurious_count);
@@ -153,6 +155,32 @@ skip:
153} 155}
154 156
155/* 157/*
158 * /proc/stat helpers
159 */
160u64 arch_irq_stat_cpu(unsigned int cpu)
161{
162 u64 sum = cpu_pda(cpu)->__nmi_count;
163
164 sum += cpu_pda(cpu)->apic_timer_irqs;
165#ifdef CONFIG_SMP
166 sum += cpu_pda(cpu)->irq_resched_count;
167 sum += cpu_pda(cpu)->irq_call_count;
168 sum += cpu_pda(cpu)->irq_tlb_count;
169#endif
170#ifdef CONFIG_X86_MCE
171 sum += cpu_pda(cpu)->irq_thermal_count;
172 sum += cpu_pda(cpu)->irq_threshold_count;
173#endif
174 sum += cpu_pda(cpu)->irq_spurious_count;
175 return sum;
176}
177
178u64 arch_irq_stat(void)
179{
180 return atomic_read(&irq_err_count);
181}
182
183/*
156 * do_IRQ handles all normal device IRQ's (the special 184 * do_IRQ handles all normal device IRQ's (the special
157 * SMP cross-CPU interrupts have their own specific 185 * SMP cross-CPU interrupts have their own specific
158 * handlers). 186 * handlers).
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
new file mode 100644
index 000000000000..d66914287ee1
--- /dev/null
+++ b/arch/x86/kernel/irqinit_32.c
@@ -0,0 +1,114 @@
1#include <linux/errno.h>
2#include <linux/signal.h>
3#include <linux/sched.h>
4#include <linux/ioport.h>
5#include <linux/interrupt.h>
6#include <linux/slab.h>
7#include <linux/random.h>
8#include <linux/init.h>
9#include <linux/kernel_stat.h>
10#include <linux/sysdev.h>
11#include <linux/bitops.h>
12
13#include <asm/atomic.h>
14#include <asm/system.h>
15#include <asm/io.h>
16#include <asm/timer.h>
17#include <asm/pgtable.h>
18#include <asm/delay.h>
19#include <asm/desc.h>
20#include <asm/apic.h>
21#include <asm/arch_hooks.h>
22#include <asm/i8259.h>
23
24
25
26/*
27 * Note that on a 486, we don't want to do a SIGFPE on an irq13
28 * as the irq is unreliable, and exception 16 works correctly
29 * (ie as explained in the intel literature). On a 386, you
30 * can't use exception 16 due to bad IBM design, so we have to
31 * rely on the less exact irq13.
32 *
33 * Careful.. Not only is IRQ13 unreliable, but it is also
34 * leads to races. IBM designers who came up with it should
35 * be shot.
36 */
37
38
39static irqreturn_t math_error_irq(int cpl, void *dev_id)
40{
41 extern void math_error(void __user *);
42 outb(0,0xF0);
43 if (ignore_fpu_irq || !boot_cpu_data.hard_math)
44 return IRQ_NONE;
45 math_error((void __user *)get_irq_regs()->ip);
46 return IRQ_HANDLED;
47}
48
49/*
50 * New motherboards sometimes make IRQ 13 be a PCI interrupt,
51 * so allow interrupt sharing.
52 */
53static struct irqaction fpu_irq = {
54 .handler = math_error_irq,
55 .mask = CPU_MASK_NONE,
56 .name = "fpu",
57};
58
59void __init init_ISA_irqs (void)
60{
61 int i;
62
63#ifdef CONFIG_X86_LOCAL_APIC
64 init_bsp_APIC();
65#endif
66 init_8259A(0);
67
68 /*
69 * 16 old-style INTA-cycle interrupts:
70 */
71 for (i = 0; i < 16; i++) {
72 set_irq_chip_and_handler_name(i, &i8259A_chip,
73 handle_level_irq, "XT");
74 }
75}
76
77/* Overridden in paravirt.c */
78void init_IRQ(void) __attribute__((weak, alias("native_init_IRQ")));
79
80void __init native_init_IRQ(void)
81{
82 int i;
83
84 /* all the set up before the call gates are initialised */
85 pre_intr_init_hook();
86
87 /*
88 * Cover the whole vector space, no vector can escape
89 * us. (some of these will be overridden and become
90 * 'special' SMP interrupts)
91 */
92 for (i = 0; i < (NR_VECTORS - FIRST_EXTERNAL_VECTOR); i++) {
93 int vector = FIRST_EXTERNAL_VECTOR + i;
94 if (i >= NR_IRQS)
95 break;
96 /* SYSCALL_VECTOR was reserved in trap_init. */
97 if (!test_bit(vector, used_vectors))
98 set_intr_gate(vector, interrupt[i]);
99 }
100
101 /* setup after call gates are initialised (usually add in
102 * the architecture specific gates)
103 */
104 intr_init_hook();
105
106 /*
107 * External FPU? Set up irq13 if so, for
108 * original braindamaged IBM FERR coupling.
109 */
110 if (boot_cpu_data.hard_math && !cpu_has_fpu)
111 setup_irq(FPU_IRQ, &fpu_irq);
112
113 irq_ctx_init(smp_processor_id());
114}
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c
new file mode 100644
index 000000000000..0373e88de95a
--- /dev/null
+++ b/arch/x86/kernel/irqinit_64.c
@@ -0,0 +1,221 @@
1#include <linux/linkage.h>
2#include <linux/errno.h>
3#include <linux/signal.h>
4#include <linux/sched.h>
5#include <linux/ioport.h>
6#include <linux/interrupt.h>
7#include <linux/timex.h>
8#include <linux/slab.h>
9#include <linux/random.h>
10#include <linux/init.h>
11#include <linux/kernel_stat.h>
12#include <linux/sysdev.h>
13#include <linux/bitops.h>
14
15#include <asm/acpi.h>
16#include <asm/atomic.h>
17#include <asm/system.h>
18#include <asm/io.h>
19#include <asm/hw_irq.h>
20#include <asm/pgtable.h>
21#include <asm/delay.h>
22#include <asm/desc.h>
23#include <asm/apic.h>
24#include <asm/i8259.h>
25
26/*
27 * Common place to define all x86 IRQ vectors
28 *
29 * This builds up the IRQ handler stubs using some ugly macros in irq.h
30 *
31 * These macros create the low-level assembly IRQ routines that save
32 * register context and call do_IRQ(). do_IRQ() then does all the
33 * operations that are needed to keep the AT (or SMP IOAPIC)
34 * interrupt-controller happy.
35 */
36
37#define IRQ_NAME2(nr) nr##_interrupt(void)
38#define IRQ_NAME(nr) IRQ_NAME2(IRQ##nr)
39
40/*
41 * SMP has a few special interrupts for IPI messages
42 */
43
44#define BUILD_IRQ(nr) \
45 asmlinkage void IRQ_NAME(nr); \
46 asm("\n.p2align\n" \
47 "IRQ" #nr "_interrupt:\n\t" \
48 "push $~(" #nr ") ; " \
49 "jmp common_interrupt");
50
51#define BI(x,y) \
52 BUILD_IRQ(x##y)
53
54#define BUILD_16_IRQS(x) \
55 BI(x,0) BI(x,1) BI(x,2) BI(x,3) \
56 BI(x,4) BI(x,5) BI(x,6) BI(x,7) \
57 BI(x,8) BI(x,9) BI(x,a) BI(x,b) \
58 BI(x,c) BI(x,d) BI(x,e) BI(x,f)
59
60/*
61 * ISA PIC or low IO-APIC triggered (INTA-cycle or APIC) interrupts:
62 * (these are usually mapped to vectors 0x30-0x3f)
63 */
64
65/*
66 * The IO-APIC gives us many more interrupt sources. Most of these
67 * are unused but an SMP system is supposed to have enough memory ...
68 * sometimes (mostly wrt. hw bugs) we get corrupted vectors all
69 * across the spectrum, so we really want to be prepared to get all
70 * of these. Plus, more powerful systems might have more than 64
71 * IO-APIC registers.
72 *
73 * (these are usually mapped into the 0x30-0xff vector range)
74 */
75 BUILD_16_IRQS(0x2) BUILD_16_IRQS(0x3)
76BUILD_16_IRQS(0x4) BUILD_16_IRQS(0x5) BUILD_16_IRQS(0x6) BUILD_16_IRQS(0x7)
77BUILD_16_IRQS(0x8) BUILD_16_IRQS(0x9) BUILD_16_IRQS(0xa) BUILD_16_IRQS(0xb)
78BUILD_16_IRQS(0xc) BUILD_16_IRQS(0xd) BUILD_16_IRQS(0xe) BUILD_16_IRQS(0xf)
79
80#undef BUILD_16_IRQS
81#undef BI
82
83
84#define IRQ(x,y) \
85 IRQ##x##y##_interrupt
86
87#define IRQLIST_16(x) \
88 IRQ(x,0), IRQ(x,1), IRQ(x,2), IRQ(x,3), \
89 IRQ(x,4), IRQ(x,5), IRQ(x,6), IRQ(x,7), \
90 IRQ(x,8), IRQ(x,9), IRQ(x,a), IRQ(x,b), \
91 IRQ(x,c), IRQ(x,d), IRQ(x,e), IRQ(x,f)
92
93/* for the irq vectors */
94static void (*__initdata interrupt[NR_VECTORS - FIRST_EXTERNAL_VECTOR])(void) = {
95 IRQLIST_16(0x2), IRQLIST_16(0x3),
96 IRQLIST_16(0x4), IRQLIST_16(0x5), IRQLIST_16(0x6), IRQLIST_16(0x7),
97 IRQLIST_16(0x8), IRQLIST_16(0x9), IRQLIST_16(0xa), IRQLIST_16(0xb),
98 IRQLIST_16(0xc), IRQLIST_16(0xd), IRQLIST_16(0xe), IRQLIST_16(0xf)
99};
100
101#undef IRQ
102#undef IRQLIST_16
103
104
105
106
107/*
108 * IRQ2 is cascade interrupt to second interrupt controller
109 */
110
111static struct irqaction irq2 = {
112 .handler = no_action,
113 .mask = CPU_MASK_NONE,
114 .name = "cascade",
115};
116DEFINE_PER_CPU(vector_irq_t, vector_irq) = {
117 [0 ... IRQ0_VECTOR - 1] = -1,
118 [IRQ0_VECTOR] = 0,
119 [IRQ1_VECTOR] = 1,
120 [IRQ2_VECTOR] = 2,
121 [IRQ3_VECTOR] = 3,
122 [IRQ4_VECTOR] = 4,
123 [IRQ5_VECTOR] = 5,
124 [IRQ6_VECTOR] = 6,
125 [IRQ7_VECTOR] = 7,
126 [IRQ8_VECTOR] = 8,
127 [IRQ9_VECTOR] = 9,
128 [IRQ10_VECTOR] = 10,
129 [IRQ11_VECTOR] = 11,
130 [IRQ12_VECTOR] = 12,
131 [IRQ13_VECTOR] = 13,
132 [IRQ14_VECTOR] = 14,
133 [IRQ15_VECTOR] = 15,
134 [IRQ15_VECTOR + 1 ... NR_VECTORS - 1] = -1
135};
136
137static void __init init_ISA_irqs (void)
138{
139 int i;
140
141 init_bsp_APIC();
142 init_8259A(0);
143
144 for (i = 0; i < NR_IRQS; i++) {
145 irq_desc[i].status = IRQ_DISABLED;
146 irq_desc[i].action = NULL;
147 irq_desc[i].depth = 1;
148
149 if (i < 16) {
150 /*
151 * 16 old-style INTA-cycle interrupts:
152 */
153 set_irq_chip_and_handler_name(i, &i8259A_chip,
154 handle_level_irq, "XT");
155 } else {
156 /*
157 * 'high' PCI IRQs filled in on demand
158 */
159 irq_desc[i].chip = &no_irq_chip;
160 }
161 }
162}
163
164void init_IRQ(void) __attribute__((weak, alias("native_init_IRQ")));
165
166void __init native_init_IRQ(void)
167{
168 int i;
169
170 init_ISA_irqs();
171 /*
172 * Cover the whole vector space, no vector can escape
173 * us. (some of these will be overridden and become
174 * 'special' SMP interrupts)
175 */
176 for (i = 0; i < (NR_VECTORS - FIRST_EXTERNAL_VECTOR); i++) {
177 int vector = FIRST_EXTERNAL_VECTOR + i;
178 if (vector != IA32_SYSCALL_VECTOR)
179 set_intr_gate(vector, interrupt[i]);
180 }
181
182#ifdef CONFIG_SMP
183 /*
184 * The reschedule interrupt is a CPU-to-CPU reschedule-helper
185 * IPI, driven by wakeup.
186 */
187 alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
188
189 /* IPIs for invalidation */
190 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
191 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
192 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
193 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
194 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
195 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
196 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
197 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
198
199 /* IPI for generic function call */
200 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
201
202 /* IPI for generic single function call */
203 alloc_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
204 call_function_single_interrupt);
205
206 /* Low priority IPI to cleanup after moving an irq */
207 set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
208#endif
209 alloc_intr_gate(THERMAL_APIC_VECTOR, thermal_interrupt);
210 alloc_intr_gate(THRESHOLD_APIC_VECTOR, threshold_interrupt);
211
212 /* self generated IPI for local APIC timer */
213 alloc_intr_gate(LOCAL_TIMER_VECTOR, apic_timer_interrupt);
214
215 /* IPI vectors for APIC spurious and error interrupts */
216 alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
217 alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
218
219 if (!acpi_ioapic)
220 setup_irq(2, &irq2);
221}
diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index 0224c3637c73..a8449571858a 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -20,9 +20,9 @@
20#include <asm/mmu_context.h> 20#include <asm/mmu_context.h>
21 21
22#ifdef CONFIG_SMP 22#ifdef CONFIG_SMP
23static void flush_ldt(void *null) 23static void flush_ldt(void *current_mm)
24{ 24{
25 if (current->active_mm) 25 if (current->active_mm == current_mm)
26 load_LDT(&current->active_mm->context); 26 load_LDT(&current->active_mm->context);
27} 27}
28#endif 28#endif
@@ -68,7 +68,7 @@ static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
68 load_LDT(pc); 68 load_LDT(pc);
69 mask = cpumask_of_cpu(smp_processor_id()); 69 mask = cpumask_of_cpu(smp_processor_id());
70 if (!cpus_equal(current->mm->cpu_vm_mask, mask)) 70 if (!cpus_equal(current->mm->cpu_vm_mask, mask))
71 smp_call_function(flush_ldt, NULL, 1, 1); 71 smp_call_function(flush_ldt, current->mm, 1);
72 preempt_enable(); 72 preempt_enable();
73#else 73#else
74 load_LDT(pc); 74 load_LDT(pc);
diff --git a/arch/x86/kernel/machine_kexec_32.c b/arch/x86/kernel/machine_kexec_32.c
index d0b234c9fc31..8864230d55af 100644
--- a/arch/x86/kernel/machine_kexec_32.c
+++ b/arch/x86/kernel/machine_kexec_32.c
@@ -11,6 +11,8 @@
11#include <linux/delay.h> 11#include <linux/delay.h>
12#include <linux/init.h> 12#include <linux/init.h>
13#include <linux/numa.h> 13#include <linux/numa.h>
14#include <linux/ftrace.h>
15
14#include <asm/pgtable.h> 16#include <asm/pgtable.h>
15#include <asm/pgalloc.h> 17#include <asm/pgalloc.h>
16#include <asm/tlbflush.h> 18#include <asm/tlbflush.h>
@@ -39,7 +41,7 @@ static void set_idt(void *newidt, __u16 limit)
39 curidt.address = (unsigned long)newidt; 41 curidt.address = (unsigned long)newidt;
40 42
41 load_idt(&curidt); 43 load_idt(&curidt);
42}; 44}
43 45
44 46
45static void set_gdt(void *newgdt, __u16 limit) 47static void set_gdt(void *newgdt, __u16 limit)
@@ -51,7 +53,7 @@ static void set_gdt(void *newgdt, __u16 limit)
51 curgdt.address = (unsigned long)newgdt; 53 curgdt.address = (unsigned long)newgdt;
52 54
53 load_gdt(&curgdt); 55 load_gdt(&curgdt);
54}; 56}
55 57
56static void load_segments(void) 58static void load_segments(void)
57{ 59{
@@ -107,6 +109,8 @@ NORET_TYPE void machine_kexec(struct kimage *image)
107 unsigned long page_list[PAGES_NR]; 109 unsigned long page_list[PAGES_NR];
108 void *control_page; 110 void *control_page;
109 111
112 tracer_disable();
113
110 /* Interrupts aren't acceptable while we reboot */ 114 /* Interrupts aren't acceptable while we reboot */
111 local_irq_disable(); 115 local_irq_disable();
112 116
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 576a03db4511..9dd9262693a3 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -11,6 +11,8 @@
11#include <linux/string.h> 11#include <linux/string.h>
12#include <linux/reboot.h> 12#include <linux/reboot.h>
13#include <linux/numa.h> 13#include <linux/numa.h>
14#include <linux/ftrace.h>
15
14#include <asm/pgtable.h> 16#include <asm/pgtable.h>
15#include <asm/tlbflush.h> 17#include <asm/tlbflush.h>
16#include <asm/mmu_context.h> 18#include <asm/mmu_context.h>
@@ -110,7 +112,7 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
110{ 112{
111 pgd_t *level4p; 113 pgd_t *level4p;
112 level4p = (pgd_t *)__va(start_pgtable); 114 level4p = (pgd_t *)__va(start_pgtable);
113 return init_level4_page(image, level4p, 0, end_pfn << PAGE_SHIFT); 115 return init_level4_page(image, level4p, 0, max_pfn << PAGE_SHIFT);
114} 116}
115 117
116static void set_idt(void *newidt, u16 limit) 118static void set_idt(void *newidt, u16 limit)
@@ -184,6 +186,8 @@ NORET_TYPE void machine_kexec(struct kimage *image)
184 unsigned long page_list[PAGES_NR]; 186 unsigned long page_list[PAGES_NR];
185 void *control_page; 187 void *control_page;
186 188
189 tracer_disable();
190
187 /* Interrupts aren't acceptable while we reboot */ 191 /* Interrupts aren't acceptable while we reboot */
188 local_irq_disable(); 192 local_irq_disable();
189 193
diff --git a/arch/x86/kernel/microcode.c b/arch/x86/kernel/microcode.c
index 69729e38b78a..56b933119a04 100644
--- a/arch/x86/kernel/microcode.c
+++ b/arch/x86/kernel/microcode.c
@@ -5,13 +5,14 @@
5 * 2006 Shaohua Li <shaohua.li@intel.com> 5 * 2006 Shaohua Li <shaohua.li@intel.com>
6 * 6 *
7 * This driver allows to upgrade microcode on Intel processors 7 * This driver allows to upgrade microcode on Intel processors
8 * belonging to IA-32 family - PentiumPro, Pentium II, 8 * belonging to IA-32 family - PentiumPro, Pentium II,
9 * Pentium III, Xeon, Pentium 4, etc. 9 * Pentium III, Xeon, Pentium 4, etc.
10 * 10 *
11 * Reference: Section 8.10 of Volume III, Intel Pentium 4 Manual, 11 * Reference: Section 8.11 of Volume 3a, IA-32 Intel? Architecture
12 * Order Number 245472 or free download from: 12 * Software Developer's Manual
13 * 13 * Order Number 253668 or free download from:
14 * http://developer.intel.com/design/pentium4/manuals/245472.htm 14 *
15 * http://developer.intel.com/design/pentium4/manuals/253668.htm
15 * 16 *
16 * For more information, go to http://www.urbanmyth.org/microcode 17 * For more information, go to http://www.urbanmyth.org/microcode
17 * 18 *
@@ -58,12 +59,12 @@
58 * nature of implementation. 59 * nature of implementation.
59 * 1.11 22 Mar 2002 Tigran Aivazian <tigran@veritas.com> 60 * 1.11 22 Mar 2002 Tigran Aivazian <tigran@veritas.com>
60 * Fix the panic when writing zero-length microcode chunk. 61 * Fix the panic when writing zero-length microcode chunk.
61 * 1.12 29 Sep 2003 Nitin Kamble <nitin.a.kamble@intel.com>, 62 * 1.12 29 Sep 2003 Nitin Kamble <nitin.a.kamble@intel.com>,
62 * Jun Nakajima <jun.nakajima@intel.com> 63 * Jun Nakajima <jun.nakajima@intel.com>
63 * Support for the microcode updates in the new format. 64 * Support for the microcode updates in the new format.
64 * 1.13 10 Oct 2003 Tigran Aivazian <tigran@veritas.com> 65 * 1.13 10 Oct 2003 Tigran Aivazian <tigran@veritas.com>
65 * Removed ->read() method and obsoleted MICROCODE_IOCFREE ioctl 66 * Removed ->read() method and obsoleted MICROCODE_IOCFREE ioctl
66 * because we no longer hold a copy of applied microcode 67 * because we no longer hold a copy of applied microcode
67 * in kernel memory. 68 * in kernel memory.
68 * 1.14 25 Jun 2004 Tigran Aivazian <tigran@veritas.com> 69 * 1.14 25 Jun 2004 Tigran Aivazian <tigran@veritas.com>
69 * Fix sigmatch() macro to handle old CPUs with pf == 0. 70 * Fix sigmatch() macro to handle old CPUs with pf == 0.
@@ -75,6 +76,7 @@
75#include <linux/kernel.h> 76#include <linux/kernel.h>
76#include <linux/init.h> 77#include <linux/init.h>
77#include <linux/sched.h> 78#include <linux/sched.h>
79#include <linux/smp_lock.h>
78#include <linux/cpumask.h> 80#include <linux/cpumask.h>
79#include <linux/module.h> 81#include <linux/module.h>
80#include <linux/slab.h> 82#include <linux/slab.h>
@@ -320,11 +322,11 @@ static void apply_microcode(int cpu)
320 return; 322 return;
321 323
322 /* serialize access to the physical write to MSR 0x79 */ 324 /* serialize access to the physical write to MSR 0x79 */
323 spin_lock_irqsave(&microcode_update_lock, flags); 325 spin_lock_irqsave(&microcode_update_lock, flags);
324 326
325 /* write microcode via MSR 0x79 */ 327 /* write microcode via MSR 0x79 */
326 wrmsr(MSR_IA32_UCODE_WRITE, 328 wrmsr(MSR_IA32_UCODE_WRITE,
327 (unsigned long) uci->mc->bits, 329 (unsigned long) uci->mc->bits,
328 (unsigned long) uci->mc->bits >> 16 >> 16); 330 (unsigned long) uci->mc->bits >> 16 >> 16);
329 wrmsr(MSR_IA32_UCODE_REV, 0, 0); 331 wrmsr(MSR_IA32_UCODE_REV, 0, 0);
330 332
@@ -341,7 +343,7 @@ static void apply_microcode(int cpu)
341 return; 343 return;
342 } 344 }
343 printk(KERN_INFO "microcode: CPU%d updated from revision " 345 printk(KERN_INFO "microcode: CPU%d updated from revision "
344 "0x%x to 0x%x, date = %08x \n", 346 "0x%x to 0x%x, date = %08x \n",
345 cpu_num, uci->rev, val[1], uci->mc->hdr.date); 347 cpu_num, uci->rev, val[1], uci->mc->hdr.date);
346 uci->rev = val[1]; 348 uci->rev = val[1];
347} 349}
@@ -422,6 +424,7 @@ out:
422 424
423static int microcode_open (struct inode *unused1, struct file *unused2) 425static int microcode_open (struct inode *unused1, struct file *unused2)
424{ 426{
427 cycle_kernel_lock();
425 return capable(CAP_SYS_RAWIO) ? 0 : -EPERM; 428 return capable(CAP_SYS_RAWIO) ? 0 : -EPERM;
426} 429}
427 430
@@ -488,7 +491,7 @@ MODULE_ALIAS_MISCDEV(MICROCODE_MINOR);
488#define microcode_dev_exit() do { } while(0) 491#define microcode_dev_exit() do { } while(0)
489#endif 492#endif
490 493
491static long get_next_ucode_from_buffer(void **mc, void *buf, 494static long get_next_ucode_from_buffer(void **mc, const u8 *buf,
492 unsigned long size, long offset) 495 unsigned long size, long offset)
493{ 496{
494 microcode_header_t *mc_header; 497 microcode_header_t *mc_header;
@@ -522,7 +525,7 @@ static int cpu_request_microcode(int cpu)
522 char name[30]; 525 char name[30];
523 struct cpuinfo_x86 *c = &cpu_data(cpu); 526 struct cpuinfo_x86 *c = &cpu_data(cpu);
524 const struct firmware *firmware; 527 const struct firmware *firmware;
525 void *buf; 528 const u8 *buf;
526 unsigned long size; 529 unsigned long size;
527 long offset = 0; 530 long offset = 0;
528 int error; 531 int error;
@@ -534,7 +537,7 @@ static int cpu_request_microcode(int cpu)
534 c->x86, c->x86_model, c->x86_mask); 537 c->x86, c->x86_model, c->x86_mask);
535 error = request_firmware(&firmware, name, &microcode_pdev->dev); 538 error = request_firmware(&firmware, name, &microcode_pdev->dev);
536 if (error) { 539 if (error) {
537 pr_debug("microcode: ucode data file %s load failed\n", name); 540 pr_debug("microcode: data file %s load failed\n", name);
538 return error; 541 return error;
539 } 542 }
540 buf = firmware->data; 543 buf = firmware->data;
@@ -805,6 +808,9 @@ static int __init microcode_init (void)
805{ 808{
806 int error; 809 int error;
807 810
811 printk(KERN_INFO
812 "IA-32 Microcode Update Driver: v" MICROCODE_VERSION " <tigran@aivazian.fsnet.co.uk>\n");
813
808 error = microcode_dev_init(); 814 error = microcode_dev_init();
809 if (error) 815 if (error)
810 return error; 816 return error;
@@ -825,9 +831,6 @@ static int __init microcode_init (void)
825 } 831 }
826 832
827 register_hotcpu_notifier(&mc_cpu_notifier); 833 register_hotcpu_notifier(&mc_cpu_notifier);
828
829 printk(KERN_INFO
830 "IA-32 Microcode Update Driver: v" MICROCODE_VERSION " <tigran@aivazian.fsnet.co.uk>\n");
831 return 0; 834 return 0;
832} 835}
833 836
diff --git a/arch/x86/kernel/mmconf-fam10h_64.c b/arch/x86/kernel/mmconf-fam10h_64.c
index edc5fbfe85c0..fdfdc550b366 100644
--- a/arch/x86/kernel/mmconf-fam10h_64.c
+++ b/arch/x86/kernel/mmconf-fam10h_64.c
@@ -12,6 +12,7 @@
12#include <asm/io.h> 12#include <asm/io.h>
13#include <asm/msr.h> 13#include <asm/msr.h>
14#include <asm/acpi.h> 14#include <asm/acpi.h>
15#include <asm/mmconfig.h>
15 16
16#include "../pci/pci.h" 17#include "../pci/pci.h"
17 18
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index 404683b94e79..3b25e49380c6 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -25,6 +25,8 @@
25#include <asm/proto.h> 25#include <asm/proto.h>
26#include <asm/acpi.h> 26#include <asm/acpi.h>
27#include <asm/bios_ebda.h> 27#include <asm/bios_ebda.h>
28#include <asm/e820.h>
29#include <asm/trampoline.h>
28 30
29#include <mach_apic.h> 31#include <mach_apic.h>
30#ifdef CONFIG_X86_32 32#ifdef CONFIG_X86_32
@@ -32,28 +34,6 @@
32#include <mach_mpparse.h> 34#include <mach_mpparse.h>
33#endif 35#endif
34 36
35/* Have we found an MP table */
36int smp_found_config;
37
38/*
39 * Various Linux-internal data structures created from the
40 * MP-table.
41 */
42#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
43int mp_bus_id_to_type[MAX_MP_BUSSES];
44#endif
45
46DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
47int mp_bus_id_to_pci_bus[MAX_MP_BUSSES] = {[0 ... MAX_MP_BUSSES - 1] = -1 };
48
49static int mp_current_pci_id;
50
51int pic_mode;
52
53/*
54 * Intel MP BIOS table parsing routines:
55 */
56
57/* 37/*
58 * Checksum an MP configuration block. 38 * Checksum an MP configuration block.
59 */ 39 */
@@ -69,15 +49,73 @@ static int __init mpf_checksum(unsigned char *mp, int len)
69} 49}
70 50
71#ifdef CONFIG_X86_NUMAQ 51#ifdef CONFIG_X86_NUMAQ
52int found_numaq;
72/* 53/*
73 * Have to match translation table entries to main table entries by counter 54 * Have to match translation table entries to main table entries by counter
74 * hence the mpc_record variable .... can't see a less disgusting way of 55 * hence the mpc_record variable .... can't see a less disgusting way of
75 * doing this .... 56 * doing this ....
76 */ 57 */
58struct mpc_config_translation {
59 unsigned char mpc_type;
60 unsigned char trans_len;
61 unsigned char trans_type;
62 unsigned char trans_quad;
63 unsigned char trans_global;
64 unsigned char trans_local;
65 unsigned short trans_reserved;
66};
67
77 68
78static int mpc_record; 69static int mpc_record;
79static struct mpc_config_translation *translation_table[MAX_MPC_ENTRY] 70static struct mpc_config_translation *translation_table[MAX_MPC_ENTRY]
80 __cpuinitdata; 71 __cpuinitdata;
72
73static inline int generate_logical_apicid(int quad, int phys_apicid)
74{
75 return (quad << 4) + (phys_apicid ? phys_apicid << 1 : 1);
76}
77
78
79static inline int mpc_apic_id(struct mpc_config_processor *m,
80 struct mpc_config_translation *translation_record)
81{
82 int quad = translation_record->trans_quad;
83 int logical_apicid = generate_logical_apicid(quad, m->mpc_apicid);
84
85 printk(KERN_DEBUG "Processor #%d %u:%u APIC version %d (quad %d, apic %d)\n",
86 m->mpc_apicid,
87 (m->mpc_cpufeature & CPU_FAMILY_MASK) >> 8,
88 (m->mpc_cpufeature & CPU_MODEL_MASK) >> 4,
89 m->mpc_apicver, quad, logical_apicid);
90 return logical_apicid;
91}
92
93int mp_bus_id_to_node[MAX_MP_BUSSES];
94
95int mp_bus_id_to_local[MAX_MP_BUSSES];
96
97static void mpc_oem_bus_info(struct mpc_config_bus *m, char *name,
98 struct mpc_config_translation *translation)
99{
100 int quad = translation->trans_quad;
101 int local = translation->trans_local;
102
103 mp_bus_id_to_node[m->mpc_busid] = quad;
104 mp_bus_id_to_local[m->mpc_busid] = local;
105 printk(KERN_INFO "Bus #%d is %s (node %d)\n",
106 m->mpc_busid, name, quad);
107}
108
109int quad_local_to_mp_bus_id [NR_CPUS/4][4];
110static void mpc_oem_pci_bus(struct mpc_config_bus *m,
111 struct mpc_config_translation *translation)
112{
113 int quad = translation->trans_quad;
114 int local = translation->trans_local;
115
116 quad_local_to_mp_bus_id[quad][local] = m->mpc_busid;
117}
118
81#endif 119#endif
82 120
83static void __cpuinit MP_processor_info(struct mpc_config_processor *m) 121static void __cpuinit MP_processor_info(struct mpc_config_processor *m)
@@ -90,7 +128,10 @@ static void __cpuinit MP_processor_info(struct mpc_config_processor *m)
90 return; 128 return;
91 } 129 }
92#ifdef CONFIG_X86_NUMAQ 130#ifdef CONFIG_X86_NUMAQ
93 apicid = mpc_apic_id(m, translation_table[mpc_record]); 131 if (found_numaq)
132 apicid = mpc_apic_id(m, translation_table[mpc_record]);
133 else
134 apicid = m->mpc_apicid;
94#else 135#else
95 apicid = m->mpc_apicid; 136 apicid = m->mpc_apicid;
96#endif 137#endif
@@ -103,17 +144,18 @@ static void __cpuinit MP_processor_info(struct mpc_config_processor *m)
103 generic_processor_info(apicid, m->mpc_apicver); 144 generic_processor_info(apicid, m->mpc_apicver);
104} 145}
105 146
147#ifdef CONFIG_X86_IO_APIC
106static void __init MP_bus_info(struct mpc_config_bus *m) 148static void __init MP_bus_info(struct mpc_config_bus *m)
107{ 149{
108 char str[7]; 150 char str[7];
109
110 memcpy(str, m->mpc_bustype, 6); 151 memcpy(str, m->mpc_bustype, 6);
111 str[6] = 0; 152 str[6] = 0;
112 153
113#ifdef CONFIG_X86_NUMAQ 154#ifdef CONFIG_X86_NUMAQ
114 mpc_oem_bus_info(m, str, translation_table[mpc_record]); 155 if (found_numaq)
156 mpc_oem_bus_info(m, str, translation_table[mpc_record]);
115#else 157#else
116 Dprintk("Bus #%d is %s\n", m->mpc_busid, str); 158 printk(KERN_INFO "Bus #%d is %s\n", m->mpc_busid, str);
117#endif 159#endif
118 160
119#if MAX_MP_BUSSES < 256 161#if MAX_MP_BUSSES < 256
@@ -132,11 +174,10 @@ static void __init MP_bus_info(struct mpc_config_bus *m)
132#endif 174#endif
133 } else if (strncmp(str, BUSTYPE_PCI, sizeof(BUSTYPE_PCI) - 1) == 0) { 175 } else if (strncmp(str, BUSTYPE_PCI, sizeof(BUSTYPE_PCI) - 1) == 0) {
134#ifdef CONFIG_X86_NUMAQ 176#ifdef CONFIG_X86_NUMAQ
135 mpc_oem_pci_bus(m, translation_table[mpc_record]); 177 if (found_numaq)
178 mpc_oem_pci_bus(m, translation_table[mpc_record]);
136#endif 179#endif
137 clear_bit(m->mpc_busid, mp_bus_not_pci); 180 clear_bit(m->mpc_busid, mp_bus_not_pci);
138 mp_bus_id_to_pci_bus[m->mpc_busid] = mp_current_pci_id;
139 mp_current_pci_id++;
140#if defined(CONFIG_EISA) || defined (CONFIG_MCA) 181#if defined(CONFIG_EISA) || defined (CONFIG_MCA)
141 mp_bus_id_to_type[m->mpc_busid] = MP_BUS_PCI; 182 mp_bus_id_to_type[m->mpc_busid] = MP_BUS_PCI;
142 } else if (strncmp(str, BUSTYPE_EISA, sizeof(BUSTYPE_EISA) - 1) == 0) { 183 } else if (strncmp(str, BUSTYPE_EISA, sizeof(BUSTYPE_EISA) - 1) == 0) {
@@ -147,6 +188,7 @@ static void __init MP_bus_info(struct mpc_config_bus *m)
147 } else 188 } else
148 printk(KERN_WARNING "Unknown bustype %s - ignoring\n", str); 189 printk(KERN_WARNING "Unknown bustype %s - ignoring\n", str);
149} 190}
191#endif
150 192
151#ifdef CONFIG_X86_IO_APIC 193#ifdef CONFIG_X86_IO_APIC
152 194
@@ -176,18 +218,89 @@ static void __init MP_ioapic_info(struct mpc_config_ioapic *m)
176 if (bad_ioapic(m->mpc_apicaddr)) 218 if (bad_ioapic(m->mpc_apicaddr))
177 return; 219 return;
178 220
179 mp_ioapics[nr_ioapics] = *m; 221 mp_ioapics[nr_ioapics].mp_apicaddr = m->mpc_apicaddr;
222 mp_ioapics[nr_ioapics].mp_apicid = m->mpc_apicid;
223 mp_ioapics[nr_ioapics].mp_type = m->mpc_type;
224 mp_ioapics[nr_ioapics].mp_apicver = m->mpc_apicver;
225 mp_ioapics[nr_ioapics].mp_flags = m->mpc_flags;
180 nr_ioapics++; 226 nr_ioapics++;
181} 227}
182 228
183static void __init MP_intsrc_info(struct mpc_config_intsrc *m) 229static void print_MP_intsrc_info(struct mpc_config_intsrc *m)
184{ 230{
185 mp_irqs[mp_irq_entries] = *m; 231 printk(KERN_CONT "Int: type %d, pol %d, trig %d, bus %02x,"
186 Dprintk("Int: type %d, pol %d, trig %d, bus %d,"
187 " IRQ %02x, APIC ID %x, APIC INT %02x\n", 232 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
188 m->mpc_irqtype, m->mpc_irqflag & 3, 233 m->mpc_irqtype, m->mpc_irqflag & 3,
189 (m->mpc_irqflag >> 2) & 3, m->mpc_srcbus, 234 (m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
190 m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq); 235 m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
236}
237
238static void __init print_mp_irq_info(struct mp_config_intsrc *mp_irq)
239{
240 printk(KERN_CONT "Int: type %d, pol %d, trig %d, bus %02x,"
241 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
242 mp_irq->mp_irqtype, mp_irq->mp_irqflag & 3,
243 (mp_irq->mp_irqflag >> 2) & 3, mp_irq->mp_srcbus,
244 mp_irq->mp_srcbusirq, mp_irq->mp_dstapic, mp_irq->mp_dstirq);
245}
246
247static void __init assign_to_mp_irq(struct mpc_config_intsrc *m,
248 struct mp_config_intsrc *mp_irq)
249{
250 mp_irq->mp_dstapic = m->mpc_dstapic;
251 mp_irq->mp_type = m->mpc_type;
252 mp_irq->mp_irqtype = m->mpc_irqtype;
253 mp_irq->mp_irqflag = m->mpc_irqflag;
254 mp_irq->mp_srcbus = m->mpc_srcbus;
255 mp_irq->mp_srcbusirq = m->mpc_srcbusirq;
256 mp_irq->mp_dstirq = m->mpc_dstirq;
257}
258
259static void __init assign_to_mpc_intsrc(struct mp_config_intsrc *mp_irq,
260 struct mpc_config_intsrc *m)
261{
262 m->mpc_dstapic = mp_irq->mp_dstapic;
263 m->mpc_type = mp_irq->mp_type;
264 m->mpc_irqtype = mp_irq->mp_irqtype;
265 m->mpc_irqflag = mp_irq->mp_irqflag;
266 m->mpc_srcbus = mp_irq->mp_srcbus;
267 m->mpc_srcbusirq = mp_irq->mp_srcbusirq;
268 m->mpc_dstirq = mp_irq->mp_dstirq;
269}
270
271static int __init mp_irq_mpc_intsrc_cmp(struct mp_config_intsrc *mp_irq,
272 struct mpc_config_intsrc *m)
273{
274 if (mp_irq->mp_dstapic != m->mpc_dstapic)
275 return 1;
276 if (mp_irq->mp_type != m->mpc_type)
277 return 2;
278 if (mp_irq->mp_irqtype != m->mpc_irqtype)
279 return 3;
280 if (mp_irq->mp_irqflag != m->mpc_irqflag)
281 return 4;
282 if (mp_irq->mp_srcbus != m->mpc_srcbus)
283 return 5;
284 if (mp_irq->mp_srcbusirq != m->mpc_srcbusirq)
285 return 6;
286 if (mp_irq->mp_dstirq != m->mpc_dstirq)
287 return 7;
288
289 return 0;
290}
291
292static void __init MP_intsrc_info(struct mpc_config_intsrc *m)
293{
294 int i;
295
296 print_MP_intsrc_info(m);
297
298 for (i = 0; i < mp_irq_entries; i++) {
299 if (!mp_irq_mpc_intsrc_cmp(&mp_irqs[i], m))
300 return;
301 }
302
303 assign_to_mp_irq(m, &mp_irqs[mp_irq_entries]);
191 if (++mp_irq_entries == MAX_IRQ_SOURCES) 304 if (++mp_irq_entries == MAX_IRQ_SOURCES)
192 panic("Max # of irq sources exceeded!!\n"); 305 panic("Max # of irq sources exceeded!!\n");
193} 306}
@@ -196,7 +309,7 @@ static void __init MP_intsrc_info(struct mpc_config_intsrc *m)
196 309
197static void __init MP_lintsrc_info(struct mpc_config_lintsrc *m) 310static void __init MP_lintsrc_info(struct mpc_config_lintsrc *m)
198{ 311{
199 Dprintk("Lint: type %d, pol %d, trig %d, bus %d," 312 printk(KERN_INFO "Lint: type %d, pol %d, trig %d, bus %02x,"
200 " IRQ %02x, APIC ID %x, APIC LINT %02x\n", 313 " IRQ %02x, APIC ID %x, APIC LINT %02x\n",
201 m->mpc_irqtype, m->mpc_irqflag & 3, 314 m->mpc_irqtype, m->mpc_irqflag & 3,
202 (m->mpc_irqflag >> 2) & 3, m->mpc_srcbusid, 315 (m->mpc_irqflag >> 2) & 3, m->mpc_srcbusid,
@@ -266,11 +379,14 @@ static void __init smp_read_mpc_oem(struct mp_config_oemtable *oemtable,
266 } 379 }
267} 380}
268 381
269static inline void mps_oem_check(struct mp_config_table *mpc, char *oem, 382void numaq_mps_oem_check(struct mp_config_table *mpc, char *oem,
270 char *productid) 383 char *productid)
271{ 384{
272 if (strncmp(oem, "IBM NUMA", 8)) 385 if (strncmp(oem, "IBM NUMA", 8))
273 printk("Warning! May not be a NUMA-Q system!\n"); 386 printk("Warning! Not a NUMA-Q system!\n");
387 else
388 found_numaq = 1;
389
274 if (mpc->mpc_oemptr) 390 if (mpc->mpc_oemptr)
275 smp_read_mpc_oem((struct mp_config_oemtable *)mpc->mpc_oemptr, 391 smp_read_mpc_oem((struct mp_config_oemtable *)mpc->mpc_oemptr,
276 mpc->mpc_oemsize); 392 mpc->mpc_oemsize);
@@ -281,12 +397,9 @@ static inline void mps_oem_check(struct mp_config_table *mpc, char *oem,
281 * Read/parse the MPC 397 * Read/parse the MPC
282 */ 398 */
283 399
284static int __init smp_read_mpc(struct mp_config_table *mpc, unsigned early) 400static int __init smp_check_mpc(struct mp_config_table *mpc, char *oem,
401 char *str)
285{ 402{
286 char str[16];
287 char oem[10];
288 int count = sizeof(*mpc);
289 unsigned char *mpt = ((unsigned char *)mpc) + count;
290 403
291 if (memcmp(mpc->mpc_signature, MPC_SIGNATURE, 4)) { 404 if (memcmp(mpc->mpc_signature, MPC_SIGNATURE, 4)) {
292 printk(KERN_ERR "MPTABLE: bad signature [%c%c%c%c]!\n", 405 printk(KERN_ERR "MPTABLE: bad signature [%c%c%c%c]!\n",
@@ -309,19 +422,42 @@ static int __init smp_read_mpc(struct mp_config_table *mpc, unsigned early)
309 } 422 }
310 memcpy(oem, mpc->mpc_oem, 8); 423 memcpy(oem, mpc->mpc_oem, 8);
311 oem[8] = 0; 424 oem[8] = 0;
312 printk(KERN_INFO "MPTABLE: OEM ID: %s ", oem); 425 printk(KERN_INFO "MPTABLE: OEM ID: %s\n", oem);
313 426
314 memcpy(str, mpc->mpc_productid, 12); 427 memcpy(str, mpc->mpc_productid, 12);
315 str[12] = 0; 428 str[12] = 0;
316 printk("Product ID: %s ", str);
317 429
318#ifdef CONFIG_X86_32 430 printk(KERN_INFO "MPTABLE: Product ID: %s\n", str);
319 mps_oem_check(mpc, oem, str);
320#endif
321 printk(KERN_INFO "MPTABLE: Product ID: %s ", str);
322 431
323 printk(KERN_INFO "MPTABLE: APIC at: 0x%X\n", mpc->mpc_lapic); 432 printk(KERN_INFO "MPTABLE: APIC at: 0x%X\n", mpc->mpc_lapic);
324 433
434 return 1;
435}
436
437static int __init smp_read_mpc(struct mp_config_table *mpc, unsigned early)
438{
439 char str[16];
440 char oem[10];
441
442 int count = sizeof(*mpc);
443 unsigned char *mpt = ((unsigned char *)mpc) + count;
444
445 if (!smp_check_mpc(mpc, oem, str))
446 return 0;
447
448#ifdef CONFIG_X86_32
449 /*
450 * need to make sure summit and es7000's mps_oem_check is safe to be
451 * called early via genericarch 's mps_oem_check
452 */
453 if (early) {
454#ifdef CONFIG_X86_NUMAQ
455 numaq_mps_oem_check(mpc, oem, str);
456#endif
457 } else
458 mps_oem_check(mpc, oem, str);
459#endif
460
325 /* save the local APIC address, it might be non-default */ 461 /* save the local APIC address, it might be non-default */
326 if (!acpi_lapic) 462 if (!acpi_lapic)
327 mp_lapic_addr = mpc->mpc_lapic; 463 mp_lapic_addr = mpc->mpc_lapic;
@@ -352,7 +488,9 @@ static int __init smp_read_mpc(struct mp_config_table *mpc, unsigned early)
352 { 488 {
353 struct mpc_config_bus *m = 489 struct mpc_config_bus *m =
354 (struct mpc_config_bus *)mpt; 490 (struct mpc_config_bus *)mpt;
491#ifdef CONFIG_X86_IO_APIC
355 MP_bus_info(m); 492 MP_bus_info(m);
493#endif
356 mpt += sizeof(*m); 494 mpt += sizeof(*m);
357 count += sizeof(*m); 495 count += sizeof(*m);
358 break; 496 break;
@@ -402,6 +540,11 @@ static int __init smp_read_mpc(struct mp_config_table *mpc, unsigned early)
402 ++mpc_record; 540 ++mpc_record;
403#endif 541#endif
404 } 542 }
543
544#ifdef CONFIG_X86_GENERICARCH
545 generic_bigsmp_probe();
546#endif
547
405 setup_apic_routing(); 548 setup_apic_routing();
406 if (!num_processors) 549 if (!num_processors)
407 printk(KERN_ERR "MPTABLE: no processors registered!\n"); 550 printk(KERN_ERR "MPTABLE: no processors registered!\n");
@@ -427,7 +570,7 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
427 intsrc.mpc_type = MP_INTSRC; 570 intsrc.mpc_type = MP_INTSRC;
428 intsrc.mpc_irqflag = 0; /* conforming */ 571 intsrc.mpc_irqflag = 0; /* conforming */
429 intsrc.mpc_srcbus = 0; 572 intsrc.mpc_srcbus = 0;
430 intsrc.mpc_dstapic = mp_ioapics[0].mpc_apicid; 573 intsrc.mpc_dstapic = mp_ioapics[0].mp_apicid;
431 574
432 intsrc.mpc_irqtype = mp_INT; 575 intsrc.mpc_irqtype = mp_INT;
433 576
@@ -488,40 +631,11 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
488 MP_intsrc_info(&intsrc); 631 MP_intsrc_info(&intsrc);
489} 632}
490 633
491#endif
492 634
493static inline void __init construct_default_ISA_mptable(int mpc_default_type) 635static void construct_ioapic_table(int mpc_default_type)
494{ 636{
495 struct mpc_config_processor processor;
496 struct mpc_config_bus bus;
497#ifdef CONFIG_X86_IO_APIC
498 struct mpc_config_ioapic ioapic; 637 struct mpc_config_ioapic ioapic;
499#endif 638 struct mpc_config_bus bus;
500 struct mpc_config_lintsrc lintsrc;
501 int linttypes[2] = { mp_ExtINT, mp_NMI };
502 int i;
503
504 /*
505 * local APIC has default address
506 */
507 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
508
509 /*
510 * 2 CPUs, numbered 0 & 1.
511 */
512 processor.mpc_type = MP_PROCESSOR;
513 /* Either an integrated APIC or a discrete 82489DX. */
514 processor.mpc_apicver = mpc_default_type > 4 ? 0x10 : 0x01;
515 processor.mpc_cpuflag = CPU_ENABLED;
516 processor.mpc_cpufeature = (boot_cpu_data.x86 << 8) |
517 (boot_cpu_data.x86_model << 4) | boot_cpu_data.x86_mask;
518 processor.mpc_featureflag = boot_cpu_data.x86_capability[0];
519 processor.mpc_reserved[0] = 0;
520 processor.mpc_reserved[1] = 0;
521 for (i = 0; i < 2; i++) {
522 processor.mpc_apicid = i;
523 MP_processor_info(&processor);
524 }
525 639
526 bus.mpc_type = MP_BUS; 640 bus.mpc_type = MP_BUS;
527 bus.mpc_busid = 0; 641 bus.mpc_busid = 0;
@@ -550,7 +664,6 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
550 MP_bus_info(&bus); 664 MP_bus_info(&bus);
551 } 665 }
552 666
553#ifdef CONFIG_X86_IO_APIC
554 ioapic.mpc_type = MP_IOAPIC; 667 ioapic.mpc_type = MP_IOAPIC;
555 ioapic.mpc_apicid = 2; 668 ioapic.mpc_apicid = 2;
556 ioapic.mpc_apicver = mpc_default_type > 4 ? 0x10 : 0x01; 669 ioapic.mpc_apicver = mpc_default_type > 4 ? 0x10 : 0x01;
@@ -562,7 +675,42 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
562 * We set up most of the low 16 IO-APIC pins according to MPS rules. 675 * We set up most of the low 16 IO-APIC pins according to MPS rules.
563 */ 676 */
564 construct_default_ioirq_mptable(mpc_default_type); 677 construct_default_ioirq_mptable(mpc_default_type);
678}
679#else
680static inline void construct_ioapic_table(int mpc_default_type) { }
565#endif 681#endif
682
683static inline void __init construct_default_ISA_mptable(int mpc_default_type)
684{
685 struct mpc_config_processor processor;
686 struct mpc_config_lintsrc lintsrc;
687 int linttypes[2] = { mp_ExtINT, mp_NMI };
688 int i;
689
690 /*
691 * local APIC has default address
692 */
693 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
694
695 /*
696 * 2 CPUs, numbered 0 & 1.
697 */
698 processor.mpc_type = MP_PROCESSOR;
699 /* Either an integrated APIC or a discrete 82489DX. */
700 processor.mpc_apicver = mpc_default_type > 4 ? 0x10 : 0x01;
701 processor.mpc_cpuflag = CPU_ENABLED;
702 processor.mpc_cpufeature = (boot_cpu_data.x86 << 8) |
703 (boot_cpu_data.x86_model << 4) | boot_cpu_data.x86_mask;
704 processor.mpc_featureflag = boot_cpu_data.x86_capability[0];
705 processor.mpc_reserved[0] = 0;
706 processor.mpc_reserved[1] = 0;
707 for (i = 0; i < 2; i++) {
708 processor.mpc_apicid = i;
709 MP_processor_info(&processor);
710 }
711
712 construct_ioapic_table(mpc_default_type);
713
566 lintsrc.mpc_type = MP_LINTSRC; 714 lintsrc.mpc_type = MP_LINTSRC;
567 lintsrc.mpc_irqflag = 0; /* conforming */ 715 lintsrc.mpc_irqflag = 0; /* conforming */
568 lintsrc.mpc_srcbusid = 0; 716 lintsrc.mpc_srcbusid = 0;
@@ -578,12 +726,22 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
578static struct intel_mp_floating *mpf_found; 726static struct intel_mp_floating *mpf_found;
579 727
580/* 728/*
729 * Machine specific quirk for finding the SMP config before other setup
730 * activities destroy the table:
731 */
732int (*mach_get_smp_config_quirk)(unsigned int early);
733
734/*
581 * Scan the memory blocks for an SMP configuration block. 735 * Scan the memory blocks for an SMP configuration block.
582 */ 736 */
583static void __init __get_smp_config(unsigned early) 737static void __init __get_smp_config(unsigned int early)
584{ 738{
585 struct intel_mp_floating *mpf = mpf_found; 739 struct intel_mp_floating *mpf = mpf_found;
586 740
741 if (mach_get_smp_config_quirk) {
742 if (mach_get_smp_config_quirk(early))
743 return;
744 }
587 if (acpi_lapic && early) 745 if (acpi_lapic && early)
588 return; 746 return;
589 /* 747 /*
@@ -600,7 +758,7 @@ static void __init __get_smp_config(unsigned early)
600 758
601 printk(KERN_INFO "Intel MultiProcessor Specification v1.%d\n", 759 printk(KERN_INFO "Intel MultiProcessor Specification v1.%d\n",
602 mpf->mpf_specification); 760 mpf->mpf_specification);
603#ifdef CONFIG_X86_32 761#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
604 if (mpf->mpf_feature2 & (1 << 7)) { 762 if (mpf->mpf_feature2 & (1 << 7)) {
605 printk(KERN_INFO " IMCR and PIC compatibility mode.\n"); 763 printk(KERN_INFO " IMCR and PIC compatibility mode.\n");
606 pic_mode = 1; 764 pic_mode = 1;
@@ -632,7 +790,9 @@ static void __init __get_smp_config(unsigned early)
632 * override the defaults. 790 * override the defaults.
633 */ 791 */
634 if (!smp_read_mpc(phys_to_virt(mpf->mpf_physptr), early)) { 792 if (!smp_read_mpc(phys_to_virt(mpf->mpf_physptr), early)) {
793#ifdef CONFIG_X86_LOCAL_APIC
635 smp_found_config = 0; 794 smp_found_config = 0;
795#endif
636 printk(KERN_ERR 796 printk(KERN_ERR
637 "BIOS bug, MP table errors detected!...\n"); 797 "BIOS bug, MP table errors detected!...\n");
638 printk(KERN_ERR "... disabling SMP support. " 798 printk(KERN_ERR "... disabling SMP support. "
@@ -689,7 +849,7 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
689 unsigned int *bp = phys_to_virt(base); 849 unsigned int *bp = phys_to_virt(base);
690 struct intel_mp_floating *mpf; 850 struct intel_mp_floating *mpf;
691 851
692 Dprintk("Scan SMP from %p for %ld bytes.\n", bp, length); 852 printk(KERN_DEBUG "Scan SMP from %p for %ld bytes.\n", bp, length);
693 BUILD_BUG_ON(sizeof(*mpf) != 16); 853 BUILD_BUG_ON(sizeof(*mpf) != 16);
694 854
695 while (length > 0) { 855 while (length > 0) {
@@ -699,15 +859,21 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
699 !mpf_checksum((unsigned char *)bp, 16) && 859 !mpf_checksum((unsigned char *)bp, 16) &&
700 ((mpf->mpf_specification == 1) 860 ((mpf->mpf_specification == 1)
701 || (mpf->mpf_specification == 4))) { 861 || (mpf->mpf_specification == 4))) {
702 862#ifdef CONFIG_X86_LOCAL_APIC
703 smp_found_config = 1; 863 smp_found_config = 1;
864#endif
704 mpf_found = mpf; 865 mpf_found = mpf;
705#ifdef CONFIG_X86_32 866
706 printk(KERN_INFO "found SMP MP-table at [%p] %08lx\n", 867 printk(KERN_INFO "found SMP MP-table at [%p] %08lx\n",
707 mpf, virt_to_phys(mpf)); 868 mpf, virt_to_phys(mpf));
708 reserve_bootmem(virt_to_phys(mpf), PAGE_SIZE, 869
870 if (!reserve)
871 return 1;
872 reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
709 BOOTMEM_DEFAULT); 873 BOOTMEM_DEFAULT);
710 if (mpf->mpf_physptr) { 874 if (mpf->mpf_physptr) {
875 unsigned long size = PAGE_SIZE;
876#ifdef CONFIG_X86_32
711 /* 877 /*
712 * We cannot access to MPC table to compute 878 * We cannot access to MPC table to compute
713 * table size yet, as only few megabytes from 879 * table size yet, as only few megabytes from
@@ -717,24 +883,15 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
717 * PAGE_SIZE from mpg->mpf_physptr yields BUG() 883 * PAGE_SIZE from mpg->mpf_physptr yields BUG()
718 * in reserve_bootmem. 884 * in reserve_bootmem.
719 */ 885 */
720 unsigned long size = PAGE_SIZE;
721 unsigned long end = max_low_pfn * PAGE_SIZE; 886 unsigned long end = max_low_pfn * PAGE_SIZE;
722 if (mpf->mpf_physptr + size > end) 887 if (mpf->mpf_physptr + size > end)
723 size = end - mpf->mpf_physptr; 888 size = end - mpf->mpf_physptr;
724 reserve_bootmem(mpf->mpf_physptr, size, 889#endif
890 reserve_bootmem_generic(mpf->mpf_physptr, size,
725 BOOTMEM_DEFAULT); 891 BOOTMEM_DEFAULT);
726 } 892 }
727 893
728#else 894 return 1;
729 if (!reserve)
730 return 1;
731
732 reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
733 if (mpf->mpf_physptr)
734 reserve_bootmem_generic(mpf->mpf_physptr,
735 PAGE_SIZE);
736#endif
737 return 1;
738 } 895 }
739 bp += 4; 896 bp += 4;
740 length -= 16; 897 length -= 16;
@@ -742,10 +899,16 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
742 return 0; 899 return 0;
743} 900}
744 901
745static void __init __find_smp_config(unsigned reserve) 902int (*mach_find_smp_config_quirk)(unsigned int reserve);
903
904static void __init __find_smp_config(unsigned int reserve)
746{ 905{
747 unsigned int address; 906 unsigned int address;
748 907
908 if (mach_find_smp_config_quirk) {
909 if (mach_find_smp_config_quirk(reserve))
910 return;
911 }
749 /* 912 /*
750 * FIXME: Linux assumes you have 640K of base ram.. 913 * FIXME: Linux assumes you have 640K of base ram..
751 * this continues the error... 914 * this continues the error...
@@ -790,298 +953,294 @@ void __init find_smp_config(void)
790 __find_smp_config(1); 953 __find_smp_config(1);
791} 954}
792 955
793/* -------------------------------------------------------------------------- 956#ifdef CONFIG_X86_IO_APIC
794 ACPI-based MP Configuration 957static u8 __initdata irq_used[MAX_IRQ_SOURCES];
795 -------------------------------------------------------------------------- */
796 958
797/* 959static int __init get_MP_intsrc_index(struct mpc_config_intsrc *m)
798 * Keep this outside and initialized to 0, for !CONFIG_ACPI builds: 960{
799 */ 961 int i;
800int es7000_plat;
801 962
802#ifdef CONFIG_ACPI 963 if (m->mpc_irqtype != mp_INT)
964 return 0;
803 965
804#ifdef CONFIG_X86_IO_APIC 966 if (m->mpc_irqflag != 0x0f)
967 return 0;
805 968
806#define MP_ISA_BUS 0 969 /* not legacy */
807 970
808extern struct mp_ioapic_routing mp_ioapic_routing[MAX_IO_APICS]; 971 for (i = 0; i < mp_irq_entries; i++) {
972 if (mp_irqs[i].mp_irqtype != mp_INT)
973 continue;
809 974
810static int mp_find_ioapic(int gsi) 975 if (mp_irqs[i].mp_irqflag != 0x0f)
811{ 976 continue;
812 int i = 0;
813 977
814 /* Find the IOAPIC that manages this GSI. */ 978 if (mp_irqs[i].mp_srcbus != m->mpc_srcbus)
815 for (i = 0; i < nr_ioapics; i++) { 979 continue;
816 if ((gsi >= mp_ioapic_routing[i].gsi_base) 980 if (mp_irqs[i].mp_srcbusirq != m->mpc_srcbusirq)
817 && (gsi <= mp_ioapic_routing[i].gsi_end)) 981 continue;
818 return i; 982 if (irq_used[i]) {
983 /* already claimed */
984 return -2;
985 }
986 irq_used[i] = 1;
987 return i;
819 } 988 }
820 989
821 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi); 990 /* not found */
822 return -1; 991 return -1;
823} 992}
824 993
825static u8 __init uniq_ioapic_id(u8 id) 994#define SPARE_SLOT_NUM 20
826{ 995
827#ifdef CONFIG_X86_32 996static struct mpc_config_intsrc __initdata *m_spare[SPARE_SLOT_NUM];
828 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
829 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
830 return io_apic_get_unique_id(nr_ioapics, id);
831 else
832 return id;
833#else
834 int i;
835 DECLARE_BITMAP(used, 256);
836 bitmap_zero(used, 256);
837 for (i = 0; i < nr_ioapics; i++) {
838 struct mpc_config_ioapic *ia = &mp_ioapics[i];
839 __set_bit(ia->mpc_apicid, used);
840 }
841 if (!test_bit(id, used))
842 return id;
843 return find_first_zero_bit(used, 256);
844#endif 997#endif
845}
846 998
847void __init mp_register_ioapic(int id, u32 address, u32 gsi_base) 999static int __init replace_intsrc_all(struct mp_config_table *mpc,
1000 unsigned long mpc_new_phys,
1001 unsigned long mpc_new_length)
848{ 1002{
849 int idx = 0; 1003#ifdef CONFIG_X86_IO_APIC
850 1004 int i;
851 if (bad_ioapic(address)) 1005 int nr_m_spare = 0;
852 return; 1006#endif
853 1007
854 idx = nr_ioapics; 1008 int count = sizeof(*mpc);
1009 unsigned char *mpt = ((unsigned char *)mpc) + count;
855 1010
856 mp_ioapics[idx].mpc_type = MP_IOAPIC; 1011 printk(KERN_INFO "mpc_length %x\n", mpc->mpc_length);
857 mp_ioapics[idx].mpc_flags = MPC_APIC_USABLE; 1012 while (count < mpc->mpc_length) {
858 mp_ioapics[idx].mpc_apicaddr = address; 1013 switch (*mpt) {
1014 case MP_PROCESSOR:
1015 {
1016 struct mpc_config_processor *m =
1017 (struct mpc_config_processor *)mpt;
1018 mpt += sizeof(*m);
1019 count += sizeof(*m);
1020 break;
1021 }
1022 case MP_BUS:
1023 {
1024 struct mpc_config_bus *m =
1025 (struct mpc_config_bus *)mpt;
1026 mpt += sizeof(*m);
1027 count += sizeof(*m);
1028 break;
1029 }
1030 case MP_IOAPIC:
1031 {
1032 mpt += sizeof(struct mpc_config_ioapic);
1033 count += sizeof(struct mpc_config_ioapic);
1034 break;
1035 }
1036 case MP_INTSRC:
1037 {
1038#ifdef CONFIG_X86_IO_APIC
1039 struct mpc_config_intsrc *m =
1040 (struct mpc_config_intsrc *)mpt;
859 1041
860 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address); 1042 printk(KERN_INFO "OLD ");
861 mp_ioapics[idx].mpc_apicid = uniq_ioapic_id(id); 1043 print_MP_intsrc_info(m);
862#ifdef CONFIG_X86_32 1044 i = get_MP_intsrc_index(m);
863 mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx); 1045 if (i > 0) {
864#else 1046 assign_to_mpc_intsrc(&mp_irqs[i], m);
865 mp_ioapics[idx].mpc_apicver = 0; 1047 printk(KERN_INFO "NEW ");
1048 print_mp_irq_info(&mp_irqs[i]);
1049 } else if (!i) {
1050 /* legacy, do nothing */
1051 } else if (nr_m_spare < SPARE_SLOT_NUM) {
1052 /*
1053 * not found (-1), or duplicated (-2)
1054 * are invalid entries,
1055 * we need to use the slot later
1056 */
1057 m_spare[nr_m_spare] = m;
1058 nr_m_spare++;
1059 }
866#endif 1060#endif
867 /* 1061 mpt += sizeof(struct mpc_config_intsrc);
868 * Build basic GSI lookup table to facilitate gsi->io_apic lookups 1062 count += sizeof(struct mpc_config_intsrc);
869 * and to prevent reprogramming of IOAPIC pins (PCI GSIs). 1063 break;
870 */ 1064 }
871 mp_ioapic_routing[idx].apic_id = mp_ioapics[idx].mpc_apicid; 1065 case MP_LINTSRC:
872 mp_ioapic_routing[idx].gsi_base = gsi_base; 1066 {
873 mp_ioapic_routing[idx].gsi_end = gsi_base + 1067 struct mpc_config_lintsrc *m =
874 io_apic_get_redir_entries(idx); 1068 (struct mpc_config_lintsrc *)mpt;
875 1069 mpt += sizeof(*m);
876 printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, " 1070 count += sizeof(*m);
877 "GSI %d-%d\n", idx, mp_ioapics[idx].mpc_apicid, 1071 break;
878 mp_ioapics[idx].mpc_apicver, mp_ioapics[idx].mpc_apicaddr, 1072 }
879 mp_ioapic_routing[idx].gsi_base, mp_ioapic_routing[idx].gsi_end); 1073 default:
880 1074 /* wrong mptable */
881 nr_ioapics++; 1075 printk(KERN_ERR "Your mptable is wrong, contact your HW vendor!\n");
882} 1076 printk(KERN_ERR "type %x\n", *mpt);
1077 print_hex_dump(KERN_ERR, " ", DUMP_PREFIX_ADDRESS, 16,
1078 1, mpc, mpc->mpc_length, 1);
1079 goto out;
1080 }
1081 }
883 1082
884void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi) 1083#ifdef CONFIG_X86_IO_APIC
885{ 1084 for (i = 0; i < mp_irq_entries; i++) {
886 struct mpc_config_intsrc intsrc; 1085 if (irq_used[i])
887 int ioapic = -1; 1086 continue;
888 int pin = -1;
889 1087
890 /* 1088 if (mp_irqs[i].mp_irqtype != mp_INT)
891 * Convert 'gsi' to 'ioapic.pin'. 1089 continue;
892 */
893 ioapic = mp_find_ioapic(gsi);
894 if (ioapic < 0)
895 return;
896 pin = gsi - mp_ioapic_routing[ioapic].gsi_base;
897 1090
898 /* 1091 if (mp_irqs[i].mp_irqflag != 0x0f)
899 * TBD: This check is for faulty timer entries, where the override 1092 continue;
900 * erroneously sets the trigger to level, resulting in a HUGE
901 * increase of timer interrupts!
902 */
903 if ((bus_irq == 0) && (trigger == 3))
904 trigger = 1;
905 1093
906 intsrc.mpc_type = MP_INTSRC; 1094 if (nr_m_spare > 0) {
907 intsrc.mpc_irqtype = mp_INT; 1095 printk(KERN_INFO "*NEW* found ");
908 intsrc.mpc_irqflag = (trigger << 2) | polarity; 1096 nr_m_spare--;
909 intsrc.mpc_srcbus = MP_ISA_BUS; 1097 assign_to_mpc_intsrc(&mp_irqs[i], m_spare[nr_m_spare]);
910 intsrc.mpc_srcbusirq = bus_irq; /* IRQ */ 1098 m_spare[nr_m_spare] = NULL;
911 intsrc.mpc_dstapic = mp_ioapics[ioapic].mpc_apicid; /* APIC ID */ 1099 } else {
912 intsrc.mpc_dstirq = pin; /* INTIN# */ 1100 struct mpc_config_intsrc *m =
1101 (struct mpc_config_intsrc *)mpt;
1102 count += sizeof(struct mpc_config_intsrc);
1103 if (!mpc_new_phys) {
1104 printk(KERN_INFO "No spare slots, try to append...take your risk, new mpc_length %x\n", count);
1105 } else {
1106 if (count <= mpc_new_length)
1107 printk(KERN_INFO "No spare slots, try to append..., new mpc_length %x\n", count);
1108 else {
1109 printk(KERN_ERR "mpc_new_length %lx is too small\n", mpc_new_length);
1110 goto out;
1111 }
1112 }
1113 assign_to_mpc_intsrc(&mp_irqs[i], m);
1114 mpc->mpc_length = count;
1115 mpt += sizeof(struct mpc_config_intsrc);
1116 }
1117 print_mp_irq_info(&mp_irqs[i]);
1118 }
1119#endif
1120out:
1121 /* update checksum */
1122 mpc->mpc_checksum = 0;
1123 mpc->mpc_checksum -= mpf_checksum((unsigned char *)mpc,
1124 mpc->mpc_length);
913 1125
914 MP_intsrc_info(&intsrc); 1126 return 0;
915} 1127}
916 1128
917void __init mp_config_acpi_legacy_irqs(void) 1129static int __initdata enable_update_mptable;
918{
919 struct mpc_config_intsrc intsrc;
920 int i = 0;
921 int ioapic = -1;
922 1130
923#if defined (CONFIG_MCA) || defined (CONFIG_EISA) 1131static int __init update_mptable_setup(char *str)
924 /* 1132{
925 * Fabricate the legacy ISA bus (bus #31). 1133 enable_update_mptable = 1;
926 */ 1134 return 0;
927 mp_bus_id_to_type[MP_ISA_BUS] = MP_BUS_ISA; 1135}
928#endif 1136early_param("update_mptable", update_mptable_setup);
929 set_bit(MP_ISA_BUS, mp_bus_not_pci);
930 Dprintk("Bus #%d is ISA\n", MP_ISA_BUS);
931 1137
932 /* 1138static unsigned long __initdata mpc_new_phys;
933 * Older generations of ES7000 have no legacy identity mappings 1139static unsigned long mpc_new_length __initdata = 4096;
934 */
935 if (es7000_plat == 1)
936 return;
937 1140
938 /* 1141/* alloc_mptable or alloc_mptable=4k */
939 * Locate the IOAPIC that manages the ISA IRQs (0-15). 1142static int __initdata alloc_mptable;
940 */ 1143static int __init parse_alloc_mptable_opt(char *p)
941 ioapic = mp_find_ioapic(0); 1144{
942 if (ioapic < 0) 1145 enable_update_mptable = 1;
943 return; 1146 alloc_mptable = 1;
1147 if (!p)
1148 return 0;
1149 mpc_new_length = memparse(p, &p);
1150 return 0;
1151}
1152early_param("alloc_mptable", parse_alloc_mptable_opt);
944 1153
945 intsrc.mpc_type = MP_INTSRC; 1154void __init early_reserve_e820_mpc_new(void)
946 intsrc.mpc_irqflag = 0; /* Conforming */ 1155{
947 intsrc.mpc_srcbus = MP_ISA_BUS; 1156 if (enable_update_mptable && alloc_mptable) {
948#ifdef CONFIG_X86_IO_APIC 1157 u64 startt = 0;
949 intsrc.mpc_dstapic = mp_ioapics[ioapic].mpc_apicid; 1158#ifdef CONFIG_X86_TRAMPOLINE
1159 startt = TRAMPOLINE_BASE;
950#endif 1160#endif
951 /* 1161 mpc_new_phys = early_reserve_e820(startt, mpc_new_length, 4);
952 * Use the default configuration for the IRQs 0-15. Unless
953 * overridden by (MADT) interrupt source override entries.
954 */
955 for (i = 0; i < 16; i++) {
956 int idx;
957
958 for (idx = 0; idx < mp_irq_entries; idx++) {
959 struct mpc_config_intsrc *irq = mp_irqs + idx;
960
961 /* Do we already have a mapping for this ISA IRQ? */
962 if (irq->mpc_srcbus == MP_ISA_BUS
963 && irq->mpc_srcbusirq == i)
964 break;
965
966 /* Do we already have a mapping for this IOAPIC pin */
967 if ((irq->mpc_dstapic == intsrc.mpc_dstapic) &&
968 (irq->mpc_dstirq == i))
969 break;
970 }
971
972 if (idx != mp_irq_entries) {
973 printk(KERN_DEBUG "ACPI: IRQ%d used by override.\n", i);
974 continue; /* IRQ already used */
975 }
976
977 intsrc.mpc_irqtype = mp_INT;
978 intsrc.mpc_srcbusirq = i; /* Identity mapped */
979 intsrc.mpc_dstirq = i;
980
981 MP_intsrc_info(&intsrc);
982 } 1162 }
983} 1163}
984 1164
985int mp_register_gsi(u32 gsi, int triggering, int polarity) 1165static int __init update_mp_table(void)
986{ 1166{
987 int ioapic; 1167 char str[16];
988 int ioapic_pin; 1168 char oem[10];
989#ifdef CONFIG_X86_32 1169 struct intel_mp_floating *mpf;
990#define MAX_GSI_NUM 4096 1170 struct mp_config_table *mpc;
991#define IRQ_COMPRESSION_START 64 1171 struct mp_config_table *mpc_new;
1172
1173 if (!enable_update_mptable)
1174 return 0;
1175
1176 mpf = mpf_found;
1177 if (!mpf)
1178 return 0;
992 1179
993 static int pci_irq = IRQ_COMPRESSION_START;
994 /* 1180 /*
995 * Mapping between Global System Interrupts, which 1181 * Now see if we need to go further.
996 * represent all possible interrupts, and IRQs
997 * assigned to actual devices.
998 */ 1182 */
999 static int gsi_to_irq[MAX_GSI_NUM]; 1183 if (mpf->mpf_feature1 != 0)
1000#else 1184 return 0;
1001
1002 if (acpi_irq_model != ACPI_IRQ_MODEL_IOAPIC)
1003 return gsi;
1004#endif
1005 1185
1006 /* Don't set up the ACPI SCI because it's already set up */ 1186 if (!mpf->mpf_physptr)
1007 if (acpi_gbl_FADT.sci_interrupt == gsi) 1187 return 0;
1008 return gsi;
1009 1188
1010 ioapic = mp_find_ioapic(gsi); 1189 mpc = phys_to_virt(mpf->mpf_physptr);
1011 if (ioapic < 0) {
1012 printk(KERN_WARNING "No IOAPIC for GSI %u\n", gsi);
1013 return gsi;
1014 }
1015 1190
1016 ioapic_pin = gsi - mp_ioapic_routing[ioapic].gsi_base; 1191 if (!smp_check_mpc(mpc, oem, str))
1192 return 0;
1017 1193
1018#ifdef CONFIG_X86_32 1194 printk(KERN_INFO "mpf: %lx\n", virt_to_phys(mpf));
1019 if (ioapic_renumber_irq) 1195 printk(KERN_INFO "mpf_physptr: %x\n", mpf->mpf_physptr);
1020 gsi = ioapic_renumber_irq(ioapic, gsi);
1021#endif
1022 1196
1023 /* 1197 if (mpc_new_phys && mpc->mpc_length > mpc_new_length) {
1024 * Avoid pin reprogramming. PRTs typically include entries 1198 mpc_new_phys = 0;
1025 * with redundant pin->gsi mappings (but unique PCI devices); 1199 printk(KERN_INFO "mpc_new_length is %ld, please use alloc_mptable=8k\n",
1026 * we only program the IOAPIC on the first. 1200 mpc_new_length);
1027 */
1028 if (ioapic_pin > MP_MAX_IOAPIC_PIN) {
1029 printk(KERN_ERR "Invalid reference to IOAPIC pin "
1030 "%d-%d\n", mp_ioapic_routing[ioapic].apic_id,
1031 ioapic_pin);
1032 return gsi;
1033 } 1201 }
1034 if (test_bit(ioapic_pin, mp_ioapic_routing[ioapic].pin_programmed)) { 1202
1035 Dprintk(KERN_DEBUG "Pin %d-%d already programmed\n", 1203 if (!mpc_new_phys) {
1036 mp_ioapic_routing[ioapic].apic_id, ioapic_pin); 1204 unsigned char old, new;
1037#ifdef CONFIG_X86_32 1205 /* check if we can change the postion */
1038 return (gsi < IRQ_COMPRESSION_START ? gsi : gsi_to_irq[gsi]); 1206 mpc->mpc_checksum = 0;
1039#else 1207 old = mpf_checksum((unsigned char *)mpc, mpc->mpc_length);
1040 return gsi; 1208 mpc->mpc_checksum = 0xff;
1041#endif 1209 new = mpf_checksum((unsigned char *)mpc, mpc->mpc_length);
1210 if (old == new) {
1211 printk(KERN_INFO "mpc is readonly, please try alloc_mptable instead\n");
1212 return 0;
1213 }
1214 printk(KERN_INFO "use in-positon replacing\n");
1215 } else {
1216 mpf->mpf_physptr = mpc_new_phys;
1217 mpc_new = phys_to_virt(mpc_new_phys);
1218 memcpy(mpc_new, mpc, mpc->mpc_length);
1219 mpc = mpc_new;
1220 /* check if we can modify that */
1221 if (mpc_new_phys - mpf->mpf_physptr) {
1222 struct intel_mp_floating *mpf_new;
1223 /* steal 16 bytes from [0, 1k) */
1224 printk(KERN_INFO "mpf new: %x\n", 0x400 - 16);
1225 mpf_new = phys_to_virt(0x400 - 16);
1226 memcpy(mpf_new, mpf, 16);
1227 mpf = mpf_new;
1228 mpf->mpf_physptr = mpc_new_phys;
1229 }
1230 mpf->mpf_checksum = 0;
1231 mpf->mpf_checksum -= mpf_checksum((unsigned char *)mpf, 16);
1232 printk(KERN_INFO "mpf_physptr new: %x\n", mpf->mpf_physptr);
1042 } 1233 }
1043 1234
1044 set_bit(ioapic_pin, mp_ioapic_routing[ioapic].pin_programmed);
1045#ifdef CONFIG_X86_32
1046 /* 1235 /*
1047 * For GSI >= 64, use IRQ compression 1236 * only replace the one with mp_INT and
1237 * MP_IRQ_TRIGGER_LEVEL|MP_IRQ_POLARITY_LOW,
1238 * already in mp_irqs , stored by ... and mp_config_acpi_gsi,
1239 * may need pci=routeirq for all coverage
1048 */ 1240 */
1049 if ((gsi >= IRQ_COMPRESSION_START) 1241 replace_intsrc_all(mpc, mpc_new_phys, mpc_new_length);
1050 && (triggering == ACPI_LEVEL_SENSITIVE)) { 1242
1051 /* 1243 return 0;
1052 * For PCI devices assign IRQs in order, avoiding gaps
1053 * due to unused I/O APIC pins.
1054 */
1055 int irq = gsi;
1056 if (gsi < MAX_GSI_NUM) {
1057 /*
1058 * Retain the VIA chipset work-around (gsi > 15), but
1059 * avoid a problem where the 8254 timer (IRQ0) is setup
1060 * via an override (so it's not on pin 0 of the ioapic),
1061 * and at the same time, the pin 0 interrupt is a PCI
1062 * type. The gsi > 15 test could cause these two pins
1063 * to be shared as IRQ0, and they are not shareable.
1064 * So test for this condition, and if necessary, avoid
1065 * the pin collision.
1066 */
1067 gsi = pci_irq++;
1068 /*
1069 * Don't assign IRQ used by ACPI SCI
1070 */
1071 if (gsi == acpi_gbl_FADT.sci_interrupt)
1072 gsi = pci_irq++;
1073 gsi_to_irq[irq] = gsi;
1074 } else {
1075 printk(KERN_ERR "GSI %u is too high\n", gsi);
1076 return gsi;
1077 }
1078 }
1079#endif
1080 io_apic_set_pci_routing(ioapic, ioapic_pin, gsi,
1081 triggering == ACPI_EDGE_SENSITIVE ? 0 : 1,
1082 polarity == ACPI_ACTIVE_HIGH ? 0 : 1);
1083 return gsi;
1084} 1244}
1085 1245
1086#endif /* CONFIG_X86_IO_APIC */ 1246late_initcall(update_mp_table);
1087#endif /* CONFIG_ACPI */
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 1f3abe048e93..a153b3905f60 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -117,12 +117,20 @@ static int msr_open(struct inode *inode, struct file *file)
117{ 117{
118 unsigned int cpu = iminor(file->f_path.dentry->d_inode); 118 unsigned int cpu = iminor(file->f_path.dentry->d_inode);
119 struct cpuinfo_x86 *c = &cpu_data(cpu); 119 struct cpuinfo_x86 *c = &cpu_data(cpu);
120 int ret = 0;
120 121
121 if (cpu >= NR_CPUS || !cpu_online(cpu)) 122 lock_kernel();
122 return -ENXIO; /* No such CPU */ 123 cpu = iminor(file->f_path.dentry->d_inode);
123 if (!cpu_has(c, X86_FEATURE_MSR))
124 return -EIO; /* MSR not supported */
125 124
125 if (cpu >= NR_CPUS || !cpu_online(cpu)) {
126 ret = -ENXIO; /* No such CPU */
127 goto out;
128 }
129 c = &cpu_data(cpu);
130 if (!cpu_has(c, X86_FEATURE_MSR))
131 ret = -EIO; /* MSR not supported */
132out:
133 unlock_kernel();
126 return 0; 134 return 0;
127} 135}
128 136
diff --git a/arch/x86/kernel/nmi_32.c b/arch/x86/kernel/nmi.c
index 84160f74eeb0..ec024b3baad0 100644
--- a/arch/x86/kernel/nmi_32.c
+++ b/arch/x86/kernel/nmi.c
@@ -11,10 +11,13 @@
11 * Mikael Pettersson : PM converted to driver model. Disable/enable API. 11 * Mikael Pettersson : PM converted to driver model. Disable/enable API.
12 */ 12 */
13 13
14#include <asm/apic.h>
15
16#include <linux/nmi.h>
17#include <linux/mm.h>
14#include <linux/delay.h> 18#include <linux/delay.h>
15#include <linux/interrupt.h> 19#include <linux/interrupt.h>
16#include <linux/module.h> 20#include <linux/module.h>
17#include <linux/nmi.h>
18#include <linux/sysdev.h> 21#include <linux/sysdev.h>
19#include <linux/sysctl.h> 22#include <linux/sysctl.h>
20#include <linux/percpu.h> 23#include <linux/percpu.h>
@@ -22,12 +25,18 @@
22#include <linux/cpumask.h> 25#include <linux/cpumask.h>
23#include <linux/kernel_stat.h> 26#include <linux/kernel_stat.h>
24#include <linux/kdebug.h> 27#include <linux/kdebug.h>
25#include <linux/slab.h> 28#include <linux/smp.h>
26 29
30#include <asm/i8259.h>
31#include <asm/io_apic.h>
27#include <asm/smp.h> 32#include <asm/smp.h>
28#include <asm/nmi.h> 33#include <asm/nmi.h>
34#include <asm/proto.h>
35#include <asm/timer.h>
29 36
30#include "mach_traps.h" 37#include <asm/mce.h>
38
39#include <mach_traps.h>
31 40
32int unknown_nmi_panic; 41int unknown_nmi_panic;
33int nmi_watchdog_enabled; 42int nmi_watchdog_enabled;
@@ -41,28 +50,65 @@ static cpumask_t backtrace_mask = CPU_MASK_NONE;
41 * 0: the lapic NMI watchdog is disabled, but can be enabled 50 * 0: the lapic NMI watchdog is disabled, but can be enabled
42 */ 51 */
43atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */ 52atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */
53EXPORT_SYMBOL(nmi_active);
44 54
45unsigned int nmi_watchdog = NMI_DEFAULT; 55unsigned int nmi_watchdog = NMI_NONE;
46static unsigned int nmi_hz = HZ; 56EXPORT_SYMBOL(nmi_watchdog);
47 57
58static int panic_on_timeout;
59
60static unsigned int nmi_hz = HZ;
48static DEFINE_PER_CPU(short, wd_enabled); 61static DEFINE_PER_CPU(short, wd_enabled);
62static int endflag __initdata;
49 63
50static int endflag __initdata = 0; 64static inline unsigned int get_nmi_count(int cpu)
65{
66#ifdef CONFIG_X86_64
67 return cpu_pda(cpu)->__nmi_count;
68#else
69 return nmi_count(cpu);
70#endif
71}
72
73static inline int mce_in_progress(void)
74{
75#if defined(CONFIG_X86_64) && defined(CONFIG_X86_MCE)
76 return atomic_read(&mce_entry) > 0;
77#endif
78 return 0;
79}
80
81/*
82 * Take the local apic timer and PIT/HPET into account. We don't
83 * know which one is active, when we have highres/dyntick on
84 */
85static inline unsigned int get_timer_irqs(int cpu)
86{
87#ifdef CONFIG_X86_64
88 return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
89#else
90 return per_cpu(irq_stat, cpu).apic_timer_irqs +
91 per_cpu(irq_stat, cpu).irq0_irqs;
92#endif
93}
51 94
52#ifdef CONFIG_SMP 95#ifdef CONFIG_SMP
53/* The performance counters used by NMI_LOCAL_APIC don't trigger when 96/*
97 * The performance counters used by NMI_LOCAL_APIC don't trigger when
54 * the CPU is idle. To make sure the NMI watchdog really ticks on all 98 * the CPU is idle. To make sure the NMI watchdog really ticks on all
55 * CPUs during the test make them busy. 99 * CPUs during the test make them busy.
56 */ 100 */
57static __init void nmi_cpu_busy(void *data) 101static __init void nmi_cpu_busy(void *data)
58{ 102{
59 local_irq_enable_in_hardirq(); 103 local_irq_enable_in_hardirq();
60 /* Intentionally don't use cpu_relax here. This is 104 /*
61 to make sure that the performance counter really ticks, 105 * Intentionally don't use cpu_relax here. This is
62 even if there is a simulator or similar that catches the 106 * to make sure that the performance counter really ticks,
63 pause instruction. On a real HT machine this is fine because 107 * even if there is a simulator or similar that catches the
64 all other CPUs are busy with "useless" delay loops and don't 108 * pause instruction. On a real HT machine this is fine because
65 care if they get somewhat less cycles. */ 109 * all other CPUs are busy with "useless" delay loops and don't
110 * care if they get somewhat less cycles.
111 */
66 while (endflag == 0) 112 while (endflag == 0)
67 mb(); 113 mb();
68} 114}
@@ -73,43 +119,34 @@ int __init check_nmi_watchdog(void)
73 unsigned int *prev_nmi_count; 119 unsigned int *prev_nmi_count;
74 int cpu; 120 int cpu;
75 121
76 if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DISABLED)) 122 if (!nmi_watchdog_active() || !atomic_read(&nmi_active))
77 return 0;
78
79 if (!atomic_read(&nmi_active))
80 return 0; 123 return 0;
81 124
82 prev_nmi_count = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL); 125 prev_nmi_count = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
83 if (!prev_nmi_count) 126 if (!prev_nmi_count)
84 return -1; 127 goto error;
85 128
86 printk(KERN_INFO "Testing NMI watchdog ... "); 129 printk(KERN_INFO "Testing NMI watchdog ... ");
87 130
88#ifdef CONFIG_SMP 131#ifdef CONFIG_SMP
89 if (nmi_watchdog == NMI_LOCAL_APIC) 132 if (nmi_watchdog == NMI_LOCAL_APIC)
90 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0); 133 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0);
91#endif 134#endif
92 135
93 for_each_possible_cpu(cpu) 136 for_each_possible_cpu(cpu)
94 prev_nmi_count[cpu] = nmi_count(cpu); 137 prev_nmi_count[cpu] = get_nmi_count(cpu);
95 local_irq_enable(); 138 local_irq_enable();
96 mdelay((20*1000)/nmi_hz); // wait 20 ticks 139 mdelay((20 * 1000) / nmi_hz); /* wait 20 ticks */
97 140
98 for_each_possible_cpu(cpu) { 141 for_each_online_cpu(cpu) {
99#ifdef CONFIG_SMP
100 /* Check cpu_callin_map here because that is set
101 after the timer is started. */
102 if (!cpu_isset(cpu, cpu_callin_map))
103 continue;
104#endif
105 if (!per_cpu(wd_enabled, cpu)) 142 if (!per_cpu(wd_enabled, cpu))
106 continue; 143 continue;
107 if (nmi_count(cpu) - prev_nmi_count[cpu] <= 5) { 144 if (get_nmi_count(cpu) - prev_nmi_count[cpu] <= 5) {
108 printk(KERN_WARNING "WARNING: CPU#%d: NMI " 145 printk(KERN_WARNING "WARNING: CPU#%d: NMI "
109 "appears to be stuck (%d->%d)!\n", 146 "appears to be stuck (%d->%d)!\n",
110 cpu, 147 cpu,
111 prev_nmi_count[cpu], 148 prev_nmi_count[cpu],
112 nmi_count(cpu)); 149 get_nmi_count(cpu));
113 per_cpu(wd_enabled, cpu) = 0; 150 per_cpu(wd_enabled, cpu) = 0;
114 atomic_dec(&nmi_active); 151 atomic_dec(&nmi_active);
115 } 152 }
@@ -118,37 +155,53 @@ int __init check_nmi_watchdog(void)
118 if (!atomic_read(&nmi_active)) { 155 if (!atomic_read(&nmi_active)) {
119 kfree(prev_nmi_count); 156 kfree(prev_nmi_count);
120 atomic_set(&nmi_active, -1); 157 atomic_set(&nmi_active, -1);
121 return -1; 158 goto error;
122 } 159 }
123 printk("OK.\n"); 160 printk("OK.\n");
124 161
125 /* now that we know it works we can reduce NMI frequency to 162 /*
126 something more reasonable; makes a difference in some configs */ 163 * now that we know it works we can reduce NMI frequency to
164 * something more reasonable; makes a difference in some configs
165 */
127 if (nmi_watchdog == NMI_LOCAL_APIC) 166 if (nmi_watchdog == NMI_LOCAL_APIC)
128 nmi_hz = lapic_adjust_nmi_hz(1); 167 nmi_hz = lapic_adjust_nmi_hz(1);
129 168
130 kfree(prev_nmi_count); 169 kfree(prev_nmi_count);
131 return 0; 170 return 0;
171error:
172 if (nmi_watchdog == NMI_IO_APIC && !timer_through_8259)
173 disable_8259A_irq(0);
174#ifdef CONFIG_X86_32
175 timer_ack = 0;
176#endif
177 return -1;
132} 178}
133 179
134static int __init setup_nmi_watchdog(char *str) 180static int __init setup_nmi_watchdog(char *str)
135{ 181{
136 int nmi; 182 unsigned int nmi;
183
184 if (!strncmp(str, "panic", 5)) {
185 panic_on_timeout = 1;
186 str = strchr(str, ',');
187 if (!str)
188 return 1;
189 ++str;
190 }
137 191
138 get_option(&str, &nmi); 192 get_option(&str, &nmi);
139 193
140 if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE)) 194 if (nmi >= NMI_INVALID)
141 return 0; 195 return 0;
142 196
143 nmi_watchdog = nmi; 197 nmi_watchdog = nmi;
144 return 1; 198 return 1;
145} 199}
146
147__setup("nmi_watchdog=", setup_nmi_watchdog); 200__setup("nmi_watchdog=", setup_nmi_watchdog);
148 201
149 202/*
150/* Suspend/resume support */ 203 * Suspend/resume support
151 204 */
152#ifdef CONFIG_PM 205#ifdef CONFIG_PM
153 206
154static int nmi_pm_active; /* nmi_active before suspend */ 207static int nmi_pm_active; /* nmi_active before suspend */
@@ -172,7 +225,6 @@ static int lapic_nmi_resume(struct sys_device *dev)
172 return 0; 225 return 0;
173} 226}
174 227
175
176static struct sysdev_class nmi_sysclass = { 228static struct sysdev_class nmi_sysclass = {
177 .name = "lapic_nmi", 229 .name = "lapic_nmi",
178 .resume = lapic_nmi_resume, 230 .resume = lapic_nmi_resume,
@@ -188,7 +240,8 @@ static int __init init_lapic_nmi_sysfs(void)
188{ 240{
189 int error; 241 int error;
190 242
191 /* should really be a BUG_ON but b/c this is an 243 /*
244 * should really be a BUG_ON but b/c this is an
192 * init call, it just doesn't work. -dcz 245 * init call, it just doesn't work. -dcz
193 */ 246 */
194 if (nmi_watchdog != NMI_LOCAL_APIC) 247 if (nmi_watchdog != NMI_LOCAL_APIC)
@@ -202,6 +255,7 @@ static int __init init_lapic_nmi_sysfs(void)
202 error = sysdev_register(&device_lapic_nmi); 255 error = sysdev_register(&device_lapic_nmi);
203 return error; 256 return error;
204} 257}
258
205/* must come after the local APIC's device_initcall() */ 259/* must come after the local APIC's device_initcall() */
206late_initcall(init_lapic_nmi_sysfs); 260late_initcall(init_lapic_nmi_sysfs);
207 261
@@ -218,12 +272,12 @@ static void __acpi_nmi_enable(void *__unused)
218void acpi_nmi_enable(void) 272void acpi_nmi_enable(void)
219{ 273{
220 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC) 274 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
221 on_each_cpu(__acpi_nmi_enable, NULL, 0, 1); 275 on_each_cpu(__acpi_nmi_enable, NULL, 1);
222} 276}
223 277
224static void __acpi_nmi_disable(void *__unused) 278static void __acpi_nmi_disable(void *__unused)
225{ 279{
226 apic_write(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED); 280 apic_write_around(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED);
227} 281}
228 282
229/* 283/*
@@ -232,7 +286,7 @@ static void __acpi_nmi_disable(void *__unused)
232void acpi_nmi_disable(void) 286void acpi_nmi_disable(void)
233{ 287{
234 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC) 288 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
235 on_each_cpu(__acpi_nmi_disable, NULL, 0, 1); 289 on_each_cpu(__acpi_nmi_disable, NULL, 1);
236} 290}
237 291
238void setup_apic_nmi_watchdog(void *unused) 292void setup_apic_nmi_watchdog(void *unused)
@@ -242,12 +296,13 @@ void setup_apic_nmi_watchdog(void *unused)
242 296
243 /* cheap hack to support suspend/resume */ 297 /* cheap hack to support suspend/resume */
244 /* if cpu0 is not active neither should the other cpus */ 298 /* if cpu0 is not active neither should the other cpus */
245 if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0)) 299 if (smp_processor_id() != 0 && atomic_read(&nmi_active) <= 0)
246 return; 300 return;
247 301
248 switch (nmi_watchdog) { 302 switch (nmi_watchdog) {
249 case NMI_LOCAL_APIC: 303 case NMI_LOCAL_APIC:
250 __get_cpu_var(wd_enabled) = 1; /* enable it before to avoid race with handler */ 304 /* enable it before to avoid race with handler */
305 __get_cpu_var(wd_enabled) = 1;
251 if (lapic_watchdog_init(nmi_hz) < 0) { 306 if (lapic_watchdog_init(nmi_hz) < 0) {
252 __get_cpu_var(wd_enabled) = 0; 307 __get_cpu_var(wd_enabled) = 0;
253 return; 308 return;
@@ -262,9 +317,8 @@ void setup_apic_nmi_watchdog(void *unused)
262void stop_apic_nmi_watchdog(void *unused) 317void stop_apic_nmi_watchdog(void *unused)
263{ 318{
264 /* only support LOCAL and IO APICs for now */ 319 /* only support LOCAL and IO APICs for now */
265 if ((nmi_watchdog != NMI_LOCAL_APIC) && 320 if (!nmi_watchdog_active())
266 (nmi_watchdog != NMI_IO_APIC)) 321 return;
267 return;
268 if (__get_cpu_var(wd_enabled) == 0) 322 if (__get_cpu_var(wd_enabled) == 0)
269 return; 323 return;
270 if (nmi_watchdog == NMI_LOCAL_APIC) 324 if (nmi_watchdog == NMI_LOCAL_APIC)
@@ -284,26 +338,26 @@ void stop_apic_nmi_watchdog(void *unused)
284 * since NMIs don't listen to _any_ locks, we have to be extremely 338 * since NMIs don't listen to _any_ locks, we have to be extremely
285 * careful not to rely on unsafe variables. The printk might lock 339 * careful not to rely on unsafe variables. The printk might lock
286 * up though, so we have to break up any console locks first ... 340 * up though, so we have to break up any console locks first ...
287 * [when there will be more tty-related locks, break them up 341 * [when there will be more tty-related locks, break them up here too!]
288 * here too!]
289 */ 342 */
290 343
291static unsigned int 344static DEFINE_PER_CPU(unsigned, last_irq_sum);
292 last_irq_sums [NR_CPUS], 345static DEFINE_PER_CPU(local_t, alert_counter);
293 alert_counter [NR_CPUS]; 346static DEFINE_PER_CPU(int, nmi_touch);
294 347
295void touch_nmi_watchdog(void) 348void touch_nmi_watchdog(void)
296{ 349{
297 if (nmi_watchdog > 0) { 350 if (nmi_watchdog_active()) {
298 unsigned cpu; 351 unsigned cpu;
299 352
300 /* 353 /*
301 * Just reset the alert counters, (other CPUs might be 354 * Tell other CPUs to reset their alert counters. We cannot
302 * spinning on locks we hold): 355 * do it ourselves because the alert count increase is not
356 * atomic.
303 */ 357 */
304 for_each_present_cpu(cpu) { 358 for_each_present_cpu(cpu) {
305 if (alert_counter[cpu]) 359 if (per_cpu(nmi_touch, cpu) != 1)
306 alert_counter[cpu] = 0; 360 per_cpu(nmi_touch, cpu) = 1;
307 } 361 }
308 } 362 }
309 363
@@ -314,12 +368,9 @@ void touch_nmi_watchdog(void)
314} 368}
315EXPORT_SYMBOL(touch_nmi_watchdog); 369EXPORT_SYMBOL(touch_nmi_watchdog);
316 370
317extern void die_nmi(struct pt_regs *, const char *msg);
318
319notrace __kprobes int 371notrace __kprobes int
320nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) 372nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
321{ 373{
322
323 /* 374 /*
324 * Since current_thread_info()-> is always on the stack, and we 375 * Since current_thread_info()-> is always on the stack, and we
325 * always switch the stack NMI-atomically, it's safe to use 376 * always switch the stack NMI-atomically, it's safe to use
@@ -337,39 +388,45 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
337 touched = 1; 388 touched = 1;
338 } 389 }
339 390
391 sum = get_timer_irqs(cpu);
392
393 if (__get_cpu_var(nmi_touch)) {
394 __get_cpu_var(nmi_touch) = 0;
395 touched = 1;
396 }
397
340 if (cpu_isset(cpu, backtrace_mask)) { 398 if (cpu_isset(cpu, backtrace_mask)) {
341 static DEFINE_SPINLOCK(lock); /* Serialise the printks */ 399 static DEFINE_SPINLOCK(lock); /* Serialise the printks */
342 400
343 spin_lock(&lock); 401 spin_lock(&lock);
344 printk("NMI backtrace for cpu %d\n", cpu); 402 printk(KERN_WARNING "NMI backtrace for cpu %d\n", cpu);
345 dump_stack(); 403 dump_stack();
346 spin_unlock(&lock); 404 spin_unlock(&lock);
347 cpu_clear(cpu, backtrace_mask); 405 cpu_clear(cpu, backtrace_mask);
348 } 406 }
349 407
350 /* 408 /* Could check oops_in_progress here too, but it's safer not to */
351 * Take the local apic timer and PIT/HPET into account. We don't 409 if (mce_in_progress())
352 * know which one is active, when we have highres/dyntick on 410 touched = 1;
353 */
354 sum = per_cpu(irq_stat, cpu).apic_timer_irqs +
355 per_cpu(irq_stat, cpu).irq0_irqs;
356 411
357 /* if the none of the timers isn't firing, this cpu isn't doing much */ 412 /* if the none of the timers isn't firing, this cpu isn't doing much */
358 if (!touched && last_irq_sums[cpu] == sum) { 413 if (!touched && __get_cpu_var(last_irq_sum) == sum) {
359 /* 414 /*
360 * Ayiee, looks like this CPU is stuck ... 415 * Ayiee, looks like this CPU is stuck ...
361 * wait a few IRQs (5 seconds) before doing the oops ... 416 * wait a few IRQs (5 seconds) before doing the oops ...
362 */ 417 */
363 alert_counter[cpu]++; 418 local_inc(&__get_cpu_var(alert_counter));
364 if (alert_counter[cpu] == 5*nmi_hz) 419 if (local_read(&__get_cpu_var(alert_counter)) == 5 * nmi_hz)
365 /* 420 /*
366 * die_nmi will return ONLY if NOTIFY_STOP happens.. 421 * die_nmi will return ONLY if NOTIFY_STOP happens..
367 */ 422 */
368 die_nmi(regs, "BUG: NMI Watchdog detected LOCKUP"); 423 die_nmi("BUG: NMI Watchdog detected LOCKUP",
424 regs, panic_on_timeout);
369 } else { 425 } else {
370 last_irq_sums[cpu] = sum; 426 __get_cpu_var(last_irq_sum) = sum;
371 alert_counter[cpu] = 0; 427 local_set(&__get_cpu_var(alert_counter), 0);
372 } 428 }
429
373 /* see if the nmi watchdog went off */ 430 /* see if the nmi watchdog went off */
374 if (!__get_cpu_var(wd_enabled)) 431 if (!__get_cpu_var(wd_enabled))
375 return rc; 432 return rc;
@@ -378,7 +435,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
378 rc |= lapic_wd_event(nmi_hz); 435 rc |= lapic_wd_event(nmi_hz);
379 break; 436 break;
380 case NMI_IO_APIC: 437 case NMI_IO_APIC:
381 /* don't know how to accurately check for this. 438 /*
439 * don't know how to accurately check for this.
382 * just assume it was a watchdog timer interrupt 440 * just assume it was a watchdog timer interrupt
383 * This matches the old behaviour. 441 * This matches the old behaviour.
384 */ 442 */
@@ -396,7 +454,7 @@ static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
396 char buf[64]; 454 char buf[64];
397 455
398 sprintf(buf, "NMI received for unknown reason %02x\n", reason); 456 sprintf(buf, "NMI received for unknown reason %02x\n", reason);
399 die_nmi(regs, buf); 457 die_nmi(buf, regs, 1); /* Always panic here */
400 return 0; 458 return 0;
401} 459}
402 460
@@ -414,32 +472,26 @@ int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
414 if (!!old_state == !!nmi_watchdog_enabled) 472 if (!!old_state == !!nmi_watchdog_enabled)
415 return 0; 473 return 0;
416 474
417 if (atomic_read(&nmi_active) < 0 || nmi_watchdog == NMI_DISABLED) { 475 if (atomic_read(&nmi_active) < 0 || !nmi_watchdog_active()) {
418 printk( KERN_WARNING "NMI watchdog is permanently disabled\n"); 476 printk(KERN_WARNING
477 "NMI watchdog is permanently disabled\n");
419 return -EIO; 478 return -EIO;
420 } 479 }
421 480
422 if (nmi_watchdog == NMI_DEFAULT) {
423 if (lapic_watchdog_ok())
424 nmi_watchdog = NMI_LOCAL_APIC;
425 else
426 nmi_watchdog = NMI_IO_APIC;
427 }
428
429 if (nmi_watchdog == NMI_LOCAL_APIC) { 481 if (nmi_watchdog == NMI_LOCAL_APIC) {
430 if (nmi_watchdog_enabled) 482 if (nmi_watchdog_enabled)
431 enable_lapic_nmi_watchdog(); 483 enable_lapic_nmi_watchdog();
432 else 484 else
433 disable_lapic_nmi_watchdog(); 485 disable_lapic_nmi_watchdog();
434 } else { 486 } else {
435 printk( KERN_WARNING 487 printk(KERN_WARNING
436 "NMI watchdog doesn't know what hardware to touch\n"); 488 "NMI watchdog doesn't know what hardware to touch\n");
437 return -EIO; 489 return -EIO;
438 } 490 }
439 return 0; 491 return 0;
440} 492}
441 493
442#endif 494#endif /* CONFIG_SYSCTL */
443 495
444int do_nmi_callback(struct pt_regs *regs, int cpu) 496int do_nmi_callback(struct pt_regs *regs, int cpu)
445{ 497{
@@ -462,6 +514,3 @@ void __trigger_all_cpu_backtrace(void)
462 mdelay(1); 514 mdelay(1);
463 } 515 }
464} 516}
465
466EXPORT_SYMBOL(nmi_active);
467EXPORT_SYMBOL(nmi_watchdog);
diff --git a/arch/x86/kernel/nmi_64.c b/arch/x86/kernel/nmi_64.c
deleted file mode 100644
index 5a29ded994fa..000000000000
--- a/arch/x86/kernel/nmi_64.c
+++ /dev/null
@@ -1,482 +0,0 @@
1/*
2 * NMI watchdog support on APIC systems
3 *
4 * Started by Ingo Molnar <mingo@redhat.com>
5 *
6 * Fixes:
7 * Mikael Pettersson : AMD K7 support for local APIC NMI watchdog.
8 * Mikael Pettersson : Power Management for local APIC NMI watchdog.
9 * Pavel Machek and
10 * Mikael Pettersson : PM converted to driver model. Disable/enable API.
11 */
12
13#include <linux/nmi.h>
14#include <linux/mm.h>
15#include <linux/delay.h>
16#include <linux/interrupt.h>
17#include <linux/module.h>
18#include <linux/sysdev.h>
19#include <linux/sysctl.h>
20#include <linux/kprobes.h>
21#include <linux/cpumask.h>
22#include <linux/kdebug.h>
23
24#include <asm/smp.h>
25#include <asm/nmi.h>
26#include <asm/proto.h>
27#include <asm/mce.h>
28
29#include <mach_traps.h>
30
31int unknown_nmi_panic;
32int nmi_watchdog_enabled;
33int panic_on_unrecovered_nmi;
34
35static cpumask_t backtrace_mask = CPU_MASK_NONE;
36
37/* nmi_active:
38 * >0: the lapic NMI watchdog is active, but can be disabled
39 * <0: the lapic NMI watchdog has not been set up, and cannot
40 * be enabled
41 * 0: the lapic NMI watchdog is disabled, but can be enabled
42 */
43atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */
44static int panic_on_timeout;
45
46unsigned int nmi_watchdog = NMI_DEFAULT;
47static unsigned int nmi_hz = HZ;
48
49static DEFINE_PER_CPU(short, wd_enabled);
50
51/* Run after command line and cpu_init init, but before all other checks */
52void nmi_watchdog_default(void)
53{
54 if (nmi_watchdog != NMI_DEFAULT)
55 return;
56 nmi_watchdog = NMI_NONE;
57}
58
59static int endflag __initdata = 0;
60
61#ifdef CONFIG_SMP
62/* The performance counters used by NMI_LOCAL_APIC don't trigger when
63 * the CPU is idle. To make sure the NMI watchdog really ticks on all
64 * CPUs during the test make them busy.
65 */
66static __init void nmi_cpu_busy(void *data)
67{
68 local_irq_enable_in_hardirq();
69 /* Intentionally don't use cpu_relax here. This is
70 to make sure that the performance counter really ticks,
71 even if there is a simulator or similar that catches the
72 pause instruction. On a real HT machine this is fine because
73 all other CPUs are busy with "useless" delay loops and don't
74 care if they get somewhat less cycles. */
75 while (endflag == 0)
76 mb();
77}
78#endif
79
80int __init check_nmi_watchdog(void)
81{
82 int *prev_nmi_count;
83 int cpu;
84
85 if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DISABLED))
86 return 0;
87
88 if (!atomic_read(&nmi_active))
89 return 0;
90
91 prev_nmi_count = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL);
92 if (!prev_nmi_count)
93 return -1;
94
95 printk(KERN_INFO "Testing NMI watchdog ... ");
96
97#ifdef CONFIG_SMP
98 if (nmi_watchdog == NMI_LOCAL_APIC)
99 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0);
100#endif
101
102 for (cpu = 0; cpu < NR_CPUS; cpu++)
103 prev_nmi_count[cpu] = cpu_pda(cpu)->__nmi_count;
104 local_irq_enable();
105 mdelay((20*1000)/nmi_hz); // wait 20 ticks
106
107 for_each_online_cpu(cpu) {
108 if (!per_cpu(wd_enabled, cpu))
109 continue;
110 if (cpu_pda(cpu)->__nmi_count - prev_nmi_count[cpu] <= 5) {
111 printk(KERN_WARNING "WARNING: CPU#%d: NMI "
112 "appears to be stuck (%d->%d)!\n",
113 cpu,
114 prev_nmi_count[cpu],
115 cpu_pda(cpu)->__nmi_count);
116 per_cpu(wd_enabled, cpu) = 0;
117 atomic_dec(&nmi_active);
118 }
119 }
120 endflag = 1;
121 if (!atomic_read(&nmi_active)) {
122 kfree(prev_nmi_count);
123 atomic_set(&nmi_active, -1);
124 return -1;
125 }
126 printk("OK.\n");
127
128 /* now that we know it works we can reduce NMI frequency to
129 something more reasonable; makes a difference in some configs */
130 if (nmi_watchdog == NMI_LOCAL_APIC)
131 nmi_hz = lapic_adjust_nmi_hz(1);
132
133 kfree(prev_nmi_count);
134 return 0;
135}
136
137static int __init setup_nmi_watchdog(char *str)
138{
139 int nmi;
140
141 if (!strncmp(str,"panic",5)) {
142 panic_on_timeout = 1;
143 str = strchr(str, ',');
144 if (!str)
145 return 1;
146 ++str;
147 }
148
149 get_option(&str, &nmi);
150
151 if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE))
152 return 0;
153
154 nmi_watchdog = nmi;
155 return 1;
156}
157
158__setup("nmi_watchdog=", setup_nmi_watchdog);
159
160#ifdef CONFIG_PM
161
162static int nmi_pm_active; /* nmi_active before suspend */
163
164static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state)
165{
166 /* only CPU0 goes here, other CPUs should be offline */
167 nmi_pm_active = atomic_read(&nmi_active);
168 stop_apic_nmi_watchdog(NULL);
169 BUG_ON(atomic_read(&nmi_active) != 0);
170 return 0;
171}
172
173static int lapic_nmi_resume(struct sys_device *dev)
174{
175 /* only CPU0 goes here, other CPUs should be offline */
176 if (nmi_pm_active > 0) {
177 setup_apic_nmi_watchdog(NULL);
178 touch_nmi_watchdog();
179 }
180 return 0;
181}
182
183static struct sysdev_class nmi_sysclass = {
184 .name = "lapic_nmi",
185 .resume = lapic_nmi_resume,
186 .suspend = lapic_nmi_suspend,
187};
188
189static struct sys_device device_lapic_nmi = {
190 .id = 0,
191 .cls = &nmi_sysclass,
192};
193
194static int __init init_lapic_nmi_sysfs(void)
195{
196 int error;
197
198 /* should really be a BUG_ON but b/c this is an
199 * init call, it just doesn't work. -dcz
200 */
201 if (nmi_watchdog != NMI_LOCAL_APIC)
202 return 0;
203
204 if (atomic_read(&nmi_active) < 0)
205 return 0;
206
207 error = sysdev_class_register(&nmi_sysclass);
208 if (!error)
209 error = sysdev_register(&device_lapic_nmi);
210 return error;
211}
212/* must come after the local APIC's device_initcall() */
213late_initcall(init_lapic_nmi_sysfs);
214
215#endif /* CONFIG_PM */
216
217static void __acpi_nmi_enable(void *__unused)
218{
219 apic_write(APIC_LVT0, APIC_DM_NMI);
220}
221
222/*
223 * Enable timer based NMIs on all CPUs:
224 */
225void acpi_nmi_enable(void)
226{
227 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
228 on_each_cpu(__acpi_nmi_enable, NULL, 0, 1);
229}
230
231static void __acpi_nmi_disable(void *__unused)
232{
233 apic_write(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED);
234}
235
236/*
237 * Disable timer based NMIs on all CPUs:
238 */
239void acpi_nmi_disable(void)
240{
241 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC)
242 on_each_cpu(__acpi_nmi_disable, NULL, 0, 1);
243}
244
245void setup_apic_nmi_watchdog(void *unused)
246{
247 if (__get_cpu_var(wd_enabled))
248 return;
249
250 /* cheap hack to support suspend/resume */
251 /* if cpu0 is not active neither should the other cpus */
252 if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0))
253 return;
254
255 switch (nmi_watchdog) {
256 case NMI_LOCAL_APIC:
257 __get_cpu_var(wd_enabled) = 1;
258 if (lapic_watchdog_init(nmi_hz) < 0) {
259 __get_cpu_var(wd_enabled) = 0;
260 return;
261 }
262 /* FALL THROUGH */
263 case NMI_IO_APIC:
264 __get_cpu_var(wd_enabled) = 1;
265 atomic_inc(&nmi_active);
266 }
267}
268
269void stop_apic_nmi_watchdog(void *unused)
270{
271 /* only support LOCAL and IO APICs for now */
272 if ((nmi_watchdog != NMI_LOCAL_APIC) &&
273 (nmi_watchdog != NMI_IO_APIC))
274 return;
275 if (__get_cpu_var(wd_enabled) == 0)
276 return;
277 if (nmi_watchdog == NMI_LOCAL_APIC)
278 lapic_watchdog_stop();
279 __get_cpu_var(wd_enabled) = 0;
280 atomic_dec(&nmi_active);
281}
282
283/*
284 * the best way to detect whether a CPU has a 'hard lockup' problem
285 * is to check it's local APIC timer IRQ counts. If they are not
286 * changing then that CPU has some problem.
287 *
288 * as these watchdog NMI IRQs are generated on every CPU, we only
289 * have to check the current processor.
290 */
291
292static DEFINE_PER_CPU(unsigned, last_irq_sum);
293static DEFINE_PER_CPU(local_t, alert_counter);
294static DEFINE_PER_CPU(int, nmi_touch);
295
296void touch_nmi_watchdog(void)
297{
298 if (nmi_watchdog > 0) {
299 unsigned cpu;
300
301 /*
302 * Tell other CPUs to reset their alert counters. We cannot
303 * do it ourselves because the alert count increase is not
304 * atomic.
305 */
306 for_each_present_cpu(cpu) {
307 if (per_cpu(nmi_touch, cpu) != 1)
308 per_cpu(nmi_touch, cpu) = 1;
309 }
310 }
311
312 touch_softlockup_watchdog();
313}
314EXPORT_SYMBOL(touch_nmi_watchdog);
315
316notrace __kprobes int
317nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
318{
319 int sum;
320 int touched = 0;
321 int cpu = smp_processor_id();
322 int rc = 0;
323
324 /* check for other users first */
325 if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT)
326 == NOTIFY_STOP) {
327 rc = 1;
328 touched = 1;
329 }
330
331 sum = read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
332 if (__get_cpu_var(nmi_touch)) {
333 __get_cpu_var(nmi_touch) = 0;
334 touched = 1;
335 }
336
337 if (cpu_isset(cpu, backtrace_mask)) {
338 static DEFINE_SPINLOCK(lock); /* Serialise the printks */
339
340 spin_lock(&lock);
341 printk("NMI backtrace for cpu %d\n", cpu);
342 dump_stack();
343 spin_unlock(&lock);
344 cpu_clear(cpu, backtrace_mask);
345 }
346
347#ifdef CONFIG_X86_MCE
348 /* Could check oops_in_progress here too, but it's safer
349 not too */
350 if (atomic_read(&mce_entry) > 0)
351 touched = 1;
352#endif
353 /* if the apic timer isn't firing, this cpu isn't doing much */
354 if (!touched && __get_cpu_var(last_irq_sum) == sum) {
355 /*
356 * Ayiee, looks like this CPU is stuck ...
357 * wait a few IRQs (5 seconds) before doing the oops ...
358 */
359 local_inc(&__get_cpu_var(alert_counter));
360 if (local_read(&__get_cpu_var(alert_counter)) == 5*nmi_hz)
361 die_nmi("NMI Watchdog detected LOCKUP on CPU %d\n", regs,
362 panic_on_timeout);
363 } else {
364 __get_cpu_var(last_irq_sum) = sum;
365 local_set(&__get_cpu_var(alert_counter), 0);
366 }
367
368 /* see if the nmi watchdog went off */
369 if (!__get_cpu_var(wd_enabled))
370 return rc;
371 switch (nmi_watchdog) {
372 case NMI_LOCAL_APIC:
373 rc |= lapic_wd_event(nmi_hz);
374 break;
375 case NMI_IO_APIC:
376 /* don't know how to accurately check for this.
377 * just assume it was a watchdog timer interrupt
378 * This matches the old behaviour.
379 */
380 rc = 1;
381 break;
382 }
383 return rc;
384}
385
386static unsigned ignore_nmis;
387
388asmlinkage notrace __kprobes void
389do_nmi(struct pt_regs *regs, long error_code)
390{
391 nmi_enter();
392 add_pda(__nmi_count,1);
393 if (!ignore_nmis)
394 default_do_nmi(regs);
395 nmi_exit();
396}
397
398void stop_nmi(void)
399{
400 acpi_nmi_disable();
401 ignore_nmis++;
402}
403
404void restart_nmi(void)
405{
406 ignore_nmis--;
407 acpi_nmi_enable();
408}
409
410#ifdef CONFIG_SYSCTL
411
412static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
413{
414 unsigned char reason = get_nmi_reason();
415 char buf[64];
416
417 sprintf(buf, "NMI received for unknown reason %02x\n", reason);
418 die_nmi(buf, regs, 1); /* Always panic here */
419 return 0;
420}
421
422/*
423 * proc handler for /proc/sys/kernel/nmi
424 */
425int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
426 void __user *buffer, size_t *length, loff_t *ppos)
427{
428 int old_state;
429
430 nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0;
431 old_state = nmi_watchdog_enabled;
432 proc_dointvec(table, write, file, buffer, length, ppos);
433 if (!!old_state == !!nmi_watchdog_enabled)
434 return 0;
435
436 if (atomic_read(&nmi_active) < 0 || nmi_watchdog == NMI_DISABLED) {
437 printk( KERN_WARNING "NMI watchdog is permanently disabled\n");
438 return -EIO;
439 }
440
441 /* if nmi_watchdog is not set yet, then set it */
442 nmi_watchdog_default();
443
444 if (nmi_watchdog == NMI_LOCAL_APIC) {
445 if (nmi_watchdog_enabled)
446 enable_lapic_nmi_watchdog();
447 else
448 disable_lapic_nmi_watchdog();
449 } else {
450 printk( KERN_WARNING
451 "NMI watchdog doesn't know what hardware to touch\n");
452 return -EIO;
453 }
454 return 0;
455}
456
457#endif
458
459int do_nmi_callback(struct pt_regs *regs, int cpu)
460{
461#ifdef CONFIG_SYSCTL
462 if (unknown_nmi_panic)
463 return unknown_nmi_panic_callback(regs, cpu);
464#endif
465 return 0;
466}
467
468void __trigger_all_cpu_backtrace(void)
469{
470 int i;
471
472 backtrace_mask = cpu_online_map;
473 /* Wait for up to 10 seconds for all CPUs to do the backtrace */
474 for (i = 0; i < 10 * 1000; i++) {
475 if (cpus_empty(backtrace_mask))
476 break;
477 mdelay(1);
478 }
479}
480
481EXPORT_SYMBOL(nmi_active);
482EXPORT_SYMBOL(nmi_watchdog);
diff --git a/arch/x86/kernel/numaq_32.c b/arch/x86/kernel/numaq_32.c
index e65281b1634b..a23e8233b9ac 100644
--- a/arch/x86/kernel/numaq_32.c
+++ b/arch/x86/kernel/numaq_32.c
@@ -31,6 +31,8 @@
31#include <asm/numaq.h> 31#include <asm/numaq.h>
32#include <asm/topology.h> 32#include <asm/topology.h>
33#include <asm/processor.h> 33#include <asm/processor.h>
34#include <asm/mpspec.h>
35#include <asm/e820.h>
34 36
35#define MB_TO_PAGES(addr) ((addr) << (20 - PAGE_SHIFT)) 37#define MB_TO_PAGES(addr) ((addr) << (20 - PAGE_SHIFT))
36 38
@@ -58,6 +60,8 @@ static void __init smp_dump_qct(void)
58 node_end_pfn[node] = MB_TO_PAGES( 60 node_end_pfn[node] = MB_TO_PAGES(
59 eq->hi_shrd_mem_start + eq->hi_shrd_mem_size); 61 eq->hi_shrd_mem_start + eq->hi_shrd_mem_size);
60 62
63 e820_register_active_regions(node, node_start_pfn[node],
64 node_end_pfn[node]);
61 memory_present(node, 65 memory_present(node,
62 node_start_pfn[node], node_end_pfn[node]); 66 node_start_pfn[node], node_end_pfn[node]);
63 node_remap_size[node] = node_memmap_size_bytes(node, 67 node_remap_size[node] = node_memmap_size_bytes(node,
@@ -67,23 +71,35 @@ static void __init smp_dump_qct(void)
67 } 71 }
68} 72}
69 73
70/* 74static __init void early_check_numaq(void)
71 * Unlike Summit, we don't really care to let the NUMA-Q 75{
72 * fall back to flat mode. Don't compile for NUMA-Q 76 /*
73 * unless you really need it! 77 * Find possible boot-time SMP configuration:
74 */ 78 */
79 early_find_smp_config();
80 /*
81 * get boot-time SMP configuration:
82 */
83 if (smp_found_config)
84 early_get_smp_config();
85}
86
75int __init get_memcfg_numaq(void) 87int __init get_memcfg_numaq(void)
76{ 88{
89 early_check_numaq();
90 if (!found_numaq)
91 return 0;
77 smp_dump_qct(); 92 smp_dump_qct();
78 return 1; 93 return 1;
79} 94}
80 95
81static int __init numaq_tsc_disable(void) 96void __init numaq_tsc_disable(void)
82{ 97{
98 if (!found_numaq)
99 return;
100
83 if (num_online_nodes() > 1) { 101 if (num_online_nodes() > 1) {
84 printk(KERN_DEBUG "NUMAQ: disabling TSC\n"); 102 printk(KERN_DEBUG "NUMAQ: disabling TSC\n");
85 setup_clear_cpu_cap(X86_FEATURE_TSC); 103 setup_clear_cpu_cap(X86_FEATURE_TSC);
86 } 104 }
87 return 0;
88} 105}
89arch_initcall(numaq_tsc_disable);
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 74f0c5ea2a03..e0f571d58c19 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -30,6 +30,7 @@
30#include <asm/setup.h> 30#include <asm/setup.h>
31#include <asm/arch_hooks.h> 31#include <asm/arch_hooks.h>
32#include <asm/time.h> 32#include <asm/time.h>
33#include <asm/pgalloc.h>
33#include <asm/irq.h> 34#include <asm/irq.h>
34#include <asm/delay.h> 35#include <asm/delay.h>
35#include <asm/fixmap.h> 36#include <asm/fixmap.h>
@@ -139,7 +140,9 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
139 /* If the operation is a nop, then nop the callsite */ 140 /* If the operation is a nop, then nop the callsite */
140 ret = paravirt_patch_nop(); 141 ret = paravirt_patch_nop();
141 else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || 142 else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
142 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret)) 143 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
144 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) ||
145 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64))
143 /* If operation requires a jmp, then jmp */ 146 /* If operation requires a jmp, then jmp */
144 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); 147 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
145 else 148 else
@@ -190,7 +193,9 @@ static void native_flush_tlb_single(unsigned long addr)
190 193
191/* These are in entry.S */ 194/* These are in entry.S */
192extern void native_iret(void); 195extern void native_iret(void);
193extern void native_irq_enable_syscall_ret(void); 196extern void native_irq_enable_sysexit(void);
197extern void native_usergs_sysret32(void);
198extern void native_usergs_sysret64(void);
194 199
195static int __init print_banner(void) 200static int __init print_banner(void)
196{ 201{
@@ -280,7 +285,7 @@ struct pv_time_ops pv_time_ops = {
280 .get_wallclock = native_get_wallclock, 285 .get_wallclock = native_get_wallclock,
281 .set_wallclock = native_set_wallclock, 286 .set_wallclock = native_set_wallclock,
282 .sched_clock = native_sched_clock, 287 .sched_clock = native_sched_clock,
283 .get_cpu_khz = native_calculate_cpu_khz, 288 .get_tsc_khz = native_calibrate_tsc,
284}; 289};
285 290
286struct pv_irq_ops pv_irq_ops = { 291struct pv_irq_ops pv_irq_ops = {
@@ -291,6 +296,9 @@ struct pv_irq_ops pv_irq_ops = {
291 .irq_enable = native_irq_enable, 296 .irq_enable = native_irq_enable,
292 .safe_halt = native_safe_halt, 297 .safe_halt = native_safe_halt,
293 .halt = native_halt, 298 .halt = native_halt,
299#ifdef CONFIG_X86_64
300 .adjust_exception_frame = paravirt_nop,
301#endif
294}; 302};
295 303
296struct pv_cpu_ops pv_cpu_ops = { 304struct pv_cpu_ops pv_cpu_ops = {
@@ -321,12 +329,23 @@ struct pv_cpu_ops pv_cpu_ops = {
321 .store_idt = native_store_idt, 329 .store_idt = native_store_idt,
322 .store_tr = native_store_tr, 330 .store_tr = native_store_tr,
323 .load_tls = native_load_tls, 331 .load_tls = native_load_tls,
332#ifdef CONFIG_X86_64
333 .load_gs_index = native_load_gs_index,
334#endif
324 .write_ldt_entry = native_write_ldt_entry, 335 .write_ldt_entry = native_write_ldt_entry,
325 .write_gdt_entry = native_write_gdt_entry, 336 .write_gdt_entry = native_write_gdt_entry,
326 .write_idt_entry = native_write_idt_entry, 337 .write_idt_entry = native_write_idt_entry,
327 .load_sp0 = native_load_sp0, 338 .load_sp0 = native_load_sp0,
328 339
329 .irq_enable_syscall_ret = native_irq_enable_syscall_ret, 340#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
341 .irq_enable_sysexit = native_irq_enable_sysexit,
342#endif
343#ifdef CONFIG_X86_64
344#ifdef CONFIG_IA32_EMULATION
345 .usergs_sysret32 = native_usergs_sysret32,
346#endif
347 .usergs_sysret64 = native_usergs_sysret64,
348#endif
330 .iret = native_iret, 349 .iret = native_iret,
331 .swapgs = native_swapgs, 350 .swapgs = native_swapgs,
332 351
@@ -366,6 +385,9 @@ struct pv_mmu_ops pv_mmu_ops = {
366 .flush_tlb_single = native_flush_tlb_single, 385 .flush_tlb_single = native_flush_tlb_single,
367 .flush_tlb_others = native_flush_tlb_others, 386 .flush_tlb_others = native_flush_tlb_others,
368 387
388 .pgd_alloc = __paravirt_pgd_alloc,
389 .pgd_free = paravirt_nop,
390
369 .alloc_pte = paravirt_nop, 391 .alloc_pte = paravirt_nop,
370 .alloc_pmd = paravirt_nop, 392 .alloc_pmd = paravirt_nop,
371 .alloc_pmd_clone = paravirt_nop, 393 .alloc_pmd_clone = paravirt_nop,
@@ -380,6 +402,9 @@ struct pv_mmu_ops pv_mmu_ops = {
380 .pte_update = paravirt_nop, 402 .pte_update = paravirt_nop,
381 .pte_update_defer = paravirt_nop, 403 .pte_update_defer = paravirt_nop,
382 404
405 .ptep_modify_prot_start = __ptep_modify_prot_start,
406 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
407
383#ifdef CONFIG_HIGHPTE 408#ifdef CONFIG_HIGHPTE
384 .kmap_atomic_pte = kmap_atomic, 409 .kmap_atomic_pte = kmap_atomic,
385#endif 410#endif
@@ -403,6 +428,7 @@ struct pv_mmu_ops pv_mmu_ops = {
403#endif /* PAGETABLE_LEVELS >= 3 */ 428#endif /* PAGETABLE_LEVELS >= 3 */
404 429
405 .pte_val = native_pte_val, 430 .pte_val = native_pte_val,
431 .pte_flags = native_pte_val,
406 .pgd_val = native_pgd_val, 432 .pgd_val = native_pgd_val,
407 433
408 .make_pte = native_make_pte, 434 .make_pte = native_make_pte,
@@ -416,6 +442,8 @@ struct pv_mmu_ops pv_mmu_ops = {
416 .enter = paravirt_nop, 442 .enter = paravirt_nop,
417 .leave = paravirt_nop, 443 .leave = paravirt_nop,
418 }, 444 },
445
446 .set_fixmap = native_set_fixmap,
419}; 447};
420 448
421EXPORT_SYMBOL_GPL(pv_time_ops); 449EXPORT_SYMBOL_GPL(pv_time_ops);
diff --git a/arch/x86/kernel/paravirt_patch_32.c b/arch/x86/kernel/paravirt_patch_32.c
index 82fc5fcab4f4..58262218781b 100644
--- a/arch/x86/kernel/paravirt_patch_32.c
+++ b/arch/x86/kernel/paravirt_patch_32.c
@@ -5,7 +5,7 @@ DEF_NATIVE(pv_irq_ops, irq_enable, "sti");
5DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf"); 5DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf");
6DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax"); 6DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax");
7DEF_NATIVE(pv_cpu_ops, iret, "iret"); 7DEF_NATIVE(pv_cpu_ops, iret, "iret");
8DEF_NATIVE(pv_cpu_ops, irq_enable_syscall_ret, "sti; sysexit"); 8DEF_NATIVE(pv_cpu_ops, irq_enable_sysexit, "sti; sysexit");
9DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax"); 9DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax");
10DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3"); 10DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3");
11DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax"); 11DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax");
@@ -29,7 +29,7 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
29 PATCH_SITE(pv_irq_ops, restore_fl); 29 PATCH_SITE(pv_irq_ops, restore_fl);
30 PATCH_SITE(pv_irq_ops, save_fl); 30 PATCH_SITE(pv_irq_ops, save_fl);
31 PATCH_SITE(pv_cpu_ops, iret); 31 PATCH_SITE(pv_cpu_ops, iret);
32 PATCH_SITE(pv_cpu_ops, irq_enable_syscall_ret); 32 PATCH_SITE(pv_cpu_ops, irq_enable_sysexit);
33 PATCH_SITE(pv_mmu_ops, read_cr2); 33 PATCH_SITE(pv_mmu_ops, read_cr2);
34 PATCH_SITE(pv_mmu_ops, read_cr3); 34 PATCH_SITE(pv_mmu_ops, read_cr3);
35 PATCH_SITE(pv_mmu_ops, write_cr3); 35 PATCH_SITE(pv_mmu_ops, write_cr3);
diff --git a/arch/x86/kernel/paravirt_patch_64.c b/arch/x86/kernel/paravirt_patch_64.c
index 7d904e138d7e..061d01df9ae6 100644
--- a/arch/x86/kernel/paravirt_patch_64.c
+++ b/arch/x86/kernel/paravirt_patch_64.c
@@ -14,8 +14,9 @@ DEF_NATIVE(pv_mmu_ops, flush_tlb_single, "invlpg (%rdi)");
14DEF_NATIVE(pv_cpu_ops, clts, "clts"); 14DEF_NATIVE(pv_cpu_ops, clts, "clts");
15DEF_NATIVE(pv_cpu_ops, wbinvd, "wbinvd"); 15DEF_NATIVE(pv_cpu_ops, wbinvd, "wbinvd");
16 16
17/* the three commands give us more control to how to return from a syscall */ 17DEF_NATIVE(pv_cpu_ops, irq_enable_sysexit, "swapgs; sti; sysexit");
18DEF_NATIVE(pv_cpu_ops, irq_enable_syscall_ret, "movq %gs:" __stringify(pda_oldrsp) ", %rsp; swapgs; sysretq;"); 18DEF_NATIVE(pv_cpu_ops, usergs_sysret64, "swapgs; sysretq");
19DEF_NATIVE(pv_cpu_ops, usergs_sysret32, "swapgs; sysretl");
19DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs"); 20DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs");
20 21
21unsigned native_patch(u8 type, u16 clobbers, void *ibuf, 22unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
@@ -35,7 +36,9 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
35 PATCH_SITE(pv_irq_ops, irq_enable); 36 PATCH_SITE(pv_irq_ops, irq_enable);
36 PATCH_SITE(pv_irq_ops, irq_disable); 37 PATCH_SITE(pv_irq_ops, irq_disable);
37 PATCH_SITE(pv_cpu_ops, iret); 38 PATCH_SITE(pv_cpu_ops, iret);
38 PATCH_SITE(pv_cpu_ops, irq_enable_syscall_ret); 39 PATCH_SITE(pv_cpu_ops, irq_enable_sysexit);
40 PATCH_SITE(pv_cpu_ops, usergs_sysret32);
41 PATCH_SITE(pv_cpu_ops, usergs_sysret64);
39 PATCH_SITE(pv_cpu_ops, swapgs); 42 PATCH_SITE(pv_cpu_ops, swapgs);
40 PATCH_SITE(pv_mmu_ops, read_cr2); 43 PATCH_SITE(pv_mmu_ops, read_cr2);
41 PATCH_SITE(pv_mmu_ops, read_cr3); 44 PATCH_SITE(pv_mmu_ops, read_cr3);
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index e28ec497e142..6959b5c45df4 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -1394,7 +1394,7 @@ void __init detect_calgary(void)
1394 return; 1394 return;
1395 } 1395 }
1396 1396
1397 specified_table_size = determine_tce_table_size(end_pfn * PAGE_SIZE); 1397 specified_table_size = determine_tce_table_size(max_pfn * PAGE_SIZE);
1398 1398
1399 for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) { 1399 for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1400 struct calgary_bus_info *info = &bus_info[bus]; 1400 struct calgary_bus_info *info = &bus_info[bus];
@@ -1459,7 +1459,7 @@ int __init calgary_iommu_init(void)
1459 if (ret) { 1459 if (ret) {
1460 printk(KERN_ERR "PCI-DMA: Calgary init failed %d, " 1460 printk(KERN_ERR "PCI-DMA: Calgary init failed %d, "
1461 "falling back to no_iommu\n", ret); 1461 "falling back to no_iommu\n", ret);
1462 if (end_pfn > MAX_DMA32_PFN) 1462 if (max_pfn > MAX_DMA32_PFN)
1463 printk(KERN_ERR "WARNING more than 4GB of memory, " 1463 printk(KERN_ERR "WARNING more than 4GB of memory, "
1464 "32bit PCI may malfunction.\n"); 1464 "32bit PCI may malfunction.\n");
1465 return ret; 1465 return ret;
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index b75c81a8d3e6..b7dd70fda031 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -7,6 +7,7 @@
7#include <asm/dma.h> 7#include <asm/dma.h>
8#include <asm/gart.h> 8#include <asm/gart.h>
9#include <asm/calgary.h> 9#include <asm/calgary.h>
10#include <asm/amd_iommu.h>
10 11
11int forbid_dac __read_mostly; 12int forbid_dac __read_mostly;
12EXPORT_SYMBOL(forbid_dac); 13EXPORT_SYMBOL(forbid_dac);
@@ -74,13 +75,17 @@ early_param("dma32_size", parse_dma32_size_opt);
74void __init dma32_reserve_bootmem(void) 75void __init dma32_reserve_bootmem(void)
75{ 76{
76 unsigned long size, align; 77 unsigned long size, align;
77 if (end_pfn <= MAX_DMA32_PFN) 78 if (max_pfn <= MAX_DMA32_PFN)
78 return; 79 return;
79 80
81 /*
82 * check aperture_64.c allocate_aperture() for reason about
83 * using 512M as goal
84 */
80 align = 64ULL<<20; 85 align = 64ULL<<20;
81 size = round_up(dma32_bootmem_size, align); 86 size = round_up(dma32_bootmem_size, align);
82 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 87 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
83 __pa(MAX_DMA_ADDRESS)); 88 512ULL<<20);
84 if (dma32_bootmem_ptr) 89 if (dma32_bootmem_ptr)
85 dma32_bootmem_size = size; 90 dma32_bootmem_size = size;
86 else 91 else
@@ -88,17 +93,14 @@ void __init dma32_reserve_bootmem(void)
88} 93}
89static void __init dma32_free_bootmem(void) 94static void __init dma32_free_bootmem(void)
90{ 95{
91 int node;
92 96
93 if (end_pfn <= MAX_DMA32_PFN) 97 if (max_pfn <= MAX_DMA32_PFN)
94 return; 98 return;
95 99
96 if (!dma32_bootmem_ptr) 100 if (!dma32_bootmem_ptr)
97 return; 101 return;
98 102
99 for_each_online_node(node) 103 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
100 free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
101 dma32_bootmem_size);
102 104
103 dma32_bootmem_ptr = NULL; 105 dma32_bootmem_ptr = NULL;
104 dma32_bootmem_size = 0; 106 dma32_bootmem_size = 0;
@@ -122,6 +124,8 @@ void __init pci_iommu_alloc(void)
122 124
123 detect_intel_iommu(); 125 detect_intel_iommu();
124 126
127 amd_iommu_detect();
128
125#ifdef CONFIG_SWIOTLB 129#ifdef CONFIG_SWIOTLB
126 pci_swiotlb_init(); 130 pci_swiotlb_init();
127#endif 131#endif
@@ -239,7 +243,7 @@ int dma_supported(struct device *dev, u64 mask)
239EXPORT_SYMBOL(dma_supported); 243EXPORT_SYMBOL(dma_supported);
240 244
241/* Allocate DMA memory on node near device */ 245/* Allocate DMA memory on node near device */
242noinline struct page * 246static noinline struct page *
243dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order) 247dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
244{ 248{
245 int node; 249 int node;
@@ -384,6 +388,8 @@ static int __init pci_iommu_init(void)
384 388
385 intel_iommu_init(); 389 intel_iommu_init();
386 390
391 amd_iommu_init();
392
387#ifdef CONFIG_GART_IOMMU 393#ifdef CONFIG_GART_IOMMU
388 gart_iommu_init(); 394 gart_iommu_init();
389#endif 395#endif
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index aa8ec928caa8..c3fe78406d18 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -104,7 +104,6 @@ static unsigned long alloc_iommu(struct device *dev, int size)
104 size, base_index, boundary_size, 0); 104 size, base_index, boundary_size, 0);
105 } 105 }
106 if (offset != -1) { 106 if (offset != -1) {
107 set_bit_string(iommu_gart_bitmap, offset, size);
108 next_bit = offset+size; 107 next_bit = offset+size;
109 if (next_bit >= iommu_pages) { 108 if (next_bit >= iommu_pages) {
110 next_bit = 0; 109 next_bit = 0;
@@ -534,8 +533,8 @@ static __init unsigned read_aperture(struct pci_dev *dev, u32 *size)
534 unsigned aper_size = 0, aper_base_32, aper_order; 533 unsigned aper_size = 0, aper_base_32, aper_order;
535 u64 aper_base; 534 u64 aper_base;
536 535
537 pci_read_config_dword(dev, 0x94, &aper_base_32); 536 pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, &aper_base_32);
538 pci_read_config_dword(dev, 0x90, &aper_order); 537 pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &aper_order);
539 aper_order = (aper_order >> 1) & 7; 538 aper_order = (aper_order >> 1) & 7;
540 539
541 aper_base = aper_base_32 & 0x7fff; 540 aper_base = aper_base_32 & 0x7fff;
@@ -549,14 +548,63 @@ static __init unsigned read_aperture(struct pci_dev *dev, u32 *size)
549 return aper_base; 548 return aper_base;
550} 549}
551 550
551static void enable_gart_translations(void)
552{
553 int i;
554
555 for (i = 0; i < num_k8_northbridges; i++) {
556 struct pci_dev *dev = k8_northbridges[i];
557
558 enable_gart_translation(dev, __pa(agp_gatt_table));
559 }
560}
561
562/*
563 * If fix_up_north_bridges is set, the north bridges have to be fixed up on
564 * resume in the same way as they are handled in gart_iommu_hole_init().
565 */
566static bool fix_up_north_bridges;
567static u32 aperture_order;
568static u32 aperture_alloc;
569
570void set_up_gart_resume(u32 aper_order, u32 aper_alloc)
571{
572 fix_up_north_bridges = true;
573 aperture_order = aper_order;
574 aperture_alloc = aper_alloc;
575}
576
552static int gart_resume(struct sys_device *dev) 577static int gart_resume(struct sys_device *dev)
553{ 578{
579 printk(KERN_INFO "PCI-DMA: Resuming GART IOMMU\n");
580
581 if (fix_up_north_bridges) {
582 int i;
583
584 printk(KERN_INFO "PCI-DMA: Restoring GART aperture settings\n");
585
586 for (i = 0; i < num_k8_northbridges; i++) {
587 struct pci_dev *dev = k8_northbridges[i];
588
589 /*
590 * Don't enable translations just yet. That is the next
591 * step. Restore the pre-suspend aperture settings.
592 */
593 pci_write_config_dword(dev, AMD64_GARTAPERTURECTL,
594 aperture_order << 1);
595 pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE,
596 aperture_alloc >> 25);
597 }
598 }
599
600 enable_gart_translations();
601
554 return 0; 602 return 0;
555} 603}
556 604
557static int gart_suspend(struct sys_device *dev, pm_message_t state) 605static int gart_suspend(struct sys_device *dev, pm_message_t state)
558{ 606{
559 return -EINVAL; 607 return 0;
560} 608}
561 609
562static struct sysdev_class gart_sysdev_class = { 610static struct sysdev_class gart_sysdev_class = {
@@ -582,6 +630,7 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
582 struct pci_dev *dev; 630 struct pci_dev *dev;
583 void *gatt; 631 void *gatt;
584 int i, error; 632 int i, error;
633 unsigned long start_pfn, end_pfn;
585 634
586 printk(KERN_INFO "PCI-DMA: Disabling AGP.\n"); 635 printk(KERN_INFO "PCI-DMA: Disabling AGP.\n");
587 aper_size = aper_base = info->aper_size = 0; 636 aper_size = aper_base = info->aper_size = 0;
@@ -614,31 +663,25 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
614 memset(gatt, 0, gatt_size); 663 memset(gatt, 0, gatt_size);
615 agp_gatt_table = gatt; 664 agp_gatt_table = gatt;
616 665
617 for (i = 0; i < num_k8_northbridges; i++) { 666 enable_gart_translations();
618 u32 gatt_reg;
619 u32 ctl;
620
621 dev = k8_northbridges[i];
622 gatt_reg = __pa(gatt) >> 12;
623 gatt_reg <<= 4;
624 pci_write_config_dword(dev, 0x98, gatt_reg);
625 pci_read_config_dword(dev, 0x90, &ctl);
626
627 ctl |= 1;
628 ctl &= ~((1<<4) | (1<<5));
629
630 pci_write_config_dword(dev, 0x90, ctl);
631 }
632 667
633 error = sysdev_class_register(&gart_sysdev_class); 668 error = sysdev_class_register(&gart_sysdev_class);
634 if (!error) 669 if (!error)
635 error = sysdev_register(&device_gart); 670 error = sysdev_register(&device_gart);
636 if (error) 671 if (error)
637 panic("Could not register gart_sysdev -- would corrupt data on next suspend"); 672 panic("Could not register gart_sysdev -- would corrupt data on next suspend");
673
638 flush_gart(); 674 flush_gart();
639 675
640 printk(KERN_INFO "PCI-DMA: aperture base @ %x size %u KB\n", 676 printk(KERN_INFO "PCI-DMA: aperture base @ %x size %u KB\n",
641 aper_base, aper_size>>10); 677 aper_base, aper_size>>10);
678
679 /* need to map that range */
680 end_pfn = (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT);
681 if (end_pfn > max_low_pfn_mapped) {
682 start_pfn = (aper_base>>PAGE_SHIFT);
683 init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
684 }
642 return 0; 685 return 0;
643 686
644 nommu: 687 nommu:
@@ -677,11 +720,11 @@ void gart_iommu_shutdown(void)
677 u32 ctl; 720 u32 ctl;
678 721
679 dev = k8_northbridges[i]; 722 dev = k8_northbridges[i];
680 pci_read_config_dword(dev, 0x90, &ctl); 723 pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl);
681 724
682 ctl &= ~1; 725 ctl &= ~GARTEN;
683 726
684 pci_write_config_dword(dev, 0x90, ctl); 727 pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl);
685 } 728 }
686} 729}
687 730
@@ -716,10 +759,10 @@ void __init gart_iommu_init(void)
716 return; 759 return;
717 760
718 if (no_iommu || 761 if (no_iommu ||
719 (!force_iommu && end_pfn <= MAX_DMA32_PFN) || 762 (!force_iommu && max_pfn <= MAX_DMA32_PFN) ||
720 !gart_iommu_aperture || 763 !gart_iommu_aperture ||
721 (no_agp && init_k8_gatt(&info) < 0)) { 764 (no_agp && init_k8_gatt(&info) < 0)) {
722 if (end_pfn > MAX_DMA32_PFN) { 765 if (max_pfn > MAX_DMA32_PFN) {
723 printk(KERN_WARNING "More than 4GB of memory " 766 printk(KERN_WARNING "More than 4GB of memory "
724 "but GART IOMMU not available.\n" 767 "but GART IOMMU not available.\n"
725 KERN_WARNING "falling back to iommu=soft.\n"); 768 KERN_WARNING "falling back to iommu=soft.\n");
@@ -788,10 +831,10 @@ void __init gart_iommu_init(void)
788 wbinvd(); 831 wbinvd();
789 832
790 /* 833 /*
791 * Try to workaround a bug (thanks to BenH) 834 * Try to workaround a bug (thanks to BenH):
792 * Set unmapped entries to a scratch page instead of 0. 835 * Set unmapped entries to a scratch page instead of 0.
793 * Any prefetches that hit unmapped entries won't get an bus abort 836 * Any prefetches that hit unmapped entries won't get an bus abort
794 * then. 837 * then. (P2P bridge may be prefetching on DMA reads).
795 */ 838 */
796 scratch = get_zeroed_page(GFP_KERNEL); 839 scratch = get_zeroed_page(GFP_KERNEL);
797 if (!scratch) 840 if (!scratch)
diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb_64.c
index 490da7f4b8d0..82299cd1d04d 100644
--- a/arch/x86/kernel/pci-swiotlb_64.c
+++ b/arch/x86/kernel/pci-swiotlb_64.c
@@ -38,7 +38,7 @@ const struct dma_mapping_ops swiotlb_dma_ops = {
38void __init pci_swiotlb_init(void) 38void __init pci_swiotlb_init(void)
39{ 39{
40 /* don't initialize swiotlb if iommu=off (no_iommu=1) */ 40 /* don't initialize swiotlb if iommu=off (no_iommu=1) */
41 if (!iommu_detected && !no_iommu && end_pfn > MAX_DMA32_PFN) 41 if (!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN)
42 swiotlb = 1; 42 swiotlb = 1;
43 if (swiotlb_force) 43 if (swiotlb_force)
44 swiotlb = 1; 44 swiotlb = 1;
diff --git a/arch/x86/kernel/probe_roms_32.c b/arch/x86/kernel/probe_roms_32.c
new file mode 100644
index 000000000000..675a48c404a5
--- /dev/null
+++ b/arch/x86/kernel/probe_roms_32.c
@@ -0,0 +1,166 @@
1#include <linux/sched.h>
2#include <linux/mm.h>
3#include <linux/uaccess.h>
4#include <linux/mmzone.h>
5#include <linux/ioport.h>
6#include <linux/seq_file.h>
7#include <linux/console.h>
8#include <linux/init.h>
9#include <linux/edd.h>
10#include <linux/dmi.h>
11#include <linux/pfn.h>
12#include <linux/pci.h>
13#include <asm/pci-direct.h>
14
15
16#include <asm/e820.h>
17#include <asm/mmzone.h>
18#include <asm/setup.h>
19#include <asm/sections.h>
20#include <asm/io.h>
21#include <setup_arch.h>
22
23static struct resource system_rom_resource = {
24 .name = "System ROM",
25 .start = 0xf0000,
26 .end = 0xfffff,
27 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
28};
29
30static struct resource extension_rom_resource = {
31 .name = "Extension ROM",
32 .start = 0xe0000,
33 .end = 0xeffff,
34 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
35};
36
37static struct resource adapter_rom_resources[] = { {
38 .name = "Adapter ROM",
39 .start = 0xc8000,
40 .end = 0,
41 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
42}, {
43 .name = "Adapter ROM",
44 .start = 0,
45 .end = 0,
46 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
47}, {
48 .name = "Adapter ROM",
49 .start = 0,
50 .end = 0,
51 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
52}, {
53 .name = "Adapter ROM",
54 .start = 0,
55 .end = 0,
56 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
57}, {
58 .name = "Adapter ROM",
59 .start = 0,
60 .end = 0,
61 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
62}, {
63 .name = "Adapter ROM",
64 .start = 0,
65 .end = 0,
66 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
67} };
68
69static struct resource video_rom_resource = {
70 .name = "Video ROM",
71 .start = 0xc0000,
72 .end = 0xc7fff,
73 .flags = IORESOURCE_BUSY | IORESOURCE_READONLY | IORESOURCE_MEM
74};
75
76#define ROMSIGNATURE 0xaa55
77
78static int __init romsignature(const unsigned char *rom)
79{
80 const unsigned short * const ptr = (const unsigned short *)rom;
81 unsigned short sig;
82
83 return probe_kernel_address(ptr, sig) == 0 && sig == ROMSIGNATURE;
84}
85
86static int __init romchecksum(const unsigned char *rom, unsigned long length)
87{
88 unsigned char sum, c;
89
90 for (sum = 0; length && probe_kernel_address(rom++, c) == 0; length--)
91 sum += c;
92 return !length && !sum;
93}
94
95void __init probe_roms(void)
96{
97 const unsigned char *rom;
98 unsigned long start, length, upper;
99 unsigned char c;
100 int i;
101
102 /* video rom */
103 upper = adapter_rom_resources[0].start;
104 for (start = video_rom_resource.start; start < upper; start += 2048) {
105 rom = isa_bus_to_virt(start);
106 if (!romsignature(rom))
107 continue;
108
109 video_rom_resource.start = start;
110
111 if (probe_kernel_address(rom + 2, c) != 0)
112 continue;
113
114 /* 0 < length <= 0x7f * 512, historically */
115 length = c * 512;
116
117 /* if checksum okay, trust length byte */
118 if (length && romchecksum(rom, length))
119 video_rom_resource.end = start + length - 1;
120
121 request_resource(&iomem_resource, &video_rom_resource);
122 break;
123 }
124
125 start = (video_rom_resource.end + 1 + 2047) & ~2047UL;
126 if (start < upper)
127 start = upper;
128
129 /* system rom */
130 request_resource(&iomem_resource, &system_rom_resource);
131 upper = system_rom_resource.start;
132
133 /* check for extension rom (ignore length byte!) */
134 rom = isa_bus_to_virt(extension_rom_resource.start);
135 if (romsignature(rom)) {
136 length = extension_rom_resource.end - extension_rom_resource.start + 1;
137 if (romchecksum(rom, length)) {
138 request_resource(&iomem_resource, &extension_rom_resource);
139 upper = extension_rom_resource.start;
140 }
141 }
142
143 /* check for adapter roms on 2k boundaries */
144 for (i = 0; i < ARRAY_SIZE(adapter_rom_resources) && start < upper; start += 2048) {
145 rom = isa_bus_to_virt(start);
146 if (!romsignature(rom))
147 continue;
148
149 if (probe_kernel_address(rom + 2, c) != 0)
150 continue;
151
152 /* 0 < length <= 0x7f * 512, historically */
153 length = c * 512;
154
155 /* but accept any length that fits if checksum okay */
156 if (!length || start + length > upper || !romchecksum(rom, length))
157 continue;
158
159 adapter_rom_resources[i].start = start;
160 adapter_rom_resources[i].end = start + length - 1;
161 request_resource(&iomem_resource, &adapter_rom_resources[i]);
162
163 start = adapter_rom_resources[i++].end & ~2047UL;
164 }
165}
166
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index ba370dc8685b..4d629c62f4f8 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -6,6 +6,13 @@
6#include <linux/sched.h> 6#include <linux/sched.h>
7#include <linux/module.h> 7#include <linux/module.h>
8#include <linux/pm.h> 8#include <linux/pm.h>
9#include <linux/clockchips.h>
10#include <asm/system.h>
11
12unsigned long idle_halt;
13EXPORT_SYMBOL(idle_halt);
14unsigned long idle_nomwait;
15EXPORT_SYMBOL(idle_nomwait);
9 16
10struct kmem_cache *task_xstate_cachep; 17struct kmem_cache *task_xstate_cachep;
11 18
@@ -45,6 +52,76 @@ void arch_task_cache_init(void)
45 SLAB_PANIC, NULL); 52 SLAB_PANIC, NULL);
46} 53}
47 54
55/*
56 * Idle related variables and functions
57 */
58unsigned long boot_option_idle_override = 0;
59EXPORT_SYMBOL(boot_option_idle_override);
60
61/*
62 * Powermanagement idle function, if any..
63 */
64void (*pm_idle)(void);
65EXPORT_SYMBOL(pm_idle);
66
67#ifdef CONFIG_X86_32
68/*
69 * This halt magic was a workaround for ancient floppy DMA
70 * wreckage. It should be safe to remove.
71 */
72static int hlt_counter;
73void disable_hlt(void)
74{
75 hlt_counter++;
76}
77EXPORT_SYMBOL(disable_hlt);
78
79void enable_hlt(void)
80{
81 hlt_counter--;
82}
83EXPORT_SYMBOL(enable_hlt);
84
85static inline int hlt_use_halt(void)
86{
87 return (!hlt_counter && boot_cpu_data.hlt_works_ok);
88}
89#else
90static inline int hlt_use_halt(void)
91{
92 return 1;
93}
94#endif
95
96/*
97 * We use this if we don't have any better
98 * idle routine..
99 */
100void default_idle(void)
101{
102 if (hlt_use_halt()) {
103 current_thread_info()->status &= ~TS_POLLING;
104 /*
105 * TS_POLLING-cleared state must be visible before we
106 * test NEED_RESCHED:
107 */
108 smp_mb();
109
110 if (!need_resched())
111 safe_halt(); /* enables interrupts racelessly */
112 else
113 local_irq_enable();
114 current_thread_info()->status |= TS_POLLING;
115 } else {
116 local_irq_enable();
117 /* loop is done by the caller */
118 cpu_relax();
119 }
120}
121#ifdef CONFIG_APM_MODULE
122EXPORT_SYMBOL(default_idle);
123#endif
124
48static void do_nothing(void *unused) 125static void do_nothing(void *unused)
49{ 126{
50} 127}
@@ -61,7 +138,7 @@ void cpu_idle_wait(void)
61{ 138{
62 smp_mb(); 139 smp_mb();
63 /* kick all the CPUs so that they exit out of pm_idle */ 140 /* kick all the CPUs so that they exit out of pm_idle */
64 smp_call_function(do_nothing, NULL, 0, 1); 141 smp_call_function(do_nothing, NULL, 1);
65} 142}
66EXPORT_SYMBOL_GPL(cpu_idle_wait); 143EXPORT_SYMBOL_GPL(cpu_idle_wait);
67 144
@@ -122,44 +199,129 @@ static void poll_idle(void)
122 * 199 *
123 * idle=mwait overrides this decision and forces the usage of mwait. 200 * idle=mwait overrides this decision and forces the usage of mwait.
124 */ 201 */
202
203#define MWAIT_INFO 0x05
204#define MWAIT_ECX_EXTENDED_INFO 0x01
205#define MWAIT_EDX_C1 0xf0
206
125static int __cpuinit mwait_usable(const struct cpuinfo_x86 *c) 207static int __cpuinit mwait_usable(const struct cpuinfo_x86 *c)
126{ 208{
209 u32 eax, ebx, ecx, edx;
210
127 if (force_mwait) 211 if (force_mwait)
128 return 1; 212 return 1;
129 213
130 if (c->x86_vendor == X86_VENDOR_AMD) { 214 if (c->cpuid_level < MWAIT_INFO)
131 switch(c->x86) { 215 return 0;
132 case 0x10: 216
133 case 0x11: 217 cpuid(MWAIT_INFO, &eax, &ebx, &ecx, &edx);
134 return 0; 218 /* Check, whether EDX has extended info about MWAIT */
135 } 219 if (!(ecx & MWAIT_ECX_EXTENDED_INFO))
136 } 220 return 1;
221
222 /*
223 * edx enumeratios MONITOR/MWAIT extensions. Check, whether
224 * C1 supports MWAIT
225 */
226 return (edx & MWAIT_EDX_C1);
227}
228
229/*
230 * Check for AMD CPUs, which have potentially C1E support
231 */
232static int __cpuinit check_c1e_idle(const struct cpuinfo_x86 *c)
233{
234 if (c->x86_vendor != X86_VENDOR_AMD)
235 return 0;
236
237 if (c->x86 < 0x0F)
238 return 0;
239
240 /* Family 0x0f models < rev F do not have C1E */
241 if (c->x86 == 0x0f && c->x86_model < 0x40)
242 return 0;
243
137 return 1; 244 return 1;
138} 245}
139 246
140void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c) 247/*
248 * C1E aware idle routine. We check for C1E active in the interrupt
249 * pending message MSR. If we detect C1E, then we handle it the same
250 * way as C3 power states (local apic timer and TSC stop)
251 */
252static void c1e_idle(void)
141{ 253{
142 static int selected; 254 static cpumask_t c1e_mask = CPU_MASK_NONE;
255 static int c1e_detected;
143 256
144 if (selected) 257 if (need_resched())
145 return; 258 return;
259
260 if (!c1e_detected) {
261 u32 lo, hi;
262
263 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
264 if (lo & K8_INTP_C1E_ACTIVE_MASK) {
265 c1e_detected = 1;
266 mark_tsc_unstable("TSC halt in C1E");
267 printk(KERN_INFO "System has C1E enabled\n");
268 }
269 }
270
271 if (c1e_detected) {
272 int cpu = smp_processor_id();
273
274 if (!cpu_isset(cpu, c1e_mask)) {
275 cpu_set(cpu, c1e_mask);
276 /*
277 * Force broadcast so ACPI can not interfere. Needs
278 * to run with interrupts enabled as it uses
279 * smp_function_call.
280 */
281 local_irq_enable();
282 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
283 &cpu);
284 printk(KERN_INFO "Switch to broadcast mode on CPU%d\n",
285 cpu);
286 local_irq_disable();
287 }
288 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
289
290 default_idle();
291
292 /*
293 * The switch back from broadcast mode needs to be
294 * called with interrupts disabled.
295 */
296 local_irq_disable();
297 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
298 local_irq_enable();
299 } else
300 default_idle();
301}
302
303void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
304{
146#ifdef CONFIG_X86_SMP 305#ifdef CONFIG_X86_SMP
147 if (pm_idle == poll_idle && smp_num_siblings > 1) { 306 if (pm_idle == poll_idle && smp_num_siblings > 1) {
148 printk(KERN_WARNING "WARNING: polling idle and HT enabled," 307 printk(KERN_WARNING "WARNING: polling idle and HT enabled,"
149 " performance may degrade.\n"); 308 " performance may degrade.\n");
150 } 309 }
151#endif 310#endif
311 if (pm_idle)
312 return;
313
152 if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) { 314 if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
153 /* 315 /*
154 * Skip, if setup has overridden idle.
155 * One CPU supports mwait => All CPUs supports mwait 316 * One CPU supports mwait => All CPUs supports mwait
156 */ 317 */
157 if (!pm_idle) { 318 printk(KERN_INFO "using mwait in idle threads.\n");
158 printk(KERN_INFO "using mwait in idle threads.\n"); 319 pm_idle = mwait_idle;
159 pm_idle = mwait_idle; 320 } else if (check_c1e_idle(c)) {
160 } 321 printk(KERN_INFO "using C1E aware idle routine\n");
161 } 322 pm_idle = c1e_idle;
162 selected = 1; 323 } else
324 pm_idle = default_idle;
163} 325}
164 326
165static int __init idle_setup(char *str) 327static int __init idle_setup(char *str)
@@ -169,7 +331,27 @@ static int __init idle_setup(char *str)
169 pm_idle = poll_idle; 331 pm_idle = poll_idle;
170 } else if (!strcmp(str, "mwait")) 332 } else if (!strcmp(str, "mwait"))
171 force_mwait = 1; 333 force_mwait = 1;
172 else 334 else if (!strcmp(str, "halt")) {
335 /*
336 * When the boot option of idle=halt is added, halt is
337 * forced to be used for CPU idle. In such case CPU C2/C3
338 * won't be used again.
339 * To continue to load the CPU idle driver, don't touch
340 * the boot_option_idle_override.
341 */
342 pm_idle = default_idle;
343 idle_halt = 1;
344 return 0;
345 } else if (!strcmp(str, "nomwait")) {
346 /*
347 * If the boot option of "idle=nomwait" is added,
348 * it means that mwait will be disabled for CPU C2/C3
349 * states. In such case it won't touch the variable
350 * of boot_option_idle_override.
351 */
352 idle_nomwait = 1;
353 return 0;
354 } else
173 return -1; 355 return -1;
174 356
175 boot_option_idle_override = 1; 357 boot_option_idle_override = 1;
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index e2db9ac5c61c..0c3927accb00 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -58,11 +58,6 @@
58 58
59asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); 59asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
60 60
61static int hlt_counter;
62
63unsigned long boot_option_idle_override = 0;
64EXPORT_SYMBOL(boot_option_idle_override);
65
66DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; 61DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
67EXPORT_PER_CPU_SYMBOL(current_task); 62EXPORT_PER_CPU_SYMBOL(current_task);
68 63
@@ -77,57 +72,24 @@ unsigned long thread_saved_pc(struct task_struct *tsk)
77 return ((unsigned long *)tsk->thread.sp)[3]; 72 return ((unsigned long *)tsk->thread.sp)[3];
78} 73}
79 74
80/* 75#ifdef CONFIG_HOTPLUG_CPU
81 * Powermanagement idle function, if any.. 76#include <asm/nmi.h>
82 */
83void (*pm_idle)(void);
84EXPORT_SYMBOL(pm_idle);
85 77
86void disable_hlt(void) 78static void cpu_exit_clear(void)
87{ 79{
88 hlt_counter++; 80 int cpu = raw_smp_processor_id();
89}
90 81
91EXPORT_SYMBOL(disable_hlt); 82 idle_task_exit();
92 83
93void enable_hlt(void) 84 cpu_uninit();
94{ 85 irq_ctx_exit(cpu);
95 hlt_counter--;
96}
97 86
98EXPORT_SYMBOL(enable_hlt); 87 cpu_clear(cpu, cpu_callout_map);
99 88 cpu_clear(cpu, cpu_callin_map);
100/*
101 * We use this if we don't have any better
102 * idle routine..
103 */
104void default_idle(void)
105{
106 if (!hlt_counter && boot_cpu_data.hlt_works_ok) {
107 current_thread_info()->status &= ~TS_POLLING;
108 /*
109 * TS_POLLING-cleared state must be visible before we
110 * test NEED_RESCHED:
111 */
112 smp_mb();
113 89
114 if (!need_resched()) 90 numa_remove_cpu(cpu);
115 safe_halt(); /* enables interrupts racelessly */
116 else
117 local_irq_enable();
118 current_thread_info()->status |= TS_POLLING;
119 } else {
120 local_irq_enable();
121 /* loop is done by the caller */
122 cpu_relax();
123 }
124} 91}
125#ifdef CONFIG_APM_MODULE
126EXPORT_SYMBOL(default_idle);
127#endif
128 92
129#ifdef CONFIG_HOTPLUG_CPU
130#include <asm/nmi.h>
131/* We don't actually take CPU down, just spin without interrupts. */ 93/* We don't actually take CPU down, just spin without interrupts. */
132static inline void play_dead(void) 94static inline void play_dead(void)
133{ 95{
@@ -168,24 +130,22 @@ void cpu_idle(void)
168 while (1) { 130 while (1) {
169 tick_nohz_stop_sched_tick(); 131 tick_nohz_stop_sched_tick();
170 while (!need_resched()) { 132 while (!need_resched()) {
171 void (*idle)(void);
172 133
173 check_pgt_cache(); 134 check_pgt_cache();
174 rmb(); 135 rmb();
175 idle = pm_idle;
176 136
177 if (rcu_pending(cpu)) 137 if (rcu_pending(cpu))
178 rcu_check_callbacks(cpu, 0); 138 rcu_check_callbacks(cpu, 0);
179 139
180 if (!idle)
181 idle = default_idle;
182
183 if (cpu_is_offline(cpu)) 140 if (cpu_is_offline(cpu))
184 play_dead(); 141 play_dead();
185 142
186 local_irq_disable(); 143 local_irq_disable();
187 __get_cpu_var(irq_stat).idle_timestamp = jiffies; 144 __get_cpu_var(irq_stat).idle_timestamp = jiffies;
188 idle(); 145 /* Don't trace irqs off for idle */
146 stop_critical_timings();
147 pm_idle();
148 start_critical_timings();
189 } 149 }
190 tick_nohz_restart_sched_tick(); 150 tick_nohz_restart_sched_tick();
191 preempt_enable_no_resched(); 151 preempt_enable_no_resched();
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index c6eb5c91e5f6..a8e53626ac9a 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -56,15 +56,6 @@ asmlinkage extern void ret_from_fork(void);
56 56
57unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED; 57unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
58 58
59unsigned long boot_option_idle_override = 0;
60EXPORT_SYMBOL(boot_option_idle_override);
61
62/*
63 * Powermanagement idle function, if any..
64 */
65void (*pm_idle)(void);
66EXPORT_SYMBOL(pm_idle);
67
68static ATOMIC_NOTIFIER_HEAD(idle_notifier); 59static ATOMIC_NOTIFIER_HEAD(idle_notifier);
69 60
70void idle_notifier_register(struct notifier_block *n) 61void idle_notifier_register(struct notifier_block *n)
@@ -94,25 +85,6 @@ void exit_idle(void)
94 __exit_idle(); 85 __exit_idle();
95} 86}
96 87
97/*
98 * We use this if we don't have any better
99 * idle routine..
100 */
101void default_idle(void)
102{
103 current_thread_info()->status &= ~TS_POLLING;
104 /*
105 * TS_POLLING-cleared state must be visible before we
106 * test NEED_RESCHED:
107 */
108 smp_mb();
109 if (!need_resched())
110 safe_halt(); /* enables interrupts racelessly */
111 else
112 local_irq_enable();
113 current_thread_info()->status |= TS_POLLING;
114}
115
116#ifdef CONFIG_HOTPLUG_CPU 88#ifdef CONFIG_HOTPLUG_CPU
117DECLARE_PER_CPU(int, cpu_state); 89DECLARE_PER_CPU(int, cpu_state);
118 90
@@ -150,12 +122,9 @@ void cpu_idle(void)
150 while (1) { 122 while (1) {
151 tick_nohz_stop_sched_tick(); 123 tick_nohz_stop_sched_tick();
152 while (!need_resched()) { 124 while (!need_resched()) {
153 void (*idle)(void);
154 125
155 rmb(); 126 rmb();
156 idle = pm_idle; 127
157 if (!idle)
158 idle = default_idle;
159 if (cpu_is_offline(smp_processor_id())) 128 if (cpu_is_offline(smp_processor_id()))
160 play_dead(); 129 play_dead();
161 /* 130 /*
@@ -165,7 +134,10 @@ void cpu_idle(void)
165 */ 134 */
166 local_irq_disable(); 135 local_irq_disable();
167 enter_idle(); 136 enter_idle();
168 idle(); 137 /* Don't trace irqs off for idle */
138 stop_critical_timings();
139 pm_idle();
140 start_critical_timings();
169 /* In many cases the interrupt that ended idle 141 /* In many cases the interrupt that ended idle
170 has already called exit_idle. But some idle 142 has already called exit_idle. But some idle
171 loops can be woken up without interrupt. */ 143 loops can be woken up without interrupt. */
@@ -366,10 +338,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
366 p->thread.fs = me->thread.fs; 338 p->thread.fs = me->thread.fs;
367 p->thread.gs = me->thread.gs; 339 p->thread.gs = me->thread.gs;
368 340
369 asm("mov %%gs,%0" : "=m" (p->thread.gsindex)); 341 savesegment(gs, p->thread.gsindex);
370 asm("mov %%fs,%0" : "=m" (p->thread.fsindex)); 342 savesegment(fs, p->thread.fsindex);
371 asm("mov %%es,%0" : "=m" (p->thread.es)); 343 savesegment(es, p->thread.es);
372 asm("mov %%ds,%0" : "=m" (p->thread.ds)); 344 savesegment(ds, p->thread.ds);
373 345
374 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) { 346 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
375 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL); 347 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
@@ -408,7 +380,9 @@ out:
408void 380void
409start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) 381start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
410{ 382{
411 asm volatile("movl %0, %%fs; movl %0, %%es; movl %0, %%ds" :: "r"(0)); 383 loadsegment(fs, 0);
384 loadsegment(es, 0);
385 loadsegment(ds, 0);
412 load_gs_index(0); 386 load_gs_index(0);
413 regs->ip = new_ip; 387 regs->ip = new_ip;
414 regs->sp = new_sp; 388 regs->sp = new_sp;
@@ -567,6 +541,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
567 *next = &next_p->thread; 541 *next = &next_p->thread;
568 int cpu = smp_processor_id(); 542 int cpu = smp_processor_id();
569 struct tss_struct *tss = &per_cpu(init_tss, cpu); 543 struct tss_struct *tss = &per_cpu(init_tss, cpu);
544 unsigned fsindex, gsindex;
570 545
571 /* we're going to use this soon, after a few expensive things */ 546 /* we're going to use this soon, after a few expensive things */
572 if (next_p->fpu_counter>5) 547 if (next_p->fpu_counter>5)
@@ -581,22 +556,38 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
581 * Switch DS and ES. 556 * Switch DS and ES.
582 * This won't pick up thread selector changes, but I guess that is ok. 557 * This won't pick up thread selector changes, but I guess that is ok.
583 */ 558 */
584 asm volatile("mov %%es,%0" : "=m" (prev->es)); 559 savesegment(es, prev->es);
585 if (unlikely(next->es | prev->es)) 560 if (unlikely(next->es | prev->es))
586 loadsegment(es, next->es); 561 loadsegment(es, next->es);
587 562
588 asm volatile ("mov %%ds,%0" : "=m" (prev->ds)); 563 savesegment(ds, prev->ds);
589 if (unlikely(next->ds | prev->ds)) 564 if (unlikely(next->ds | prev->ds))
590 loadsegment(ds, next->ds); 565 loadsegment(ds, next->ds);
591 566
567
568 /* We must save %fs and %gs before load_TLS() because
569 * %fs and %gs may be cleared by load_TLS().
570 *
571 * (e.g. xen_load_tls())
572 */
573 savesegment(fs, fsindex);
574 savesegment(gs, gsindex);
575
592 load_TLS(next, cpu); 576 load_TLS(next, cpu);
593 577
578 /*
579 * Leave lazy mode, flushing any hypercalls made here.
580 * This must be done before restoring TLS segments so
581 * the GDT and LDT are properly updated, and must be
582 * done before math_state_restore, so the TS bit is up
583 * to date.
584 */
585 arch_leave_lazy_cpu_mode();
586
594 /* 587 /*
595 * Switch FS and GS. 588 * Switch FS and GS.
596 */ 589 */
597 { 590 {
598 unsigned fsindex;
599 asm volatile("movl %%fs,%0" : "=r" (fsindex));
600 /* segment register != 0 always requires a reload. 591 /* segment register != 0 always requires a reload.
601 also reload when it has changed. 592 also reload when it has changed.
602 when prev process used 64bit base always reload 593 when prev process used 64bit base always reload
@@ -614,10 +605,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
614 if (next->fs) 605 if (next->fs)
615 wrmsrl(MSR_FS_BASE, next->fs); 606 wrmsrl(MSR_FS_BASE, next->fs);
616 prev->fsindex = fsindex; 607 prev->fsindex = fsindex;
617 } 608
618 {
619 unsigned gsindex;
620 asm volatile("movl %%gs,%0" : "=r" (gsindex));
621 if (unlikely(gsindex | next->gsindex | prev->gs)) { 609 if (unlikely(gsindex | next->gsindex | prev->gs)) {
622 load_gs_index(next->gsindex); 610 load_gs_index(next->gsindex);
623 if (gsindex) 611 if (gsindex)
@@ -798,7 +786,7 @@ long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
798 set_32bit_tls(task, FS_TLS, addr); 786 set_32bit_tls(task, FS_TLS, addr);
799 if (doit) { 787 if (doit) {
800 load_TLS(&task->thread, cpu); 788 load_TLS(&task->thread, cpu);
801 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL)); 789 loadsegment(fs, FS_TLS_SEL);
802 } 790 }
803 task->thread.fsindex = FS_TLS_SEL; 791 task->thread.fsindex = FS_TLS_SEL;
804 task->thread.fs = 0; 792 task->thread.fs = 0;
@@ -808,7 +796,7 @@ long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
808 if (doit) { 796 if (doit) {
809 /* set the selector to 0 to not confuse 797 /* set the selector to 0 to not confuse
810 __switch_to */ 798 __switch_to */
811 asm volatile("movl %0,%%fs" :: "r" (0)); 799 loadsegment(fs, 0);
812 ret = checking_wrmsrl(MSR_FS_BASE, addr); 800 ret = checking_wrmsrl(MSR_FS_BASE, addr);
813 } 801 }
814 } 802 }
@@ -831,7 +819,7 @@ long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
831 if (task->thread.gsindex == GS_TLS_SEL) 819 if (task->thread.gsindex == GS_TLS_SEL)
832 base = read_32bit_tls(task, GS_TLS); 820 base = read_32bit_tls(task, GS_TLS);
833 else if (doit) { 821 else if (doit) {
834 asm("movl %%gs,%0" : "=r" (gsindex)); 822 savesegment(gs, gsindex);
835 if (gsindex) 823 if (gsindex)
836 rdmsrl(MSR_KERNEL_GS_BASE, base); 824 rdmsrl(MSR_KERNEL_GS_BASE, base);
837 else 825 else
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index a7835f282936..77040b6070e1 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -943,13 +943,13 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data)
943 return copy_regset_to_user(child, &user_x86_32_view, 943 return copy_regset_to_user(child, &user_x86_32_view,
944 REGSET_XFP, 944 REGSET_XFP,
945 0, sizeof(struct user_fxsr_struct), 945 0, sizeof(struct user_fxsr_struct),
946 datap); 946 datap) ? -EIO : 0;
947 947
948 case PTRACE_SETFPXREGS: /* Set the child extended FPU state. */ 948 case PTRACE_SETFPXREGS: /* Set the child extended FPU state. */
949 return copy_regset_from_user(child, &user_x86_32_view, 949 return copy_regset_from_user(child, &user_x86_32_view,
950 REGSET_XFP, 950 REGSET_XFP,
951 0, sizeof(struct user_fxsr_struct), 951 0, sizeof(struct user_fxsr_struct),
952 datap); 952 datap) ? -EIO : 0;
953#endif 953#endif
954 954
955#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION 955#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index d89a648fe710..d13858818100 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -65,6 +65,7 @@ static enum {
65 ICH_FORCE_HPET_RESUME, 65 ICH_FORCE_HPET_RESUME,
66 VT8237_FORCE_HPET_RESUME, 66 VT8237_FORCE_HPET_RESUME,
67 NVIDIA_FORCE_HPET_RESUME, 67 NVIDIA_FORCE_HPET_RESUME,
68 ATI_FORCE_HPET_RESUME,
68} force_hpet_resume_type; 69} force_hpet_resume_type;
69 70
70static void __iomem *rcba_base; 71static void __iomem *rcba_base;
@@ -158,6 +159,8 @@ static void ich_force_enable_hpet(struct pci_dev *dev)
158 159
159DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ESB2_0, 160DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ESB2_0,
160 ich_force_enable_hpet); 161 ich_force_enable_hpet);
162DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH6_0,
163 ich_force_enable_hpet);
161DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH6_1, 164DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH6_1,
162 ich_force_enable_hpet); 165 ich_force_enable_hpet);
163DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH7_0, 166DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH7_0,
@@ -174,6 +177,12 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH9_7,
174 177
175static struct pci_dev *cached_dev; 178static struct pci_dev *cached_dev;
176 179
180static void hpet_print_force_info(void)
181{
182 printk(KERN_INFO "HPET not enabled in BIOS. "
183 "You might try hpet=force boot option\n");
184}
185
177static void old_ich_force_hpet_resume(void) 186static void old_ich_force_hpet_resume(void)
178{ 187{
179 u32 val; 188 u32 val;
@@ -253,8 +262,12 @@ static void old_ich_force_enable_hpet_user(struct pci_dev *dev)
253{ 262{
254 if (hpet_force_user) 263 if (hpet_force_user)
255 old_ich_force_enable_hpet(dev); 264 old_ich_force_enable_hpet(dev);
265 else
266 hpet_print_force_info();
256} 267}
257 268
269DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ESB_1,
270 old_ich_force_enable_hpet_user);
258DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_0, 271DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_0,
259 old_ich_force_enable_hpet_user); 272 old_ich_force_enable_hpet_user);
260DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_12, 273DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_12,
@@ -290,9 +303,14 @@ static void vt8237_force_enable_hpet(struct pci_dev *dev)
290{ 303{
291 u32 uninitialized_var(val); 304 u32 uninitialized_var(val);
292 305
293 if (!hpet_force_user || hpet_address || force_hpet_address) 306 if (hpet_address || force_hpet_address)
294 return; 307 return;
295 308
309 if (!hpet_force_user) {
310 hpet_print_force_info();
311 return;
312 }
313
296 pci_read_config_dword(dev, 0x68, &val); 314 pci_read_config_dword(dev, 0x68, &val);
297 /* 315 /*
298 * Bit 7 is HPET enable bit. 316 * Bit 7 is HPET enable bit.
@@ -330,6 +348,36 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8235,
330DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8237, 348DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8237,
331 vt8237_force_enable_hpet); 349 vt8237_force_enable_hpet);
332 350
351static void ati_force_hpet_resume(void)
352{
353 pci_write_config_dword(cached_dev, 0x14, 0xfed00000);
354 printk(KERN_DEBUG "Force enabled HPET at resume\n");
355}
356
357static void ati_force_enable_hpet(struct pci_dev *dev)
358{
359 u32 uninitialized_var(val);
360
361 if (hpet_address || force_hpet_address)
362 return;
363
364 if (!hpet_force_user) {
365 hpet_print_force_info();
366 return;
367 }
368
369 pci_write_config_dword(dev, 0x14, 0xfed00000);
370 pci_read_config_dword(dev, 0x14, &val);
371 force_hpet_address = val;
372 force_hpet_resume_type = ATI_FORCE_HPET_RESUME;
373 dev_printk(KERN_DEBUG, &dev->dev, "Force enabled HPET at 0x%lx\n",
374 force_hpet_address);
375 cached_dev = dev;
376 return;
377}
378DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_IXP400_SMBUS,
379 ati_force_enable_hpet);
380
333/* 381/*
334 * Undocumented chipset feature taken from LinuxBIOS. 382 * Undocumented chipset feature taken from LinuxBIOS.
335 */ 383 */
@@ -343,8 +391,13 @@ static void nvidia_force_enable_hpet(struct pci_dev *dev)
343{ 391{
344 u32 uninitialized_var(val); 392 u32 uninitialized_var(val);
345 393
346 if (!hpet_force_user || hpet_address || force_hpet_address) 394 if (hpet_address || force_hpet_address)
395 return;
396
397 if (!hpet_force_user) {
398 hpet_print_force_info();
347 return; 399 return;
400 }
348 401
349 pci_write_config_dword(dev, 0x44, 0xfed00001); 402 pci_write_config_dword(dev, 0x44, 0xfed00001);
350 pci_read_config_dword(dev, 0x44, &val); 403 pci_read_config_dword(dev, 0x44, &val);
@@ -397,6 +450,9 @@ void force_hpet_resume(void)
397 case NVIDIA_FORCE_HPET_RESUME: 450 case NVIDIA_FORCE_HPET_RESUME:
398 nvidia_force_hpet_resume(); 451 nvidia_force_hpet_resume();
399 return; 452 return;
453 case ATI_FORCE_HPET_RESUME:
454 ati_force_hpet_resume();
455 return;
400 default: 456 default:
401 break; 457 break;
402 } 458 }
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index f6be7d5f82f8..f8a62160e151 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -27,7 +27,7 @@
27void (*pm_power_off)(void); 27void (*pm_power_off)(void);
28EXPORT_SYMBOL(pm_power_off); 28EXPORT_SYMBOL(pm_power_off);
29 29
30static long no_idt[3]; 30static const struct desc_ptr no_idt = {};
31static int reboot_mode; 31static int reboot_mode;
32enum reboot_type reboot_type = BOOT_KBD; 32enum reboot_type reboot_type = BOOT_KBD;
33int reboot_force; 33int reboot_force;
@@ -201,15 +201,15 @@ core_initcall(reboot_init);
201 controller to pulse the CPU reset line, which is more thorough, but 201 controller to pulse the CPU reset line, which is more thorough, but
202 doesn't work with at least one type of 486 motherboard. It is easy 202 doesn't work with at least one type of 486 motherboard. It is easy
203 to stop this code working; hence the copious comments. */ 203 to stop this code working; hence the copious comments. */
204static unsigned long long 204static const unsigned long long
205real_mode_gdt_entries [3] = 205real_mode_gdt_entries [3] =
206{ 206{
207 0x0000000000000000ULL, /* Null descriptor */ 207 0x0000000000000000ULL, /* Null descriptor */
208 0x00009a000000ffffULL, /* 16-bit real-mode 64k code at 0x00000000 */ 208 0x00009b000000ffffULL, /* 16-bit real-mode 64k code at 0x00000000 */
209 0x000092000100ffffULL /* 16-bit real-mode 64k data at 0x00000100 */ 209 0x000093000100ffffULL /* 16-bit real-mode 64k data at 0x00000100 */
210}; 210};
211 211
212static struct desc_ptr 212static const struct desc_ptr
213real_mode_gdt = { sizeof (real_mode_gdt_entries) - 1, (long)real_mode_gdt_entries }, 213real_mode_gdt = { sizeof (real_mode_gdt_entries) - 1, (long)real_mode_gdt_entries },
214real_mode_idt = { 0x3ff, 0 }; 214real_mode_idt = { 0x3ff, 0 };
215 215
@@ -231,7 +231,7 @@ real_mode_idt = { 0x3ff, 0 };
231 231
232 More could be done here to set up the registers as if a CPU reset had 232 More could be done here to set up the registers as if a CPU reset had
233 occurred; hopefully real BIOSs don't assume much. */ 233 occurred; hopefully real BIOSs don't assume much. */
234static unsigned char real_mode_switch [] = 234static const unsigned char real_mode_switch [] =
235{ 235{
236 0x66, 0x0f, 0x20, 0xc0, /* movl %cr0,%eax */ 236 0x66, 0x0f, 0x20, 0xc0, /* movl %cr0,%eax */
237 0x66, 0x83, 0xe0, 0x11, /* andl $0x00000011,%eax */ 237 0x66, 0x83, 0xe0, 0x11, /* andl $0x00000011,%eax */
@@ -245,7 +245,7 @@ static unsigned char real_mode_switch [] =
245 0x24, 0x10, /* f: andb $0x10,al */ 245 0x24, 0x10, /* f: andb $0x10,al */
246 0x66, 0x0f, 0x22, 0xc0 /* movl %eax,%cr0 */ 246 0x66, 0x0f, 0x22, 0xc0 /* movl %eax,%cr0 */
247}; 247};
248static unsigned char jump_to_bios [] = 248static const unsigned char jump_to_bios [] =
249{ 249{
250 0xea, 0x00, 0x00, 0xff, 0xff /* ljmp $0xffff,$0x0000 */ 250 0xea, 0x00, 0x00, 0xff, 0xff /* ljmp $0xffff,$0x0000 */
251}; 251};
@@ -255,7 +255,7 @@ static unsigned char jump_to_bios [] =
255 * specified by the code and length parameters. 255 * specified by the code and length parameters.
256 * We assume that length will aways be less that 100! 256 * We assume that length will aways be less that 100!
257 */ 257 */
258void machine_real_restart(unsigned char *code, int length) 258void machine_real_restart(const unsigned char *code, int length)
259{ 259{
260 local_irq_disable(); 260 local_irq_disable();
261 261
@@ -368,7 +368,7 @@ static void native_machine_emergency_restart(void)
368 } 368 }
369 369
370 case BOOT_TRIPLE: 370 case BOOT_TRIPLE:
371 load_idt((const struct desc_ptr *)&no_idt); 371 load_idt(&no_idt);
372 __asm__ __volatile__("int3"); 372 __asm__ __volatile__("int3");
373 373
374 reboot_type = BOOT_KBD; 374 reboot_type = BOOT_KBD;
diff --git a/arch/x86/kernel/reboot_fixups_32.c b/arch/x86/kernel/reboot_fixups_32.c
index dec0b5ec25c2..61a837743fe5 100644
--- a/arch/x86/kernel/reboot_fixups_32.c
+++ b/arch/x86/kernel/reboot_fixups_32.c
@@ -49,7 +49,7 @@ struct device_fixup {
49 void (*reboot_fixup)(struct pci_dev *); 49 void (*reboot_fixup)(struct pci_dev *);
50}; 50};
51 51
52static struct device_fixup fixups_table[] = { 52static const struct device_fixup fixups_table[] = {
53{ PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5530_LEGACY, cs5530a_warm_reset }, 53{ PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5530_LEGACY, cs5530a_warm_reset },
54{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CS5536_ISA, cs5536_warm_reset }, 54{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CS5536_ISA, cs5536_warm_reset },
55{ PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SC1100_BRIDGE, cs5530a_warm_reset }, 55{ PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SC1100_BRIDGE, cs5530a_warm_reset },
@@ -64,7 +64,7 @@ static struct device_fixup fixups_table[] = {
64 */ 64 */
65void mach_reboot_fixups(void) 65void mach_reboot_fixups(void)
66{ 66{
67 struct device_fixup *cur; 67 const struct device_fixup *cur;
68 struct pci_dev *dev; 68 struct pci_dev *dev;
69 int i; 69 int i;
70 70
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 6f80b852a196..531b55b8e81a 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1,139 +1,894 @@
1#include <linux/kernel.h> 1/*
2 * Copyright (C) 1995 Linus Torvalds
3 *
4 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
5 *
6 * Memory region support
7 * David Parsons <orc@pell.chi.il.us>, July-August 1999
8 *
9 * Added E820 sanitization routine (removes overlapping memory regions);
10 * Brian Moyle <bmoyle@mvista.com>, February 2001
11 *
12 * Moved CPU detection code to cpu/${cpu}.c
13 * Patrick Mochel <mochel@osdl.org>, March 2002
14 *
15 * Provisions for empty E820 memory regions (reported by certain BIOSes).
16 * Alex Achenbach <xela@slit.de>, December 2002.
17 *
18 */
19
20/*
21 * This file handles the architecture-dependent parts of initialization
22 */
23
24#include <linux/sched.h>
25#include <linux/mm.h>
26#include <linux/mmzone.h>
27#include <linux/screen_info.h>
28#include <linux/ioport.h>
29#include <linux/acpi.h>
30#include <linux/apm_bios.h>
31#include <linux/initrd.h>
32#include <linux/bootmem.h>
33#include <linux/seq_file.h>
34#include <linux/console.h>
35#include <linux/mca.h>
36#include <linux/root_dev.h>
37#include <linux/highmem.h>
2#include <linux/module.h> 38#include <linux/module.h>
39#include <linux/efi.h>
3#include <linux/init.h> 40#include <linux/init.h>
4#include <linux/bootmem.h> 41#include <linux/edd.h>
42#include <linux/iscsi_ibft.h>
43#include <linux/nodemask.h>
44#include <linux/kexec.h>
45#include <linux/dmi.h>
46#include <linux/pfn.h>
47#include <linux/pci.h>
48#include <asm/pci-direct.h>
49#include <linux/init_ohci1394_dma.h>
50#include <linux/kvm_para.h>
51
52#include <linux/errno.h>
53#include <linux/kernel.h>
54#include <linux/stddef.h>
55#include <linux/unistd.h>
56#include <linux/ptrace.h>
57#include <linux/slab.h>
58#include <linux/user.h>
59#include <linux/delay.h>
60#include <linux/highmem.h>
61
62#include <linux/kallsyms.h>
63#include <linux/edd.h>
64#include <linux/iscsi_ibft.h>
65#include <linux/kexec.h>
66#include <linux/cpufreq.h>
67#include <linux/dma-mapping.h>
68#include <linux/ctype.h>
69#include <linux/uaccess.h>
70
5#include <linux/percpu.h> 71#include <linux/percpu.h>
72#include <linux/crash_dump.h>
73
74#include <video/edid.h>
75
76#include <asm/mtrr.h>
77#include <asm/apic.h>
78#include <asm/e820.h>
79#include <asm/mpspec.h>
80#include <asm/setup.h>
81#include <asm/arch_hooks.h>
82#include <asm/efi.h>
83#include <asm/sections.h>
84#include <asm/dmi.h>
85#include <asm/io_apic.h>
86#include <asm/ist.h>
87#include <asm/vmi.h>
88#include <setup_arch.h>
89#include <asm/bios_ebda.h>
90#include <asm/cacheflush.h>
91#include <asm/processor.h>
92#include <asm/bugs.h>
93
94#include <asm/system.h>
95#include <asm/vsyscall.h>
6#include <asm/smp.h> 96#include <asm/smp.h>
97#include <asm/desc.h>
98#include <asm/dma.h>
99#include <asm/gart.h>
100#include <asm/mmu_context.h>
101#include <asm/proto.h>
102
103#include <mach_apic.h>
104#include <asm/paravirt.h>
105
7#include <asm/percpu.h> 106#include <asm/percpu.h>
8#include <asm/sections.h> 107#include <asm/sections.h>
9#include <asm/processor.h>
10#include <asm/setup.h>
11#include <asm/topology.h> 108#include <asm/topology.h>
12#include <asm/mpspec.h>
13#include <asm/apicdef.h> 109#include <asm/apicdef.h>
110#ifdef CONFIG_X86_64
111#include <asm/numa_64.h>
112#endif
14 113
15#ifdef CONFIG_X86_LOCAL_APIC 114#ifndef ARCH_SETUP
16unsigned int num_processors; 115#define ARCH_SETUP
17unsigned disabled_cpus __cpuinitdata; 116#endif
18/* Processor that is doing the boot up */ 117
19unsigned int boot_cpu_physical_apicid = -1U; 118#ifndef CONFIG_DEBUG_BOOT_PARAMS
20EXPORT_SYMBOL(boot_cpu_physical_apicid); 119struct boot_params __initdata boot_params;
120#else
121struct boot_params boot_params;
122#endif
123
124/*
125 * Machine setup..
126 */
127static struct resource data_resource = {
128 .name = "Kernel data",
129 .start = 0,
130 .end = 0,
131 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
132};
133
134static struct resource code_resource = {
135 .name = "Kernel code",
136 .start = 0,
137 .end = 0,
138 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
139};
140
141static struct resource bss_resource = {
142 .name = "Kernel bss",
143 .start = 0,
144 .end = 0,
145 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
146};
147
148
149#ifdef CONFIG_X86_32
150/* This value is set up by the early boot code to point to the value
151 immediately after the boot time page tables. It contains a *physical*
152 address, and must not be in the .bss segment! */
153unsigned long init_pg_tables_start __initdata = ~0UL;
154unsigned long init_pg_tables_end __initdata = ~0UL;
155
156static struct resource video_ram_resource = {
157 .name = "Video RAM area",
158 .start = 0xa0000,
159 .end = 0xbffff,
160 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
161};
162
163/* cpu data as detected by the assembly code in head.S */
164struct cpuinfo_x86 new_cpu_data __cpuinitdata = {0, 0, 0, 0, -1, 1, 0, 0, -1};
165/* common cpu data for all cpus */
166struct cpuinfo_x86 boot_cpu_data __read_mostly = {0, 0, 0, 0, -1, 1, 0, 0, -1};
167EXPORT_SYMBOL(boot_cpu_data);
168static void set_mca_bus(int x)
169{
170#ifdef CONFIG_MCA
171 MCA_bus = x;
172#endif
173}
174
175unsigned int def_to_bigsmp;
176
177/* for MCA, but anyone else can use it if they want */
178unsigned int machine_id;
179unsigned int machine_submodel_id;
180unsigned int BIOS_revision;
181
182struct apm_info apm_info;
183EXPORT_SYMBOL(apm_info);
184
185#if defined(CONFIG_X86_SPEEDSTEP_SMI) || \
186 defined(CONFIG_X86_SPEEDSTEP_SMI_MODULE)
187struct ist_info ist_info;
188EXPORT_SYMBOL(ist_info);
189#else
190struct ist_info ist_info;
191#endif
192
193#else
194struct cpuinfo_x86 boot_cpu_data __read_mostly;
195EXPORT_SYMBOL(boot_cpu_data);
196#endif
21 197
22DEFINE_PER_CPU(u16, x86_cpu_to_apicid) = BAD_APICID;
23EXPORT_PER_CPU_SYMBOL(x86_cpu_to_apicid);
24 198
25/* Bitmask of physically existing CPUs */ 199#if !defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
26physid_mask_t phys_cpu_present_map; 200unsigned long mmu_cr4_features;
201#else
202unsigned long mmu_cr4_features = X86_CR4_PAE;
27#endif 203#endif
28 204
29#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_X86_SMP) 205/* Boot loader ID as an integer, for the benefit of proc_dointvec */
206int bootloader_type;
207
30/* 208/*
31 * Copy data used in early init routines from the initial arrays to the 209 * Early DMI memory
32 * per cpu data areas. These arrays then become expendable and the 210 */
33 * *_early_ptr's are zeroed indicating that the static arrays are gone. 211int dmi_alloc_index;
212char dmi_alloc_data[DMI_MAX_DATA];
213
214/*
215 * Setup options
216 */
217struct screen_info screen_info;
218EXPORT_SYMBOL(screen_info);
219struct edid_info edid_info;
220EXPORT_SYMBOL_GPL(edid_info);
221
222extern int root_mountflags;
223
224unsigned long saved_video_mode;
225
226#define RAMDISK_IMAGE_START_MASK 0x07FF
227#define RAMDISK_PROMPT_FLAG 0x8000
228#define RAMDISK_LOAD_FLAG 0x4000
229
230static char __initdata command_line[COMMAND_LINE_SIZE];
231
232#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE)
233struct edd edd;
234#ifdef CONFIG_EDD_MODULE
235EXPORT_SYMBOL(edd);
236#endif
237/**
238 * copy_edd() - Copy the BIOS EDD information
239 * from boot_params into a safe place.
240 *
34 */ 241 */
35static void __init setup_per_cpu_maps(void) 242static inline void copy_edd(void)
243{
244 memcpy(edd.mbr_signature, boot_params.edd_mbr_sig_buffer,
245 sizeof(edd.mbr_signature));
246 memcpy(edd.edd_info, boot_params.eddbuf, sizeof(edd.edd_info));
247 edd.mbr_signature_nr = boot_params.edd_mbr_sig_buf_entries;
248 edd.edd_info_nr = boot_params.eddbuf_entries;
249}
250#else
251static inline void copy_edd(void)
252{
253}
254#endif
255
256#ifdef CONFIG_BLK_DEV_INITRD
257
258#ifdef CONFIG_X86_32
259
260#define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT)
261static void __init relocate_initrd(void)
36{ 262{
37 int cpu;
38 263
39 for_each_possible_cpu(cpu) { 264 u64 ramdisk_image = boot_params.hdr.ramdisk_image;
40 per_cpu(x86_cpu_to_apicid, cpu) = x86_cpu_to_apicid_init[cpu]; 265 u64 ramdisk_size = boot_params.hdr.ramdisk_size;
41 per_cpu(x86_bios_cpu_apicid, cpu) = 266 u64 end_of_lowmem = max_low_pfn << PAGE_SHIFT;
42 x86_bios_cpu_apicid_init[cpu]; 267 u64 ramdisk_here;
43#ifdef CONFIG_NUMA 268 unsigned long slop, clen, mapaddr;
44 per_cpu(x86_cpu_to_node_map, cpu) = 269 char *p, *q;
45 x86_cpu_to_node_map_init[cpu]; 270
271 /* We need to move the initrd down into lowmem */
272 ramdisk_here = find_e820_area(0, end_of_lowmem, ramdisk_size,
273 PAGE_SIZE);
274
275 if (ramdisk_here == -1ULL)
276 panic("Cannot find place for new RAMDISK of size %lld\n",
277 ramdisk_size);
278
279 /* Note: this includes all the lowmem currently occupied by
280 the initrd, we rely on that fact to keep the data intact. */
281 reserve_early(ramdisk_here, ramdisk_here + ramdisk_size,
282 "NEW RAMDISK");
283 initrd_start = ramdisk_here + PAGE_OFFSET;
284 initrd_end = initrd_start + ramdisk_size;
285 printk(KERN_INFO "Allocated new RAMDISK: %08llx - %08llx\n",
286 ramdisk_here, ramdisk_here + ramdisk_size);
287
288 q = (char *)initrd_start;
289
290 /* Copy any lowmem portion of the initrd */
291 if (ramdisk_image < end_of_lowmem) {
292 clen = end_of_lowmem - ramdisk_image;
293 p = (char *)__va(ramdisk_image);
294 memcpy(q, p, clen);
295 q += clen;
296 ramdisk_image += clen;
297 ramdisk_size -= clen;
298 }
299
300 /* Copy the highmem portion of the initrd */
301 while (ramdisk_size) {
302 slop = ramdisk_image & ~PAGE_MASK;
303 clen = ramdisk_size;
304 if (clen > MAX_MAP_CHUNK-slop)
305 clen = MAX_MAP_CHUNK-slop;
306 mapaddr = ramdisk_image & PAGE_MASK;
307 p = early_ioremap(mapaddr, clen+slop);
308 memcpy(q, p+slop, clen);
309 early_iounmap(p, clen+slop);
310 q += clen;
311 ramdisk_image += clen;
312 ramdisk_size -= clen;
313 }
314 /* high pages is not converted by early_res_to_bootmem */
315 ramdisk_image = boot_params.hdr.ramdisk_image;
316 ramdisk_size = boot_params.hdr.ramdisk_size;
317 printk(KERN_INFO "Move RAMDISK from %016llx - %016llx to"
318 " %08llx - %08llx\n",
319 ramdisk_image, ramdisk_image + ramdisk_size - 1,
320 ramdisk_here, ramdisk_here + ramdisk_size - 1);
321}
46#endif 322#endif
323
324static void __init reserve_initrd(void)
325{
326 u64 ramdisk_image = boot_params.hdr.ramdisk_image;
327 u64 ramdisk_size = boot_params.hdr.ramdisk_size;
328 u64 ramdisk_end = ramdisk_image + ramdisk_size;
329 u64 end_of_lowmem = max_low_pfn << PAGE_SHIFT;
330
331 if (!boot_params.hdr.type_of_loader ||
332 !ramdisk_image || !ramdisk_size)
333 return; /* No initrd provided by bootloader */
334
335 initrd_start = 0;
336
337 if (ramdisk_size >= (end_of_lowmem>>1)) {
338 free_early(ramdisk_image, ramdisk_end);
339 printk(KERN_ERR "initrd too large to handle, "
340 "disabling initrd\n");
341 return;
342 }
343
344 printk(KERN_INFO "RAMDISK: %08llx - %08llx\n", ramdisk_image,
345 ramdisk_end);
346
347
348 if (ramdisk_end <= end_of_lowmem) {
349 /* All in lowmem, easy case */
350 /*
351 * don't need to reserve again, already reserved early
352 * in i386_start_kernel
353 */
354 initrd_start = ramdisk_image + PAGE_OFFSET;
355 initrd_end = initrd_start + ramdisk_size;
356 return;
47 } 357 }
48 358
49 /* indicate the early static arrays will soon be gone */ 359#ifdef CONFIG_X86_32
50 x86_cpu_to_apicid_early_ptr = NULL; 360 relocate_initrd();
51 x86_bios_cpu_apicid_early_ptr = NULL; 361#else
52#ifdef CONFIG_NUMA 362 printk(KERN_ERR "initrd extends beyond end of memory "
53 x86_cpu_to_node_map_early_ptr = NULL; 363 "(0x%08llx > 0x%08llx)\ndisabling initrd\n",
364 ramdisk_end, end_of_lowmem);
365 initrd_start = 0;
54#endif 366#endif
367 free_early(ramdisk_image, ramdisk_end);
55} 368}
369#else
370static void __init reserve_initrd(void)
371{
372}
373#endif /* CONFIG_BLK_DEV_INITRD */
374
375static void __init parse_setup_data(void)
376{
377 struct setup_data *data;
378 u64 pa_data;
379
380 if (boot_params.hdr.version < 0x0209)
381 return;
382 pa_data = boot_params.hdr.setup_data;
383 while (pa_data) {
384 data = early_ioremap(pa_data, PAGE_SIZE);
385 switch (data->type) {
386 case SETUP_E820_EXT:
387 parse_e820_ext(data, pa_data);
388 break;
389 default:
390 break;
391 }
392 pa_data = data->next;
393 early_iounmap(data, PAGE_SIZE);
394 }
395}
396
397static void __init e820_reserve_setup_data(void)
398{
399 struct setup_data *data;
400 u64 pa_data;
401 int found = 0;
402
403 if (boot_params.hdr.version < 0x0209)
404 return;
405 pa_data = boot_params.hdr.setup_data;
406 while (pa_data) {
407 data = early_ioremap(pa_data, sizeof(*data));
408 e820_update_range(pa_data, sizeof(*data)+data->len,
409 E820_RAM, E820_RESERVED_KERN);
410 found = 1;
411 pa_data = data->next;
412 early_iounmap(data, sizeof(*data));
413 }
414 if (!found)
415 return;
56 416
57#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP 417 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
58cpumask_t *cpumask_of_cpu_map __read_mostly; 418 memcpy(&e820_saved, &e820, sizeof(struct e820map));
59EXPORT_SYMBOL(cpumask_of_cpu_map); 419 printk(KERN_INFO "extended physical RAM map:\n");
420 e820_print_map("reserve setup_data");
421}
60 422
61/* requires nr_cpu_ids to be initialized */ 423static void __init reserve_early_setup_data(void)
62static void __init setup_cpumask_of_cpu(void)
63{ 424{
64 int i; 425 struct setup_data *data;
426 u64 pa_data;
427 char buf[32];
428
429 if (boot_params.hdr.version < 0x0209)
430 return;
431 pa_data = boot_params.hdr.setup_data;
432 while (pa_data) {
433 data = early_ioremap(pa_data, sizeof(*data));
434 sprintf(buf, "setup data %x", data->type);
435 reserve_early(pa_data, pa_data+sizeof(*data)+data->len, buf);
436 pa_data = data->next;
437 early_iounmap(data, sizeof(*data));
438 }
439}
440
441/*
442 * --------- Crashkernel reservation ------------------------------
443 */
444
445#ifdef CONFIG_KEXEC
446
447/**
448 * Reserve @size bytes of crashkernel memory at any suitable offset.
449 *
450 * @size: Size of the crashkernel memory to reserve.
451 * Returns the base address on success, and -1ULL on failure.
452 */
453unsigned long long find_and_reserve_crashkernel(unsigned long long size)
454{
455 const unsigned long long alignment = 16<<20; /* 16M */
456 unsigned long long start = 0LL;
457
458 while (1) {
459 int ret;
460
461 start = find_e820_area(start, ULONG_MAX, size, alignment);
462 if (start == -1ULL)
463 return start;
464
465 /* try to reserve it */
466 ret = reserve_bootmem_generic(start, size, BOOTMEM_EXCLUSIVE);
467 if (ret >= 0)
468 return start;
469
470 start += alignment;
471 }
472}
473
474static inline unsigned long long get_total_mem(void)
475{
476 unsigned long long total;
477
478 total = max_low_pfn - min_low_pfn;
479#ifdef CONFIG_HIGHMEM
480 total += highend_pfn - highstart_pfn;
481#endif
482
483 return total << PAGE_SHIFT;
484}
485
486static void __init reserve_crashkernel(void)
487{
488 unsigned long long total_mem;
489 unsigned long long crash_size, crash_base;
490 int ret;
491
492 total_mem = get_total_mem();
65 493
66 /* alloc_bootmem zeroes memory */ 494 ret = parse_crashkernel(boot_command_line, total_mem,
67 cpumask_of_cpu_map = alloc_bootmem_low(sizeof(cpumask_t) * nr_cpu_ids); 495 &crash_size, &crash_base);
68 for (i = 0; i < nr_cpu_ids; i++) 496 if (ret != 0 || crash_size <= 0)
69 cpu_set(i, cpumask_of_cpu_map[i]); 497 return;
498
499 /* 0 means: find the address automatically */
500 if (crash_base <= 0) {
501 crash_base = find_and_reserve_crashkernel(crash_size);
502 if (crash_base == -1ULL) {
503 pr_info("crashkernel reservation failed. "
504 "No suitable area found.\n");
505 return;
506 }
507 } else {
508 ret = reserve_bootmem_generic(crash_base, crash_size,
509 BOOTMEM_EXCLUSIVE);
510 if (ret < 0) {
511 pr_info("crashkernel reservation failed - "
512 "memory is in use\n");
513 return;
514 }
515 }
516
517 printk(KERN_INFO "Reserving %ldMB of memory at %ldMB "
518 "for crashkernel (System RAM: %ldMB)\n",
519 (unsigned long)(crash_size >> 20),
520 (unsigned long)(crash_base >> 20),
521 (unsigned long)(total_mem >> 20));
522
523 crashk_res.start = crash_base;
524 crashk_res.end = crash_base + crash_size - 1;
525 insert_resource(&iomem_resource, &crashk_res);
70} 526}
71#else 527#else
72static inline void setup_cpumask_of_cpu(void) { } 528static void __init reserve_crashkernel(void)
529{
530}
73#endif 531#endif
74 532
75#ifdef CONFIG_X86_32 533static struct resource standard_io_resources[] = {
76/* 534 { .name = "dma1", .start = 0x00, .end = 0x1f,
77 * Great future not-so-futuristic plan: make i386 and x86_64 do it 535 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
78 * the same way 536 { .name = "pic1", .start = 0x20, .end = 0x21,
537 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
538 { .name = "timer0", .start = 0x40, .end = 0x43,
539 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
540 { .name = "timer1", .start = 0x50, .end = 0x53,
541 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
542 { .name = "keyboard", .start = 0x60, .end = 0x60,
543 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
544 { .name = "keyboard", .start = 0x64, .end = 0x64,
545 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
546 { .name = "dma page reg", .start = 0x80, .end = 0x8f,
547 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
548 { .name = "pic2", .start = 0xa0, .end = 0xa1,
549 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
550 { .name = "dma2", .start = 0xc0, .end = 0xdf,
551 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
552 { .name = "fpu", .start = 0xf0, .end = 0xff,
553 .flags = IORESOURCE_BUSY | IORESOURCE_IO }
554};
555
556static void __init reserve_standard_io_resources(void)
557{
558 int i;
559
560 /* request I/O space for devices used on all i[345]86 PCs */
561 for (i = 0; i < ARRAY_SIZE(standard_io_resources); i++)
562 request_resource(&ioport_resource, &standard_io_resources[i]);
563
564}
565
566#ifdef CONFIG_PROC_VMCORE
567/* elfcorehdr= specifies the location of elf core header
568 * stored by the crashed kernel. This option will be passed
569 * by kexec loader to the capture kernel.
79 */ 570 */
80unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 571static int __init setup_elfcorehdr(char *arg)
81EXPORT_SYMBOL(__per_cpu_offset); 572{
573 char *end;
574 if (!arg)
575 return -EINVAL;
576 elfcorehdr_addr = memparse(arg, &end);
577 return end > arg ? 0 : -EINVAL;
578}
579early_param("elfcorehdr", setup_elfcorehdr);
82#endif 580#endif
83 581
84/* 582/*
85 * Great future plan: 583 * Determine if we were loaded by an EFI loader. If so, then we have also been
86 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data. 584 * passed the efi memmap, systab, etc., so we should use these data structures
87 * Always point %gs to its beginning 585 * for initialization. Note, the efi init code path is determined by the
586 * global efi_enabled. This allows the same kernel image to be used on existing
587 * systems (with a traditional BIOS) as well as on EFI systems.
588 */
589/*
590 * setup_arch - architecture-specific boot-time initializations
591 *
592 * Note: On x86_64, fixmaps are ready for use even before this is called.
88 */ 593 */
89void __init setup_per_cpu_areas(void) 594
595void __init setup_arch(char **cmdline_p)
90{ 596{
91 int i, highest_cpu = 0; 597#ifdef CONFIG_X86_32
92 unsigned long size; 598 memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
599 visws_early_detect();
600 pre_setup_arch_hook();
601 early_cpu_init();
602#else
603 printk(KERN_INFO "Command line: %s\n", boot_command_line);
604#endif
93 605
94#ifdef CONFIG_HOTPLUG_CPU 606 early_ioremap_init();
95 prefill_possible_map(); 607
608 ROOT_DEV = old_decode_dev(boot_params.hdr.root_dev);
609 screen_info = boot_params.screen_info;
610 edid_info = boot_params.edid_info;
611#ifdef CONFIG_X86_32
612 apm_info.bios = boot_params.apm_bios_info;
613 ist_info = boot_params.ist_info;
614 if (boot_params.sys_desc_table.length != 0) {
615 set_mca_bus(boot_params.sys_desc_table.table[3] & 0x2);
616 machine_id = boot_params.sys_desc_table.table[0];
617 machine_submodel_id = boot_params.sys_desc_table.table[1];
618 BIOS_revision = boot_params.sys_desc_table.table[2];
619 }
620#endif
621 saved_video_mode = boot_params.hdr.vid_mode;
622 bootloader_type = boot_params.hdr.type_of_loader;
623
624#ifdef CONFIG_BLK_DEV_RAM
625 rd_image_start = boot_params.hdr.ram_size & RAMDISK_IMAGE_START_MASK;
626 rd_prompt = ((boot_params.hdr.ram_size & RAMDISK_PROMPT_FLAG) != 0);
627 rd_doload = ((boot_params.hdr.ram_size & RAMDISK_LOAD_FLAG) != 0);
628#endif
629#ifdef CONFIG_EFI
630 if (!strncmp((char *)&boot_params.efi_info.efi_loader_signature,
631#ifdef CONFIG_X86_32
632 "EL32",
633#else
634 "EL64",
635#endif
636 4)) {
637 efi_enabled = 1;
638 efi_reserve_early();
639 }
96#endif 640#endif
97 641
98 /* Copy section for each CPU (we discard the original) */ 642 ARCH_SETUP
99 size = PERCPU_ENOUGH_ROOM;
100 printk(KERN_INFO "PERCPU: Allocating %lu bytes of per cpu data\n",
101 size);
102 643
103 for_each_possible_cpu(i) { 644 setup_memory_map();
104 char *ptr; 645 parse_setup_data();
105#ifndef CONFIG_NEED_MULTIPLE_NODES 646 /* update the e820_saved too */
106 ptr = alloc_bootmem_pages(size); 647 e820_reserve_setup_data();
648
649 copy_edd();
650
651 if (!boot_params.hdr.root_flags)
652 root_mountflags &= ~MS_RDONLY;
653 init_mm.start_code = (unsigned long) _text;
654 init_mm.end_code = (unsigned long) _etext;
655 init_mm.end_data = (unsigned long) _edata;
656#ifdef CONFIG_X86_32
657 init_mm.brk = init_pg_tables_end + PAGE_OFFSET;
107#else 658#else
108 int node = early_cpu_to_node(i); 659 init_mm.brk = (unsigned long) &_end;
109 if (!node_online(node) || !NODE_DATA(node)) {
110 ptr = alloc_bootmem_pages(size);
111 printk(KERN_INFO
112 "cpu %d has no node or node-local memory\n", i);
113 }
114 else
115 ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
116#endif 660#endif
117 if (!ptr) 661
118 panic("Cannot allocate cpu data for CPU %d\n", i); 662 code_resource.start = virt_to_phys(_text);
663 code_resource.end = virt_to_phys(_etext)-1;
664 data_resource.start = virt_to_phys(_etext);
665 data_resource.end = virt_to_phys(_edata)-1;
666 bss_resource.start = virt_to_phys(&__bss_start);
667 bss_resource.end = virt_to_phys(&__bss_stop)-1;
668
119#ifdef CONFIG_X86_64 669#ifdef CONFIG_X86_64
120 cpu_pda(i)->data_offset = ptr - __per_cpu_start; 670 early_cpu_init();
671#endif
672 strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
673 *cmdline_p = command_line;
674
675 parse_early_param();
676
677 /* after early param, so could get panic from serial */
678 reserve_early_setup_data();
679
680 if (acpi_mps_check()) {
681#ifdef CONFIG_X86_LOCAL_APIC
682 disable_apic = 1;
683#endif
684 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
685 }
686
687#ifdef CONFIG_PCI
688 if (pci_early_dump_regs)
689 early_dump_pci_devices();
690#endif
691
692 finish_e820_parsing();
693
694#ifdef CONFIG_X86_32
695 probe_roms();
696#endif
697
698 /* after parse_early_param, so could debug it */
699 insert_resource(&iomem_resource, &code_resource);
700 insert_resource(&iomem_resource, &data_resource);
701 insert_resource(&iomem_resource, &bss_resource);
702
703 if (efi_enabled)
704 efi_init();
705
706#ifdef CONFIG_X86_32
707 if (ppro_with_ram_bug()) {
708 e820_update_range(0x70000000ULL, 0x40000ULL, E820_RAM,
709 E820_RESERVED);
710 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
711 printk(KERN_INFO "fixed physical RAM map:\n");
712 e820_print_map("bad_ppro");
713 }
714#else
715 early_gart_iommu_check();
716#endif
717
718 /*
719 * partially used pages are not usable - thus
720 * we are rounding upwards:
721 */
722 max_pfn = e820_end_of_ram_pfn();
723
724 /* preallocate 4k for mptable mpc */
725 early_reserve_e820_mpc_new();
726 /* update e820 for memory not covered by WB MTRRs */
727 mtrr_bp_init();
728 if (mtrr_trim_uncached_memory(max_pfn))
729 max_pfn = e820_end_of_ram_pfn();
730
731#ifdef CONFIG_X86_32
732 /* max_low_pfn get updated here */
733 find_low_pfn_range();
121#else 734#else
122 __per_cpu_offset[i] = ptr - __per_cpu_start; 735 num_physpages = max_pfn;
736
737 check_efer();
738
739 /* How many end-of-memory variables you have, grandma! */
740 /* need this before calling reserve_initrd */
741 if (max_pfn > (1UL<<(32 - PAGE_SHIFT)))
742 max_low_pfn = e820_end_of_low_ram_pfn();
743 else
744 max_low_pfn = max_pfn;
745
746 high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1;
123#endif 747#endif
124 memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
125 748
126 highest_cpu = i; 749 /* max_pfn_mapped is updated here */
750 max_low_pfn_mapped = init_memory_mapping(0, max_low_pfn<<PAGE_SHIFT);
751 max_pfn_mapped = max_low_pfn_mapped;
752
753#ifdef CONFIG_X86_64
754 if (max_pfn > max_low_pfn) {
755 max_pfn_mapped = init_memory_mapping(1UL<<32,
756 max_pfn<<PAGE_SHIFT);
757 /* can we preseve max_low_pfn ?*/
758 max_low_pfn = max_pfn;
127 } 759 }
760#endif
128 761
129 nr_cpu_ids = highest_cpu + 1; 762 /*
130 printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d\n", NR_CPUS, nr_cpu_ids); 763 * NOTE: On x86-32, only from this point on, fixmaps are ready for use.
764 */
131 765
132 /* Setup percpu data maps */ 766#ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT
133 setup_per_cpu_maps(); 767 if (init_ohci1394_dma_early)
768 init_ohci1394_dma_on_all_controllers();
769#endif
134 770
135 /* Setup cpumask_of_cpu map */ 771 reserve_initrd();
136 setup_cpumask_of_cpu(); 772
137} 773#ifdef CONFIG_X86_64
774 vsmp_init();
775#endif
776
777 dmi_scan_machine();
778
779 io_delay_init();
780
781 /*
782 * Parse the ACPI tables for possible boot-time SMP configuration.
783 */
784 acpi_boot_table_init();
785
786#ifdef CONFIG_ACPI_NUMA
787 /*
788 * Parse SRAT to discover nodes.
789 */
790 acpi_numa_init();
791#endif
792
793 initmem_init(0, max_pfn);
794
795#ifdef CONFIG_X86_64
796 dma32_reserve_bootmem();
797#endif
798
799#ifdef CONFIG_ACPI_SLEEP
800 /*
801 * Reserve low memory region for sleep support.
802 */
803 acpi_reserve_bootmem();
804#endif
805#ifdef CONFIG_X86_FIND_SMP_CONFIG
806 /*
807 * Find and reserve possible boot-time SMP configuration:
808 */
809 find_smp_config();
810#endif
811 reserve_crashkernel();
812
813 reserve_ibft_region();
814
815#ifdef CONFIG_KVM_CLOCK
816 kvmclock_init();
817#endif
818
819#if defined(CONFIG_VMI) && defined(CONFIG_X86_32)
820 /*
821 * Must be after max_low_pfn is determined, and before kernel
822 * pagetables are setup.
823 */
824 vmi_init();
825#endif
826
827 paging_init();
828
829#ifdef CONFIG_X86_64
830 map_vsyscall();
831#endif
832
833#ifdef CONFIG_X86_GENERICARCH
834 generic_apic_probe();
835#endif
836
837 early_quirks();
838
839 /*
840 * Read APIC and some other early information from ACPI tables.
841 */
842 acpi_boot_init();
843
844#if defined(CONFIG_X86_MPPARSE) || defined(CONFIG_X86_VISWS)
845 /*
846 * get boot-time SMP configuration:
847 */
848 if (smp_found_config)
849 get_smp_config();
850#endif
138 851
852 prefill_possible_map();
853#ifdef CONFIG_X86_64
854 init_cpu_to_node();
855#endif
856
857#ifdef CONFIG_X86_NUMAQ
858 /*
859 * need to check online nodes num, call it
860 * here before time_init/tsc_init
861 */
862 numaq_tsc_disable();
863#endif
864
865 init_apic_mappings();
866 ioapic_init_mappings();
867
868#if defined(CONFIG_SMP) && defined(CONFIG_X86_PC) && defined(CONFIG_X86_32)
869 if (def_to_bigsmp)
870 printk(KERN_WARNING "More than 8 CPUs detected and "
871 "CONFIG_X86_PC cannot handle it.\nUse "
872 "CONFIG_X86_GENERICARCH or CONFIG_X86_BIGSMP.\n");
873#endif
874 kvm_guest_init();
875
876 e820_reserve_resources();
877 e820_mark_nosave_regions(max_low_pfn);
878
879#ifdef CONFIG_X86_32
880 request_resource(&iomem_resource, &video_ram_resource);
881#endif
882 reserve_standard_io_resources();
883
884 e820_setup_gap();
885
886#ifdef CONFIG_VT
887#if defined(CONFIG_VGA_CONSOLE)
888 if (!efi_enabled || (efi_mem_type(0xa0000) != EFI_CONVENTIONAL_MEMORY))
889 conswitchp = &vga_con;
890#elif defined(CONFIG_DUMMY_CONSOLE)
891 conswitchp = &dummy_con;
139#endif 892#endif
893#endif
894}
diff --git a/arch/x86/kernel/setup64.c b/arch/x86/kernel/setup64.c
deleted file mode 100644
index aee0e8200777..000000000000
--- a/arch/x86/kernel/setup64.c
+++ /dev/null
@@ -1,287 +0,0 @@
1/*
2 * X86-64 specific CPU setup.
3 * Copyright (C) 1995 Linus Torvalds
4 * Copyright 2001, 2002, 2003 SuSE Labs / Andi Kleen.
5 * See setup.c for older changelog.
6 */
7#include <linux/init.h>
8#include <linux/kernel.h>
9#include <linux/sched.h>
10#include <linux/string.h>
11#include <linux/bootmem.h>
12#include <linux/bitops.h>
13#include <linux/module.h>
14#include <linux/kgdb.h>
15#include <asm/pda.h>
16#include <asm/pgtable.h>
17#include <asm/processor.h>
18#include <asm/desc.h>
19#include <asm/atomic.h>
20#include <asm/mmu_context.h>
21#include <asm/smp.h>
22#include <asm/i387.h>
23#include <asm/percpu.h>
24#include <asm/proto.h>
25#include <asm/sections.h>
26#include <asm/setup.h>
27#include <asm/genapic.h>
28
29#ifndef CONFIG_DEBUG_BOOT_PARAMS
30struct boot_params __initdata boot_params;
31#else
32struct boot_params boot_params;
33#endif
34
35cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE;
36
37struct x8664_pda *_cpu_pda[NR_CPUS] __read_mostly;
38EXPORT_SYMBOL(_cpu_pda);
39struct x8664_pda boot_cpu_pda[NR_CPUS] __cacheline_aligned;
40
41struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
42
43char boot_cpu_stack[IRQSTACKSIZE] __attribute__((section(".bss.page_aligned")));
44
45unsigned long __supported_pte_mask __read_mostly = ~0UL;
46EXPORT_SYMBOL_GPL(__supported_pte_mask);
47
48static int do_not_nx __cpuinitdata = 0;
49
50/* noexec=on|off
51Control non executable mappings for 64bit processes.
52
53on Enable(default)
54off Disable
55*/
56static int __init nonx_setup(char *str)
57{
58 if (!str)
59 return -EINVAL;
60 if (!strncmp(str, "on", 2)) {
61 __supported_pte_mask |= _PAGE_NX;
62 do_not_nx = 0;
63 } else if (!strncmp(str, "off", 3)) {
64 do_not_nx = 1;
65 __supported_pte_mask &= ~_PAGE_NX;
66 }
67 return 0;
68}
69early_param("noexec", nonx_setup);
70
71int force_personality32 = 0;
72
73/* noexec32=on|off
74Control non executable heap for 32bit processes.
75To control the stack too use noexec=off
76
77on PROT_READ does not imply PROT_EXEC for 32bit processes (default)
78off PROT_READ implies PROT_EXEC
79*/
80static int __init nonx32_setup(char *str)
81{
82 if (!strcmp(str, "on"))
83 force_personality32 &= ~READ_IMPLIES_EXEC;
84 else if (!strcmp(str, "off"))
85 force_personality32 |= READ_IMPLIES_EXEC;
86 return 1;
87}
88__setup("noexec32=", nonx32_setup);
89
90void pda_init(int cpu)
91{
92 struct x8664_pda *pda = cpu_pda(cpu);
93
94 /* Setup up data that may be needed in __get_free_pages early */
95 asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0));
96 /* Memory clobbers used to order PDA accessed */
97 mb();
98 wrmsrl(MSR_GS_BASE, pda);
99 mb();
100
101 pda->cpunumber = cpu;
102 pda->irqcount = -1;
103 pda->kernelstack =
104 (unsigned long)stack_thread_info() - PDA_STACKOFFSET + THREAD_SIZE;
105 pda->active_mm = &init_mm;
106 pda->mmu_state = 0;
107
108 if (cpu == 0) {
109 /* others are initialized in smpboot.c */
110 pda->pcurrent = &init_task;
111 pda->irqstackptr = boot_cpu_stack;
112 } else {
113 pda->irqstackptr = (char *)
114 __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
115 if (!pda->irqstackptr)
116 panic("cannot allocate irqstack for cpu %d", cpu);
117 }
118
119
120 pda->irqstackptr += IRQSTACKSIZE-64;
121}
122
123char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]
124__attribute__((section(".bss.page_aligned")));
125
126extern asmlinkage void ignore_sysret(void);
127
128/* May not be marked __init: used by software suspend */
129void syscall_init(void)
130{
131 /*
132 * LSTAR and STAR live in a bit strange symbiosis.
133 * They both write to the same internal register. STAR allows to set CS/DS
134 * but only a 32bit target. LSTAR sets the 64bit rip.
135 */
136 wrmsrl(MSR_STAR, ((u64)__USER32_CS)<<48 | ((u64)__KERNEL_CS)<<32);
137 wrmsrl(MSR_LSTAR, system_call);
138 wrmsrl(MSR_CSTAR, ignore_sysret);
139
140#ifdef CONFIG_IA32_EMULATION
141 syscall32_cpu_init ();
142#endif
143
144 /* Flags to clear on syscall */
145 wrmsrl(MSR_SYSCALL_MASK,
146 X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL);
147}
148
149void __cpuinit check_efer(void)
150{
151 unsigned long efer;
152
153 rdmsrl(MSR_EFER, efer);
154 if (!(efer & EFER_NX) || do_not_nx) {
155 __supported_pte_mask &= ~_PAGE_NX;
156 }
157}
158
159unsigned long kernel_eflags;
160
161/*
162 * Copies of the original ist values from the tss are only accessed during
163 * debugging, no special alignment required.
164 */
165DEFINE_PER_CPU(struct orig_ist, orig_ist);
166
167/*
168 * cpu_init() initializes state that is per-CPU. Some data is already
169 * initialized (naturally) in the bootstrap process, such as the GDT
170 * and IDT. We reload them nevertheless, this function acts as a
171 * 'CPU state barrier', nothing should get across.
172 * A lot of state is already set up in PDA init.
173 */
174void __cpuinit cpu_init (void)
175{
176 int cpu = stack_smp_processor_id();
177 struct tss_struct *t = &per_cpu(init_tss, cpu);
178 struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu);
179 unsigned long v;
180 char *estacks = NULL;
181 struct task_struct *me;
182 int i;
183
184 /* CPU 0 is initialised in head64.c */
185 if (cpu != 0) {
186 pda_init(cpu);
187 } else
188 estacks = boot_exception_stacks;
189
190 me = current;
191
192 if (cpu_test_and_set(cpu, cpu_initialized))
193 panic("CPU#%d already initialized!\n", cpu);
194
195 printk("Initializing CPU#%d\n", cpu);
196
197 clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
198
199 /*
200 * Initialize the per-CPU GDT with the boot GDT,
201 * and set up the GDT descriptor:
202 */
203 if (cpu)
204 memcpy(get_cpu_gdt_table(cpu), cpu_gdt_table, GDT_SIZE);
205
206 cpu_gdt_descr[cpu].size = GDT_SIZE;
207 load_gdt((const struct desc_ptr *)&cpu_gdt_descr[cpu]);
208 load_idt((const struct desc_ptr *)&idt_descr);
209
210 memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
211 syscall_init();
212
213 wrmsrl(MSR_FS_BASE, 0);
214 wrmsrl(MSR_KERNEL_GS_BASE, 0);
215 barrier();
216
217 check_efer();
218
219 /*
220 * set up and load the per-CPU TSS
221 */
222 for (v = 0; v < N_EXCEPTION_STACKS; v++) {
223 static const unsigned int order[N_EXCEPTION_STACKS] = {
224 [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
225 [DEBUG_STACK - 1] = DEBUG_STACK_ORDER
226 };
227 if (cpu) {
228 estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
229 if (!estacks)
230 panic("Cannot allocate exception stack %ld %d\n",
231 v, cpu);
232 }
233 estacks += PAGE_SIZE << order[v];
234 orig_ist->ist[v] = t->x86_tss.ist[v] = (unsigned long)estacks;
235 }
236
237 t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
238 /*
239 * <= is required because the CPU will access up to
240 * 8 bits beyond the end of the IO permission bitmap.
241 */
242 for (i = 0; i <= IO_BITMAP_LONGS; i++)
243 t->io_bitmap[i] = ~0UL;
244
245 atomic_inc(&init_mm.mm_count);
246 me->active_mm = &init_mm;
247 if (me->mm)
248 BUG();
249 enter_lazy_tlb(&init_mm, me);
250
251 set_tss_desc(cpu, t);
252 load_TR_desc();
253 load_LDT(&init_mm.context);
254
255#ifdef CONFIG_KGDB
256 /*
257 * If the kgdb is connected no debug regs should be altered. This
258 * is only applicable when KGDB and a KGDB I/O module are built
259 * into the kernel and you are using early debugging with
260 * kgdbwait. KGDB will control the kernel HW breakpoint registers.
261 */
262 if (kgdb_connected && arch_kgdb_ops.correct_hw_break)
263 arch_kgdb_ops.correct_hw_break();
264 else {
265#endif
266 /*
267 * Clear all 6 debug registers:
268 */
269
270 set_debugreg(0UL, 0);
271 set_debugreg(0UL, 1);
272 set_debugreg(0UL, 2);
273 set_debugreg(0UL, 3);
274 set_debugreg(0UL, 6);
275 set_debugreg(0UL, 7);
276#ifdef CONFIG_KGDB
277 /* If the kgdb is connected no debug regs should be altered. */
278 }
279#endif
280
281 fpu_init();
282
283 raw_local_save_flags(kernel_eflags);
284
285 if (is_uv_system())
286 uv_cpu_init();
287}
diff --git a/arch/x86/kernel/setup_32.c b/arch/x86/kernel/setup_32.c
deleted file mode 100644
index 5a2f8e063887..000000000000
--- a/arch/x86/kernel/setup_32.c
+++ /dev/null
@@ -1,964 +0,0 @@
1/*
2 * Copyright (C) 1995 Linus Torvalds
3 *
4 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
5 *
6 * Memory region support
7 * David Parsons <orc@pell.chi.il.us>, July-August 1999
8 *
9 * Added E820 sanitization routine (removes overlapping memory regions);
10 * Brian Moyle <bmoyle@mvista.com>, February 2001
11 *
12 * Moved CPU detection code to cpu/${cpu}.c
13 * Patrick Mochel <mochel@osdl.org>, March 2002
14 *
15 * Provisions for empty E820 memory regions (reported by certain BIOSes).
16 * Alex Achenbach <xela@slit.de>, December 2002.
17 *
18 */
19
20/*
21 * This file handles the architecture-dependent parts of initialization
22 */
23
24#include <linux/sched.h>
25#include <linux/mm.h>
26#include <linux/mmzone.h>
27#include <linux/screen_info.h>
28#include <linux/ioport.h>
29#include <linux/acpi.h>
30#include <linux/apm_bios.h>
31#include <linux/initrd.h>
32#include <linux/bootmem.h>
33#include <linux/seq_file.h>
34#include <linux/console.h>
35#include <linux/mca.h>
36#include <linux/root_dev.h>
37#include <linux/highmem.h>
38#include <linux/module.h>
39#include <linux/efi.h>
40#include <linux/init.h>
41#include <linux/edd.h>
42#include <linux/iscsi_ibft.h>
43#include <linux/nodemask.h>
44#include <linux/kexec.h>
45#include <linux/crash_dump.h>
46#include <linux/dmi.h>
47#include <linux/pfn.h>
48#include <linux/pci.h>
49#include <linux/init_ohci1394_dma.h>
50#include <linux/kvm_para.h>
51
52#include <video/edid.h>
53
54#include <asm/mtrr.h>
55#include <asm/apic.h>
56#include <asm/e820.h>
57#include <asm/mpspec.h>
58#include <asm/mmzone.h>
59#include <asm/setup.h>
60#include <asm/arch_hooks.h>
61#include <asm/sections.h>
62#include <asm/io_apic.h>
63#include <asm/ist.h>
64#include <asm/io.h>
65#include <asm/vmi.h>
66#include <setup_arch.h>
67#include <asm/bios_ebda.h>
68#include <asm/cacheflush.h>
69#include <asm/processor.h>
70
71/* This value is set up by the early boot code to point to the value
72 immediately after the boot time page tables. It contains a *physical*
73 address, and must not be in the .bss segment! */
74unsigned long init_pg_tables_end __initdata = ~0UL;
75
76/*
77 * Machine setup..
78 */
79static struct resource data_resource = {
80 .name = "Kernel data",
81 .start = 0,
82 .end = 0,
83 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
84};
85
86static struct resource code_resource = {
87 .name = "Kernel code",
88 .start = 0,
89 .end = 0,
90 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
91};
92
93static struct resource bss_resource = {
94 .name = "Kernel bss",
95 .start = 0,
96 .end = 0,
97 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
98};
99
100static struct resource video_ram_resource = {
101 .name = "Video RAM area",
102 .start = 0xa0000,
103 .end = 0xbffff,
104 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
105};
106
107static struct resource standard_io_resources[] = { {
108 .name = "dma1",
109 .start = 0x0000,
110 .end = 0x001f,
111 .flags = IORESOURCE_BUSY | IORESOURCE_IO
112}, {
113 .name = "pic1",
114 .start = 0x0020,
115 .end = 0x0021,
116 .flags = IORESOURCE_BUSY | IORESOURCE_IO
117}, {
118 .name = "timer0",
119 .start = 0x0040,
120 .end = 0x0043,
121 .flags = IORESOURCE_BUSY | IORESOURCE_IO
122}, {
123 .name = "timer1",
124 .start = 0x0050,
125 .end = 0x0053,
126 .flags = IORESOURCE_BUSY | IORESOURCE_IO
127}, {
128 .name = "keyboard",
129 .start = 0x0060,
130 .end = 0x0060,
131 .flags = IORESOURCE_BUSY | IORESOURCE_IO
132}, {
133 .name = "keyboard",
134 .start = 0x0064,
135 .end = 0x0064,
136 .flags = IORESOURCE_BUSY | IORESOURCE_IO
137}, {
138 .name = "dma page reg",
139 .start = 0x0080,
140 .end = 0x008f,
141 .flags = IORESOURCE_BUSY | IORESOURCE_IO
142}, {
143 .name = "pic2",
144 .start = 0x00a0,
145 .end = 0x00a1,
146 .flags = IORESOURCE_BUSY | IORESOURCE_IO
147}, {
148 .name = "dma2",
149 .start = 0x00c0,
150 .end = 0x00df,
151 .flags = IORESOURCE_BUSY | IORESOURCE_IO
152}, {
153 .name = "fpu",
154 .start = 0x00f0,
155 .end = 0x00ff,
156 .flags = IORESOURCE_BUSY | IORESOURCE_IO
157} };
158
159/* cpu data as detected by the assembly code in head.S */
160struct cpuinfo_x86 new_cpu_data __cpuinitdata = { 0, 0, 0, 0, -1, 1, 0, 0, -1 };
161/* common cpu data for all cpus */
162struct cpuinfo_x86 boot_cpu_data __read_mostly = { 0, 0, 0, 0, -1, 1, 0, 0, -1 };
163EXPORT_SYMBOL(boot_cpu_data);
164
165unsigned int def_to_bigsmp;
166
167#ifndef CONFIG_X86_PAE
168unsigned long mmu_cr4_features;
169#else
170unsigned long mmu_cr4_features = X86_CR4_PAE;
171#endif
172
173/* for MCA, but anyone else can use it if they want */
174unsigned int machine_id;
175unsigned int machine_submodel_id;
176unsigned int BIOS_revision;
177
178/* Boot loader ID as an integer, for the benefit of proc_dointvec */
179int bootloader_type;
180
181/* user-defined highmem size */
182static unsigned int highmem_pages = -1;
183
184/*
185 * Setup options
186 */
187struct screen_info screen_info;
188EXPORT_SYMBOL(screen_info);
189struct apm_info apm_info;
190EXPORT_SYMBOL(apm_info);
191struct edid_info edid_info;
192EXPORT_SYMBOL_GPL(edid_info);
193struct ist_info ist_info;
194#if defined(CONFIG_X86_SPEEDSTEP_SMI) || \
195 defined(CONFIG_X86_SPEEDSTEP_SMI_MODULE)
196EXPORT_SYMBOL(ist_info);
197#endif
198
199extern void early_cpu_init(void);
200extern int root_mountflags;
201
202unsigned long saved_video_mode;
203
204#define RAMDISK_IMAGE_START_MASK 0x07FF
205#define RAMDISK_PROMPT_FLAG 0x8000
206#define RAMDISK_LOAD_FLAG 0x4000
207
208static char __initdata command_line[COMMAND_LINE_SIZE];
209
210#ifndef CONFIG_DEBUG_BOOT_PARAMS
211struct boot_params __initdata boot_params;
212#else
213struct boot_params boot_params;
214#endif
215
216#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE)
217struct edd edd;
218#ifdef CONFIG_EDD_MODULE
219EXPORT_SYMBOL(edd);
220#endif
221/**
222 * copy_edd() - Copy the BIOS EDD information
223 * from boot_params into a safe place.
224 *
225 */
226static inline void copy_edd(void)
227{
228 memcpy(edd.mbr_signature, boot_params.edd_mbr_sig_buffer,
229 sizeof(edd.mbr_signature));
230 memcpy(edd.edd_info, boot_params.eddbuf, sizeof(edd.edd_info));
231 edd.mbr_signature_nr = boot_params.edd_mbr_sig_buf_entries;
232 edd.edd_info_nr = boot_params.eddbuf_entries;
233}
234#else
235static inline void copy_edd(void)
236{
237}
238#endif
239
240int __initdata user_defined_memmap;
241
242/*
243 * "mem=nopentium" disables the 4MB page tables.
244 * "mem=XXX[kKmM]" defines a memory region from HIGH_MEM
245 * to <mem>, overriding the bios size.
246 * "memmap=XXX[KkmM]@XXX[KkmM]" defines a memory region from
247 * <start> to <start>+<mem>, overriding the bios size.
248 *
249 * HPA tells me bootloaders need to parse mem=, so no new
250 * option should be mem= [also see Documentation/i386/boot.txt]
251 */
252static int __init parse_mem(char *arg)
253{
254 if (!arg)
255 return -EINVAL;
256
257 if (strcmp(arg, "nopentium") == 0) {
258 setup_clear_cpu_cap(X86_FEATURE_PSE);
259 } else {
260 /* If the user specifies memory size, we
261 * limit the BIOS-provided memory map to
262 * that size. exactmap can be used to specify
263 * the exact map. mem=number can be used to
264 * trim the existing memory map.
265 */
266 unsigned long long mem_size;
267
268 mem_size = memparse(arg, &arg);
269 limit_regions(mem_size);
270 user_defined_memmap = 1;
271 }
272 return 0;
273}
274early_param("mem", parse_mem);
275
276#ifdef CONFIG_PROC_VMCORE
277/* elfcorehdr= specifies the location of elf core header
278 * stored by the crashed kernel.
279 */
280static int __init parse_elfcorehdr(char *arg)
281{
282 if (!arg)
283 return -EINVAL;
284
285 elfcorehdr_addr = memparse(arg, &arg);
286 return 0;
287}
288early_param("elfcorehdr", parse_elfcorehdr);
289#endif /* CONFIG_PROC_VMCORE */
290
291/*
292 * highmem=size forces highmem to be exactly 'size' bytes.
293 * This works even on boxes that have no highmem otherwise.
294 * This also works to reduce highmem size on bigger boxes.
295 */
296static int __init parse_highmem(char *arg)
297{
298 if (!arg)
299 return -EINVAL;
300
301 highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT;
302 return 0;
303}
304early_param("highmem", parse_highmem);
305
306/*
307 * vmalloc=size forces the vmalloc area to be exactly 'size'
308 * bytes. This can be used to increase (or decrease) the
309 * vmalloc area - the default is 128m.
310 */
311static int __init parse_vmalloc(char *arg)
312{
313 if (!arg)
314 return -EINVAL;
315
316 __VMALLOC_RESERVE = memparse(arg, &arg);
317 return 0;
318}
319early_param("vmalloc", parse_vmalloc);
320
321/*
322 * reservetop=size reserves a hole at the top of the kernel address space which
323 * a hypervisor can load into later. Needed for dynamically loaded hypervisors,
324 * so relocating the fixmap can be done before paging initialization.
325 */
326static int __init parse_reservetop(char *arg)
327{
328 unsigned long address;
329
330 if (!arg)
331 return -EINVAL;
332
333 address = memparse(arg, &arg);
334 reserve_top_address(address);
335 return 0;
336}
337early_param("reservetop", parse_reservetop);
338
339/*
340 * Determine low and high memory ranges:
341 */
342unsigned long __init find_max_low_pfn(void)
343{
344 unsigned long max_low_pfn;
345
346 max_low_pfn = max_pfn;
347 if (max_low_pfn > MAXMEM_PFN) {
348 if (highmem_pages == -1)
349 highmem_pages = max_pfn - MAXMEM_PFN;
350 if (highmem_pages + MAXMEM_PFN < max_pfn)
351 max_pfn = MAXMEM_PFN + highmem_pages;
352 if (highmem_pages + MAXMEM_PFN > max_pfn) {
353 printk("only %luMB highmem pages available, ignoring highmem size of %uMB.\n", pages_to_mb(max_pfn - MAXMEM_PFN), pages_to_mb(highmem_pages));
354 highmem_pages = 0;
355 }
356 max_low_pfn = MAXMEM_PFN;
357#ifndef CONFIG_HIGHMEM
358 /* Maximum memory usable is what is directly addressable */
359 printk(KERN_WARNING "Warning only %ldMB will be used.\n",
360 MAXMEM>>20);
361 if (max_pfn > MAX_NONPAE_PFN)
362 printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n");
363 else
364 printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n");
365 max_pfn = MAXMEM_PFN;
366#else /* !CONFIG_HIGHMEM */
367#ifndef CONFIG_HIGHMEM64G
368 if (max_pfn > MAX_NONPAE_PFN) {
369 max_pfn = MAX_NONPAE_PFN;
370 printk(KERN_WARNING "Warning only 4GB will be used.\n");
371 printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n");
372 }
373#endif /* !CONFIG_HIGHMEM64G */
374#endif /* !CONFIG_HIGHMEM */
375 } else {
376 if (highmem_pages == -1)
377 highmem_pages = 0;
378#ifdef CONFIG_HIGHMEM
379 if (highmem_pages >= max_pfn) {
380 printk(KERN_ERR "highmem size specified (%uMB) is bigger than pages available (%luMB)!.\n", pages_to_mb(highmem_pages), pages_to_mb(max_pfn));
381 highmem_pages = 0;
382 }
383 if (highmem_pages) {
384 if (max_low_pfn-highmem_pages < 64*1024*1024/PAGE_SIZE){
385 printk(KERN_ERR "highmem size %uMB results in smaller than 64MB lowmem, ignoring it.\n", pages_to_mb(highmem_pages));
386 highmem_pages = 0;
387 }
388 max_low_pfn -= highmem_pages;
389 }
390#else
391 if (highmem_pages)
392 printk(KERN_ERR "ignoring highmem size on non-highmem kernel!\n");
393#endif
394 }
395 return max_low_pfn;
396}
397
398#define BIOS_LOWMEM_KILOBYTES 0x413
399
400/*
401 * The BIOS places the EBDA/XBDA at the top of conventional
402 * memory, and usually decreases the reported amount of
403 * conventional memory (int 0x12) too. This also contains a
404 * workaround for Dell systems that neglect to reserve EBDA.
405 * The same workaround also avoids a problem with the AMD768MPX
406 * chipset: reserve a page before VGA to prevent PCI prefetch
407 * into it (errata #56). Usually the page is reserved anyways,
408 * unless you have no PS/2 mouse plugged in.
409 */
410static void __init reserve_ebda_region(void)
411{
412 unsigned int lowmem, ebda_addr;
413
414 /* To determine the position of the EBDA and the */
415 /* end of conventional memory, we need to look at */
416 /* the BIOS data area. In a paravirtual environment */
417 /* that area is absent. We'll just have to assume */
418 /* that the paravirt case can handle memory setup */
419 /* correctly, without our help. */
420 if (paravirt_enabled())
421 return;
422
423 /* end of low (conventional) memory */
424 lowmem = *(unsigned short *)__va(BIOS_LOWMEM_KILOBYTES);
425 lowmem <<= 10;
426
427 /* start of EBDA area */
428 ebda_addr = get_bios_ebda();
429
430 /* Fixup: bios puts an EBDA in the top 64K segment */
431 /* of conventional memory, but does not adjust lowmem. */
432 if ((lowmem - ebda_addr) <= 0x10000)
433 lowmem = ebda_addr;
434
435 /* Fixup: bios does not report an EBDA at all. */
436 /* Some old Dells seem to need 4k anyhow (bugzilla 2990) */
437 if ((ebda_addr == 0) && (lowmem >= 0x9f000))
438 lowmem = 0x9f000;
439
440 /* Paranoia: should never happen, but... */
441 if ((lowmem == 0) || (lowmem >= 0x100000))
442 lowmem = 0x9f000;
443
444 /* reserve all memory between lowmem and the 1MB mark */
445 reserve_bootmem(lowmem, 0x100000 - lowmem, BOOTMEM_DEFAULT);
446}
447
448#ifndef CONFIG_NEED_MULTIPLE_NODES
449static void __init setup_bootmem_allocator(void);
450static unsigned long __init setup_memory(void)
451{
452 /*
453 * partially used pages are not usable - thus
454 * we are rounding upwards:
455 */
456 min_low_pfn = PFN_UP(init_pg_tables_end);
457
458 max_low_pfn = find_max_low_pfn();
459
460#ifdef CONFIG_HIGHMEM
461 highstart_pfn = highend_pfn = max_pfn;
462 if (max_pfn > max_low_pfn) {
463 highstart_pfn = max_low_pfn;
464 }
465 printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
466 pages_to_mb(highend_pfn - highstart_pfn));
467 num_physpages = highend_pfn;
468 high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
469#else
470 num_physpages = max_low_pfn;
471 high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
472#endif
473#ifdef CONFIG_FLATMEM
474 max_mapnr = num_physpages;
475#endif
476 printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
477 pages_to_mb(max_low_pfn));
478
479 setup_bootmem_allocator();
480
481 return max_low_pfn;
482}
483
484static void __init zone_sizes_init(void)
485{
486 unsigned long max_zone_pfns[MAX_NR_ZONES];
487 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
488 max_zone_pfns[ZONE_DMA] =
489 virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
490 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
491#ifdef CONFIG_HIGHMEM
492 max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
493 add_active_range(0, 0, highend_pfn);
494#else
495 add_active_range(0, 0, max_low_pfn);
496#endif
497
498 free_area_init_nodes(max_zone_pfns);
499}
500#else
501extern unsigned long __init setup_memory(void);
502extern void zone_sizes_init(void);
503#endif /* !CONFIG_NEED_MULTIPLE_NODES */
504
505static inline unsigned long long get_total_mem(void)
506{
507 unsigned long long total;
508
509 total = max_low_pfn - min_low_pfn;
510#ifdef CONFIG_HIGHMEM
511 total += highend_pfn - highstart_pfn;
512#endif
513
514 return total << PAGE_SHIFT;
515}
516
517#ifdef CONFIG_KEXEC
518static void __init reserve_crashkernel(void)
519{
520 unsigned long long total_mem;
521 unsigned long long crash_size, crash_base;
522 int ret;
523
524 total_mem = get_total_mem();
525
526 ret = parse_crashkernel(boot_command_line, total_mem,
527 &crash_size, &crash_base);
528 if (ret == 0 && crash_size > 0) {
529 if (crash_base > 0) {
530 printk(KERN_INFO "Reserving %ldMB of memory at %ldMB "
531 "for crashkernel (System RAM: %ldMB)\n",
532 (unsigned long)(crash_size >> 20),
533 (unsigned long)(crash_base >> 20),
534 (unsigned long)(total_mem >> 20));
535
536 if (reserve_bootmem(crash_base, crash_size,
537 BOOTMEM_EXCLUSIVE) < 0) {
538 printk(KERN_INFO "crashkernel reservation "
539 "failed - memory is in use\n");
540 return;
541 }
542
543 crashk_res.start = crash_base;
544 crashk_res.end = crash_base + crash_size - 1;
545 } else
546 printk(KERN_INFO "crashkernel reservation failed - "
547 "you have to specify a base address\n");
548 }
549}
550#else
551static inline void __init reserve_crashkernel(void)
552{}
553#endif
554
555#ifdef CONFIG_BLK_DEV_INITRD
556
557static bool do_relocate_initrd = false;
558
559static void __init reserve_initrd(void)
560{
561 unsigned long ramdisk_image = boot_params.hdr.ramdisk_image;
562 unsigned long ramdisk_size = boot_params.hdr.ramdisk_size;
563 unsigned long ramdisk_end = ramdisk_image + ramdisk_size;
564 unsigned long end_of_lowmem = max_low_pfn << PAGE_SHIFT;
565 unsigned long ramdisk_here;
566
567 initrd_start = 0;
568
569 if (!boot_params.hdr.type_of_loader ||
570 !ramdisk_image || !ramdisk_size)
571 return; /* No initrd provided by bootloader */
572
573 if (ramdisk_end < ramdisk_image) {
574 printk(KERN_ERR "initrd wraps around end of memory, "
575 "disabling initrd\n");
576 return;
577 }
578 if (ramdisk_size >= end_of_lowmem/2) {
579 printk(KERN_ERR "initrd too large to handle, "
580 "disabling initrd\n");
581 return;
582 }
583 if (ramdisk_end <= end_of_lowmem) {
584 /* All in lowmem, easy case */
585 reserve_bootmem(ramdisk_image, ramdisk_size, BOOTMEM_DEFAULT);
586 initrd_start = ramdisk_image + PAGE_OFFSET;
587 initrd_end = initrd_start+ramdisk_size;
588 return;
589 }
590
591 /* We need to move the initrd down into lowmem */
592 ramdisk_here = (end_of_lowmem - ramdisk_size) & PAGE_MASK;
593
594 /* Note: this includes all the lowmem currently occupied by
595 the initrd, we rely on that fact to keep the data intact. */
596 reserve_bootmem(ramdisk_here, ramdisk_size, BOOTMEM_DEFAULT);
597 initrd_start = ramdisk_here + PAGE_OFFSET;
598 initrd_end = initrd_start + ramdisk_size;
599
600 do_relocate_initrd = true;
601}
602
603#define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT)
604
605static void __init relocate_initrd(void)
606{
607 unsigned long ramdisk_image = boot_params.hdr.ramdisk_image;
608 unsigned long ramdisk_size = boot_params.hdr.ramdisk_size;
609 unsigned long end_of_lowmem = max_low_pfn << PAGE_SHIFT;
610 unsigned long ramdisk_here;
611 unsigned long slop, clen, mapaddr;
612 char *p, *q;
613
614 if (!do_relocate_initrd)
615 return;
616
617 ramdisk_here = initrd_start - PAGE_OFFSET;
618
619 q = (char *)initrd_start;
620
621 /* Copy any lowmem portion of the initrd */
622 if (ramdisk_image < end_of_lowmem) {
623 clen = end_of_lowmem - ramdisk_image;
624 p = (char *)__va(ramdisk_image);
625 memcpy(q, p, clen);
626 q += clen;
627 ramdisk_image += clen;
628 ramdisk_size -= clen;
629 }
630
631 /* Copy the highmem portion of the initrd */
632 while (ramdisk_size) {
633 slop = ramdisk_image & ~PAGE_MASK;
634 clen = ramdisk_size;
635 if (clen > MAX_MAP_CHUNK-slop)
636 clen = MAX_MAP_CHUNK-slop;
637 mapaddr = ramdisk_image & PAGE_MASK;
638 p = early_ioremap(mapaddr, clen+slop);
639 memcpy(q, p+slop, clen);
640 early_iounmap(p, clen+slop);
641 q += clen;
642 ramdisk_image += clen;
643 ramdisk_size -= clen;
644 }
645}
646
647#endif /* CONFIG_BLK_DEV_INITRD */
648
649void __init setup_bootmem_allocator(void)
650{
651 unsigned long bootmap_size;
652 /*
653 * Initialize the boot-time allocator (with low memory only):
654 */
655 bootmap_size = init_bootmem(min_low_pfn, max_low_pfn);
656
657 register_bootmem_low_pages(max_low_pfn);
658
659 /*
660 * Reserve the bootmem bitmap itself as well. We do this in two
661 * steps (first step was init_bootmem()) because this catches
662 * the (very unlikely) case of us accidentally initializing the
663 * bootmem allocator with an invalid RAM area.
664 */
665 reserve_bootmem(__pa_symbol(_text), (PFN_PHYS(min_low_pfn) +
666 bootmap_size + PAGE_SIZE-1) - __pa_symbol(_text),
667 BOOTMEM_DEFAULT);
668
669 /*
670 * reserve physical page 0 - it's a special BIOS page on many boxes,
671 * enabling clean reboots, SMP operation, laptop functions.
672 */
673 reserve_bootmem(0, PAGE_SIZE, BOOTMEM_DEFAULT);
674
675 /* reserve EBDA region */
676 reserve_ebda_region();
677
678#ifdef CONFIG_SMP
679 /*
680 * But first pinch a few for the stack/trampoline stuff
681 * FIXME: Don't need the extra page at 4K, but need to fix
682 * trampoline before removing it. (see the GDT stuff)
683 */
684 reserve_bootmem(PAGE_SIZE, PAGE_SIZE, BOOTMEM_DEFAULT);
685#endif
686#ifdef CONFIG_ACPI_SLEEP
687 /*
688 * Reserve low memory region for sleep support.
689 */
690 acpi_reserve_bootmem();
691#endif
692#ifdef CONFIG_X86_FIND_SMP_CONFIG
693 /*
694 * Find and reserve possible boot-time SMP configuration:
695 */
696 find_smp_config();
697#endif
698#ifdef CONFIG_BLK_DEV_INITRD
699 reserve_initrd();
700#endif
701 numa_kva_reserve();
702 reserve_crashkernel();
703
704 reserve_ibft_region();
705}
706
707/*
708 * The node 0 pgdat is initialized before all of these because
709 * it's needed for bootmem. node>0 pgdats have their virtual
710 * space allocated before the pagetables are in place to access
711 * them, so they can't be cleared then.
712 *
713 * This should all compile down to nothing when NUMA is off.
714 */
715static void __init remapped_pgdat_init(void)
716{
717 int nid;
718
719 for_each_online_node(nid) {
720 if (nid != 0)
721 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
722 }
723}
724
725#ifdef CONFIG_MCA
726static void set_mca_bus(int x)
727{
728 MCA_bus = x;
729}
730#else
731static void set_mca_bus(int x) { }
732#endif
733
734/* Overridden in paravirt.c if CONFIG_PARAVIRT */
735char * __init __attribute__((weak)) memory_setup(void)
736{
737 return machine_specific_memory_setup();
738}
739
740#ifdef CONFIG_NUMA
741/*
742 * In the golden day, when everything among i386 and x86_64 will be
743 * integrated, this will not live here
744 */
745void *x86_cpu_to_node_map_early_ptr;
746int x86_cpu_to_node_map_init[NR_CPUS] = {
747 [0 ... NR_CPUS-1] = NUMA_NO_NODE
748};
749DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE;
750#endif
751
752/*
753 * Determine if we were loaded by an EFI loader. If so, then we have also been
754 * passed the efi memmap, systab, etc., so we should use these data structures
755 * for initialization. Note, the efi init code path is determined by the
756 * global efi_enabled. This allows the same kernel image to be used on existing
757 * systems (with a traditional BIOS) as well as on EFI systems.
758 */
759void __init setup_arch(char **cmdline_p)
760{
761 unsigned long max_low_pfn;
762
763 memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
764 pre_setup_arch_hook();
765 early_cpu_init();
766 early_ioremap_init();
767
768#ifdef CONFIG_EFI
769 if (!strncmp((char *)&boot_params.efi_info.efi_loader_signature,
770 "EL32", 4))
771 efi_enabled = 1;
772#endif
773
774 ROOT_DEV = old_decode_dev(boot_params.hdr.root_dev);
775 screen_info = boot_params.screen_info;
776 edid_info = boot_params.edid_info;
777 apm_info.bios = boot_params.apm_bios_info;
778 ist_info = boot_params.ist_info;
779 saved_video_mode = boot_params.hdr.vid_mode;
780 if( boot_params.sys_desc_table.length != 0 ) {
781 set_mca_bus(boot_params.sys_desc_table.table[3] & 0x2);
782 machine_id = boot_params.sys_desc_table.table[0];
783 machine_submodel_id = boot_params.sys_desc_table.table[1];
784 BIOS_revision = boot_params.sys_desc_table.table[2];
785 }
786 bootloader_type = boot_params.hdr.type_of_loader;
787
788#ifdef CONFIG_BLK_DEV_RAM
789 rd_image_start = boot_params.hdr.ram_size & RAMDISK_IMAGE_START_MASK;
790 rd_prompt = ((boot_params.hdr.ram_size & RAMDISK_PROMPT_FLAG) != 0);
791 rd_doload = ((boot_params.hdr.ram_size & RAMDISK_LOAD_FLAG) != 0);
792#endif
793 ARCH_SETUP
794
795 printk(KERN_INFO "BIOS-provided physical RAM map:\n");
796 print_memory_map(memory_setup());
797
798 copy_edd();
799
800 if (!boot_params.hdr.root_flags)
801 root_mountflags &= ~MS_RDONLY;
802 init_mm.start_code = (unsigned long) _text;
803 init_mm.end_code = (unsigned long) _etext;
804 init_mm.end_data = (unsigned long) _edata;
805 init_mm.brk = init_pg_tables_end + PAGE_OFFSET;
806
807 code_resource.start = virt_to_phys(_text);
808 code_resource.end = virt_to_phys(_etext)-1;
809 data_resource.start = virt_to_phys(_etext);
810 data_resource.end = virt_to_phys(_edata)-1;
811 bss_resource.start = virt_to_phys(&__bss_start);
812 bss_resource.end = virt_to_phys(&__bss_stop)-1;
813
814 parse_early_param();
815
816 if (user_defined_memmap) {
817 printk(KERN_INFO "user-defined physical RAM map:\n");
818 print_memory_map("user");
819 }
820
821 strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
822 *cmdline_p = command_line;
823
824 if (efi_enabled)
825 efi_init();
826
827 /* update e820 for memory not covered by WB MTRRs */
828 propagate_e820_map();
829 mtrr_bp_init();
830 if (mtrr_trim_uncached_memory(max_pfn))
831 propagate_e820_map();
832
833 max_low_pfn = setup_memory();
834
835#ifdef CONFIG_KVM_CLOCK
836 kvmclock_init();
837#endif
838
839#ifdef CONFIG_VMI
840 /*
841 * Must be after max_low_pfn is determined, and before kernel
842 * pagetables are setup.
843 */
844 vmi_init();
845#endif
846 kvm_guest_init();
847
848 /*
849 * NOTE: before this point _nobody_ is allowed to allocate
850 * any memory using the bootmem allocator. Although the
851 * allocator is now initialised only the first 8Mb of the kernel
852 * virtual address space has been mapped. All allocations before
853 * paging_init() has completed must use the alloc_bootmem_low_pages()
854 * variant (which allocates DMA'able memory) and care must be taken
855 * not to exceed the 8Mb limit.
856 */
857
858#ifdef CONFIG_SMP
859 smp_alloc_memory(); /* AP processor realmode stacks in low memory*/
860#endif
861 paging_init();
862
863 /*
864 * NOTE: On x86-32, only from this point on, fixmaps are ready for use.
865 */
866
867#ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT
868 if (init_ohci1394_dma_early)
869 init_ohci1394_dma_on_all_controllers();
870#endif
871
872 remapped_pgdat_init();
873 sparse_init();
874 zone_sizes_init();
875
876 /*
877 * NOTE: at this point the bootmem allocator is fully available.
878 */
879
880#ifdef CONFIG_BLK_DEV_INITRD
881 relocate_initrd();
882#endif
883
884 paravirt_post_allocator_init();
885
886 dmi_scan_machine();
887
888 io_delay_init();
889
890#ifdef CONFIG_X86_SMP
891 /*
892 * setup to use the early static init tables during kernel startup
893 * X86_SMP will exclude sub-arches that don't deal well with it.
894 */
895 x86_cpu_to_apicid_early_ptr = (void *)x86_cpu_to_apicid_init;
896 x86_bios_cpu_apicid_early_ptr = (void *)x86_bios_cpu_apicid_init;
897#ifdef CONFIG_NUMA
898 x86_cpu_to_node_map_early_ptr = (void *)x86_cpu_to_node_map_init;
899#endif
900#endif
901
902#ifdef CONFIG_X86_GENERICARCH
903 generic_apic_probe();
904#endif
905
906#ifdef CONFIG_ACPI
907 /*
908 * Parse the ACPI tables for possible boot-time SMP configuration.
909 */
910 acpi_boot_table_init();
911#endif
912
913 early_quirks();
914
915#ifdef CONFIG_ACPI
916 acpi_boot_init();
917
918#if defined(CONFIG_SMP) && defined(CONFIG_X86_PC)
919 if (def_to_bigsmp)
920 printk(KERN_WARNING "More than 8 CPUs detected and "
921 "CONFIG_X86_PC cannot handle it.\nUse "
922 "CONFIG_X86_GENERICARCH or CONFIG_X86_BIGSMP.\n");
923#endif
924#endif
925#ifdef CONFIG_X86_LOCAL_APIC
926 if (smp_found_config)
927 get_smp_config();
928#endif
929
930 e820_register_memory();
931 e820_mark_nosave_regions();
932
933#ifdef CONFIG_VT
934#if defined(CONFIG_VGA_CONSOLE)
935 if (!efi_enabled || (efi_mem_type(0xa0000) != EFI_CONVENTIONAL_MEMORY))
936 conswitchp = &vga_con;
937#elif defined(CONFIG_DUMMY_CONSOLE)
938 conswitchp = &dummy_con;
939#endif
940#endif
941}
942
943/*
944 * Request address space for all standard resources
945 *
946 * This is called just before pcibios_init(), which is also a
947 * subsys_initcall, but is linked in later (in arch/i386/pci/common.c).
948 */
949static int __init request_standard_resources(void)
950{
951 int i;
952
953 printk(KERN_INFO "Setting up standard PCI resources\n");
954 init_iomem_resources(&code_resource, &data_resource, &bss_resource);
955
956 request_resource(&iomem_resource, &video_ram_resource);
957
958 /* request I/O space for devices used on all i[345]86 PCs */
959 for (i = 0; i < ARRAY_SIZE(standard_io_resources); i++)
960 request_resource(&ioport_resource, &standard_io_resources[i]);
961 return 0;
962}
963
964subsys_initcall(request_standard_resources);
diff --git a/arch/x86/kernel/setup_64.c b/arch/x86/kernel/setup_64.c
deleted file mode 100644
index 6dff1286ad8a..000000000000
--- a/arch/x86/kernel/setup_64.c
+++ /dev/null
@@ -1,1194 +0,0 @@
1/*
2 * Copyright (C) 1995 Linus Torvalds
3 */
4
5/*
6 * This file handles the architecture-dependent parts of initialization
7 */
8
9#include <linux/errno.h>
10#include <linux/sched.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/stddef.h>
14#include <linux/unistd.h>
15#include <linux/ptrace.h>
16#include <linux/slab.h>
17#include <linux/user.h>
18#include <linux/screen_info.h>
19#include <linux/ioport.h>
20#include <linux/delay.h>
21#include <linux/init.h>
22#include <linux/initrd.h>
23#include <linux/highmem.h>
24#include <linux/bootmem.h>
25#include <linux/module.h>
26#include <asm/processor.h>
27#include <linux/console.h>
28#include <linux/seq_file.h>
29#include <linux/crash_dump.h>
30#include <linux/root_dev.h>
31#include <linux/pci.h>
32#include <asm/pci-direct.h>
33#include <linux/efi.h>
34#include <linux/acpi.h>
35#include <linux/kallsyms.h>
36#include <linux/edd.h>
37#include <linux/iscsi_ibft.h>
38#include <linux/mmzone.h>
39#include <linux/kexec.h>
40#include <linux/cpufreq.h>
41#include <linux/dmi.h>
42#include <linux/dma-mapping.h>
43#include <linux/ctype.h>
44#include <linux/sort.h>
45#include <linux/uaccess.h>
46#include <linux/init_ohci1394_dma.h>
47#include <linux/kvm_para.h>
48
49#include <asm/mtrr.h>
50#include <asm/uaccess.h>
51#include <asm/system.h>
52#include <asm/vsyscall.h>
53#include <asm/io.h>
54#include <asm/smp.h>
55#include <asm/msr.h>
56#include <asm/desc.h>
57#include <video/edid.h>
58#include <asm/e820.h>
59#include <asm/dma.h>
60#include <asm/gart.h>
61#include <asm/mpspec.h>
62#include <asm/mmu_context.h>
63#include <asm/proto.h>
64#include <asm/setup.h>
65#include <asm/numa.h>
66#include <asm/sections.h>
67#include <asm/dmi.h>
68#include <asm/cacheflush.h>
69#include <asm/mce.h>
70#include <asm/ds.h>
71#include <asm/topology.h>
72#include <asm/trampoline.h>
73#include <asm/pat.h>
74
75#include <mach_apic.h>
76#ifdef CONFIG_PARAVIRT
77#include <asm/paravirt.h>
78#else
79#define ARCH_SETUP
80#endif
81
82/*
83 * Machine setup..
84 */
85
86struct cpuinfo_x86 boot_cpu_data __read_mostly;
87EXPORT_SYMBOL(boot_cpu_data);
88
89__u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;
90
91unsigned long mmu_cr4_features;
92
93/* Boot loader ID as an integer, for the benefit of proc_dointvec */
94int bootloader_type;
95
96unsigned long saved_video_mode;
97
98int force_mwait __cpuinitdata;
99
100/*
101 * Early DMI memory
102 */
103int dmi_alloc_index;
104char dmi_alloc_data[DMI_MAX_DATA];
105
106/*
107 * Setup options
108 */
109struct screen_info screen_info;
110EXPORT_SYMBOL(screen_info);
111struct sys_desc_table_struct {
112 unsigned short length;
113 unsigned char table[0];
114};
115
116struct edid_info edid_info;
117EXPORT_SYMBOL_GPL(edid_info);
118
119extern int root_mountflags;
120
121char __initdata command_line[COMMAND_LINE_SIZE];
122
123static struct resource standard_io_resources[] = {
124 { .name = "dma1", .start = 0x00, .end = 0x1f,
125 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
126 { .name = "pic1", .start = 0x20, .end = 0x21,
127 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
128 { .name = "timer0", .start = 0x40, .end = 0x43,
129 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
130 { .name = "timer1", .start = 0x50, .end = 0x53,
131 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
132 { .name = "keyboard", .start = 0x60, .end = 0x60,
133 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
134 { .name = "keyboard", .start = 0x64, .end = 0x64,
135 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
136 { .name = "dma page reg", .start = 0x80, .end = 0x8f,
137 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
138 { .name = "pic2", .start = 0xa0, .end = 0xa1,
139 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
140 { .name = "dma2", .start = 0xc0, .end = 0xdf,
141 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
142 { .name = "fpu", .start = 0xf0, .end = 0xff,
143 .flags = IORESOURCE_BUSY | IORESOURCE_IO }
144};
145
146#define IORESOURCE_RAM (IORESOURCE_BUSY | IORESOURCE_MEM)
147
148static struct resource data_resource = {
149 .name = "Kernel data",
150 .start = 0,
151 .end = 0,
152 .flags = IORESOURCE_RAM,
153};
154static struct resource code_resource = {
155 .name = "Kernel code",
156 .start = 0,
157 .end = 0,
158 .flags = IORESOURCE_RAM,
159};
160static struct resource bss_resource = {
161 .name = "Kernel bss",
162 .start = 0,
163 .end = 0,
164 .flags = IORESOURCE_RAM,
165};
166
167static void __cpuinit early_identify_cpu(struct cpuinfo_x86 *c);
168
169#ifdef CONFIG_PROC_VMCORE
170/* elfcorehdr= specifies the location of elf core header
171 * stored by the crashed kernel. This option will be passed
172 * by kexec loader to the capture kernel.
173 */
174static int __init setup_elfcorehdr(char *arg)
175{
176 char *end;
177 if (!arg)
178 return -EINVAL;
179 elfcorehdr_addr = memparse(arg, &end);
180 return end > arg ? 0 : -EINVAL;
181}
182early_param("elfcorehdr", setup_elfcorehdr);
183#endif
184
185#ifndef CONFIG_NUMA
186static void __init
187contig_initmem_init(unsigned long start_pfn, unsigned long end_pfn)
188{
189 unsigned long bootmap_size, bootmap;
190
191 bootmap_size = bootmem_bootmap_pages(end_pfn)<<PAGE_SHIFT;
192 bootmap = find_e820_area(0, end_pfn<<PAGE_SHIFT, bootmap_size,
193 PAGE_SIZE);
194 if (bootmap == -1L)
195 panic("Cannot find bootmem map of size %ld\n", bootmap_size);
196 bootmap_size = init_bootmem(bootmap >> PAGE_SHIFT, end_pfn);
197 e820_register_active_regions(0, start_pfn, end_pfn);
198 free_bootmem_with_active_regions(0, end_pfn);
199 early_res_to_bootmem(0, end_pfn<<PAGE_SHIFT);
200 reserve_bootmem(bootmap, bootmap_size, BOOTMEM_DEFAULT);
201}
202#endif
203
204#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE)
205struct edd edd;
206#ifdef CONFIG_EDD_MODULE
207EXPORT_SYMBOL(edd);
208#endif
209/**
210 * copy_edd() - Copy the BIOS EDD information
211 * from boot_params into a safe place.
212 *
213 */
214static inline void copy_edd(void)
215{
216 memcpy(edd.mbr_signature, boot_params.edd_mbr_sig_buffer,
217 sizeof(edd.mbr_signature));
218 memcpy(edd.edd_info, boot_params.eddbuf, sizeof(edd.edd_info));
219 edd.mbr_signature_nr = boot_params.edd_mbr_sig_buf_entries;
220 edd.edd_info_nr = boot_params.eddbuf_entries;
221}
222#else
223static inline void copy_edd(void)
224{
225}
226#endif
227
228#ifdef CONFIG_KEXEC
229static void __init reserve_crashkernel(void)
230{
231 unsigned long long total_mem;
232 unsigned long long crash_size, crash_base;
233 int ret;
234
235 total_mem = ((unsigned long long)max_low_pfn - min_low_pfn) << PAGE_SHIFT;
236
237 ret = parse_crashkernel(boot_command_line, total_mem,
238 &crash_size, &crash_base);
239 if (ret == 0 && crash_size) {
240 if (crash_base <= 0) {
241 printk(KERN_INFO "crashkernel reservation failed - "
242 "you have to specify a base address\n");
243 return;
244 }
245
246 if (reserve_bootmem(crash_base, crash_size,
247 BOOTMEM_EXCLUSIVE) < 0) {
248 printk(KERN_INFO "crashkernel reservation failed - "
249 "memory is in use\n");
250 return;
251 }
252
253 printk(KERN_INFO "Reserving %ldMB of memory at %ldMB "
254 "for crashkernel (System RAM: %ldMB)\n",
255 (unsigned long)(crash_size >> 20),
256 (unsigned long)(crash_base >> 20),
257 (unsigned long)(total_mem >> 20));
258 crashk_res.start = crash_base;
259 crashk_res.end = crash_base + crash_size - 1;
260 insert_resource(&iomem_resource, &crashk_res);
261 }
262}
263#else
264static inline void __init reserve_crashkernel(void)
265{}
266#endif
267
268/* Overridden in paravirt.c if CONFIG_PARAVIRT */
269void __attribute__((weak)) __init memory_setup(void)
270{
271 machine_specific_memory_setup();
272}
273
274static void __init parse_setup_data(void)
275{
276 struct setup_data *data;
277 unsigned long pa_data;
278
279 if (boot_params.hdr.version < 0x0209)
280 return;
281 pa_data = boot_params.hdr.setup_data;
282 while (pa_data) {
283 data = early_ioremap(pa_data, PAGE_SIZE);
284 switch (data->type) {
285 default:
286 break;
287 }
288#ifndef CONFIG_DEBUG_BOOT_PARAMS
289 free_early(pa_data, pa_data+sizeof(*data)+data->len);
290#endif
291 pa_data = data->next;
292 early_iounmap(data, PAGE_SIZE);
293 }
294}
295
296#ifdef CONFIG_PCI_MMCONFIG
297extern void __cpuinit fam10h_check_enable_mmcfg(void);
298extern void __init check_enable_amd_mmconf_dmi(void);
299#else
300void __cpuinit fam10h_check_enable_mmcfg(void)
301{
302}
303void __init check_enable_amd_mmconf_dmi(void)
304{
305}
306#endif
307
308/*
309 * setup_arch - architecture-specific boot-time initializations
310 *
311 * Note: On x86_64, fixmaps are ready for use even before this is called.
312 */
313void __init setup_arch(char **cmdline_p)
314{
315 unsigned i;
316
317 printk(KERN_INFO "Command line: %s\n", boot_command_line);
318
319 ROOT_DEV = old_decode_dev(boot_params.hdr.root_dev);
320 screen_info = boot_params.screen_info;
321 edid_info = boot_params.edid_info;
322 saved_video_mode = boot_params.hdr.vid_mode;
323 bootloader_type = boot_params.hdr.type_of_loader;
324
325#ifdef CONFIG_BLK_DEV_RAM
326 rd_image_start = boot_params.hdr.ram_size & RAMDISK_IMAGE_START_MASK;
327 rd_prompt = ((boot_params.hdr.ram_size & RAMDISK_PROMPT_FLAG) != 0);
328 rd_doload = ((boot_params.hdr.ram_size & RAMDISK_LOAD_FLAG) != 0);
329#endif
330#ifdef CONFIG_EFI
331 if (!strncmp((char *)&boot_params.efi_info.efi_loader_signature,
332 "EL64", 4))
333 efi_enabled = 1;
334#endif
335
336 ARCH_SETUP
337
338 memory_setup();
339 copy_edd();
340
341 if (!boot_params.hdr.root_flags)
342 root_mountflags &= ~MS_RDONLY;
343 init_mm.start_code = (unsigned long) &_text;
344 init_mm.end_code = (unsigned long) &_etext;
345 init_mm.end_data = (unsigned long) &_edata;
346 init_mm.brk = (unsigned long) &_end;
347
348 code_resource.start = virt_to_phys(&_text);
349 code_resource.end = virt_to_phys(&_etext)-1;
350 data_resource.start = virt_to_phys(&_etext);
351 data_resource.end = virt_to_phys(&_edata)-1;
352 bss_resource.start = virt_to_phys(&__bss_start);
353 bss_resource.end = virt_to_phys(&__bss_stop)-1;
354
355 early_identify_cpu(&boot_cpu_data);
356
357 strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
358 *cmdline_p = command_line;
359
360 parse_setup_data();
361
362 parse_early_param();
363
364#ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT
365 if (init_ohci1394_dma_early)
366 init_ohci1394_dma_on_all_controllers();
367#endif
368
369 finish_e820_parsing();
370
371 /* after parse_early_param, so could debug it */
372 insert_resource(&iomem_resource, &code_resource);
373 insert_resource(&iomem_resource, &data_resource);
374 insert_resource(&iomem_resource, &bss_resource);
375
376 early_gart_iommu_check();
377
378 e820_register_active_regions(0, 0, -1UL);
379 /*
380 * partially used pages are not usable - thus
381 * we are rounding upwards:
382 */
383 end_pfn = e820_end_of_ram();
384 /* update e820 for memory not covered by WB MTRRs */
385 mtrr_bp_init();
386 if (mtrr_trim_uncached_memory(end_pfn)) {
387 e820_register_active_regions(0, 0, -1UL);
388 end_pfn = e820_end_of_ram();
389 }
390
391 num_physpages = end_pfn;
392
393 check_efer();
394
395 max_pfn_mapped = init_memory_mapping(0, (max_pfn_mapped << PAGE_SHIFT));
396 if (efi_enabled)
397 efi_init();
398
399 vsmp_init();
400
401 dmi_scan_machine();
402
403 io_delay_init();
404
405#ifdef CONFIG_KVM_CLOCK
406 kvmclock_init();
407#endif
408
409#ifdef CONFIG_SMP
410 /* setup to use the early static init tables during kernel startup */
411 x86_cpu_to_apicid_early_ptr = (void *)x86_cpu_to_apicid_init;
412 x86_bios_cpu_apicid_early_ptr = (void *)x86_bios_cpu_apicid_init;
413#ifdef CONFIG_NUMA
414 x86_cpu_to_node_map_early_ptr = (void *)x86_cpu_to_node_map_init;
415#endif
416#endif
417
418#ifdef CONFIG_ACPI
419 /*
420 * Initialize the ACPI boot-time table parser (gets the RSDP and SDT).
421 * Call this early for SRAT node setup.
422 */
423 acpi_boot_table_init();
424#endif
425
426 /* How many end-of-memory variables you have, grandma! */
427 max_low_pfn = end_pfn;
428 max_pfn = end_pfn;
429 high_memory = (void *)__va(end_pfn * PAGE_SIZE - 1) + 1;
430
431 /* Remove active ranges so rediscovery with NUMA-awareness happens */
432 remove_all_active_ranges();
433
434#ifdef CONFIG_ACPI_NUMA
435 /*
436 * Parse SRAT to discover nodes.
437 */
438 acpi_numa_init();
439#endif
440
441#ifdef CONFIG_NUMA
442 numa_initmem_init(0, end_pfn);
443#else
444 contig_initmem_init(0, end_pfn);
445#endif
446
447 dma32_reserve_bootmem();
448
449#ifdef CONFIG_ACPI_SLEEP
450 /*
451 * Reserve low memory region for sleep support.
452 */
453 acpi_reserve_bootmem();
454#endif
455
456 if (efi_enabled)
457 efi_reserve_bootmem();
458
459 /*
460 * Find and reserve possible boot-time SMP configuration:
461 */
462 find_smp_config();
463#ifdef CONFIG_BLK_DEV_INITRD
464 if (boot_params.hdr.type_of_loader && boot_params.hdr.ramdisk_image) {
465 unsigned long ramdisk_image = boot_params.hdr.ramdisk_image;
466 unsigned long ramdisk_size = boot_params.hdr.ramdisk_size;
467 unsigned long ramdisk_end = ramdisk_image + ramdisk_size;
468 unsigned long end_of_mem = end_pfn << PAGE_SHIFT;
469
470 if (ramdisk_end <= end_of_mem) {
471 /*
472 * don't need to reserve again, already reserved early
473 * in x86_64_start_kernel, and early_res_to_bootmem
474 * convert that to reserved in bootmem
475 */
476 initrd_start = ramdisk_image + PAGE_OFFSET;
477 initrd_end = initrd_start+ramdisk_size;
478 } else {
479 free_bootmem(ramdisk_image, ramdisk_size);
480 printk(KERN_ERR "initrd extends beyond end of memory "
481 "(0x%08lx > 0x%08lx)\ndisabling initrd\n",
482 ramdisk_end, end_of_mem);
483 initrd_start = 0;
484 }
485 }
486#endif
487 reserve_crashkernel();
488
489 reserve_ibft_region();
490
491 paging_init();
492 map_vsyscall();
493
494 early_quirks();
495
496#ifdef CONFIG_ACPI
497 /*
498 * Read APIC and some other early information from ACPI tables.
499 */
500 acpi_boot_init();
501#endif
502
503 init_cpu_to_node();
504
505 /*
506 * get boot-time SMP configuration:
507 */
508 if (smp_found_config)
509 get_smp_config();
510 init_apic_mappings();
511 ioapic_init_mappings();
512
513 kvm_guest_init();
514
515 /*
516 * We trust e820 completely. No explicit ROM probing in memory.
517 */
518 e820_reserve_resources();
519 e820_mark_nosave_regions();
520
521 /* request I/O space for devices used on all i[345]86 PCs */
522 for (i = 0; i < ARRAY_SIZE(standard_io_resources); i++)
523 request_resource(&ioport_resource, &standard_io_resources[i]);
524
525 e820_setup_gap();
526
527#ifdef CONFIG_VT
528#if defined(CONFIG_VGA_CONSOLE)
529 if (!efi_enabled || (efi_mem_type(0xa0000) != EFI_CONVENTIONAL_MEMORY))
530 conswitchp = &vga_con;
531#elif defined(CONFIG_DUMMY_CONSOLE)
532 conswitchp = &dummy_con;
533#endif
534#endif
535
536 /* do this before identify_cpu for boot cpu */
537 check_enable_amd_mmconf_dmi();
538}
539
540static int __cpuinit get_model_name(struct cpuinfo_x86 *c)
541{
542 unsigned int *v;
543
544 if (c->extended_cpuid_level < 0x80000004)
545 return 0;
546
547 v = (unsigned int *) c->x86_model_id;
548 cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
549 cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
550 cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
551 c->x86_model_id[48] = 0;
552 return 1;
553}
554
555
556static void __cpuinit display_cacheinfo(struct cpuinfo_x86 *c)
557{
558 unsigned int n, dummy, eax, ebx, ecx, edx;
559
560 n = c->extended_cpuid_level;
561
562 if (n >= 0x80000005) {
563 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
564 printk(KERN_INFO "CPU: L1 I Cache: %dK (%d bytes/line), "
565 "D cache %dK (%d bytes/line)\n",
566 edx>>24, edx&0xFF, ecx>>24, ecx&0xFF);
567 c->x86_cache_size = (ecx>>24) + (edx>>24);
568 /* On K8 L1 TLB is inclusive, so don't count it */
569 c->x86_tlbsize = 0;
570 }
571
572 if (n >= 0x80000006) {
573 cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
574 ecx = cpuid_ecx(0x80000006);
575 c->x86_cache_size = ecx >> 16;
576 c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
577
578 printk(KERN_INFO "CPU: L2 Cache: %dK (%d bytes/line)\n",
579 c->x86_cache_size, ecx & 0xFF);
580 }
581 if (n >= 0x80000008) {
582 cpuid(0x80000008, &eax, &dummy, &dummy, &dummy);
583 c->x86_virt_bits = (eax >> 8) & 0xff;
584 c->x86_phys_bits = eax & 0xff;
585 }
586}
587
588#ifdef CONFIG_NUMA
589static int __cpuinit nearby_node(int apicid)
590{
591 int i, node;
592
593 for (i = apicid - 1; i >= 0; i--) {
594 node = apicid_to_node[i];
595 if (node != NUMA_NO_NODE && node_online(node))
596 return node;
597 }
598 for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
599 node = apicid_to_node[i];
600 if (node != NUMA_NO_NODE && node_online(node))
601 return node;
602 }
603 return first_node(node_online_map); /* Shouldn't happen */
604}
605#endif
606
607/*
608 * On a AMD dual core setup the lower bits of the APIC id distingush the cores.
609 * Assumes number of cores is a power of two.
610 */
611static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
612{
613#ifdef CONFIG_SMP
614 unsigned bits;
615#ifdef CONFIG_NUMA
616 int cpu = smp_processor_id();
617 int node = 0;
618 unsigned apicid = hard_smp_processor_id();
619#endif
620 bits = c->x86_coreid_bits;
621
622 /* Low order bits define the core id (index of core in socket) */
623 c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
624 /* Convert the initial APIC ID into the socket ID */
625 c->phys_proc_id = c->initial_apicid >> bits;
626
627#ifdef CONFIG_NUMA
628 node = c->phys_proc_id;
629 if (apicid_to_node[apicid] != NUMA_NO_NODE)
630 node = apicid_to_node[apicid];
631 if (!node_online(node)) {
632 /* Two possibilities here:
633 - The CPU is missing memory and no node was created.
634 In that case try picking one from a nearby CPU
635 - The APIC IDs differ from the HyperTransport node IDs
636 which the K8 northbridge parsing fills in.
637 Assume they are all increased by a constant offset,
638 but in the same order as the HT nodeids.
639 If that doesn't result in a usable node fall back to the
640 path for the previous case. */
641
642 int ht_nodeid = c->initial_apicid;
643
644 if (ht_nodeid >= 0 &&
645 apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
646 node = apicid_to_node[ht_nodeid];
647 /* Pick a nearby node */
648 if (!node_online(node))
649 node = nearby_node(apicid);
650 }
651 numa_set_node(cpu, node);
652
653 printk(KERN_INFO "CPU %d/%x -> Node %d\n", cpu, apicid, node);
654#endif
655#endif
656}
657
658static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c)
659{
660#ifdef CONFIG_SMP
661 unsigned bits, ecx;
662
663 /* Multi core CPU? */
664 if (c->extended_cpuid_level < 0x80000008)
665 return;
666
667 ecx = cpuid_ecx(0x80000008);
668
669 c->x86_max_cores = (ecx & 0xff) + 1;
670
671 /* CPU telling us the core id bits shift? */
672 bits = (ecx >> 12) & 0xF;
673
674 /* Otherwise recompute */
675 if (bits == 0) {
676 while ((1 << bits) < c->x86_max_cores)
677 bits++;
678 }
679
680 c->x86_coreid_bits = bits;
681
682#endif
683}
684
685#define ENABLE_C1E_MASK 0x18000000
686#define CPUID_PROCESSOR_SIGNATURE 1
687#define CPUID_XFAM 0x0ff00000
688#define CPUID_XFAM_K8 0x00000000
689#define CPUID_XFAM_10H 0x00100000
690#define CPUID_XFAM_11H 0x00200000
691#define CPUID_XMOD 0x000f0000
692#define CPUID_XMOD_REV_F 0x00040000
693
694/* AMD systems with C1E don't have a working lAPIC timer. Check for that. */
695static __cpuinit int amd_apic_timer_broken(void)
696{
697 u32 lo, hi, eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
698
699 switch (eax & CPUID_XFAM) {
700 case CPUID_XFAM_K8:
701 if ((eax & CPUID_XMOD) < CPUID_XMOD_REV_F)
702 break;
703 case CPUID_XFAM_10H:
704 case CPUID_XFAM_11H:
705 rdmsr(MSR_K8_ENABLE_C1E, lo, hi);
706 if (lo & ENABLE_C1E_MASK)
707 return 1;
708 break;
709 default:
710 /* err on the side of caution */
711 return 1;
712 }
713 return 0;
714}
715
716static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
717{
718 early_init_amd_mc(c);
719
720 /* c->x86_power is 8000_0007 edx. Bit 8 is constant TSC */
721 if (c->x86_power & (1<<8))
722 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
723}
724
725static void __cpuinit init_amd(struct cpuinfo_x86 *c)
726{
727 unsigned level;
728
729#ifdef CONFIG_SMP
730 unsigned long value;
731
732 /*
733 * Disable TLB flush filter by setting HWCR.FFDIS on K8
734 * bit 6 of msr C001_0015
735 *
736 * Errata 63 for SH-B3 steppings
737 * Errata 122 for all steppings (F+ have it disabled by default)
738 */
739 if (c->x86 == 15) {
740 rdmsrl(MSR_K8_HWCR, value);
741 value |= 1 << 6;
742 wrmsrl(MSR_K8_HWCR, value);
743 }
744#endif
745
746 /* Bit 31 in normal CPUID used for nonstandard 3DNow ID;
747 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway */
748 clear_cpu_cap(c, 0*32+31);
749
750 /* On C+ stepping K8 rep microcode works well for copy/memset */
751 level = cpuid_eax(1);
752 if (c->x86 == 15 && ((level >= 0x0f48 && level < 0x0f50) ||
753 level >= 0x0f58))
754 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
755 if (c->x86 == 0x10 || c->x86 == 0x11)
756 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
757
758 /* Enable workaround for FXSAVE leak */
759 if (c->x86 >= 6)
760 set_cpu_cap(c, X86_FEATURE_FXSAVE_LEAK);
761
762 level = get_model_name(c);
763 if (!level) {
764 switch (c->x86) {
765 case 15:
766 /* Should distinguish Models here, but this is only
767 a fallback anyways. */
768 strcpy(c->x86_model_id, "Hammer");
769 break;
770 }
771 }
772 display_cacheinfo(c);
773
774 /* Multi core CPU? */
775 if (c->extended_cpuid_level >= 0x80000008)
776 amd_detect_cmp(c);
777
778 if (c->extended_cpuid_level >= 0x80000006 &&
779 (cpuid_edx(0x80000006) & 0xf000))
780 num_cache_leaves = 4;
781 else
782 num_cache_leaves = 3;
783
784 if (c->x86 == 0xf || c->x86 == 0x10 || c->x86 == 0x11)
785 set_cpu_cap(c, X86_FEATURE_K8);
786
787 /* MFENCE stops RDTSC speculation */
788 set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
789
790 if (c->x86 == 0x10)
791 fam10h_check_enable_mmcfg();
792
793 if (amd_apic_timer_broken())
794 disable_apic_timer = 1;
795
796 if (c == &boot_cpu_data && c->x86 >= 0xf && c->x86 <= 0x11) {
797 unsigned long long tseg;
798
799 /*
800 * Split up direct mapping around the TSEG SMM area.
801 * Don't do it for gbpages because there seems very little
802 * benefit in doing so.
803 */
804 if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg) &&
805 (tseg >> PMD_SHIFT) < (max_pfn_mapped >> (PMD_SHIFT-PAGE_SHIFT)))
806 set_memory_4k((unsigned long)__va(tseg), 1);
807 }
808}
809
810void __cpuinit detect_ht(struct cpuinfo_x86 *c)
811{
812#ifdef CONFIG_SMP
813 u32 eax, ebx, ecx, edx;
814 int index_msb, core_bits;
815
816 cpuid(1, &eax, &ebx, &ecx, &edx);
817
818
819 if (!cpu_has(c, X86_FEATURE_HT))
820 return;
821 if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
822 goto out;
823
824 smp_num_siblings = (ebx & 0xff0000) >> 16;
825
826 if (smp_num_siblings == 1) {
827 printk(KERN_INFO "CPU: Hyper-Threading is disabled\n");
828 } else if (smp_num_siblings > 1) {
829
830 if (smp_num_siblings > NR_CPUS) {
831 printk(KERN_WARNING "CPU: Unsupported number of "
832 "siblings %d", smp_num_siblings);
833 smp_num_siblings = 1;
834 return;
835 }
836
837 index_msb = get_count_order(smp_num_siblings);
838 c->phys_proc_id = phys_pkg_id(index_msb);
839
840 smp_num_siblings = smp_num_siblings / c->x86_max_cores;
841
842 index_msb = get_count_order(smp_num_siblings);
843
844 core_bits = get_count_order(c->x86_max_cores);
845
846 c->cpu_core_id = phys_pkg_id(index_msb) &
847 ((1 << core_bits) - 1);
848 }
849out:
850 if ((c->x86_max_cores * smp_num_siblings) > 1) {
851 printk(KERN_INFO "CPU: Physical Processor ID: %d\n",
852 c->phys_proc_id);
853 printk(KERN_INFO "CPU: Processor Core ID: %d\n",
854 c->cpu_core_id);
855 }
856
857#endif
858}
859
860/*
861 * find out the number of processor cores on the die
862 */
863static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c)
864{
865 unsigned int eax, t;
866
867 if (c->cpuid_level < 4)
868 return 1;
869
870 cpuid_count(4, 0, &eax, &t, &t, &t);
871
872 if (eax & 0x1f)
873 return ((eax >> 26) + 1);
874 else
875 return 1;
876}
877
878static void __cpuinit srat_detect_node(void)
879{
880#ifdef CONFIG_NUMA
881 unsigned node;
882 int cpu = smp_processor_id();
883 int apicid = hard_smp_processor_id();
884
885 /* Don't do the funky fallback heuristics the AMD version employs
886 for now. */
887 node = apicid_to_node[apicid];
888 if (node == NUMA_NO_NODE || !node_online(node))
889 node = first_node(node_online_map);
890 numa_set_node(cpu, node);
891
892 printk(KERN_INFO "CPU %d/%x -> Node %d\n", cpu, apicid, node);
893#endif
894}
895
896static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
897{
898 if ((c->x86 == 0xf && c->x86_model >= 0x03) ||
899 (c->x86 == 0x6 && c->x86_model >= 0x0e))
900 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
901}
902
903static void __cpuinit init_intel(struct cpuinfo_x86 *c)
904{
905 /* Cache sizes */
906 unsigned n;
907
908 init_intel_cacheinfo(c);
909 if (c->cpuid_level > 9) {
910 unsigned eax = cpuid_eax(10);
911 /* Check for version and the number of counters */
912 if ((eax & 0xff) && (((eax>>8) & 0xff) > 1))
913 set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON);
914 }
915
916 if (cpu_has_ds) {
917 unsigned int l1, l2;
918 rdmsr(MSR_IA32_MISC_ENABLE, l1, l2);
919 if (!(l1 & (1<<11)))
920 set_cpu_cap(c, X86_FEATURE_BTS);
921 if (!(l1 & (1<<12)))
922 set_cpu_cap(c, X86_FEATURE_PEBS);
923 }
924
925
926 if (cpu_has_bts)
927 ds_init_intel(c);
928
929 n = c->extended_cpuid_level;
930 if (n >= 0x80000008) {
931 unsigned eax = cpuid_eax(0x80000008);
932 c->x86_virt_bits = (eax >> 8) & 0xff;
933 c->x86_phys_bits = eax & 0xff;
934 /* CPUID workaround for Intel 0F34 CPU */
935 if (c->x86_vendor == X86_VENDOR_INTEL &&
936 c->x86 == 0xF && c->x86_model == 0x3 &&
937 c->x86_mask == 0x4)
938 c->x86_phys_bits = 36;
939 }
940
941 if (c->x86 == 15)
942 c->x86_cache_alignment = c->x86_clflush_size * 2;
943 if (c->x86 == 6)
944 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
945 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
946 c->x86_max_cores = intel_num_cpu_cores(c);
947
948 srat_detect_node();
949}
950
951static void __cpuinit early_init_centaur(struct cpuinfo_x86 *c)
952{
953 if (c->x86 == 0x6 && c->x86_model >= 0xf)
954 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
955}
956
957static void __cpuinit init_centaur(struct cpuinfo_x86 *c)
958{
959 /* Cache sizes */
960 unsigned n;
961
962 n = c->extended_cpuid_level;
963 if (n >= 0x80000008) {
964 unsigned eax = cpuid_eax(0x80000008);
965 c->x86_virt_bits = (eax >> 8) & 0xff;
966 c->x86_phys_bits = eax & 0xff;
967 }
968
969 if (c->x86 == 0x6 && c->x86_model >= 0xf) {
970 c->x86_cache_alignment = c->x86_clflush_size * 2;
971 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
972 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
973 }
974 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
975}
976
977static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
978{
979 char *v = c->x86_vendor_id;
980
981 if (!strcmp(v, "AuthenticAMD"))
982 c->x86_vendor = X86_VENDOR_AMD;
983 else if (!strcmp(v, "GenuineIntel"))
984 c->x86_vendor = X86_VENDOR_INTEL;
985 else if (!strcmp(v, "CentaurHauls"))
986 c->x86_vendor = X86_VENDOR_CENTAUR;
987 else
988 c->x86_vendor = X86_VENDOR_UNKNOWN;
989}
990
991/* Do some early cpuid on the boot CPU to get some parameter that are
992 needed before check_bugs. Everything advanced is in identify_cpu
993 below. */
994static void __cpuinit early_identify_cpu(struct cpuinfo_x86 *c)
995{
996 u32 tfms, xlvl;
997
998 c->loops_per_jiffy = loops_per_jiffy;
999 c->x86_cache_size = -1;
1000 c->x86_vendor = X86_VENDOR_UNKNOWN;
1001 c->x86_model = c->x86_mask = 0; /* So far unknown... */
1002 c->x86_vendor_id[0] = '\0'; /* Unset */
1003 c->x86_model_id[0] = '\0'; /* Unset */
1004 c->x86_clflush_size = 64;
1005 c->x86_cache_alignment = c->x86_clflush_size;
1006 c->x86_max_cores = 1;
1007 c->x86_coreid_bits = 0;
1008 c->extended_cpuid_level = 0;
1009 memset(&c->x86_capability, 0, sizeof c->x86_capability);
1010
1011 /* Get vendor name */
1012 cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
1013 (unsigned int *)&c->x86_vendor_id[0],
1014 (unsigned int *)&c->x86_vendor_id[8],
1015 (unsigned int *)&c->x86_vendor_id[4]);
1016
1017 get_cpu_vendor(c);
1018
1019 /* Initialize the standard set of capabilities */
1020 /* Note that the vendor-specific code below might override */
1021
1022 /* Intel-defined flags: level 0x00000001 */
1023 if (c->cpuid_level >= 0x00000001) {
1024 __u32 misc;
1025 cpuid(0x00000001, &tfms, &misc, &c->x86_capability[4],
1026 &c->x86_capability[0]);
1027 c->x86 = (tfms >> 8) & 0xf;
1028 c->x86_model = (tfms >> 4) & 0xf;
1029 c->x86_mask = tfms & 0xf;
1030 if (c->x86 == 0xf)
1031 c->x86 += (tfms >> 20) & 0xff;
1032 if (c->x86 >= 0x6)
1033 c->x86_model += ((tfms >> 16) & 0xF) << 4;
1034 if (test_cpu_cap(c, X86_FEATURE_CLFLSH))
1035 c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
1036 } else {
1037 /* Have CPUID level 0 only - unheard of */
1038 c->x86 = 4;
1039 }
1040
1041 c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xff;
1042#ifdef CONFIG_SMP
1043 c->phys_proc_id = c->initial_apicid;
1044#endif
1045 /* AMD-defined flags: level 0x80000001 */
1046 xlvl = cpuid_eax(0x80000000);
1047 c->extended_cpuid_level = xlvl;
1048 if ((xlvl & 0xffff0000) == 0x80000000) {
1049 if (xlvl >= 0x80000001) {
1050 c->x86_capability[1] = cpuid_edx(0x80000001);
1051 c->x86_capability[6] = cpuid_ecx(0x80000001);
1052 }
1053 if (xlvl >= 0x80000004)
1054 get_model_name(c); /* Default name */
1055 }
1056
1057 /* Transmeta-defined flags: level 0x80860001 */
1058 xlvl = cpuid_eax(0x80860000);
1059 if ((xlvl & 0xffff0000) == 0x80860000) {
1060 /* Don't set x86_cpuid_level here for now to not confuse. */
1061 if (xlvl >= 0x80860001)
1062 c->x86_capability[2] = cpuid_edx(0x80860001);
1063 }
1064
1065 c->extended_cpuid_level = cpuid_eax(0x80000000);
1066 if (c->extended_cpuid_level >= 0x80000007)
1067 c->x86_power = cpuid_edx(0x80000007);
1068
1069 switch (c->x86_vendor) {
1070 case X86_VENDOR_AMD:
1071 early_init_amd(c);
1072 break;
1073 case X86_VENDOR_INTEL:
1074 early_init_intel(c);
1075 break;
1076 case X86_VENDOR_CENTAUR:
1077 early_init_centaur(c);
1078 break;
1079 }
1080
1081 validate_pat_support(c);
1082}
1083
1084/*
1085 * This does the hard work of actually picking apart the CPU stuff...
1086 */
1087void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
1088{
1089 int i;
1090
1091 early_identify_cpu(c);
1092
1093 init_scattered_cpuid_features(c);
1094
1095 c->apicid = phys_pkg_id(0);
1096
1097 /*
1098 * Vendor-specific initialization. In this section we
1099 * canonicalize the feature flags, meaning if there are
1100 * features a certain CPU supports which CPUID doesn't
1101 * tell us, CPUID claiming incorrect flags, or other bugs,
1102 * we handle them here.
1103 *
1104 * At the end of this section, c->x86_capability better
1105 * indicate the features this CPU genuinely supports!
1106 */
1107 switch (c->x86_vendor) {
1108 case X86_VENDOR_AMD:
1109 init_amd(c);
1110 break;
1111
1112 case X86_VENDOR_INTEL:
1113 init_intel(c);
1114 break;
1115
1116 case X86_VENDOR_CENTAUR:
1117 init_centaur(c);
1118 break;
1119
1120 case X86_VENDOR_UNKNOWN:
1121 default:
1122 display_cacheinfo(c);
1123 break;
1124 }
1125
1126 detect_ht(c);
1127
1128 /*
1129 * On SMP, boot_cpu_data holds the common feature set between
1130 * all CPUs; so make sure that we indicate which features are
1131 * common between the CPUs. The first time this routine gets
1132 * executed, c == &boot_cpu_data.
1133 */
1134 if (c != &boot_cpu_data) {
1135 /* AND the already accumulated flags with these */
1136 for (i = 0; i < NCAPINTS; i++)
1137 boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1138 }
1139
1140 /* Clear all flags overriden by options */
1141 for (i = 0; i < NCAPINTS; i++)
1142 c->x86_capability[i] &= ~cleared_cpu_caps[i];
1143
1144#ifdef CONFIG_X86_MCE
1145 mcheck_init(c);
1146#endif
1147 select_idle_routine(c);
1148
1149#ifdef CONFIG_NUMA
1150 numa_add_cpu(smp_processor_id());
1151#endif
1152
1153}
1154
1155void __cpuinit identify_boot_cpu(void)
1156{
1157 identify_cpu(&boot_cpu_data);
1158}
1159
1160void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
1161{
1162 BUG_ON(c == &boot_cpu_data);
1163 identify_cpu(c);
1164 mtrr_ap_init();
1165}
1166
1167static __init int setup_noclflush(char *arg)
1168{
1169 setup_clear_cpu_cap(X86_FEATURE_CLFLSH);
1170 return 1;
1171}
1172__setup("noclflush", setup_noclflush);
1173
1174void __cpuinit print_cpu_info(struct cpuinfo_x86 *c)
1175{
1176 if (c->x86_model_id[0])
1177 printk(KERN_CONT "%s", c->x86_model_id);
1178
1179 if (c->x86_mask || c->cpuid_level >= 0)
1180 printk(KERN_CONT " stepping %02x\n", c->x86_mask);
1181 else
1182 printk(KERN_CONT "\n");
1183}
1184
1185static __init int setup_disablecpuid(char *arg)
1186{
1187 int bit;
1188 if (get_option(&arg, &bit) && bit < NCAPINTS*32)
1189 setup_clear_cpu_cap(bit);
1190 else
1191 return 0;
1192 return 1;
1193}
1194__setup("clearcpuid=", setup_disablecpuid);
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
new file mode 100644
index 000000000000..cac68430d31f
--- /dev/null
+++ b/arch/x86/kernel/setup_percpu.c
@@ -0,0 +1,399 @@
1#include <linux/kernel.h>
2#include <linux/module.h>
3#include <linux/init.h>
4#include <linux/bootmem.h>
5#include <linux/percpu.h>
6#include <linux/kexec.h>
7#include <linux/crash_dump.h>
8#include <asm/smp.h>
9#include <asm/percpu.h>
10#include <asm/sections.h>
11#include <asm/processor.h>
12#include <asm/setup.h>
13#include <asm/topology.h>
14#include <asm/mpspec.h>
15#include <asm/apicdef.h>
16#include <asm/highmem.h>
17
18#ifdef CONFIG_X86_LOCAL_APIC
19unsigned int num_processors;
20unsigned disabled_cpus __cpuinitdata;
21/* Processor that is doing the boot up */
22unsigned int boot_cpu_physical_apicid = -1U;
23unsigned int max_physical_apicid;
24EXPORT_SYMBOL(boot_cpu_physical_apicid);
25
26/* Bitmask of physically existing CPUs */
27physid_mask_t phys_cpu_present_map;
28#endif
29
30/* map cpu index to physical APIC ID */
31DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
32DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
33EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
34EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
35
36#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
37#define X86_64_NUMA 1
38
39/* map cpu index to node index */
40DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
41EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
42
43/* which logical CPUs are on which nodes */
44cpumask_t *node_to_cpumask_map;
45EXPORT_SYMBOL(node_to_cpumask_map);
46
47/* setup node_to_cpumask_map */
48static void __init setup_node_to_cpumask_map(void);
49
50#else
51static inline void setup_node_to_cpumask_map(void) { }
52#endif
53
54#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_X86_SMP)
55/*
56 * Copy data used in early init routines from the initial arrays to the
57 * per cpu data areas. These arrays then become expendable and the
58 * *_early_ptr's are zeroed indicating that the static arrays are gone.
59 */
60static void __init setup_per_cpu_maps(void)
61{
62 int cpu;
63
64 for_each_possible_cpu(cpu) {
65 per_cpu(x86_cpu_to_apicid, cpu) =
66 early_per_cpu_map(x86_cpu_to_apicid, cpu);
67 per_cpu(x86_bios_cpu_apicid, cpu) =
68 early_per_cpu_map(x86_bios_cpu_apicid, cpu);
69#ifdef X86_64_NUMA
70 per_cpu(x86_cpu_to_node_map, cpu) =
71 early_per_cpu_map(x86_cpu_to_node_map, cpu);
72#endif
73 }
74
75 /* indicate the early static arrays will soon be gone */
76 early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
77 early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
78#ifdef X86_64_NUMA
79 early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
80#endif
81}
82
83#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP
84cpumask_t *cpumask_of_cpu_map __read_mostly;
85EXPORT_SYMBOL(cpumask_of_cpu_map);
86
87/* requires nr_cpu_ids to be initialized */
88static void __init setup_cpumask_of_cpu(void)
89{
90 int i;
91
92 /* alloc_bootmem zeroes memory */
93 cpumask_of_cpu_map = alloc_bootmem_low(sizeof(cpumask_t) * nr_cpu_ids);
94 for (i = 0; i < nr_cpu_ids; i++)
95 cpu_set(i, cpumask_of_cpu_map[i]);
96}
97#else
98static inline void setup_cpumask_of_cpu(void) { }
99#endif
100
101#ifdef CONFIG_X86_32
102/*
103 * Great future not-so-futuristic plan: make i386 and x86_64 do it
104 * the same way
105 */
106unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
107EXPORT_SYMBOL(__per_cpu_offset);
108static inline void setup_cpu_pda_map(void) { }
109
110#elif !defined(CONFIG_SMP)
111static inline void setup_cpu_pda_map(void) { }
112
113#else /* CONFIG_SMP && CONFIG_X86_64 */
114
115/*
116 * Allocate cpu_pda pointer table and array via alloc_bootmem.
117 */
118static void __init setup_cpu_pda_map(void)
119{
120 char *pda;
121 struct x8664_pda **new_cpu_pda;
122 unsigned long size;
123 int cpu;
124
125 size = roundup(sizeof(struct x8664_pda), cache_line_size());
126
127 /* allocate cpu_pda array and pointer table */
128 {
129 unsigned long tsize = nr_cpu_ids * sizeof(void *);
130 unsigned long asize = size * (nr_cpu_ids - 1);
131
132 tsize = roundup(tsize, cache_line_size());
133 new_cpu_pda = alloc_bootmem(tsize + asize);
134 pda = (char *)new_cpu_pda + tsize;
135 }
136
137 /* initialize pointer table to static pda's */
138 for_each_possible_cpu(cpu) {
139 if (cpu == 0) {
140 /* leave boot cpu pda in place */
141 new_cpu_pda[0] = cpu_pda(0);
142 continue;
143 }
144 new_cpu_pda[cpu] = (struct x8664_pda *)pda;
145 new_cpu_pda[cpu]->in_bootmem = 1;
146 pda += size;
147 }
148
149 /* point to new pointer table */
150 _cpu_pda = new_cpu_pda;
151}
152#endif
153
154/*
155 * Great future plan:
156 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
157 * Always point %gs to its beginning
158 */
159void __init setup_per_cpu_areas(void)
160{
161 ssize_t size = PERCPU_ENOUGH_ROOM;
162 char *ptr;
163 int cpu;
164
165 /* Setup cpu_pda map */
166 setup_cpu_pda_map();
167
168 /* Copy section for each CPU (we discard the original) */
169 size = PERCPU_ENOUGH_ROOM;
170 printk(KERN_INFO "PERCPU: Allocating %zd bytes of per cpu data\n",
171 size);
172
173 for_each_possible_cpu(cpu) {
174#ifndef CONFIG_NEED_MULTIPLE_NODES
175 ptr = alloc_bootmem_pages(size);
176#else
177 int node = early_cpu_to_node(cpu);
178 if (!node_online(node) || !NODE_DATA(node)) {
179 ptr = alloc_bootmem_pages(size);
180 printk(KERN_INFO
181 "cpu %d has no node %d or node-local memory\n",
182 cpu, node);
183 }
184 else
185 ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
186#endif
187 per_cpu_offset(cpu) = ptr - __per_cpu_start;
188 memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
189
190 }
191
192 printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d, nr_node_ids %d\n",
193 NR_CPUS, nr_cpu_ids, nr_node_ids);
194
195 /* Setup percpu data maps */
196 setup_per_cpu_maps();
197
198 /* Setup node to cpumask map */
199 setup_node_to_cpumask_map();
200
201 /* Setup cpumask_of_cpu map */
202 setup_cpumask_of_cpu();
203}
204
205#endif
206
207#ifdef X86_64_NUMA
208
209/*
210 * Allocate node_to_cpumask_map based on number of available nodes
211 * Requires node_possible_map to be valid.
212 *
213 * Note: node_to_cpumask() is not valid until after this is done.
214 */
215static void __init setup_node_to_cpumask_map(void)
216{
217 unsigned int node, num = 0;
218 cpumask_t *map;
219
220 /* setup nr_node_ids if not done yet */
221 if (nr_node_ids == MAX_NUMNODES) {
222 for_each_node_mask(node, node_possible_map)
223 num = node;
224 nr_node_ids = num + 1;
225 }
226
227 /* allocate the map */
228 map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
229
230 Dprintk(KERN_DEBUG "Node to cpumask map at %p for %d nodes\n",
231 map, nr_node_ids);
232
233 /* node_to_cpumask() will now work */
234 node_to_cpumask_map = map;
235}
236
237void __cpuinit numa_set_node(int cpu, int node)
238{
239 int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
240
241 if (cpu_pda(cpu) && node != NUMA_NO_NODE)
242 cpu_pda(cpu)->nodenumber = node;
243
244 if (cpu_to_node_map)
245 cpu_to_node_map[cpu] = node;
246
247 else if (per_cpu_offset(cpu))
248 per_cpu(x86_cpu_to_node_map, cpu) = node;
249
250 else
251 Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu);
252}
253
254void __cpuinit numa_clear_node(int cpu)
255{
256 numa_set_node(cpu, NUMA_NO_NODE);
257}
258
259#ifndef CONFIG_DEBUG_PER_CPU_MAPS
260
261void __cpuinit numa_add_cpu(int cpu)
262{
263 cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
264}
265
266void __cpuinit numa_remove_cpu(int cpu)
267{
268 cpu_clear(cpu, node_to_cpumask_map[cpu_to_node(cpu)]);
269}
270
271#else /* CONFIG_DEBUG_PER_CPU_MAPS */
272
273/*
274 * --------- debug versions of the numa functions ---------
275 */
276static void __cpuinit numa_set_cpumask(int cpu, int enable)
277{
278 int node = cpu_to_node(cpu);
279 cpumask_t *mask;
280 char buf[64];
281
282 if (node_to_cpumask_map == NULL) {
283 printk(KERN_ERR "node_to_cpumask_map NULL\n");
284 dump_stack();
285 return;
286 }
287
288 mask = &node_to_cpumask_map[node];
289 if (enable)
290 cpu_set(cpu, *mask);
291 else
292 cpu_clear(cpu, *mask);
293
294 cpulist_scnprintf(buf, sizeof(buf), *mask);
295 printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
296 enable? "numa_add_cpu":"numa_remove_cpu", cpu, node, buf);
297 }
298
299void __cpuinit numa_add_cpu(int cpu)
300{
301 numa_set_cpumask(cpu, 1);
302}
303
304void __cpuinit numa_remove_cpu(int cpu)
305{
306 numa_set_cpumask(cpu, 0);
307}
308
309int cpu_to_node(int cpu)
310{
311 if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
312 printk(KERN_WARNING
313 "cpu_to_node(%d): usage too early!\n", cpu);
314 dump_stack();
315 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
316 }
317 return per_cpu(x86_cpu_to_node_map, cpu);
318}
319EXPORT_SYMBOL(cpu_to_node);
320
321/*
322 * Same function as cpu_to_node() but used if called before the
323 * per_cpu areas are setup.
324 */
325int early_cpu_to_node(int cpu)
326{
327 if (early_per_cpu_ptr(x86_cpu_to_node_map))
328 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
329
330 if (!per_cpu_offset(cpu)) {
331 printk(KERN_WARNING
332 "early_cpu_to_node(%d): no per_cpu area!\n", cpu);
333 dump_stack();
334 return NUMA_NO_NODE;
335 }
336 return per_cpu(x86_cpu_to_node_map, cpu);
337}
338
339
340/* empty cpumask */
341static const cpumask_t cpu_mask_none;
342
343/*
344 * Returns a pointer to the bitmask of CPUs on Node 'node'.
345 */
346const cpumask_t *_node_to_cpumask_ptr(int node)
347{
348 if (node_to_cpumask_map == NULL) {
349 printk(KERN_WARNING
350 "_node_to_cpumask_ptr(%d): no node_to_cpumask_map!\n",
351 node);
352 dump_stack();
353 return (const cpumask_t *)&cpu_online_map;
354 }
355 if (node >= nr_node_ids) {
356 printk(KERN_WARNING
357 "_node_to_cpumask_ptr(%d): node > nr_node_ids(%d)\n",
358 node, nr_node_ids);
359 dump_stack();
360 return &cpu_mask_none;
361 }
362 return &node_to_cpumask_map[node];
363}
364EXPORT_SYMBOL(_node_to_cpumask_ptr);
365
366/*
367 * Returns a bitmask of CPUs on Node 'node'.
368 *
369 * Side note: this function creates the returned cpumask on the stack
370 * so with a high NR_CPUS count, excessive stack space is used. The
371 * node_to_cpumask_ptr function should be used whenever possible.
372 */
373cpumask_t node_to_cpumask(int node)
374{
375 if (node_to_cpumask_map == NULL) {
376 printk(KERN_WARNING
377 "node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
378 dump_stack();
379 return cpu_online_map;
380 }
381 if (node >= nr_node_ids) {
382 printk(KERN_WARNING
383 "node_to_cpumask(%d): node > nr_node_ids(%d)\n",
384 node, nr_node_ids);
385 dump_stack();
386 return cpu_mask_none;
387 }
388 return node_to_cpumask_map[node];
389}
390EXPORT_SYMBOL(node_to_cpumask);
391
392/*
393 * --------- end of debug versions of the numa functions ---------
394 */
395
396#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
397
398#endif /* X86_64_NUMA */
399
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index 0cb7aadc87cd..361b7a4c640c 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -121,132 +121,23 @@ static void native_smp_send_reschedule(int cpu)
121 send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); 121 send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
122} 122}
123 123
124/* 124void native_send_call_func_single_ipi(int cpu)
125 * Structure and data for smp_call_function(). This is designed to minimise
126 * static memory requirements. It also looks cleaner.
127 */
128static DEFINE_SPINLOCK(call_lock);
129
130struct call_data_struct {
131 void (*func) (void *info);
132 void *info;
133 atomic_t started;
134 atomic_t finished;
135 int wait;
136};
137
138void lock_ipi_call_lock(void)
139{ 125{
140 spin_lock_irq(&call_lock); 126 send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_SINGLE_VECTOR);
141}
142
143void unlock_ipi_call_lock(void)
144{
145 spin_unlock_irq(&call_lock);
146}
147
148static struct call_data_struct *call_data;
149
150static void __smp_call_function(void (*func) (void *info), void *info,
151 int nonatomic, int wait)
152{
153 struct call_data_struct data;
154 int cpus = num_online_cpus() - 1;
155
156 if (!cpus)
157 return;
158
159 data.func = func;
160 data.info = info;
161 atomic_set(&data.started, 0);
162 data.wait = wait;
163 if (wait)
164 atomic_set(&data.finished, 0);
165
166 call_data = &data;
167 mb();
168
169 /* Send a message to all other CPUs and wait for them to respond */
170 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
171
172 /* Wait for response */
173 while (atomic_read(&data.started) != cpus)
174 cpu_relax();
175
176 if (wait)
177 while (atomic_read(&data.finished) != cpus)
178 cpu_relax();
179} 127}
180 128
181 129void native_send_call_func_ipi(cpumask_t mask)
182/**
183 * smp_call_function_mask(): Run a function on a set of other CPUs.
184 * @mask: The set of cpus to run on. Must not include the current cpu.
185 * @func: The function to run. This must be fast and non-blocking.
186 * @info: An arbitrary pointer to pass to the function.
187 * @wait: If true, wait (atomically) until function has completed on other CPUs.
188 *
189 * Returns 0 on success, else a negative status code.
190 *
191 * If @wait is true, then returns once @func has returned; otherwise
192 * it returns just before the target cpu calls @func.
193 *
194 * You must not call this function with disabled interrupts or from a
195 * hardware interrupt handler or from a bottom half handler.
196 */
197static int
198native_smp_call_function_mask(cpumask_t mask,
199 void (*func)(void *), void *info,
200 int wait)
201{ 130{
202 struct call_data_struct data;
203 cpumask_t allbutself; 131 cpumask_t allbutself;
204 int cpus;
205
206 /* Can deadlock when called with interrupts disabled */
207 WARN_ON(irqs_disabled());
208
209 /* Holding any lock stops cpus from going down. */
210 spin_lock(&call_lock);
211 132
212 allbutself = cpu_online_map; 133 allbutself = cpu_online_map;
213 cpu_clear(smp_processor_id(), allbutself); 134 cpu_clear(smp_processor_id(), allbutself);
214 135
215 cpus_and(mask, mask, allbutself);
216 cpus = cpus_weight(mask);
217
218 if (!cpus) {
219 spin_unlock(&call_lock);
220 return 0;
221 }
222
223 data.func = func;
224 data.info = info;
225 atomic_set(&data.started, 0);
226 data.wait = wait;
227 if (wait)
228 atomic_set(&data.finished, 0);
229
230 call_data = &data;
231 wmb();
232
233 /* Send a message to other CPUs */
234 if (cpus_equal(mask, allbutself) && 136 if (cpus_equal(mask, allbutself) &&
235 cpus_equal(cpu_online_map, cpu_callout_map)) 137 cpus_equal(cpu_online_map, cpu_callout_map))
236 send_IPI_allbutself(CALL_FUNCTION_VECTOR); 138 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
237 else 139 else
238 send_IPI_mask(mask, CALL_FUNCTION_VECTOR); 140 send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
239
240 /* Wait for response */
241 while (atomic_read(&data.started) != cpus)
242 cpu_relax();
243
244 if (wait)
245 while (atomic_read(&data.finished) != cpus)
246 cpu_relax();
247 spin_unlock(&call_lock);
248
249 return 0;
250} 141}
251 142
252static void stop_this_cpu(void *dummy) 143static void stop_this_cpu(void *dummy)
@@ -268,18 +159,13 @@ static void stop_this_cpu(void *dummy)
268 159
269static void native_smp_send_stop(void) 160static void native_smp_send_stop(void)
270{ 161{
271 int nolock;
272 unsigned long flags; 162 unsigned long flags;
273 163
274 if (reboot_force) 164 if (reboot_force)
275 return; 165 return;
276 166
277 /* Don't deadlock on the call lock in panic */ 167 smp_call_function(stop_this_cpu, NULL, 0);
278 nolock = !spin_trylock(&call_lock);
279 local_irq_save(flags); 168 local_irq_save(flags);
280 __smp_call_function(stop_this_cpu, NULL, 0, 0);
281 if (!nolock)
282 spin_unlock(&call_lock);
283 disable_local_APIC(); 169 disable_local_APIC();
284 local_irq_restore(flags); 170 local_irq_restore(flags);
285} 171}
@@ -301,33 +187,28 @@ void smp_reschedule_interrupt(struct pt_regs *regs)
301 187
302void smp_call_function_interrupt(struct pt_regs *regs) 188void smp_call_function_interrupt(struct pt_regs *regs)
303{ 189{
304 void (*func) (void *info) = call_data->func;
305 void *info = call_data->info;
306 int wait = call_data->wait;
307
308 ack_APIC_irq(); 190 ack_APIC_irq();
309 /*
310 * Notify initiating CPU that I've grabbed the data and am
311 * about to execute the function
312 */
313 mb();
314 atomic_inc(&call_data->started);
315 /*
316 * At this point the info structure may be out of scope unless wait==1
317 */
318 irq_enter(); 191 irq_enter();
319 (*func)(info); 192 generic_smp_call_function_interrupt();
320#ifdef CONFIG_X86_32 193#ifdef CONFIG_X86_32
321 __get_cpu_var(irq_stat).irq_call_count++; 194 __get_cpu_var(irq_stat).irq_call_count++;
322#else 195#else
323 add_pda(irq_call_count, 1); 196 add_pda(irq_call_count, 1);
324#endif 197#endif
325 irq_exit(); 198 irq_exit();
199}
326 200
327 if (wait) { 201void smp_call_function_single_interrupt(struct pt_regs *regs)
328 mb(); 202{
329 atomic_inc(&call_data->finished); 203 ack_APIC_irq();
330 } 204 irq_enter();
205 generic_smp_call_function_single_interrupt();
206#ifdef CONFIG_X86_32
207 __get_cpu_var(irq_stat).irq_call_count++;
208#else
209 add_pda(irq_call_count, 1);
210#endif
211 irq_exit();
331} 212}
332 213
333struct smp_ops smp_ops = { 214struct smp_ops smp_ops = {
@@ -338,7 +219,8 @@ struct smp_ops smp_ops = {
338 219
339 .smp_send_stop = native_smp_send_stop, 220 .smp_send_stop = native_smp_send_stop,
340 .smp_send_reschedule = native_smp_send_reschedule, 221 .smp_send_reschedule = native_smp_send_reschedule,
341 .smp_call_function_mask = native_smp_call_function_mask, 222
223 .send_call_func_ipi = native_send_call_func_ipi,
224 .send_call_func_single_ipi = native_send_call_func_single_ipi,
342}; 225};
343EXPORT_SYMBOL_GPL(smp_ops); 226EXPORT_SYMBOL_GPL(smp_ops);
344
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 56078d61c793..687376ab07e8 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -59,7 +59,6 @@
59#include <asm/pgtable.h> 59#include <asm/pgtable.h>
60#include <asm/tlbflush.h> 60#include <asm/tlbflush.h>
61#include <asm/mtrr.h> 61#include <asm/mtrr.h>
62#include <asm/nmi.h>
63#include <asm/vmi.h> 62#include <asm/vmi.h>
64#include <asm/genapic.h> 63#include <asm/genapic.h>
65#include <linux/mc146818rtc.h> 64#include <linux/mc146818rtc.h>
@@ -68,22 +67,6 @@
68#include <mach_wakecpu.h> 67#include <mach_wakecpu.h>
69#include <smpboot_hooks.h> 68#include <smpboot_hooks.h>
70 69
71/*
72 * FIXME: For x86_64, those are defined in other files. But moving them here,
73 * would make the setup areas dependent on smp, which is a loss. When we
74 * integrate apic between arches, we can probably do a better job, but
75 * right now, they'll stay here -- glommer
76 */
77
78/* which logical CPU number maps to which CPU (physical APIC ID) */
79u16 x86_cpu_to_apicid_init[NR_CPUS] __initdata =
80 { [0 ... NR_CPUS-1] = BAD_APICID };
81void *x86_cpu_to_apicid_early_ptr;
82
83u16 x86_bios_cpu_apicid_init[NR_CPUS] __initdata
84 = { [0 ... NR_CPUS-1] = BAD_APICID };
85void *x86_bios_cpu_apicid_early_ptr;
86
87#ifdef CONFIG_X86_32 70#ifdef CONFIG_X86_32
88u8 apicid_2_node[MAX_APICID]; 71u8 apicid_2_node[MAX_APICID];
89static int low_mappings; 72static int low_mappings;
@@ -198,13 +181,12 @@ static void map_cpu_to_logical_apicid(void)
198 map_cpu_to_node(cpu, node); 181 map_cpu_to_node(cpu, node);
199} 182}
200 183
201static void unmap_cpu_to_logical_apicid(int cpu) 184void numa_remove_cpu(int cpu)
202{ 185{
203 cpu_2_logical_apicid[cpu] = BAD_APICID; 186 cpu_2_logical_apicid[cpu] = BAD_APICID;
204 unmap_cpu_to_node(cpu); 187 unmap_cpu_to_node(cpu);
205} 188}
206#else 189#else
207#define unmap_cpu_to_logical_apicid(cpu) do {} while (0)
208#define map_cpu_to_logical_apicid() do {} while (0) 190#define map_cpu_to_logical_apicid() do {} while (0)
209#endif 191#endif
210 192
@@ -345,19 +327,12 @@ static void __cpuinit start_secondary(void *unused)
345 * lock helps us to not include this cpu in a currently in progress 327 * lock helps us to not include this cpu in a currently in progress
346 * smp_call_function(). 328 * smp_call_function().
347 */ 329 */
348 lock_ipi_call_lock(); 330 ipi_call_lock_irq();
349#ifdef CONFIG_X86_64 331#ifdef CONFIG_X86_IO_APIC
350 spin_lock(&vector_lock); 332 setup_vector_irq(smp_processor_id());
351
352 /* Setup the per cpu irq handling data structures */
353 __setup_vector_irq(smp_processor_id());
354 /*
355 * Allow the master to continue.
356 */
357 spin_unlock(&vector_lock);
358#endif 333#endif
359 cpu_set(smp_processor_id(), cpu_online_map); 334 cpu_set(smp_processor_id(), cpu_online_map);
360 unlock_ipi_call_lock(); 335 ipi_call_unlock_irq();
361 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 336 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
362 337
363 setup_secondary_clock(); 338 setup_secondary_clock();
@@ -366,31 +341,8 @@ static void __cpuinit start_secondary(void *unused)
366 cpu_idle(); 341 cpu_idle();
367} 342}
368 343
369#ifdef CONFIG_X86_32
370/*
371 * Everything has been set up for the secondary
372 * CPUs - they just need to reload everything
373 * from the task structure
374 * This function must not return.
375 */
376void __devinit initialize_secondary(void)
377{
378 /*
379 * We don't actually need to load the full TSS,
380 * basically just the stack pointer and the ip.
381 */
382
383 asm volatile(
384 "movl %0,%%esp\n\t"
385 "jmp *%1"
386 :
387 :"m" (current->thread.sp), "m" (current->thread.ip));
388}
389#endif
390
391static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c) 344static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c)
392{ 345{
393#ifdef CONFIG_X86_32
394 /* 346 /*
395 * Mask B, Pentium, but not Pentium MMX 347 * Mask B, Pentium, but not Pentium MMX
396 */ 348 */
@@ -440,7 +392,6 @@ static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c)
440 392
441valid_k7: 393valid_k7:
442 ; 394 ;
443#endif
444} 395}
445 396
446static void __cpuinit smp_checks(void) 397static void __cpuinit smp_checks(void)
@@ -555,23 +506,6 @@ cpumask_t cpu_coregroup_map(int cpu)
555 return c->llc_shared_map; 506 return c->llc_shared_map;
556} 507}
557 508
558#ifdef CONFIG_X86_32
559/*
560 * We are called very early to get the low memory for the
561 * SMP bootup trampoline page.
562 */
563void __init smp_alloc_memory(void)
564{
565 trampoline_base = alloc_bootmem_low_pages(PAGE_SIZE);
566 /*
567 * Has to be in very low memory so we can execute
568 * real-mode AP code.
569 */
570 if (__pa(trampoline_base) >= 0x9F000)
571 BUG();
572}
573#endif
574
575static void impress_friends(void) 509static void impress_friends(void)
576{ 510{
577 int cpu; 511 int cpu;
@@ -748,11 +682,7 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
748 * target processor state. 682 * target processor state.
749 */ 683 */
750 startup_ipi_hook(phys_apicid, (unsigned long) start_secondary, 684 startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
751#ifdef CONFIG_X86_64
752 (unsigned long)init_rsp);
753#else
754 (unsigned long)stack_start.sp); 685 (unsigned long)stack_start.sp);
755#endif
756 686
757 /* 687 /*
758 * Run STARTUP IPI loop. 688 * Run STARTUP IPI loop.
@@ -832,6 +762,45 @@ static void __cpuinit do_fork_idle(struct work_struct *work)
832 complete(&c_idle->done); 762 complete(&c_idle->done);
833} 763}
834 764
765#ifdef CONFIG_X86_64
766/*
767 * Allocate node local memory for the AP pda.
768 *
769 * Must be called after the _cpu_pda pointer table is initialized.
770 */
771static int __cpuinit get_local_pda(int cpu)
772{
773 struct x8664_pda *oldpda, *newpda;
774 unsigned long size = sizeof(struct x8664_pda);
775 int node = cpu_to_node(cpu);
776
777 if (cpu_pda(cpu) && !cpu_pda(cpu)->in_bootmem)
778 return 0;
779
780 oldpda = cpu_pda(cpu);
781 newpda = kmalloc_node(size, GFP_ATOMIC, node);
782 if (!newpda) {
783 printk(KERN_ERR "Could not allocate node local PDA "
784 "for CPU %d on node %d\n", cpu, node);
785
786 if (oldpda)
787 return 0; /* have a usable pda */
788 else
789 return -1;
790 }
791
792 if (oldpda) {
793 memcpy(newpda, oldpda, size);
794 if (!after_bootmem)
795 free_bootmem((unsigned long)oldpda, size);
796 }
797
798 newpda->in_bootmem = 0;
799 cpu_pda(cpu) = newpda;
800 return 0;
801}
802#endif /* CONFIG_X86_64 */
803
835static int __cpuinit do_boot_cpu(int apicid, int cpu) 804static int __cpuinit do_boot_cpu(int apicid, int cpu)
836/* 805/*
837 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad 806 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
@@ -848,28 +817,14 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
848 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done), 817 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
849 }; 818 };
850 INIT_WORK(&c_idle.work, do_fork_idle); 819 INIT_WORK(&c_idle.work, do_fork_idle);
851#ifdef CONFIG_X86_64
852 /* allocate memory for gdts of secondary cpus. Hotplug is considered */
853 if (!cpu_gdt_descr[cpu].address &&
854 !(cpu_gdt_descr[cpu].address = get_zeroed_page(GFP_KERNEL))) {
855 printk(KERN_ERR "Failed to allocate GDT for CPU %d\n", cpu);
856 return -1;
857 }
858 820
821#ifdef CONFIG_X86_64
859 /* Allocate node local memory for AP pdas */ 822 /* Allocate node local memory for AP pdas */
860 if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) { 823 if (cpu > 0) {
861 struct x8664_pda *newpda, *pda; 824 boot_error = get_local_pda(cpu);
862 int node = cpu_to_node(cpu); 825 if (boot_error)
863 pda = cpu_pda(cpu); 826 goto restore_state;
864 newpda = kmalloc_node(sizeof(struct x8664_pda), GFP_ATOMIC, 827 /* if can't get pda memory, can't start cpu */
865 node);
866 if (newpda) {
867 memcpy(newpda, pda, sizeof(struct x8664_pda));
868 cpu_pda(cpu) = newpda;
869 } else
870 printk(KERN_ERR
871 "Could not allocate node local PDA for CPU %d on node %d\n",
872 cpu, node);
873 } 828 }
874#endif 829#endif
875 830
@@ -905,18 +860,15 @@ do_rest:
905#ifdef CONFIG_X86_32 860#ifdef CONFIG_X86_32
906 per_cpu(current_task, cpu) = c_idle.idle; 861 per_cpu(current_task, cpu) = c_idle.idle;
907 init_gdt(cpu); 862 init_gdt(cpu);
908 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
909 c_idle.idle->thread.ip = (unsigned long) start_secondary;
910 /* Stack for startup_32 can be just as for start_secondary onwards */ 863 /* Stack for startup_32 can be just as for start_secondary onwards */
911 stack_start.sp = (void *) c_idle.idle->thread.sp;
912 irq_ctx_init(cpu); 864 irq_ctx_init(cpu);
913#else 865#else
914 cpu_pda(cpu)->pcurrent = c_idle.idle; 866 cpu_pda(cpu)->pcurrent = c_idle.idle;
915 init_rsp = c_idle.idle->thread.sp;
916 load_sp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
917 initial_code = (unsigned long)start_secondary;
918 clear_tsk_thread_flag(c_idle.idle, TIF_FORK); 867 clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
919#endif 868#endif
869 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
870 initial_code = (unsigned long)start_secondary;
871 stack_start.sp = (void *) c_idle.idle->thread.sp;
920 872
921 /* start_ip had better be page-aligned! */ 873 /* start_ip had better be page-aligned! */
922 start_ip = setup_trampoline(); 874 start_ip = setup_trampoline();
@@ -987,16 +939,14 @@ do_rest:
987 inquire_remote_apic(apicid); 939 inquire_remote_apic(apicid);
988 } 940 }
989 } 941 }
990
991 if (boot_error) {
992 /* Try to put things back the way they were before ... */
993 unmap_cpu_to_logical_apicid(cpu);
994#ifdef CONFIG_X86_64 942#ifdef CONFIG_X86_64
995 clear_node_cpumask(cpu); /* was set by numa_add_cpu */ 943restore_state:
996#endif 944#endif
945 if (boot_error) {
946 /* Try to put things back the way they were before ... */
947 numa_remove_cpu(cpu); /* was set by numa_add_cpu */
997 cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */ 948 cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */
998 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */ 949 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
999 cpu_clear(cpu, cpu_possible_map);
1000 cpu_clear(cpu, cpu_present_map); 950 cpu_clear(cpu, cpu_present_map);
1001 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID; 951 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
1002 } 952 }
@@ -1088,14 +1038,12 @@ static __init void disable_smp(void)
1088{ 1038{
1089 cpu_present_map = cpumask_of_cpu(0); 1039 cpu_present_map = cpumask_of_cpu(0);
1090 cpu_possible_map = cpumask_of_cpu(0); 1040 cpu_possible_map = cpumask_of_cpu(0);
1091#ifdef CONFIG_X86_32
1092 smpboot_clear_io_apic_irqs(); 1041 smpboot_clear_io_apic_irqs();
1093#endif 1042
1094 if (smp_found_config) 1043 if (smp_found_config)
1095 phys_cpu_present_map = 1044 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1096 physid_mask_of_physid(boot_cpu_physical_apicid);
1097 else 1045 else
1098 phys_cpu_present_map = physid_mask_of_physid(0); 1046 physid_set_mask_of_physid(0, &phys_cpu_present_map);
1099 map_cpu_to_logical_apicid(); 1047 map_cpu_to_logical_apicid();
1100 cpu_set(0, per_cpu(cpu_sibling_map, 0)); 1048 cpu_set(0, per_cpu(cpu_sibling_map, 0));
1101 cpu_set(0, per_cpu(cpu_core_map, 0)); 1049 cpu_set(0, per_cpu(cpu_core_map, 0));
@@ -1158,12 +1106,12 @@ static int __init smp_sanity_check(unsigned max_cpus)
1158 * If SMP should be disabled, then really disable it! 1106 * If SMP should be disabled, then really disable it!
1159 */ 1107 */
1160 if (!max_cpus) { 1108 if (!max_cpus) {
1161 printk(KERN_INFO "SMP mode deactivated," 1109 printk(KERN_INFO "SMP mode deactivated.\n");
1162 "forcing use of dummy APIC emulation.\n");
1163 smpboot_clear_io_apic(); 1110 smpboot_clear_io_apic();
1164#ifdef CONFIG_X86_32 1111
1112 localise_nmi_watchdog();
1113
1165 connect_bsp_APIC(); 1114 connect_bsp_APIC();
1166#endif
1167 setup_local_APIC(); 1115 setup_local_APIC();
1168 end_local_APIC_setup(); 1116 end_local_APIC_setup();
1169 return -1; 1117 return -1;
@@ -1191,7 +1139,6 @@ static void __init smp_cpu_index_default(void)
1191void __init native_smp_prepare_cpus(unsigned int max_cpus) 1139void __init native_smp_prepare_cpus(unsigned int max_cpus)
1192{ 1140{
1193 preempt_disable(); 1141 preempt_disable();
1194 nmi_watchdog_default();
1195 smp_cpu_index_default(); 1142 smp_cpu_index_default();
1196 current_cpu_data = boot_cpu_data; 1143 current_cpu_data = boot_cpu_data;
1197 cpu_callin_map = cpumask_of_cpu(0); 1144 cpu_callin_map = cpumask_of_cpu(0);
@@ -1218,9 +1165,8 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
1218 } 1165 }
1219 preempt_enable(); 1166 preempt_enable();
1220 1167
1221#ifdef CONFIG_X86_32
1222 connect_bsp_APIC(); 1168 connect_bsp_APIC();
1223#endif 1169
1224 /* 1170 /*
1225 * Switch from PIC to APIC mode. 1171 * Switch from PIC to APIC mode.
1226 */ 1172 */
@@ -1258,8 +1204,8 @@ void __init native_smp_prepare_boot_cpu(void)
1258 int me = smp_processor_id(); 1204 int me = smp_processor_id();
1259#ifdef CONFIG_X86_32 1205#ifdef CONFIG_X86_32
1260 init_gdt(me); 1206 init_gdt(me);
1261 switch_to_new_gdt();
1262#endif 1207#endif
1208 switch_to_new_gdt();
1263 /* already set me in cpu_online_map in boot_cpu_init() */ 1209 /* already set me in cpu_online_map in boot_cpu_init() */
1264 cpu_set(me, cpu_callout_map); 1210 cpu_set(me, cpu_callout_map);
1265 per_cpu(cpu_state, me) = CPU_ONLINE; 1211 per_cpu(cpu_state, me) = CPU_ONLINE;
@@ -1279,23 +1225,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
1279 1225
1280#ifdef CONFIG_HOTPLUG_CPU 1226#ifdef CONFIG_HOTPLUG_CPU
1281 1227
1282# ifdef CONFIG_X86_32
1283void cpu_exit_clear(void)
1284{
1285 int cpu = raw_smp_processor_id();
1286
1287 idle_task_exit();
1288
1289 cpu_uninit();
1290 irq_ctx_exit(cpu);
1291
1292 cpu_clear(cpu, cpu_callout_map);
1293 cpu_clear(cpu, cpu_callin_map);
1294
1295 unmap_cpu_to_logical_apicid(cpu);
1296}
1297# endif /* CONFIG_X86_32 */
1298
1299static void remove_siblinginfo(int cpu) 1228static void remove_siblinginfo(int cpu)
1300{ 1229{
1301 int sibling; 1230 int sibling;
@@ -1349,12 +1278,20 @@ __init void prefill_possible_map(void)
1349 int i; 1278 int i;
1350 int possible; 1279 int possible;
1351 1280
1281 /* no processor from mptable or madt */
1282 if (!num_processors)
1283 num_processors = 1;
1284
1285#ifdef CONFIG_HOTPLUG_CPU
1352 if (additional_cpus == -1) { 1286 if (additional_cpus == -1) {
1353 if (disabled_cpus > 0) 1287 if (disabled_cpus > 0)
1354 additional_cpus = disabled_cpus; 1288 additional_cpus = disabled_cpus;
1355 else 1289 else
1356 additional_cpus = 0; 1290 additional_cpus = 0;
1357 } 1291 }
1292#else
1293 additional_cpus = 0;
1294#endif
1358 possible = num_processors + additional_cpus; 1295 possible = num_processors + additional_cpus;
1359 if (possible > NR_CPUS) 1296 if (possible > NR_CPUS)
1360 possible = NR_CPUS; 1297 possible = NR_CPUS;
@@ -1364,18 +1301,18 @@ __init void prefill_possible_map(void)
1364 1301
1365 for (i = 0; i < possible; i++) 1302 for (i = 0; i < possible; i++)
1366 cpu_set(i, cpu_possible_map); 1303 cpu_set(i, cpu_possible_map);
1304
1305 nr_cpu_ids = possible;
1367} 1306}
1368 1307
1369static void __ref remove_cpu_from_maps(int cpu) 1308static void __ref remove_cpu_from_maps(int cpu)
1370{ 1309{
1371 cpu_clear(cpu, cpu_online_map); 1310 cpu_clear(cpu, cpu_online_map);
1372#ifdef CONFIG_X86_64
1373 cpu_clear(cpu, cpu_callout_map); 1311 cpu_clear(cpu, cpu_callout_map);
1374 cpu_clear(cpu, cpu_callin_map); 1312 cpu_clear(cpu, cpu_callin_map);
1375 /* was set by cpu_init() */ 1313 /* was set by cpu_init() */
1376 clear_bit(cpu, (unsigned long *)&cpu_initialized); 1314 clear_bit(cpu, (unsigned long *)&cpu_initialized);
1377 clear_node_cpumask(cpu); 1315 numa_remove_cpu(cpu);
1378#endif
1379} 1316}
1380 1317
1381int __cpu_disable(void) 1318int __cpu_disable(void)
diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c
index 3449064d141a..99941b37eca0 100644
--- a/arch/x86/kernel/smpcommon.c
+++ b/arch/x86/kernel/smpcommon.c
@@ -25,59 +25,3 @@ __cpuinit void init_gdt(int cpu)
25 per_cpu(cpu_number, cpu) = cpu; 25 per_cpu(cpu_number, cpu) = cpu;
26} 26}
27#endif 27#endif
28
29/**
30 * smp_call_function(): Run a function on all other CPUs.
31 * @func: The function to run. This must be fast and non-blocking.
32 * @info: An arbitrary pointer to pass to the function.
33 * @nonatomic: Unused.
34 * @wait: If true, wait (atomically) until function has completed on other CPUs.
35 *
36 * Returns 0 on success, else a negative status code.
37 *
38 * If @wait is true, then returns once @func has returned; otherwise
39 * it returns just before the target cpu calls @func.
40 *
41 * You must not call this function with disabled interrupts or from a
42 * hardware interrupt handler or from a bottom half handler.
43 */
44int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
45 int wait)
46{
47 return smp_call_function_mask(cpu_online_map, func, info, wait);
48}
49EXPORT_SYMBOL(smp_call_function);
50
51/**
52 * smp_call_function_single - Run a function on a specific CPU
53 * @cpu: The target CPU. Cannot be the calling CPU.
54 * @func: The function to run. This must be fast and non-blocking.
55 * @info: An arbitrary pointer to pass to the function.
56 * @nonatomic: Unused.
57 * @wait: If true, wait until function has completed on other CPUs.
58 *
59 * Returns 0 on success, else a negative status code.
60 *
61 * If @wait is true, then returns once @func has returned; otherwise
62 * it returns just before the target cpu calls @func.
63 */
64int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
65 int nonatomic, int wait)
66{
67 /* prevent preemption and reschedule on another processor */
68 int ret;
69 int me = get_cpu();
70 if (cpu == me) {
71 local_irq_disable();
72 func(info);
73 local_irq_enable();
74 put_cpu();
75 return 0;
76 }
77
78 ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
79
80 put_cpu();
81 return ret;
82}
83EXPORT_SYMBOL(smp_call_function_single);
diff --git a/arch/x86/kernel/srat_32.c b/arch/x86/kernel/srat_32.c
deleted file mode 100644
index 70e4a374b4e8..000000000000
--- a/arch/x86/kernel/srat_32.c
+++ /dev/null
@@ -1,358 +0,0 @@
1/*
2 * Some of the code in this file has been gleaned from the 64 bit
3 * discontigmem support code base.
4 *
5 * Copyright (C) 2002, IBM Corp.
6 *
7 * All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
17 * NON INFRINGEMENT. See the GNU General Public License for more
18 * details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 *
24 * Send feedback to Pat Gaughen <gone@us.ibm.com>
25 */
26#include <linux/mm.h>
27#include <linux/bootmem.h>
28#include <linux/mmzone.h>
29#include <linux/acpi.h>
30#include <linux/nodemask.h>
31#include <asm/srat.h>
32#include <asm/topology.h>
33#include <asm/smp.h>
34
35/*
36 * proximity macros and definitions
37 */
38#define NODE_ARRAY_INDEX(x) ((x) / 8) /* 8 bits/char */
39#define NODE_ARRAY_OFFSET(x) ((x) % 8) /* 8 bits/char */
40#define BMAP_SET(bmap, bit) ((bmap)[NODE_ARRAY_INDEX(bit)] |= 1 << NODE_ARRAY_OFFSET(bit))
41#define BMAP_TEST(bmap, bit) ((bmap)[NODE_ARRAY_INDEX(bit)] & (1 << NODE_ARRAY_OFFSET(bit)))
42/* bitmap length; _PXM is at most 255 */
43#define PXM_BITMAP_LEN (MAX_PXM_DOMAINS / 8)
44static u8 pxm_bitmap[PXM_BITMAP_LEN]; /* bitmap of proximity domains */
45
46#define MAX_CHUNKS_PER_NODE 3
47#define MAXCHUNKS (MAX_CHUNKS_PER_NODE * MAX_NUMNODES)
48struct node_memory_chunk_s {
49 unsigned long start_pfn;
50 unsigned long end_pfn;
51 u8 pxm; // proximity domain of node
52 u8 nid; // which cnode contains this chunk?
53 u8 bank; // which mem bank on this node
54};
55static struct node_memory_chunk_s node_memory_chunk[MAXCHUNKS];
56
57static int num_memory_chunks; /* total number of memory chunks */
58static u8 __initdata apicid_to_pxm[MAX_APICID];
59
60/* Identify CPU proximity domains */
61static void __init parse_cpu_affinity_structure(char *p)
62{
63 struct acpi_srat_cpu_affinity *cpu_affinity =
64 (struct acpi_srat_cpu_affinity *) p;
65
66 if ((cpu_affinity->flags & ACPI_SRAT_CPU_ENABLED) == 0)
67 return; /* empty entry */
68
69 /* mark this node as "seen" in node bitmap */
70 BMAP_SET(pxm_bitmap, cpu_affinity->proximity_domain_lo);
71
72 apicid_to_pxm[cpu_affinity->apic_id] = cpu_affinity->proximity_domain_lo;
73
74 printk("CPU 0x%02X in proximity domain 0x%02X\n",
75 cpu_affinity->apic_id, cpu_affinity->proximity_domain_lo);
76}
77
78/*
79 * Identify memory proximity domains and hot-remove capabilities.
80 * Fill node memory chunk list structure.
81 */
82static void __init parse_memory_affinity_structure (char *sratp)
83{
84 unsigned long long paddr, size;
85 unsigned long start_pfn, end_pfn;
86 u8 pxm;
87 struct node_memory_chunk_s *p, *q, *pend;
88 struct acpi_srat_mem_affinity *memory_affinity =
89 (struct acpi_srat_mem_affinity *) sratp;
90
91 if ((memory_affinity->flags & ACPI_SRAT_MEM_ENABLED) == 0)
92 return; /* empty entry */
93
94 pxm = memory_affinity->proximity_domain & 0xff;
95
96 /* mark this node as "seen" in node bitmap */
97 BMAP_SET(pxm_bitmap, pxm);
98
99 /* calculate info for memory chunk structure */
100 paddr = memory_affinity->base_address;
101 size = memory_affinity->length;
102
103 start_pfn = paddr >> PAGE_SHIFT;
104 end_pfn = (paddr + size) >> PAGE_SHIFT;
105
106
107 if (num_memory_chunks >= MAXCHUNKS) {
108 printk("Too many mem chunks in SRAT. Ignoring %lld MBytes at %llx\n",
109 size/(1024*1024), paddr);
110 return;
111 }
112
113 /* Insertion sort based on base address */
114 pend = &node_memory_chunk[num_memory_chunks];
115 for (p = &node_memory_chunk[0]; p < pend; p++) {
116 if (start_pfn < p->start_pfn)
117 break;
118 }
119 if (p < pend) {
120 for (q = pend; q >= p; q--)
121 *(q + 1) = *q;
122 }
123 p->start_pfn = start_pfn;
124 p->end_pfn = end_pfn;
125 p->pxm = pxm;
126
127 num_memory_chunks++;
128
129 printk("Memory range 0x%lX to 0x%lX (type 0x%X) in proximity domain 0x%02X %s\n",
130 start_pfn, end_pfn,
131 memory_affinity->memory_type,
132 pxm,
133 ((memory_affinity->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ?
134 "enabled and removable" : "enabled" ) );
135}
136
137/*
138 * The SRAT table always lists ascending addresses, so can always
139 * assume that the first "start" address that you see is the real
140 * start of the node, and that the current "end" address is after
141 * the previous one.
142 */
143static __init void node_read_chunk(int nid, struct node_memory_chunk_s *memory_chunk)
144{
145 /*
146 * Only add present memory as told by the e820.
147 * There is no guarantee from the SRAT that the memory it
148 * enumerates is present at boot time because it represents
149 * *possible* memory hotplug areas the same as normal RAM.
150 */
151 if (memory_chunk->start_pfn >= max_pfn) {
152 printk (KERN_INFO "Ignoring SRAT pfns: 0x%08lx -> %08lx\n",
153 memory_chunk->start_pfn, memory_chunk->end_pfn);
154 return;
155 }
156 if (memory_chunk->nid != nid)
157 return;
158
159 if (!node_has_online_mem(nid))
160 node_start_pfn[nid] = memory_chunk->start_pfn;
161
162 if (node_start_pfn[nid] > memory_chunk->start_pfn)
163 node_start_pfn[nid] = memory_chunk->start_pfn;
164
165 if (node_end_pfn[nid] < memory_chunk->end_pfn)
166 node_end_pfn[nid] = memory_chunk->end_pfn;
167}
168
169/* Parse the ACPI Static Resource Affinity Table */
170static int __init acpi20_parse_srat(struct acpi_table_srat *sratp)
171{
172 u8 *start, *end, *p;
173 int i, j, nid;
174
175 start = (u8 *)(&(sratp->reserved) + 1); /* skip header */
176 p = start;
177 end = (u8 *)sratp + sratp->header.length;
178
179 memset(pxm_bitmap, 0, sizeof(pxm_bitmap)); /* init proximity domain bitmap */
180 memset(node_memory_chunk, 0, sizeof(node_memory_chunk));
181
182 num_memory_chunks = 0;
183 while (p < end) {
184 switch (*p) {
185 case ACPI_SRAT_TYPE_CPU_AFFINITY:
186 parse_cpu_affinity_structure(p);
187 break;
188 case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
189 parse_memory_affinity_structure(p);
190 break;
191 default:
192 printk("ACPI 2.0 SRAT: unknown entry skipped: type=0x%02X, len=%d\n", p[0], p[1]);
193 break;
194 }
195 p += p[1];
196 if (p[1] == 0) {
197 printk("acpi20_parse_srat: Entry length value is zero;"
198 " can't parse any further!\n");
199 break;
200 }
201 }
202
203 if (num_memory_chunks == 0) {
204 printk("could not finy any ACPI SRAT memory areas.\n");
205 goto out_fail;
206 }
207
208 /* Calculate total number of nodes in system from PXM bitmap and create
209 * a set of sequential node IDs starting at zero. (ACPI doesn't seem
210 * to specify the range of _PXM values.)
211 */
212 /*
213 * MCD - we no longer HAVE to number nodes sequentially. PXM domain
214 * numbers could go as high as 256, and MAX_NUMNODES for i386 is typically
215 * 32, so we will continue numbering them in this manner until MAX_NUMNODES
216 * approaches MAX_PXM_DOMAINS for i386.
217 */
218 nodes_clear(node_online_map);
219 for (i = 0; i < MAX_PXM_DOMAINS; i++) {
220 if (BMAP_TEST(pxm_bitmap, i)) {
221 int nid = acpi_map_pxm_to_node(i);
222 node_set_online(nid);
223 }
224 }
225 BUG_ON(num_online_nodes() == 0);
226
227 /* set cnode id in memory chunk structure */
228 for (i = 0; i < num_memory_chunks; i++)
229 node_memory_chunk[i].nid = pxm_to_node(node_memory_chunk[i].pxm);
230
231 printk("pxm bitmap: ");
232 for (i = 0; i < sizeof(pxm_bitmap); i++) {
233 printk("%02X ", pxm_bitmap[i]);
234 }
235 printk("\n");
236 printk("Number of logical nodes in system = %d\n", num_online_nodes());
237 printk("Number of memory chunks in system = %d\n", num_memory_chunks);
238
239 for (i = 0; i < MAX_APICID; i++)
240 apicid_2_node[i] = pxm_to_node(apicid_to_pxm[i]);
241
242 for (j = 0; j < num_memory_chunks; j++){
243 struct node_memory_chunk_s * chunk = &node_memory_chunk[j];
244 printk("chunk %d nid %d start_pfn %08lx end_pfn %08lx\n",
245 j, chunk->nid, chunk->start_pfn, chunk->end_pfn);
246 node_read_chunk(chunk->nid, chunk);
247 add_active_range(chunk->nid, chunk->start_pfn, chunk->end_pfn);
248 }
249
250 for_each_online_node(nid) {
251 unsigned long start = node_start_pfn[nid];
252 unsigned long end = node_end_pfn[nid];
253
254 memory_present(nid, start, end);
255 node_remap_size[nid] = node_memmap_size_bytes(nid, start, end);
256 }
257 return 1;
258out_fail:
259 return 0;
260}
261
262struct acpi_static_rsdt {
263 struct acpi_table_rsdt table;
264 u32 padding[7]; /* Allow for 7 more table entries */
265};
266
267int __init get_memcfg_from_srat(void)
268{
269 struct acpi_table_header *header = NULL;
270 struct acpi_table_rsdp *rsdp = NULL;
271 struct acpi_table_rsdt *rsdt = NULL;
272 acpi_native_uint rsdp_address = 0;
273 struct acpi_static_rsdt saved_rsdt;
274 int tables = 0;
275 int i = 0;
276
277 rsdp_address = acpi_os_get_root_pointer();
278 if (!rsdp_address) {
279 printk("%s: System description tables not found\n",
280 __func__);
281 goto out_err;
282 }
283
284 printk("%s: assigning address to rsdp\n", __func__);
285 rsdp = (struct acpi_table_rsdp *)(u32)rsdp_address;
286 if (!rsdp) {
287 printk("%s: Didn't find ACPI root!\n", __func__);
288 goto out_err;
289 }
290
291 printk(KERN_INFO "%.8s v%d [%.6s]\n", rsdp->signature, rsdp->revision,
292 rsdp->oem_id);
293
294 if (strncmp(rsdp->signature, ACPI_SIG_RSDP,strlen(ACPI_SIG_RSDP))) {
295 printk(KERN_WARNING "%s: RSDP table signature incorrect\n", __func__);
296 goto out_err;
297 }
298
299 rsdt = (struct acpi_table_rsdt *)
300 early_ioremap(rsdp->rsdt_physical_address, sizeof(struct acpi_table_rsdt));
301
302 if (!rsdt) {
303 printk(KERN_WARNING
304 "%s: ACPI: Invalid root system description tables (RSDT)\n",
305 __func__);
306 goto out_err;
307 }
308
309 header = &rsdt->header;
310
311 if (strncmp(header->signature, ACPI_SIG_RSDT, strlen(ACPI_SIG_RSDT))) {
312 printk(KERN_WARNING "ACPI: RSDT signature incorrect\n");
313 goto out_err;
314 }
315
316 /*
317 * The number of tables is computed by taking the
318 * size of all entries (header size minus total
319 * size of RSDT) divided by the size of each entry
320 * (4-byte table pointers).
321 */
322 tables = (header->length - sizeof(struct acpi_table_header)) / 4;
323
324 if (!tables)
325 goto out_err;
326
327 memcpy(&saved_rsdt, rsdt, sizeof(saved_rsdt));
328
329 if (saved_rsdt.table.header.length > sizeof(saved_rsdt)) {
330 printk(KERN_WARNING "ACPI: Too big length in RSDT: %d\n",
331 saved_rsdt.table.header.length);
332 goto out_err;
333 }
334
335 printk("Begin SRAT table scan....\n");
336
337 for (i = 0; i < tables; i++) {
338 /* Map in header, then map in full table length. */
339 header = (struct acpi_table_header *)
340 early_ioremap(saved_rsdt.table.table_offset_entry[i], sizeof(struct acpi_table_header));
341 if (!header)
342 break;
343 header = (struct acpi_table_header *)
344 early_ioremap(saved_rsdt.table.table_offset_entry[i], header->length);
345 if (!header)
346 break;
347
348 if (strncmp((char *) &header->signature, ACPI_SIG_SRAT, 4))
349 continue;
350
351 /* we've found the srat table. don't need to look at any more tables */
352 return acpi20_parse_srat((struct acpi_table_srat *)header);
353 }
354out_err:
355 remove_all_active_ranges();
356 printk("failed to get NUMA memory information from SRAT table\n");
357 return 0;
358}
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index c28c342c162f..a03e7f6d90c3 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -74,6 +74,7 @@ void save_stack_trace(struct stack_trace *trace)
74 if (trace->nr_entries < trace->max_entries) 74 if (trace->nr_entries < trace->max_entries)
75 trace->entries[trace->nr_entries++] = ULONG_MAX; 75 trace->entries[trace->nr_entries++] = ULONG_MAX;
76} 76}
77EXPORT_SYMBOL_GPL(save_stack_trace);
77 78
78void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) 79void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
79{ 80{
@@ -81,3 +82,4 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
81 if (trace->nr_entries < trace->max_entries) 82 if (trace->nr_entries < trace->max_entries)
82 trace->entries[trace->nr_entries++] = ULONG_MAX; 83 trace->entries[trace->nr_entries++] = ULONG_MAX;
83} 84}
85EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
diff --git a/arch/x86/kernel/summit_32.c b/arch/x86/kernel/summit_32.c
index ae751094eba9..d67ce5f044ba 100644
--- a/arch/x86/kernel/summit_32.c
+++ b/arch/x86/kernel/summit_32.c
@@ -36,7 +36,9 @@ static struct rio_table_hdr *rio_table_hdr __initdata;
36static struct scal_detail *scal_devs[MAX_NUMNODES] __initdata; 36static struct scal_detail *scal_devs[MAX_NUMNODES] __initdata;
37static struct rio_detail *rio_devs[MAX_NUMNODES*4] __initdata; 37static struct rio_detail *rio_devs[MAX_NUMNODES*4] __initdata;
38 38
39#ifndef CONFIG_X86_NUMAQ
39static int mp_bus_id_to_node[MAX_MP_BUSSES] __initdata; 40static int mp_bus_id_to_node[MAX_MP_BUSSES] __initdata;
41#endif
40 42
41static int __init setup_pci_node_map_for_wpeg(int wpeg_num, int last_bus) 43static int __init setup_pci_node_map_for_wpeg(int wpeg_num, int last_bus)
42{ 44{
diff --git a/arch/x86/kernel/sys_i386_32.c b/arch/x86/kernel/sys_i386_32.c
index d2ab52cc1d6b..7066cb855a60 100644
--- a/arch/x86/kernel/sys_i386_32.c
+++ b/arch/x86/kernel/sys_i386_32.c
@@ -19,8 +19,8 @@
19#include <linux/utsname.h> 19#include <linux/utsname.h>
20#include <linux/ipc.h> 20#include <linux/ipc.h>
21 21
22#include <asm/uaccess.h> 22#include <linux/uaccess.h>
23#include <asm/unistd.h> 23#include <linux/unistd.h>
24 24
25asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, 25asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
26 unsigned long prot, unsigned long flags, 26 unsigned long prot, unsigned long flags,
@@ -103,7 +103,7 @@ asmlinkage int old_select(struct sel_arg_struct __user *arg)
103 * 103 *
104 * This is really horribly ugly. 104 * This is really horribly ugly.
105 */ 105 */
106asmlinkage int sys_ipc (uint call, int first, int second, 106asmlinkage int sys_ipc(uint call, int first, int second,
107 int third, void __user *ptr, long fifth) 107 int third, void __user *ptr, long fifth)
108{ 108{
109 int version, ret; 109 int version, ret;
@@ -113,24 +113,24 @@ asmlinkage int sys_ipc (uint call, int first, int second,
113 113
114 switch (call) { 114 switch (call) {
115 case SEMOP: 115 case SEMOP:
116 return sys_semtimedop (first, (struct sembuf __user *)ptr, second, NULL); 116 return sys_semtimedop(first, (struct sembuf __user *)ptr, second, NULL);
117 case SEMTIMEDOP: 117 case SEMTIMEDOP:
118 return sys_semtimedop(first, (struct sembuf __user *)ptr, second, 118 return sys_semtimedop(first, (struct sembuf __user *)ptr, second,
119 (const struct timespec __user *)fifth); 119 (const struct timespec __user *)fifth);
120 120
121 case SEMGET: 121 case SEMGET:
122 return sys_semget (first, second, third); 122 return sys_semget(first, second, third);
123 case SEMCTL: { 123 case SEMCTL: {
124 union semun fourth; 124 union semun fourth;
125 if (!ptr) 125 if (!ptr)
126 return -EINVAL; 126 return -EINVAL;
127 if (get_user(fourth.__pad, (void __user * __user *) ptr)) 127 if (get_user(fourth.__pad, (void __user * __user *) ptr))
128 return -EFAULT; 128 return -EFAULT;
129 return sys_semctl (first, second, third, fourth); 129 return sys_semctl(first, second, third, fourth);
130 } 130 }
131 131
132 case MSGSND: 132 case MSGSND:
133 return sys_msgsnd (first, (struct msgbuf __user *) ptr, 133 return sys_msgsnd(first, (struct msgbuf __user *) ptr,
134 second, third); 134 second, third);
135 case MSGRCV: 135 case MSGRCV:
136 switch (version) { 136 switch (version) {
@@ -138,45 +138,45 @@ asmlinkage int sys_ipc (uint call, int first, int second,
138 struct ipc_kludge tmp; 138 struct ipc_kludge tmp;
139 if (!ptr) 139 if (!ptr)
140 return -EINVAL; 140 return -EINVAL;
141 141
142 if (copy_from_user(&tmp, 142 if (copy_from_user(&tmp,
143 (struct ipc_kludge __user *) ptr, 143 (struct ipc_kludge __user *) ptr,
144 sizeof (tmp))) 144 sizeof(tmp)))
145 return -EFAULT; 145 return -EFAULT;
146 return sys_msgrcv (first, tmp.msgp, second, 146 return sys_msgrcv(first, tmp.msgp, second,
147 tmp.msgtyp, third); 147 tmp.msgtyp, third);
148 } 148 }
149 default: 149 default:
150 return sys_msgrcv (first, 150 return sys_msgrcv(first,
151 (struct msgbuf __user *) ptr, 151 (struct msgbuf __user *) ptr,
152 second, fifth, third); 152 second, fifth, third);
153 } 153 }
154 case MSGGET: 154 case MSGGET:
155 return sys_msgget ((key_t) first, second); 155 return sys_msgget((key_t) first, second);
156 case MSGCTL: 156 case MSGCTL:
157 return sys_msgctl (first, second, (struct msqid_ds __user *) ptr); 157 return sys_msgctl(first, second, (struct msqid_ds __user *) ptr);
158 158
159 case SHMAT: 159 case SHMAT:
160 switch (version) { 160 switch (version) {
161 default: { 161 default: {
162 ulong raddr; 162 ulong raddr;
163 ret = do_shmat (first, (char __user *) ptr, second, &raddr); 163 ret = do_shmat(first, (char __user *) ptr, second, &raddr);
164 if (ret) 164 if (ret)
165 return ret; 165 return ret;
166 return put_user (raddr, (ulong __user *) third); 166 return put_user(raddr, (ulong __user *) third);
167 } 167 }
168 case 1: /* iBCS2 emulator entry point */ 168 case 1: /* iBCS2 emulator entry point */
169 if (!segment_eq(get_fs(), get_ds())) 169 if (!segment_eq(get_fs(), get_ds()))
170 return -EINVAL; 170 return -EINVAL;
171 /* The "(ulong *) third" is valid _only_ because of the kernel segment thing */ 171 /* The "(ulong *) third" is valid _only_ because of the kernel segment thing */
172 return do_shmat (first, (char __user *) ptr, second, (ulong *) third); 172 return do_shmat(first, (char __user *) ptr, second, (ulong *) third);
173 } 173 }
174 case SHMDT: 174 case SHMDT:
175 return sys_shmdt ((char __user *)ptr); 175 return sys_shmdt((char __user *)ptr);
176 case SHMGET: 176 case SHMGET:
177 return sys_shmget (first, second, third); 177 return sys_shmget(first, second, third);
178 case SHMCTL: 178 case SHMCTL:
179 return sys_shmctl (first, second, 179 return sys_shmctl(first, second,
180 (struct shmid_ds __user *) ptr); 180 (struct shmid_ds __user *) ptr);
181 default: 181 default:
182 return -ENOSYS; 182 return -ENOSYS;
@@ -186,28 +186,28 @@ asmlinkage int sys_ipc (uint call, int first, int second,
186/* 186/*
187 * Old cruft 187 * Old cruft
188 */ 188 */
189asmlinkage int sys_uname(struct old_utsname __user * name) 189asmlinkage int sys_uname(struct old_utsname __user *name)
190{ 190{
191 int err; 191 int err;
192 if (!name) 192 if (!name)
193 return -EFAULT; 193 return -EFAULT;
194 down_read(&uts_sem); 194 down_read(&uts_sem);
195 err = copy_to_user(name, utsname(), sizeof (*name)); 195 err = copy_to_user(name, utsname(), sizeof(*name));
196 up_read(&uts_sem); 196 up_read(&uts_sem);
197 return err?-EFAULT:0; 197 return err? -EFAULT:0;
198} 198}
199 199
200asmlinkage int sys_olduname(struct oldold_utsname __user * name) 200asmlinkage int sys_olduname(struct oldold_utsname __user *name)
201{ 201{
202 int error; 202 int error;
203 203
204 if (!name) 204 if (!name)
205 return -EFAULT; 205 return -EFAULT;
206 if (!access_ok(VERIFY_WRITE,name,sizeof(struct oldold_utsname))) 206 if (!access_ok(VERIFY_WRITE, name, sizeof(struct oldold_utsname)))
207 return -EFAULT; 207 return -EFAULT;
208 208
209 down_read(&uts_sem); 209 down_read(&uts_sem);
210 210
211 error = __copy_to_user(&name->sysname, &utsname()->sysname, 211 error = __copy_to_user(&name->sysname, &utsname()->sysname,
212 __OLD_UTS_LEN); 212 __OLD_UTS_LEN);
213 error |= __put_user(0, name->sysname + __OLD_UTS_LEN); 213 error |= __put_user(0, name->sysname + __OLD_UTS_LEN);
@@ -223,9 +223,9 @@ asmlinkage int sys_olduname(struct oldold_utsname __user * name)
223 error |= __copy_to_user(&name->machine, &utsname()->machine, 223 error |= __copy_to_user(&name->machine, &utsname()->machine,
224 __OLD_UTS_LEN); 224 __OLD_UTS_LEN);
225 error |= __put_user(0, name->machine + __OLD_UTS_LEN); 225 error |= __put_user(0, name->machine + __OLD_UTS_LEN);
226 226
227 up_read(&uts_sem); 227 up_read(&uts_sem);
228 228
229 error = error ? -EFAULT : 0; 229 error = error ? -EFAULT : 0;
230 230
231 return error; 231 return error;
@@ -241,6 +241,6 @@ int kernel_execve(const char *filename, char *const argv[], char *const envp[])
241 long __res; 241 long __res;
242 asm volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" 242 asm volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx"
243 : "=a" (__res) 243 : "=a" (__res)
244 : "0" (__NR_execve),"ri" (filename),"c" (argv), "d" (envp) : "memory"); 244 : "0" (__NR_execve), "ri" (filename), "c" (argv), "d" (envp) : "memory");
245 return __res; 245 return __res;
246} 246}
diff --git a/arch/x86/kernel/time_32.c b/arch/x86/kernel/time_32.c
index 2ff21f398934..059ca6ee59b4 100644
--- a/arch/x86/kernel/time_32.c
+++ b/arch/x86/kernel/time_32.c
@@ -39,9 +39,6 @@
39 39
40#include "do_timer.h" 40#include "do_timer.h"
41 41
42unsigned int cpu_khz; /* Detected as we calibrate the TSC */
43EXPORT_SYMBOL(cpu_khz);
44
45int timer_ack; 42int timer_ack;
46 43
47unsigned long profile_pc(struct pt_regs *regs) 44unsigned long profile_pc(struct pt_regs *regs)
@@ -84,8 +81,7 @@ irqreturn_t timer_interrupt(int irq, void *dev_id)
84 if (timer_ack) { 81 if (timer_ack) {
85 /* 82 /*
86 * Subtle, when I/O APICs are used we have to ack timer IRQ 83 * Subtle, when I/O APICs are used we have to ack timer IRQ
87 * manually to reset the IRR bit for do_slow_gettimeoffset(). 84 * manually to deassert NMI lines for the watchdog if run
88 * This will also deassert NMI lines for the watchdog if run
89 * on an 82489DX-based system. 85 * on an 82489DX-based system.
90 */ 86 */
91 spin_lock(&i8259A_lock); 87 spin_lock(&i8259A_lock);
diff --git a/arch/x86/kernel/time_64.c b/arch/x86/kernel/time_64.c
index c737849e2ef7..e3d49c553af2 100644
--- a/arch/x86/kernel/time_64.c
+++ b/arch/x86/kernel/time_64.c
@@ -56,7 +56,7 @@ static irqreturn_t timer_event_interrupt(int irq, void *dev_id)
56/* calibrate_cpu is used on systems with fixed rate TSCs to determine 56/* calibrate_cpu is used on systems with fixed rate TSCs to determine
57 * processor frequency */ 57 * processor frequency */
58#define TICK_COUNT 100000000 58#define TICK_COUNT 100000000
59unsigned long __init native_calculate_cpu_khz(void) 59unsigned long __init calibrate_cpu(void)
60{ 60{
61 int tsc_start, tsc_now; 61 int tsc_start, tsc_now;
62 int i, no_ctr_free; 62 int i, no_ctr_free;
@@ -116,23 +116,11 @@ void __init hpet_time_init(void)
116 116
117void __init time_init(void) 117void __init time_init(void)
118{ 118{
119 tsc_calibrate(); 119 tsc_init();
120
121 cpu_khz = tsc_khz;
122 if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) &&
123 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD))
124 cpu_khz = calculate_cpu_khz();
125
126 if (unsynchronized_tsc())
127 mark_tsc_unstable("TSCs unsynchronized");
128
129 if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP)) 120 if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
130 vgetcpu_mode = VGETCPU_RDTSCP; 121 vgetcpu_mode = VGETCPU_RDTSCP;
131 else 122 else
132 vgetcpu_mode = VGETCPU_LSL; 123 vgetcpu_mode = VGETCPU_LSL;
133 124
134 printk(KERN_INFO "time.c: Detected %d.%03d MHz processor.\n",
135 cpu_khz / 1000, cpu_khz % 1000);
136 init_tsc_clocksource();
137 late_time_init = choose_time_init(); 125 late_time_init = choose_time_init();
138} 126}
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
index 9bb2363851af..fec1ecedc9b7 100644
--- a/arch/x86/kernel/tlb_32.c
+++ b/arch/x86/kernel/tlb_32.c
@@ -238,6 +238,6 @@ static void do_flush_tlb_all(void *info)
238 238
239void flush_tlb_all(void) 239void flush_tlb_all(void)
240{ 240{
241 on_each_cpu(do_flush_tlb_all, NULL, 1, 1); 241 on_each_cpu(do_flush_tlb_all, NULL, 1);
242} 242}
243 243
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c
index a1f07d793202..dcbf7a1159ea 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/kernel/tlb_64.c
@@ -15,6 +15,8 @@
15#include <asm/proto.h> 15#include <asm/proto.h>
16#include <asm/apicdef.h> 16#include <asm/apicdef.h>
17#include <asm/idle.h> 17#include <asm/idle.h>
18#include <asm/uv/uv_hub.h>
19#include <asm/uv/uv_bau.h>
18 20
19#include <mach_ipi.h> 21#include <mach_ipi.h>
20/* 22/*
@@ -162,6 +164,9 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
162 union smp_flush_state *f; 164 union smp_flush_state *f;
163 cpumask_t cpumask = *cpumaskp; 165 cpumask_t cpumask = *cpumaskp;
164 166
167 if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va))
168 return;
169
165 /* Caller has disabled preemption */ 170 /* Caller has disabled preemption */
166 sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; 171 sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
167 f = &per_cpu(flush_state, sender); 172 f = &per_cpu(flush_state, sender);
@@ -270,5 +275,5 @@ static void do_flush_tlb_all(void *info)
270 275
271void flush_tlb_all(void) 276void flush_tlb_all(void)
272{ 277{
273 on_each_cpu(do_flush_tlb_all, NULL, 1, 1); 278 on_each_cpu(do_flush_tlb_all, NULL, 1);
274} 279}
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
new file mode 100644
index 000000000000..d0fbb7712ab0
--- /dev/null
+++ b/arch/x86/kernel/tlb_uv.c
@@ -0,0 +1,792 @@
1/*
2 * SGI UltraViolet TLB flush routines.
3 *
4 * (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI.
5 *
6 * This code is released under the GNU General Public License version 2 or
7 * later.
8 */
9#include <linux/mc146818rtc.h>
10#include <linux/proc_fs.h>
11#include <linux/kernel.h>
12
13#include <asm/mmu_context.h>
14#include <asm/uv/uv_mmrs.h>
15#include <asm/uv/uv_hub.h>
16#include <asm/uv/uv_bau.h>
17#include <asm/genapic.h>
18#include <asm/idle.h>
19#include <asm/tsc.h>
20
21#include <mach_apic.h>
22
23static struct bau_control **uv_bau_table_bases __read_mostly;
24static int uv_bau_retry_limit __read_mostly;
25
26/* position of pnode (which is nasid>>1): */
27static int uv_nshift __read_mostly;
28
29static unsigned long uv_mmask __read_mostly;
30
31static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
32static DEFINE_PER_CPU(struct bau_control, bau_control);
33
34/*
35 * Free a software acknowledge hardware resource by clearing its Pending
36 * bit. This will return a reply to the sender.
37 * If the message has timed out, a reply has already been sent by the
38 * hardware but the resource has not been released. In that case our
39 * clear of the Timeout bit (as well) will free the resource. No reply will
40 * be sent (the hardware will only do one reply per message).
41 */
42static void uv_reply_to_message(int resource,
43 struct bau_payload_queue_entry *msg,
44 struct bau_msg_status *msp)
45{
46 unsigned long dw;
47
48 dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource);
49 msg->replied_to = 1;
50 msg->sw_ack_vector = 0;
51 if (msp)
52 msp->seen_by.bits = 0;
53 uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
54}
55
56/*
57 * Do all the things a cpu should do for a TLB shootdown message.
58 * Other cpu's may come here at the same time for this message.
59 */
60static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
61 int msg_slot, int sw_ack_slot)
62{
63 unsigned long this_cpu_mask;
64 struct bau_msg_status *msp;
65 int cpu;
66
67 msp = __get_cpu_var(bau_control).msg_statuses + msg_slot;
68 cpu = uv_blade_processor_id();
69 msg->number_of_cpus =
70 uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
71 this_cpu_mask = 1UL << cpu;
72 if (msp->seen_by.bits & this_cpu_mask)
73 return;
74 atomic_or_long(&msp->seen_by.bits, this_cpu_mask);
75
76 if (msg->replied_to == 1)
77 return;
78
79 if (msg->address == TLB_FLUSH_ALL) {
80 local_flush_tlb();
81 __get_cpu_var(ptcstats).alltlb++;
82 } else {
83 __flush_tlb_one(msg->address);
84 __get_cpu_var(ptcstats).onetlb++;
85 }
86
87 __get_cpu_var(ptcstats).requestee++;
88
89 atomic_inc_short(&msg->acknowledge_count);
90 if (msg->number_of_cpus == msg->acknowledge_count)
91 uv_reply_to_message(sw_ack_slot, msg, msp);
92}
93
94/*
95 * Examine the payload queue on one distribution node to see
96 * which messages have not been seen, and which cpu(s) have not seen them.
97 *
98 * Returns the number of cpu's that have not responded.
99 */
100static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)
101{
102 struct bau_payload_queue_entry *msg;
103 struct bau_msg_status *msp;
104 int count = 0;
105 int i;
106 int j;
107
108 for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;
109 msg++, i++) {
110 if ((msg->sending_cpu == sender) && (!msg->replied_to)) {
111 msp = bau_tablesp->msg_statuses + i;
112 printk(KERN_DEBUG
113 "blade %d: address:%#lx %d of %d, not cpu(s): ",
114 i, msg->address, msg->acknowledge_count,
115 msg->number_of_cpus);
116 for (j = 0; j < msg->number_of_cpus; j++) {
117 if (!((1L << j) & msp->seen_by.bits)) {
118 count++;
119 printk("%d ", j);
120 }
121 }
122 printk("\n");
123 }
124 }
125 return count;
126}
127
128/*
129 * Examine the payload queue on all the distribution nodes to see
130 * which messages have not been seen, and which cpu(s) have not seen them.
131 *
132 * Returns the number of cpu's that have not responded.
133 */
134static int uv_examine_destinations(struct bau_target_nodemask *distribution)
135{
136 int sender;
137 int i;
138 int count = 0;
139
140 sender = smp_processor_id();
141 for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) {
142 if (!bau_node_isset(i, distribution))
143 continue;
144 count += uv_examine_destination(uv_bau_table_bases[i], sender);
145 }
146 return count;
147}
148
149/*
150 * wait for completion of a broadcast message
151 *
152 * return COMPLETE, RETRY or GIVEUP
153 */
154static int uv_wait_completion(struct bau_desc *bau_desc,
155 unsigned long mmr_offset, int right_shift)
156{
157 int exams = 0;
158 long destination_timeouts = 0;
159 long source_timeouts = 0;
160 unsigned long descriptor_status;
161
162 while ((descriptor_status = (((unsigned long)
163 uv_read_local_mmr(mmr_offset) >>
164 right_shift) & UV_ACT_STATUS_MASK)) !=
165 DESC_STATUS_IDLE) {
166 if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
167 source_timeouts++;
168 if (source_timeouts > SOURCE_TIMEOUT_LIMIT)
169 source_timeouts = 0;
170 __get_cpu_var(ptcstats).s_retry++;
171 return FLUSH_RETRY;
172 }
173 /*
174 * spin here looking for progress at the destinations
175 */
176 if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) {
177 destination_timeouts++;
178 if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) {
179 /*
180 * returns number of cpus not responding
181 */
182 if (uv_examine_destinations
183 (&bau_desc->distribution) == 0) {
184 __get_cpu_var(ptcstats).d_retry++;
185 return FLUSH_RETRY;
186 }
187 exams++;
188 if (exams >= uv_bau_retry_limit) {
189 printk(KERN_DEBUG
190 "uv_flush_tlb_others");
191 printk("giving up on cpu %d\n",
192 smp_processor_id());
193 return FLUSH_GIVEUP;
194 }
195 /*
196 * delays can hang the simulator
197 udelay(1000);
198 */
199 destination_timeouts = 0;
200 }
201 }
202 }
203 return FLUSH_COMPLETE;
204}
205
206/**
207 * uv_flush_send_and_wait
208 *
209 * Send a broadcast and wait for a broadcast message to complete.
210 *
211 * The cpumaskp mask contains the cpus the broadcast was sent to.
212 *
213 * Returns 1 if all remote flushing was done. The mask is zeroed.
214 * Returns 0 if some remote flushing remains to be done. The mask is left
215 * unchanged.
216 */
217int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
218 cpumask_t *cpumaskp)
219{
220 int completion_status = 0;
221 int right_shift;
222 int tries = 0;
223 int blade;
224 int bit;
225 unsigned long mmr_offset;
226 unsigned long index;
227 cycles_t time1;
228 cycles_t time2;
229
230 if (cpu < UV_CPUS_PER_ACT_STATUS) {
231 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
232 right_shift = cpu * UV_ACT_STATUS_SIZE;
233 } else {
234 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1;
235 right_shift =
236 ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE);
237 }
238 time1 = get_cycles();
239 do {
240 tries++;
241 index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |
242 cpu;
243 uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
244 completion_status = uv_wait_completion(bau_desc, mmr_offset,
245 right_shift);
246 } while (completion_status == FLUSH_RETRY);
247 time2 = get_cycles();
248 __get_cpu_var(ptcstats).sflush += (time2 - time1);
249 if (tries > 1)
250 __get_cpu_var(ptcstats).retriesok++;
251
252 if (completion_status == FLUSH_GIVEUP) {
253 /*
254 * Cause the caller to do an IPI-style TLB shootdown on
255 * the cpu's, all of which are still in the mask.
256 */
257 __get_cpu_var(ptcstats).ptc_i++;
258 return 0;
259 }
260
261 /*
262 * Success, so clear the remote cpu's from the mask so we don't
263 * use the IPI method of shootdown on them.
264 */
265 for_each_cpu_mask(bit, *cpumaskp) {
266 blade = uv_cpu_to_blade_id(bit);
267 if (blade == this_blade)
268 continue;
269 cpu_clear(bit, *cpumaskp);
270 }
271 if (!cpus_empty(*cpumaskp))
272 return 0;
273 return 1;
274}
275
276/**
277 * uv_flush_tlb_others - globally purge translation cache of a virtual
278 * address or all TLB's
279 * @cpumaskp: mask of all cpu's in which the address is to be removed
280 * @mm: mm_struct containing virtual address range
281 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
282 *
283 * This is the entry point for initiating any UV global TLB shootdown.
284 *
285 * Purges the translation caches of all specified processors of the given
286 * virtual address, or purges all TLB's on specified processors.
287 *
288 * The caller has derived the cpumaskp from the mm_struct and has subtracted
289 * the local cpu from the mask. This function is called only if there
290 * are bits set in the mask. (e.g. flush_tlb_page())
291 *
292 * The cpumaskp is converted into a nodemask of the nodes containing
293 * the cpus.
294 *
295 * Returns 1 if all remote flushing was done.
296 * Returns 0 if some remote flushing remains to be done.
297 */
298int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
299 unsigned long va)
300{
301 int i;
302 int bit;
303 int blade;
304 int cpu;
305 int this_blade;
306 int locals = 0;
307 struct bau_desc *bau_desc;
308
309 cpu = uv_blade_processor_id();
310 this_blade = uv_numa_blade_id();
311 bau_desc = __get_cpu_var(bau_control).descriptor_base;
312 bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu;
313
314 bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
315
316 i = 0;
317 for_each_cpu_mask(bit, *cpumaskp) {
318 blade = uv_cpu_to_blade_id(bit);
319 BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
320 if (blade == this_blade) {
321 locals++;
322 continue;
323 }
324 bau_node_set(blade, &bau_desc->distribution);
325 i++;
326 }
327 if (i == 0) {
328 /*
329 * no off_node flushing; return status for local node
330 */
331 if (locals)
332 return 0;
333 else
334 return 1;
335 }
336 __get_cpu_var(ptcstats).requestor++;
337 __get_cpu_var(ptcstats).ntargeted += i;
338
339 bau_desc->payload.address = va;
340 bau_desc->payload.sending_cpu = smp_processor_id();
341
342 return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp);
343}
344
345/*
346 * The BAU message interrupt comes here. (registered by set_intr_gate)
347 * See entry_64.S
348 *
349 * We received a broadcast assist message.
350 *
351 * Interrupts may have been disabled; this interrupt could represent
352 * the receipt of several messages.
353 *
354 * All cores/threads on this node get this interrupt.
355 * The last one to see it does the s/w ack.
356 * (the resource will not be freed until noninterruptable cpus see this
357 * interrupt; hardware will timeout the s/w ack and reply ERROR)
358 */
359void uv_bau_message_interrupt(struct pt_regs *regs)
360{
361 struct bau_payload_queue_entry *va_queue_first;
362 struct bau_payload_queue_entry *va_queue_last;
363 struct bau_payload_queue_entry *msg;
364 struct pt_regs *old_regs = set_irq_regs(regs);
365 cycles_t time1;
366 cycles_t time2;
367 int msg_slot;
368 int sw_ack_slot;
369 int fw;
370 int count = 0;
371 unsigned long local_pnode;
372
373 ack_APIC_irq();
374 exit_idle();
375 irq_enter();
376
377 time1 = get_cycles();
378
379 local_pnode = uv_blade_to_pnode(uv_numa_blade_id());
380
381 va_queue_first = __get_cpu_var(bau_control).va_queue_first;
382 va_queue_last = __get_cpu_var(bau_control).va_queue_last;
383
384 msg = __get_cpu_var(bau_control).bau_msg_head;
385 while (msg->sw_ack_vector) {
386 count++;
387 fw = msg->sw_ack_vector;
388 msg_slot = msg - va_queue_first;
389 sw_ack_slot = ffs(fw) - 1;
390
391 uv_bau_process_message(msg, msg_slot, sw_ack_slot);
392
393 msg++;
394 if (msg > va_queue_last)
395 msg = va_queue_first;
396 __get_cpu_var(bau_control).bau_msg_head = msg;
397 }
398 if (!count)
399 __get_cpu_var(ptcstats).nomsg++;
400 else if (count > 1)
401 __get_cpu_var(ptcstats).multmsg++;
402
403 time2 = get_cycles();
404 __get_cpu_var(ptcstats).dflush += (time2 - time1);
405
406 irq_exit();
407 set_irq_regs(old_regs);
408}
409
410static void uv_enable_timeouts(void)
411{
412 int i;
413 int blade;
414 int last_blade;
415 int pnode;
416 int cur_cpu = 0;
417 unsigned long apicid;
418
419 last_blade = -1;
420 for_each_online_node(i) {
421 blade = uv_node_to_blade_id(i);
422 if (blade == last_blade)
423 continue;
424 last_blade = blade;
425 apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
426 pnode = uv_blade_to_pnode(blade);
427 cur_cpu += uv_blade_nr_possible_cpus(i);
428 }
429}
430
431static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
432{
433 if (*offset < num_possible_cpus())
434 return offset;
435 return NULL;
436}
437
438static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
439{
440 (*offset)++;
441 if (*offset < num_possible_cpus())
442 return offset;
443 return NULL;
444}
445
446static void uv_ptc_seq_stop(struct seq_file *file, void *data)
447{
448}
449
450/*
451 * Display the statistics thru /proc
452 * data points to the cpu number
453 */
454static int uv_ptc_seq_show(struct seq_file *file, void *data)
455{
456 struct ptc_stats *stat;
457 int cpu;
458
459 cpu = *(loff_t *)data;
460
461 if (!cpu) {
462 seq_printf(file,
463 "# cpu requestor requestee one all sretry dretry ptc_i ");
464 seq_printf(file,
465 "sw_ack sflush dflush sok dnomsg dmult starget\n");
466 }
467 if (cpu < num_possible_cpus() && cpu_online(cpu)) {
468 stat = &per_cpu(ptcstats, cpu);
469 seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ",
470 cpu, stat->requestor,
471 stat->requestee, stat->onetlb, stat->alltlb,
472 stat->s_retry, stat->d_retry, stat->ptc_i);
473 seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n",
474 uv_read_global_mmr64(uv_blade_to_pnode
475 (uv_cpu_to_blade_id(cpu)),
476 UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE),
477 stat->sflush, stat->dflush,
478 stat->retriesok, stat->nomsg,
479 stat->multmsg, stat->ntargeted);
480 }
481
482 return 0;
483}
484
485/*
486 * 0: display meaning of the statistics
487 * >0: retry limit
488 */
489static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
490 size_t count, loff_t *data)
491{
492 long newmode;
493 char optstr[64];
494
495 if (count == 0 || count > sizeof(optstr))
496 return -EINVAL;
497 if (copy_from_user(optstr, user, count))
498 return -EFAULT;
499 optstr[count - 1] = '\0';
500 if (strict_strtoul(optstr, 10, &newmode) < 0) {
501 printk(KERN_DEBUG "%s is invalid\n", optstr);
502 return -EINVAL;
503 }
504
505 if (newmode == 0) {
506 printk(KERN_DEBUG "# cpu: cpu number\n");
507 printk(KERN_DEBUG
508 "requestor: times this cpu was the flush requestor\n");
509 printk(KERN_DEBUG
510 "requestee: times this cpu was requested to flush its TLBs\n");
511 printk(KERN_DEBUG
512 "one: times requested to flush a single address\n");
513 printk(KERN_DEBUG
514 "all: times requested to flush all TLB's\n");
515 printk(KERN_DEBUG
516 "sretry: number of retries of source-side timeouts\n");
517 printk(KERN_DEBUG
518 "dretry: number of retries of destination-side timeouts\n");
519 printk(KERN_DEBUG
520 "ptc_i: times UV fell through to IPI-style flushes\n");
521 printk(KERN_DEBUG
522 "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
523 printk(KERN_DEBUG
524 "sflush_us: cycles spent in uv_flush_tlb_others()\n");
525 printk(KERN_DEBUG
526 "dflush_us: cycles spent in handling flush requests\n");
527 printk(KERN_DEBUG "sok: successes on retry\n");
528 printk(KERN_DEBUG "dnomsg: interrupts with no message\n");
529 printk(KERN_DEBUG
530 "dmult: interrupts with multiple messages\n");
531 printk(KERN_DEBUG "starget: nodes targeted\n");
532 } else {
533 uv_bau_retry_limit = newmode;
534 printk(KERN_DEBUG "timeout retry limit:%d\n",
535 uv_bau_retry_limit);
536 }
537
538 return count;
539}
540
541static const struct seq_operations uv_ptc_seq_ops = {
542 .start = uv_ptc_seq_start,
543 .next = uv_ptc_seq_next,
544 .stop = uv_ptc_seq_stop,
545 .show = uv_ptc_seq_show
546};
547
548static int uv_ptc_proc_open(struct inode *inode, struct file *file)
549{
550 return seq_open(file, &uv_ptc_seq_ops);
551}
552
553static const struct file_operations proc_uv_ptc_operations = {
554 .open = uv_ptc_proc_open,
555 .read = seq_read,
556 .write = uv_ptc_proc_write,
557 .llseek = seq_lseek,
558 .release = seq_release,
559};
560
561static int __init uv_ptc_init(void)
562{
563 struct proc_dir_entry *proc_uv_ptc;
564
565 if (!is_uv_system())
566 return 0;
567
568 if (!proc_mkdir("sgi_uv", NULL))
569 return -EINVAL;
570
571 proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL);
572 if (!proc_uv_ptc) {
573 printk(KERN_ERR "unable to create %s proc entry\n",
574 UV_PTC_BASENAME);
575 remove_proc_entry("sgi_uv", NULL);
576 return -EINVAL;
577 }
578 proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
579 return 0;
580}
581
582/*
583 * begin the initialization of the per-blade control structures
584 */
585static struct bau_control * __init uv_table_bases_init(int blade, int node)
586{
587 int i;
588 int *ip;
589 struct bau_msg_status *msp;
590 struct bau_control *bau_tabp;
591
592 bau_tabp =
593 kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
594 BUG_ON(!bau_tabp);
595
596 bau_tabp->msg_statuses =
597 kmalloc_node(sizeof(struct bau_msg_status) *
598 DEST_Q_SIZE, GFP_KERNEL, node);
599 BUG_ON(!bau_tabp->msg_statuses);
600
601 for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)
602 bau_cpubits_clear(&msp->seen_by, (int)
603 uv_blade_nr_possible_cpus(blade));
604
605 bau_tabp->watching =
606 kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node);
607 BUG_ON(!bau_tabp->watching);
608
609 for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++)
610 *ip = 0;
611
612 uv_bau_table_bases[blade] = bau_tabp;
613
614 return bau_tabp;
615}
616
617/*
618 * finish the initialization of the per-blade control structures
619 */
620static void __init
621uv_table_bases_finish(int blade, int node, int cur_cpu,
622 struct bau_control *bau_tablesp,
623 struct bau_desc *adp)
624{
625 struct bau_control *bcp;
626 int i;
627
628 for (i = cur_cpu; i < cur_cpu + uv_blade_nr_possible_cpus(blade); i++) {
629 bcp = (struct bau_control *)&per_cpu(bau_control, i);
630
631 bcp->bau_msg_head = bau_tablesp->va_queue_first;
632 bcp->va_queue_first = bau_tablesp->va_queue_first;
633 bcp->va_queue_last = bau_tablesp->va_queue_last;
634 bcp->watching = bau_tablesp->watching;
635 bcp->msg_statuses = bau_tablesp->msg_statuses;
636 bcp->descriptor_base = adp;
637 }
638}
639
640/*
641 * initialize the sending side's sending buffers
642 */
643static struct bau_desc * __init
644uv_activation_descriptor_init(int node, int pnode)
645{
646 int i;
647 unsigned long pa;
648 unsigned long m;
649 unsigned long n;
650 unsigned long mmr_image;
651 struct bau_desc *adp;
652 struct bau_desc *ad2;
653
654 adp = (struct bau_desc *)
655 kmalloc_node(16384, GFP_KERNEL, node);
656 BUG_ON(!adp);
657
658 pa = __pa((unsigned long)adp);
659 n = pa >> uv_nshift;
660 m = pa & uv_mmask;
661
662 mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE);
663 if (mmr_image) {
664 uv_write_global_mmr64(pnode, (unsigned long)
665 UVH_LB_BAU_SB_DESCRIPTOR_BASE,
666 (n << UV_DESC_BASE_PNODE_SHIFT | m));
667 }
668
669 for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
670 memset(ad2, 0, sizeof(struct bau_desc));
671 ad2->header.sw_ack_flag = 1;
672 ad2->header.base_dest_nodeid =
673 uv_blade_to_pnode(uv_cpu_to_blade_id(0));
674 ad2->header.command = UV_NET_ENDPOINT_INTD;
675 ad2->header.int_both = 1;
676 /*
677 * all others need to be set to zero:
678 * fairness chaining multilevel count replied_to
679 */
680 }
681 return adp;
682}
683
684/*
685 * initialize the destination side's receiving buffers
686 */
687static struct bau_payload_queue_entry * __init
688uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp)
689{
690 struct bau_payload_queue_entry *pqp;
691 char *cp;
692
693 pqp = (struct bau_payload_queue_entry *) kmalloc_node(
694 (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),
695 GFP_KERNEL, node);
696 BUG_ON(!pqp);
697
698 cp = (char *)pqp + 31;
699 pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
700 bau_tablesp->va_queue_first = pqp;
701 uv_write_global_mmr64(pnode,
702 UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST,
703 ((unsigned long)pnode <<
704 UV_PAYLOADQ_PNODE_SHIFT) |
705 uv_physnodeaddr(pqp));
706 uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
707 uv_physnodeaddr(pqp));
708 bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
709 uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
710 (unsigned long)
711 uv_physnodeaddr(bau_tablesp->va_queue_last));
712 memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);
713
714 return pqp;
715}
716
717/*
718 * Initialization of each UV blade's structures
719 */
720static int __init uv_init_blade(int blade, int node, int cur_cpu)
721{
722 int pnode;
723 unsigned long pa;
724 unsigned long apicid;
725 struct bau_desc *adp;
726 struct bau_payload_queue_entry *pqp;
727 struct bau_control *bau_tablesp;
728
729 bau_tablesp = uv_table_bases_init(blade, node);
730 pnode = uv_blade_to_pnode(blade);
731 adp = uv_activation_descriptor_init(node, pnode);
732 pqp = uv_payload_queue_init(node, pnode, bau_tablesp);
733 uv_table_bases_finish(blade, node, cur_cpu, bau_tablesp, adp);
734 /*
735 * the below initialization can't be in firmware because the
736 * messaging IRQ will be determined by the OS
737 */
738 apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
739 pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG);
740 if ((pa & 0xff) != UV_BAU_MESSAGE) {
741 uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG,
742 ((apicid << 32) | UV_BAU_MESSAGE));
743 }
744 return 0;
745}
746
747/*
748 * Initialization of BAU-related structures
749 */
750static int __init uv_bau_init(void)
751{
752 int blade;
753 int node;
754 int nblades;
755 int last_blade;
756 int cur_cpu = 0;
757
758 if (!is_uv_system())
759 return 0;
760
761 uv_bau_retry_limit = 1;
762 uv_nshift = uv_hub_info->n_val;
763 uv_mmask = (1UL << uv_hub_info->n_val) - 1;
764 nblades = 0;
765 last_blade = -1;
766 for_each_online_node(node) {
767 blade = uv_node_to_blade_id(node);
768 if (blade == last_blade)
769 continue;
770 last_blade = blade;
771 nblades++;
772 }
773 uv_bau_table_bases = (struct bau_control **)
774 kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
775 BUG_ON(!uv_bau_table_bases);
776
777 last_blade = -1;
778 for_each_online_node(node) {
779 blade = uv_node_to_blade_id(node);
780 if (blade == last_blade)
781 continue;
782 last_blade = blade;
783 uv_init_blade(blade, node, cur_cpu);
784 cur_cpu += uv_blade_nr_possible_cpus(blade);
785 }
786 set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1);
787 uv_enable_timeouts();
788
789 return 0;
790}
791__initcall(uv_bau_init);
792__initcall(uv_ptc_init);
diff --git a/arch/x86/kernel/trampoline.c b/arch/x86/kernel/trampoline.c
index abbf199adebb..1106fac6024d 100644
--- a/arch/x86/kernel/trampoline.c
+++ b/arch/x86/kernel/trampoline.c
@@ -2,7 +2,7 @@
2 2
3#include <asm/trampoline.h> 3#include <asm/trampoline.h>
4 4
5/* ready for x86_64, no harm for x86, since it will overwrite after alloc */ 5/* ready for x86_64 and x86 */
6unsigned char *trampoline_base = __va(TRAMPOLINE_BASE); 6unsigned char *trampoline_base = __va(TRAMPOLINE_BASE);
7 7
8/* 8/*
diff --git a/arch/x86/kernel/traps_32.c b/arch/x86/kernel/traps_32.c
index 08d752de4eee..8a768973c4f0 100644
--- a/arch/x86/kernel/traps_32.c
+++ b/arch/x86/kernel/traps_32.c
@@ -1,5 +1,6 @@
1/* 1/*
2 * Copyright (C) 1991, 1992 Linus Torvalds 2 * Copyright (C) 1991, 1992 Linus Torvalds
3 * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
3 * 4 *
4 * Pentium III FXSR, SSE support 5 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000 6 * Gareth Hughes <gareth@valinux.com>, May 2000
@@ -60,8 +61,6 @@
60 61
61#include "mach_traps.h" 62#include "mach_traps.h"
62 63
63int panic_on_unrecovered_nmi;
64
65DECLARE_BITMAP(used_vectors, NR_VECTORS); 64DECLARE_BITMAP(used_vectors, NR_VECTORS);
66EXPORT_SYMBOL_GPL(used_vectors); 65EXPORT_SYMBOL_GPL(used_vectors);
67 66
@@ -98,19 +97,22 @@ asmlinkage void alignment_check(void);
98asmlinkage void spurious_interrupt_bug(void); 97asmlinkage void spurious_interrupt_bug(void);
99asmlinkage void machine_check(void); 98asmlinkage void machine_check(void);
100 99
100int panic_on_unrecovered_nmi;
101int kstack_depth_to_print = 24; 101int kstack_depth_to_print = 24;
102static unsigned int code_bytes = 64; 102static unsigned int code_bytes = 64;
103static int ignore_nmis;
104static int die_counter;
103 105
104void printk_address(unsigned long address, int reliable) 106void printk_address(unsigned long address, int reliable)
105{ 107{
106#ifdef CONFIG_KALLSYMS 108#ifdef CONFIG_KALLSYMS
107 char namebuf[KSYM_NAME_LEN];
108 unsigned long offset = 0; 109 unsigned long offset = 0;
109 unsigned long symsize; 110 unsigned long symsize;
110 const char *symname; 111 const char *symname;
111 char reliab[4] = "";
112 char *delim = ":";
113 char *modname; 112 char *modname;
113 char *delim = ":";
114 char namebuf[KSYM_NAME_LEN];
115 char reliab[4] = "";
114 116
115 symname = kallsyms_lookup(address, &symsize, &offset, 117 symname = kallsyms_lookup(address, &symsize, &offset,
116 &modname, namebuf); 118 &modname, namebuf);
@@ -130,22 +132,23 @@ void printk_address(unsigned long address, int reliable)
130#endif 132#endif
131} 133}
132 134
133static inline int valid_stack_ptr(struct thread_info *tinfo, void *p, unsigned size) 135static inline int valid_stack_ptr(struct thread_info *tinfo,
136 void *p, unsigned int size)
134{ 137{
135 return p > (void *)tinfo && 138 void *t = tinfo;
136 p <= (void *)tinfo + THREAD_SIZE - size; 139 return p > t && p <= t + THREAD_SIZE - size;
137} 140}
138 141
139/* The form of the top of the frame on the stack */ 142/* The form of the top of the frame on the stack */
140struct stack_frame { 143struct stack_frame {
141 struct stack_frame *next_frame; 144 struct stack_frame *next_frame;
142 unsigned long return_address; 145 unsigned long return_address;
143}; 146};
144 147
145static inline unsigned long 148static inline unsigned long
146print_context_stack(struct thread_info *tinfo, 149print_context_stack(struct thread_info *tinfo,
147 unsigned long *stack, unsigned long bp, 150 unsigned long *stack, unsigned long bp,
148 const struct stacktrace_ops *ops, void *data) 151 const struct stacktrace_ops *ops, void *data)
149{ 152{
150 struct stack_frame *frame = (struct stack_frame *)bp; 153 struct stack_frame *frame = (struct stack_frame *)bp;
151 154
@@ -167,8 +170,6 @@ print_context_stack(struct thread_info *tinfo,
167 return bp; 170 return bp;
168} 171}
169 172
170#define MSG(msg) ops->warning(data, msg)
171
172void dump_trace(struct task_struct *task, struct pt_regs *regs, 173void dump_trace(struct task_struct *task, struct pt_regs *regs,
173 unsigned long *stack, unsigned long bp, 174 unsigned long *stack, unsigned long bp,
174 const struct stacktrace_ops *ops, void *data) 175 const struct stacktrace_ops *ops, void *data)
@@ -178,7 +179,6 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
178 179
179 if (!stack) { 180 if (!stack) {
180 unsigned long dummy; 181 unsigned long dummy;
181
182 stack = &dummy; 182 stack = &dummy;
183 if (task != current) 183 if (task != current)
184 stack = (unsigned long *)task->thread.sp; 184 stack = (unsigned long *)task->thread.sp;
@@ -196,7 +196,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
196 } 196 }
197#endif 197#endif
198 198
199 while (1) { 199 for (;;) {
200 struct thread_info *context; 200 struct thread_info *context;
201 201
202 context = (struct thread_info *) 202 context = (struct thread_info *)
@@ -248,10 +248,10 @@ static void print_trace_address(void *data, unsigned long addr, int reliable)
248} 248}
249 249
250static const struct stacktrace_ops print_trace_ops = { 250static const struct stacktrace_ops print_trace_ops = {
251 .warning = print_trace_warning, 251 .warning = print_trace_warning,
252 .warning_symbol = print_trace_warning_symbol, 252 .warning_symbol = print_trace_warning_symbol,
253 .stack = print_trace_stack, 253 .stack = print_trace_stack,
254 .address = print_trace_address, 254 .address = print_trace_address,
255}; 255};
256 256
257static void 257static void
@@ -351,15 +351,14 @@ void show_registers(struct pt_regs *regs)
351 printk(KERN_EMERG "Code: "); 351 printk(KERN_EMERG "Code: ");
352 352
353 ip = (u8 *)regs->ip - code_prologue; 353 ip = (u8 *)regs->ip - code_prologue;
354 if (ip < (u8 *)PAGE_OFFSET || 354 if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) {
355 probe_kernel_address(ip, c)) {
356 /* try starting at EIP */ 355 /* try starting at EIP */
357 ip = (u8 *)regs->ip; 356 ip = (u8 *)regs->ip;
358 code_len = code_len - code_prologue + 1; 357 code_len = code_len - code_prologue + 1;
359 } 358 }
360 for (i = 0; i < code_len; i++, ip++) { 359 for (i = 0; i < code_len; i++, ip++) {
361 if (ip < (u8 *)PAGE_OFFSET || 360 if (ip < (u8 *)PAGE_OFFSET ||
362 probe_kernel_address(ip, c)) { 361 probe_kernel_address(ip, c)) {
363 printk(" Bad EIP value."); 362 printk(" Bad EIP value.");
364 break; 363 break;
365 } 364 }
@@ -384,8 +383,6 @@ int is_valid_bugaddr(unsigned long ip)
384 return ud2 == 0x0b0f; 383 return ud2 == 0x0b0f;
385} 384}
386 385
387static int die_counter;
388
389int __kprobes __die(const char *str, struct pt_regs *regs, long err) 386int __kprobes __die(const char *str, struct pt_regs *regs, long err)
390{ 387{
391 unsigned short ss; 388 unsigned short ss;
@@ -402,26 +399,22 @@ int __kprobes __die(const char *str, struct pt_regs *regs, long err)
402 printk("DEBUG_PAGEALLOC"); 399 printk("DEBUG_PAGEALLOC");
403#endif 400#endif
404 printk("\n"); 401 printk("\n");
405
406 if (notify_die(DIE_OOPS, str, regs, err, 402 if (notify_die(DIE_OOPS, str, regs, err,
407 current->thread.trap_no, SIGSEGV) != NOTIFY_STOP) { 403 current->thread.trap_no, SIGSEGV) == NOTIFY_STOP)
408 404 return 1;
409 show_registers(regs);
410 /* Executive summary in case the oops scrolled away */
411 sp = (unsigned long) (&regs->sp);
412 savesegment(ss, ss);
413 if (user_mode(regs)) {
414 sp = regs->sp;
415 ss = regs->ss & 0xffff;
416 }
417 printk(KERN_EMERG "EIP: [<%08lx>] ", regs->ip);
418 print_symbol("%s", regs->ip);
419 printk(" SS:ESP %04x:%08lx\n", ss, sp);
420 405
421 return 0; 406 show_registers(regs);
407 /* Executive summary in case the oops scrolled away */
408 sp = (unsigned long) (&regs->sp);
409 savesegment(ss, ss);
410 if (user_mode(regs)) {
411 sp = regs->sp;
412 ss = regs->ss & 0xffff;
422 } 413 }
423 414 printk(KERN_EMERG "EIP: [<%08lx>] ", regs->ip);
424 return 1; 415 print_symbol("%s", regs->ip);
416 printk(" SS:ESP %04x:%08lx\n", ss, sp);
417 return 0;
425} 418}
426 419
427/* 420/*
@@ -546,7 +539,7 @@ void do_##name(struct pt_regs *regs, long error_code) \
546{ \ 539{ \
547 trace_hardirqs_fixup(); \ 540 trace_hardirqs_fixup(); \
548 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 541 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
549 == NOTIFY_STOP) \ 542 == NOTIFY_STOP) \
550 return; \ 543 return; \
551 do_trap(trapnr, signr, str, 0, regs, error_code, NULL); \ 544 do_trap(trapnr, signr, str, 0, regs, error_code, NULL); \
552} 545}
@@ -562,7 +555,7 @@ void do_##name(struct pt_regs *regs, long error_code) \
562 info.si_code = sicode; \ 555 info.si_code = sicode; \
563 info.si_addr = (void __user *)siaddr; \ 556 info.si_addr = (void __user *)siaddr; \
564 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 557 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
565 == NOTIFY_STOP) \ 558 == NOTIFY_STOP) \
566 return; \ 559 return; \
567 do_trap(trapnr, signr, str, 0, regs, error_code, &info); \ 560 do_trap(trapnr, signr, str, 0, regs, error_code, &info); \
568} 561}
@@ -571,7 +564,7 @@ void do_##name(struct pt_regs *regs, long error_code) \
571void do_##name(struct pt_regs *regs, long error_code) \ 564void do_##name(struct pt_regs *regs, long error_code) \
572{ \ 565{ \
573 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 566 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
574 == NOTIFY_STOP) \ 567 == NOTIFY_STOP) \
575 return; \ 568 return; \
576 do_trap(trapnr, signr, str, 1, regs, error_code, NULL); \ 569 do_trap(trapnr, signr, str, 1, regs, error_code, NULL); \
577} 570}
@@ -586,27 +579,29 @@ void do_##name(struct pt_regs *regs, long error_code) \
586 info.si_addr = (void __user *)siaddr; \ 579 info.si_addr = (void __user *)siaddr; \
587 trace_hardirqs_fixup(); \ 580 trace_hardirqs_fixup(); \
588 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 581 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
589 == NOTIFY_STOP) \ 582 == NOTIFY_STOP) \
590 return; \ 583 return; \
591 do_trap(trapnr, signr, str, 1, regs, error_code, &info); \ 584 do_trap(trapnr, signr, str, 1, regs, error_code, &info); \
592} 585}
593 586
594DO_VM86_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip) 587DO_VM86_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip)
595#ifndef CONFIG_KPROBES 588#ifndef CONFIG_KPROBES
596DO_VM86_ERROR(3, SIGTRAP, "int3", int3) 589DO_VM86_ERROR(3, SIGTRAP, "int3", int3)
597#endif 590#endif
598DO_VM86_ERROR(4, SIGSEGV, "overflow", overflow) 591DO_VM86_ERROR(4, SIGSEGV, "overflow", overflow)
599DO_VM86_ERROR(5, SIGSEGV, "bounds", bounds) 592DO_VM86_ERROR(5, SIGSEGV, "bounds", bounds)
600DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip, 0) 593DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip, 0)
601DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun) 594DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
602DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS) 595DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS)
603DO_ERROR(11, SIGBUS, "segment not present", segment_not_present) 596DO_ERROR(11, SIGBUS, "segment not present", segment_not_present)
604DO_ERROR(12, SIGBUS, "stack segment", stack_segment) 597DO_ERROR(12, SIGBUS, "stack segment", stack_segment)
605DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0, 0) 598DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0, 0)
606DO_ERROR_INFO(32, SIGILL, "iret exception", iret_error, ILL_BADSTK, 0, 1) 599DO_ERROR_INFO(32, SIGILL, "iret exception", iret_error, ILL_BADSTK, 0, 1)
607 600
608void __kprobes do_general_protection(struct pt_regs *regs, long error_code) 601void __kprobes
602do_general_protection(struct pt_regs *regs, long error_code)
609{ 603{
604 struct task_struct *tsk;
610 struct thread_struct *thread; 605 struct thread_struct *thread;
611 struct tss_struct *tss; 606 struct tss_struct *tss;
612 int cpu; 607 int cpu;
@@ -647,23 +642,24 @@ void __kprobes do_general_protection(struct pt_regs *regs, long error_code)
647 if (regs->flags & X86_VM_MASK) 642 if (regs->flags & X86_VM_MASK)
648 goto gp_in_vm86; 643 goto gp_in_vm86;
649 644
645 tsk = current;
650 if (!user_mode(regs)) 646 if (!user_mode(regs))
651 goto gp_in_kernel; 647 goto gp_in_kernel;
652 648
653 current->thread.error_code = error_code; 649 tsk->thread.error_code = error_code;
654 current->thread.trap_no = 13; 650 tsk->thread.trap_no = 13;
655 651
656 if (show_unhandled_signals && unhandled_signal(current, SIGSEGV) && 652 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
657 printk_ratelimit()) { 653 printk_ratelimit()) {
658 printk(KERN_INFO 654 printk(KERN_INFO
659 "%s[%d] general protection ip:%lx sp:%lx error:%lx", 655 "%s[%d] general protection ip:%lx sp:%lx error:%lx",
660 current->comm, task_pid_nr(current), 656 tsk->comm, task_pid_nr(tsk),
661 regs->ip, regs->sp, error_code); 657 regs->ip, regs->sp, error_code);
662 print_vma_addr(" in ", regs->ip); 658 print_vma_addr(" in ", regs->ip);
663 printk("\n"); 659 printk("\n");
664 } 660 }
665 661
666 force_sig(SIGSEGV, current); 662 force_sig(SIGSEGV, tsk);
667 return; 663 return;
668 664
669gp_in_vm86: 665gp_in_vm86:
@@ -672,14 +668,15 @@ gp_in_vm86:
672 return; 668 return;
673 669
674gp_in_kernel: 670gp_in_kernel:
675 if (!fixup_exception(regs)) { 671 if (fixup_exception(regs))
676 current->thread.error_code = error_code; 672 return;
677 current->thread.trap_no = 13; 673
678 if (notify_die(DIE_GPF, "general protection fault", regs, 674 tsk->thread.error_code = error_code;
675 tsk->thread.trap_no = 13;
676 if (notify_die(DIE_GPF, "general protection fault", regs,
679 error_code, 13, SIGSEGV) == NOTIFY_STOP) 677 error_code, 13, SIGSEGV) == NOTIFY_STOP)
680 return; 678 return;
681 die("general protection fault", regs, error_code); 679 die("general protection fault", regs, error_code);
682 }
683} 680}
684 681
685static notrace __kprobes void 682static notrace __kprobes void
@@ -756,9 +753,9 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
756 753
757static DEFINE_SPINLOCK(nmi_print_lock); 754static DEFINE_SPINLOCK(nmi_print_lock);
758 755
759void notrace __kprobes die_nmi(struct pt_regs *regs, const char *msg) 756void notrace __kprobes die_nmi(char *str, struct pt_regs *regs, int do_panic)
760{ 757{
761 if (notify_die(DIE_NMIWATCHDOG, msg, regs, 0, 2, SIGINT) == NOTIFY_STOP) 758 if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
762 return; 759 return;
763 760
764 spin_lock(&nmi_print_lock); 761 spin_lock(&nmi_print_lock);
@@ -767,10 +764,12 @@ void notrace __kprobes die_nmi(struct pt_regs *regs, const char *msg)
767 * to get a message out: 764 * to get a message out:
768 */ 765 */
769 bust_spinlocks(1); 766 bust_spinlocks(1);
770 printk(KERN_EMERG "%s", msg); 767 printk(KERN_EMERG "%s", str);
771 printk(" on CPU%d, ip %08lx, registers:\n", 768 printk(" on CPU%d, ip %08lx, registers:\n",
772 smp_processor_id(), regs->ip); 769 smp_processor_id(), regs->ip);
773 show_registers(regs); 770 show_registers(regs);
771 if (do_panic)
772 panic("Non maskable interrupt");
774 console_silent(); 773 console_silent();
775 spin_unlock(&nmi_print_lock); 774 spin_unlock(&nmi_print_lock);
776 bust_spinlocks(0); 775 bust_spinlocks(0);
@@ -790,14 +789,17 @@ void notrace __kprobes die_nmi(struct pt_regs *regs, const char *msg)
790static notrace __kprobes void default_do_nmi(struct pt_regs *regs) 789static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
791{ 790{
792 unsigned char reason = 0; 791 unsigned char reason = 0;
792 int cpu;
793
794 cpu = smp_processor_id();
793 795
794 /* Only the BSP gets external NMIs from the system: */ 796 /* Only the BSP gets external NMIs from the system. */
795 if (!smp_processor_id()) 797 if (!cpu)
796 reason = get_nmi_reason(); 798 reason = get_nmi_reason();
797 799
798 if (!(reason & 0xc0)) { 800 if (!(reason & 0xc0)) {
799 if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT) 801 if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT)
800 == NOTIFY_STOP) 802 == NOTIFY_STOP)
801 return; 803 return;
802#ifdef CONFIG_X86_LOCAL_APIC 804#ifdef CONFIG_X86_LOCAL_APIC
803 /* 805 /*
@@ -806,7 +808,7 @@ static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
806 */ 808 */
807 if (nmi_watchdog_tick(regs, reason)) 809 if (nmi_watchdog_tick(regs, reason))
808 return; 810 return;
809 if (!do_nmi_callback(regs, smp_processor_id())) 811 if (!do_nmi_callback(regs, cpu))
810 unknown_nmi_error(reason, regs); 812 unknown_nmi_error(reason, regs);
811#else 813#else
812 unknown_nmi_error(reason, regs); 814 unknown_nmi_error(reason, regs);
@@ -816,6 +818,8 @@ static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
816 } 818 }
817 if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) 819 if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP)
818 return; 820 return;
821
822 /* AK: following checks seem to be broken on modern chipsets. FIXME */
819 if (reason & 0x80) 823 if (reason & 0x80)
820 mem_parity_error(reason, regs); 824 mem_parity_error(reason, regs);
821 if (reason & 0x40) 825 if (reason & 0x40)
@@ -827,8 +831,6 @@ static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
827 reassert_nmi(); 831 reassert_nmi();
828} 832}
829 833
830static int ignore_nmis;
831
832notrace __kprobes void do_nmi(struct pt_regs *regs, long error_code) 834notrace __kprobes void do_nmi(struct pt_regs *regs, long error_code)
833{ 835{
834 int cpu; 836 int cpu;
@@ -913,7 +915,7 @@ void __kprobes do_debug(struct pt_regs *regs, long error_code)
913 tsk->thread.debugctlmsr = 0; 915 tsk->thread.debugctlmsr = 0;
914 916
915 if (notify_die(DIE_DEBUG, "debug", regs, condition, error_code, 917 if (notify_die(DIE_DEBUG, "debug", regs, condition, error_code,
916 SIGTRAP) == NOTIFY_STOP) 918 SIGTRAP) == NOTIFY_STOP)
917 return; 919 return;
918 /* It's safe to allow irq's after DR6 has been saved */ 920 /* It's safe to allow irq's after DR6 has been saved */
919 if (regs->flags & X86_EFLAGS_IF) 921 if (regs->flags & X86_EFLAGS_IF)
@@ -974,9 +976,8 @@ clear_TF_reenable:
974void math_error(void __user *ip) 976void math_error(void __user *ip)
975{ 977{
976 struct task_struct *task; 978 struct task_struct *task;
977 unsigned short cwd;
978 unsigned short swd;
979 siginfo_t info; 979 siginfo_t info;
980 unsigned short cwd, swd;
980 981
981 /* 982 /*
982 * Save the info for the exception handler and clear the error. 983 * Save the info for the exception handler and clear the error.
@@ -995,7 +996,7 @@ void math_error(void __user *ip)
995 * C1 reg you need in case of a stack fault, 0x040 is the stack 996 * C1 reg you need in case of a stack fault, 0x040 is the stack
996 * fault bit. We should only be taking one exception at a time, 997 * fault bit. We should only be taking one exception at a time,
997 * so if this combination doesn't produce any single exception, 998 * so if this combination doesn't produce any single exception,
998 * then we have a bad program that isn't syncronizing its FPU usage 999 * then we have a bad program that isn't synchronizing its FPU usage
999 * and it will suffer the consequences since we won't be able to 1000 * and it will suffer the consequences since we won't be able to
1000 * fully reproduce the context of the exception 1001 * fully reproduce the context of the exception
1001 */ 1002 */
@@ -1004,7 +1005,7 @@ void math_error(void __user *ip)
1004 switch (swd & ~cwd & 0x3f) { 1005 switch (swd & ~cwd & 0x3f) {
1005 case 0x000: /* No unmasked exception */ 1006 case 0x000: /* No unmasked exception */
1006 return; 1007 return;
1007 default: /* Multiple exceptions */ 1008 default: /* Multiple exceptions */
1008 break; 1009 break;
1009 case 0x001: /* Invalid Op */ 1010 case 0x001: /* Invalid Op */
1010 /* 1011 /*
@@ -1040,8 +1041,8 @@ void do_coprocessor_error(struct pt_regs *regs, long error_code)
1040static void simd_math_error(void __user *ip) 1041static void simd_math_error(void __user *ip)
1041{ 1042{
1042 struct task_struct *task; 1043 struct task_struct *task;
1043 unsigned short mxcsr;
1044 siginfo_t info; 1044 siginfo_t info;
1045 unsigned short mxcsr;
1045 1046
1046 /* 1047 /*
1047 * Save the info for the exception handler and clear the error. 1048 * Save the info for the exception handler and clear the error.
@@ -1117,7 +1118,7 @@ void do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
1117 1118
1118unsigned long patch_espfix_desc(unsigned long uesp, unsigned long kesp) 1119unsigned long patch_espfix_desc(unsigned long uesp, unsigned long kesp)
1119{ 1120{
1120 struct desc_struct *gdt = __get_cpu_var(gdt_page).gdt; 1121 struct desc_struct *gdt = get_cpu_gdt_table(smp_processor_id());
1121 unsigned long base = (kesp - uesp) & -THREAD_SIZE; 1122 unsigned long base = (kesp - uesp) & -THREAD_SIZE;
1122 unsigned long new_kesp = kesp - base; 1123 unsigned long new_kesp = kesp - base;
1123 unsigned long lim_pages = (new_kesp | (THREAD_SIZE - 1)) >> PAGE_SHIFT; 1124 unsigned long lim_pages = (new_kesp | (THREAD_SIZE - 1)) >> PAGE_SHIFT;
@@ -1196,19 +1197,16 @@ void __init trap_init(void)
1196 early_iounmap(p, 4); 1197 early_iounmap(p, 4);
1197#endif 1198#endif
1198 1199
1199#ifdef CONFIG_X86_LOCAL_APIC 1200 set_trap_gate(0, &divide_error);
1200 init_apic_mappings(); 1201 set_intr_gate(1, &debug);
1201#endif 1202 set_intr_gate(2, &nmi);
1202 set_trap_gate(0, &divide_error); 1203 set_system_intr_gate(3, &int3); /* int3 can be called from all */
1203 set_intr_gate(1, &debug); 1204 set_system_gate(4, &overflow); /* int4 can be called from all */
1204 set_intr_gate(2, &nmi); 1205 set_trap_gate(5, &bounds);
1205 set_system_intr_gate(3, &int3); /* int3/4 can be called from all */ 1206 set_trap_gate(6, &invalid_op);
1206 set_system_gate(4, &overflow); 1207 set_trap_gate(7, &device_not_available);
1207 set_trap_gate(5, &bounds); 1208 set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS);
1208 set_trap_gate(6, &invalid_op); 1209 set_trap_gate(9, &coprocessor_segment_overrun);
1209 set_trap_gate(7, &device_not_available);
1210 set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS);
1211 set_trap_gate(9, &coprocessor_segment_overrun);
1212 set_trap_gate(10, &invalid_TSS); 1210 set_trap_gate(10, &invalid_TSS);
1213 set_trap_gate(11, &segment_not_present); 1211 set_trap_gate(11, &segment_not_present);
1214 set_trap_gate(12, &stack_segment); 1212 set_trap_gate(12, &stack_segment);
diff --git a/arch/x86/kernel/traps_64.c b/arch/x86/kernel/traps_64.c
index adff76ea97c4..2696a6837782 100644
--- a/arch/x86/kernel/traps_64.c
+++ b/arch/x86/kernel/traps_64.c
@@ -10,49 +10,49 @@
10 * 'Traps.c' handles hardware traps and faults after we have saved some 10 * 'Traps.c' handles hardware traps and faults after we have saved some
11 * state in 'entry.S'. 11 * state in 'entry.S'.
12 */ 12 */
13#include <linux/sched.h> 13#include <linux/moduleparam.h>
14#include <linux/interrupt.h>
15#include <linux/kallsyms.h>
16#include <linux/spinlock.h>
17#include <linux/kprobes.h>
18#include <linux/uaccess.h>
19#include <linux/utsname.h>
20#include <linux/kdebug.h>
14#include <linux/kernel.h> 21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/ptrace.h>
15#include <linux/string.h> 24#include <linux/string.h>
25#include <linux/unwind.h>
26#include <linux/delay.h>
16#include <linux/errno.h> 27#include <linux/errno.h>
17#include <linux/ptrace.h> 28#include <linux/kexec.h>
29#include <linux/sched.h>
18#include <linux/timer.h> 30#include <linux/timer.h>
19#include <linux/mm.h>
20#include <linux/init.h> 31#include <linux/init.h>
21#include <linux/delay.h>
22#include <linux/spinlock.h>
23#include <linux/interrupt.h>
24#include <linux/kallsyms.h>
25#include <linux/module.h>
26#include <linux/moduleparam.h>
27#include <linux/nmi.h>
28#include <linux/kprobes.h>
29#include <linux/kexec.h>
30#include <linux/unwind.h>
31#include <linux/uaccess.h>
32#include <linux/bug.h> 32#include <linux/bug.h>
33#include <linux/kdebug.h> 33#include <linux/nmi.h>
34#include <linux/utsname.h> 34#include <linux/mm.h>
35
36#include <mach_traps.h>
37 35
38#if defined(CONFIG_EDAC) 36#if defined(CONFIG_EDAC)
39#include <linux/edac.h> 37#include <linux/edac.h>
40#endif 38#endif
41 39
42#include <asm/system.h> 40#include <asm/stacktrace.h>
43#include <asm/io.h> 41#include <asm/processor.h>
44#include <asm/atomic.h>
45#include <asm/debugreg.h> 42#include <asm/debugreg.h>
43#include <asm/atomic.h>
44#include <asm/system.h>
45#include <asm/unwind.h>
46#include <asm/desc.h> 46#include <asm/desc.h>
47#include <asm/i387.h> 47#include <asm/i387.h>
48#include <asm/processor.h> 48#include <asm/nmi.h>
49#include <asm/unwind.h>
50#include <asm/smp.h> 49#include <asm/smp.h>
50#include <asm/io.h>
51#include <asm/pgalloc.h> 51#include <asm/pgalloc.h>
52#include <asm/pda.h>
53#include <asm/proto.h> 52#include <asm/proto.h>
54#include <asm/nmi.h> 53#include <asm/pda.h>
55#include <asm/stacktrace.h> 54
55#include <mach_traps.h>
56 56
57asmlinkage void divide_error(void); 57asmlinkage void divide_error(void);
58asmlinkage void debug(void); 58asmlinkage void debug(void);
@@ -71,12 +71,15 @@ asmlinkage void general_protection(void);
71asmlinkage void page_fault(void); 71asmlinkage void page_fault(void);
72asmlinkage void coprocessor_error(void); 72asmlinkage void coprocessor_error(void);
73asmlinkage void simd_coprocessor_error(void); 73asmlinkage void simd_coprocessor_error(void);
74asmlinkage void reserved(void);
75asmlinkage void alignment_check(void); 74asmlinkage void alignment_check(void);
76asmlinkage void machine_check(void);
77asmlinkage void spurious_interrupt_bug(void); 75asmlinkage void spurious_interrupt_bug(void);
76asmlinkage void machine_check(void);
78 77
78int panic_on_unrecovered_nmi;
79int kstack_depth_to_print = 12;
79static unsigned int code_bytes = 64; 80static unsigned int code_bytes = 64;
81static int ignore_nmis;
82static int die_counter;
80 83
81static inline void conditional_sti(struct pt_regs *regs) 84static inline void conditional_sti(struct pt_regs *regs)
82{ 85{
@@ -100,34 +103,9 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
100 dec_preempt_count(); 103 dec_preempt_count();
101} 104}
102 105
103int kstack_depth_to_print = 12;
104
105void printk_address(unsigned long address, int reliable) 106void printk_address(unsigned long address, int reliable)
106{ 107{
107#ifdef CONFIG_KALLSYMS 108 printk(" [<%016lx>] %s%pS\n", address, reliable ? "": "? ", (void *) address);
108 unsigned long offset = 0, symsize;
109 const char *symname;
110 char *modname;
111 char *delim = ":";
112 char namebuf[KSYM_NAME_LEN];
113 char reliab[4] = "";
114
115 symname = kallsyms_lookup(address, &symsize, &offset,
116 &modname, namebuf);
117 if (!symname) {
118 printk(" [<%016lx>]\n", address);
119 return;
120 }
121 if (!reliable)
122 strcpy(reliab, "? ");
123
124 if (!modname)
125 modname = delim = "";
126 printk(" [<%016lx>] %s%s%s%s%s+0x%lx/0x%lx\n",
127 address, reliab, delim, modname, delim, symname, offset, symsize);
128#else
129 printk(" [<%016lx>]\n", address);
130#endif
131} 109}
132 110
133static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack, 111static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
@@ -204,8 +182,6 @@ static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
204 return NULL; 182 return NULL;
205} 183}
206 184
207#define MSG(txt) ops->warning(data, txt)
208
209/* 185/*
210 * x86-64 can have up to three kernel stacks: 186 * x86-64 can have up to three kernel stacks:
211 * process stack 187 * process stack
@@ -232,11 +208,11 @@ struct stack_frame {
232 unsigned long return_address; 208 unsigned long return_address;
233}; 209};
234 210
235 211static inline unsigned long
236static inline unsigned long print_context_stack(struct thread_info *tinfo, 212print_context_stack(struct thread_info *tinfo,
237 unsigned long *stack, unsigned long bp, 213 unsigned long *stack, unsigned long bp,
238 const struct stacktrace_ops *ops, void *data, 214 const struct stacktrace_ops *ops, void *data,
239 unsigned long *end) 215 unsigned long *end)
240{ 216{
241 struct stack_frame *frame = (struct stack_frame *)bp; 217 struct stack_frame *frame = (struct stack_frame *)bp;
242 218
@@ -258,7 +234,7 @@ static inline unsigned long print_context_stack(struct thread_info *tinfo,
258 return bp; 234 return bp;
259} 235}
260 236
261void dump_trace(struct task_struct *tsk, struct pt_regs *regs, 237void dump_trace(struct task_struct *task, struct pt_regs *regs,
262 unsigned long *stack, unsigned long bp, 238 unsigned long *stack, unsigned long bp,
263 const struct stacktrace_ops *ops, void *data) 239 const struct stacktrace_ops *ops, void *data)
264{ 240{
@@ -267,36 +243,34 @@ void dump_trace(struct task_struct *tsk, struct pt_regs *regs,
267 unsigned used = 0; 243 unsigned used = 0;
268 struct thread_info *tinfo; 244 struct thread_info *tinfo;
269 245
270 if (!tsk) 246 if (!task)
271 tsk = current; 247 task = current;
272 tinfo = task_thread_info(tsk);
273 248
274 if (!stack) { 249 if (!stack) {
275 unsigned long dummy; 250 unsigned long dummy;
276 stack = &dummy; 251 stack = &dummy;
277 if (tsk && tsk != current) 252 if (task && task != current)
278 stack = (unsigned long *)tsk->thread.sp; 253 stack = (unsigned long *)task->thread.sp;
279 } 254 }
280 255
281#ifdef CONFIG_FRAME_POINTER 256#ifdef CONFIG_FRAME_POINTER
282 if (!bp) { 257 if (!bp) {
283 if (tsk == current) { 258 if (task == current) {
284 /* Grab bp right from our regs */ 259 /* Grab bp right from our regs */
285 asm("movq %%rbp, %0" : "=r" (bp):); 260 asm("movq %%rbp, %0" : "=r" (bp) :);
286 } else { 261 } else {
287 /* bp is the last reg pushed by switch_to */ 262 /* bp is the last reg pushed by switch_to */
288 bp = *(unsigned long *) tsk->thread.sp; 263 bp = *(unsigned long *) task->thread.sp;
289 } 264 }
290 } 265 }
291#endif 266#endif
292 267
293
294
295 /* 268 /*
296 * Print function call entries in all stacks, starting at the 269 * Print function call entries in all stacks, starting at the
297 * current stack address. If the stacks consist of nested 270 * current stack address. If the stacks consist of nested
298 * exceptions 271 * exceptions
299 */ 272 */
273 tinfo = task_thread_info(task);
300 for (;;) { 274 for (;;) {
301 char *id; 275 char *id;
302 unsigned long *estack_end; 276 unsigned long *estack_end;
@@ -381,18 +355,17 @@ static const struct stacktrace_ops print_trace_ops = {
381 .address = print_trace_address, 355 .address = print_trace_address,
382}; 356};
383 357
384void 358void show_trace(struct task_struct *task, struct pt_regs *regs,
385show_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long *stack, 359 unsigned long *stack, unsigned long bp)
386 unsigned long bp)
387{ 360{
388 printk("\nCall Trace:\n"); 361 printk("\nCall Trace:\n");
389 dump_trace(tsk, regs, stack, bp, &print_trace_ops, NULL); 362 dump_trace(task, regs, stack, bp, &print_trace_ops, NULL);
390 printk("\n"); 363 printk("\n");
391} 364}
392 365
393static void 366static void
394_show_stack(struct task_struct *tsk, struct pt_regs *regs, unsigned long *sp, 367_show_stack(struct task_struct *task, struct pt_regs *regs,
395 unsigned long bp) 368 unsigned long *sp, unsigned long bp)
396{ 369{
397 unsigned long *stack; 370 unsigned long *stack;
398 int i; 371 int i;
@@ -404,14 +377,14 @@ _show_stack(struct task_struct *tsk, struct pt_regs *regs, unsigned long *sp,
404 // back trace for this cpu. 377 // back trace for this cpu.
405 378
406 if (sp == NULL) { 379 if (sp == NULL) {
407 if (tsk) 380 if (task)
408 sp = (unsigned long *)tsk->thread.sp; 381 sp = (unsigned long *)task->thread.sp;
409 else 382 else
410 sp = (unsigned long *)&sp; 383 sp = (unsigned long *)&sp;
411 } 384 }
412 385
413 stack = sp; 386 stack = sp;
414 for(i=0; i < kstack_depth_to_print; i++) { 387 for (i = 0; i < kstack_depth_to_print; i++) {
415 if (stack >= irqstack && stack <= irqstack_end) { 388 if (stack >= irqstack && stack <= irqstack_end) {
416 if (stack == irqstack_end) { 389 if (stack == irqstack_end) {
417 stack = (unsigned long *) (irqstack_end[-1]); 390 stack = (unsigned long *) (irqstack_end[-1]);
@@ -426,12 +399,12 @@ _show_stack(struct task_struct *tsk, struct pt_regs *regs, unsigned long *sp,
426 printk(" %016lx", *stack++); 399 printk(" %016lx", *stack++);
427 touch_nmi_watchdog(); 400 touch_nmi_watchdog();
428 } 401 }
429 show_trace(tsk, regs, sp, bp); 402 show_trace(task, regs, sp, bp);
430} 403}
431 404
432void show_stack(struct task_struct *tsk, unsigned long * sp) 405void show_stack(struct task_struct *task, unsigned long *sp)
433{ 406{
434 _show_stack(tsk, NULL, sp, 0); 407 _show_stack(task, NULL, sp, 0);
435} 408}
436 409
437/* 410/*
@@ -439,8 +412,8 @@ void show_stack(struct task_struct *tsk, unsigned long * sp)
439 */ 412 */
440void dump_stack(void) 413void dump_stack(void)
441{ 414{
442 unsigned long dummy;
443 unsigned long bp = 0; 415 unsigned long bp = 0;
416 unsigned long stack;
444 417
445#ifdef CONFIG_FRAME_POINTER 418#ifdef CONFIG_FRAME_POINTER
446 if (!bp) 419 if (!bp)
@@ -452,7 +425,7 @@ void dump_stack(void)
452 init_utsname()->release, 425 init_utsname()->release,
453 (int)strcspn(init_utsname()->version, " "), 426 (int)strcspn(init_utsname()->version, " "),
454 init_utsname()->version); 427 init_utsname()->version);
455 show_trace(NULL, NULL, &dummy, bp); 428 show_trace(NULL, NULL, &stack, bp);
456} 429}
457 430
458EXPORT_SYMBOL(dump_stack); 431EXPORT_SYMBOL(dump_stack);
@@ -463,12 +436,8 @@ void show_registers(struct pt_regs *regs)
463 unsigned long sp; 436 unsigned long sp;
464 const int cpu = smp_processor_id(); 437 const int cpu = smp_processor_id();
465 struct task_struct *cur = cpu_pda(cpu)->pcurrent; 438 struct task_struct *cur = cpu_pda(cpu)->pcurrent;
466 u8 *ip;
467 unsigned int code_prologue = code_bytes * 43 / 64;
468 unsigned int code_len = code_bytes;
469 439
470 sp = regs->sp; 440 sp = regs->sp;
471 ip = (u8 *) regs->ip - code_prologue;
472 printk("CPU %d ", cpu); 441 printk("CPU %d ", cpu);
473 __show_regs(regs); 442 __show_regs(regs);
474 printk("Process %s (pid: %d, threadinfo %p, task %p)\n", 443 printk("Process %s (pid: %d, threadinfo %p, task %p)\n",
@@ -479,15 +448,21 @@ void show_registers(struct pt_regs *regs)
479 * time of the fault.. 448 * time of the fault..
480 */ 449 */
481 if (!user_mode(regs)) { 450 if (!user_mode(regs)) {
451 unsigned int code_prologue = code_bytes * 43 / 64;
452 unsigned int code_len = code_bytes;
482 unsigned char c; 453 unsigned char c;
454 u8 *ip;
455
483 printk("Stack: "); 456 printk("Stack: ");
484 _show_stack(NULL, regs, (unsigned long *)sp, regs->bp); 457 _show_stack(NULL, regs, (unsigned long *)sp, regs->bp);
485 printk("\n"); 458 printk("\n");
486 459
487 printk(KERN_EMERG "Code: "); 460 printk(KERN_EMERG "Code: ");
461
462 ip = (u8 *)regs->ip - code_prologue;
488 if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) { 463 if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) {
489 /* try starting at RIP */ 464 /* try starting at RIP */
490 ip = (u8 *) regs->ip; 465 ip = (u8 *)regs->ip;
491 code_len = code_len - code_prologue + 1; 466 code_len = code_len - code_prologue + 1;
492 } 467 }
493 for (i = 0; i < code_len; i++, ip++) { 468 for (i = 0; i < code_len; i++, ip++) {
@@ -503,7 +478,7 @@ void show_registers(struct pt_regs *regs)
503 } 478 }
504 } 479 }
505 printk("\n"); 480 printk("\n");
506} 481}
507 482
508int is_valid_bugaddr(unsigned long ip) 483int is_valid_bugaddr(unsigned long ip)
509{ 484{
@@ -561,10 +536,9 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
561 do_exit(signr); 536 do_exit(signr);
562} 537}
563 538
564int __kprobes __die(const char * str, struct pt_regs * regs, long err) 539int __kprobes __die(const char *str, struct pt_regs *regs, long err)
565{ 540{
566 static int die_counter; 541 printk(KERN_EMERG "%s: %04lx [%u] ", str, err & 0xffff, ++die_counter);
567 printk(KERN_EMERG "%s: %04lx [%u] ", str, err & 0xffff,++die_counter);
568#ifdef CONFIG_PREEMPT 542#ifdef CONFIG_PREEMPT
569 printk("PREEMPT "); 543 printk("PREEMPT ");
570#endif 544#endif
@@ -575,8 +549,10 @@ int __kprobes __die(const char * str, struct pt_regs * regs, long err)
575 printk("DEBUG_PAGEALLOC"); 549 printk("DEBUG_PAGEALLOC");
576#endif 550#endif
577 printk("\n"); 551 printk("\n");
578 if (notify_die(DIE_OOPS, str, regs, err, current->thread.trap_no, SIGSEGV) == NOTIFY_STOP) 552 if (notify_die(DIE_OOPS, str, regs, err,
553 current->thread.trap_no, SIGSEGV) == NOTIFY_STOP)
579 return 1; 554 return 1;
555
580 show_registers(regs); 556 show_registers(regs);
581 add_taint(TAINT_DIE); 557 add_taint(TAINT_DIE);
582 /* Executive summary in case the oops scrolled away */ 558 /* Executive summary in case the oops scrolled away */
@@ -588,7 +564,7 @@ int __kprobes __die(const char * str, struct pt_regs * regs, long err)
588 return 0; 564 return 0;
589} 565}
590 566
591void die(const char * str, struct pt_regs * regs, long err) 567void die(const char *str, struct pt_regs *regs, long err)
592{ 568{
593 unsigned long flags = oops_begin(); 569 unsigned long flags = oops_begin();
594 570
@@ -605,8 +581,7 @@ die_nmi(char *str, struct pt_regs *regs, int do_panic)
605{ 581{
606 unsigned long flags; 582 unsigned long flags;
607 583
608 if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == 584 if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
609 NOTIFY_STOP)
610 return; 585 return;
611 586
612 flags = oops_begin(); 587 flags = oops_begin();
@@ -614,7 +589,9 @@ die_nmi(char *str, struct pt_regs *regs, int do_panic)
614 * We are in trouble anyway, lets at least try 589 * We are in trouble anyway, lets at least try
615 * to get a message out. 590 * to get a message out.
616 */ 591 */
617 printk(str, smp_processor_id()); 592 printk(KERN_EMERG "%s", str);
593 printk(" on CPU%d, ip %08lx, registers:\n",
594 smp_processor_id(), regs->ip);
618 show_registers(regs); 595 show_registers(regs);
619 if (kexec_should_crash(current)) 596 if (kexec_should_crash(current))
620 crash_kexec(regs); 597 crash_kexec(regs);
@@ -626,44 +603,44 @@ die_nmi(char *str, struct pt_regs *regs, int do_panic)
626 do_exit(SIGBUS); 603 do_exit(SIGBUS);
627} 604}
628 605
629static void __kprobes do_trap(int trapnr, int signr, char *str, 606static void __kprobes
630 struct pt_regs * regs, long error_code, 607do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
631 siginfo_t *info) 608 long error_code, siginfo_t *info)
632{ 609{
633 struct task_struct *tsk = current; 610 struct task_struct *tsk = current;
634 611
635 if (user_mode(regs)) { 612 if (!user_mode(regs))
636 /* 613 goto kernel_trap;
637 * We want error_code and trap_no set for userspace
638 * faults and kernelspace faults which result in
639 * die(), but not kernelspace faults which are fixed
640 * up. die() gives the process no chance to handle
641 * the signal and notice the kernel fault information,
642 * so that won't result in polluting the information
643 * about previously queued, but not yet delivered,
644 * faults. See also do_general_protection below.
645 */
646 tsk->thread.error_code = error_code;
647 tsk->thread.trap_no = trapnr;
648
649 if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
650 printk_ratelimit()) {
651 printk(KERN_INFO
652 "%s[%d] trap %s ip:%lx sp:%lx error:%lx",
653 tsk->comm, tsk->pid, str,
654 regs->ip, regs->sp, error_code);
655 print_vma_addr(" in ", regs->ip);
656 printk("\n");
657 }
658 614
659 if (info) 615 /*
660 force_sig_info(signr, info, tsk); 616 * We want error_code and trap_no set for userspace faults and
661 else 617 * kernelspace faults which result in die(), but not
662 force_sig(signr, tsk); 618 * kernelspace faults which are fixed up. die() gives the
663 return; 619 * process no chance to handle the signal and notice the
620 * kernel fault information, so that won't result in polluting
621 * the information about previously queued, but not yet
622 * delivered, faults. See also do_general_protection below.
623 */
624 tsk->thread.error_code = error_code;
625 tsk->thread.trap_no = trapnr;
626
627 if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
628 printk_ratelimit()) {
629 printk(KERN_INFO
630 "%s[%d] trap %s ip:%lx sp:%lx error:%lx",
631 tsk->comm, tsk->pid, str,
632 regs->ip, regs->sp, error_code);
633 print_vma_addr(" in ", regs->ip);
634 printk("\n");
664 } 635 }
665 636
637 if (info)
638 force_sig_info(signr, info, tsk);
639 else
640 force_sig(signr, tsk);
641 return;
666 642
643kernel_trap:
667 if (!fixup_exception(regs)) { 644 if (!fixup_exception(regs)) {
668 tsk->thread.error_code = error_code; 645 tsk->thread.error_code = error_code;
669 tsk->thread.trap_no = trapnr; 646 tsk->thread.trap_no = trapnr;
@@ -673,41 +650,39 @@ static void __kprobes do_trap(int trapnr, int signr, char *str,
673} 650}
674 651
675#define DO_ERROR(trapnr, signr, str, name) \ 652#define DO_ERROR(trapnr, signr, str, name) \
676asmlinkage void do_##name(struct pt_regs * regs, long error_code) \ 653asmlinkage void do_##name(struct pt_regs * regs, long error_code) \
677{ \ 654{ \
678 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 655 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
679 == NOTIFY_STOP) \ 656 == NOTIFY_STOP) \
680 return; \ 657 return; \
681 conditional_sti(regs); \ 658 conditional_sti(regs); \
682 do_trap(trapnr, signr, str, regs, error_code, NULL); \ 659 do_trap(trapnr, signr, str, regs, error_code, NULL); \
683} 660}
684 661
685#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \ 662#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
686asmlinkage void do_##name(struct pt_regs * regs, long error_code) \ 663asmlinkage void do_##name(struct pt_regs * regs, long error_code) \
687{ \ 664{ \
688 siginfo_t info; \ 665 siginfo_t info; \
689 info.si_signo = signr; \ 666 info.si_signo = signr; \
690 info.si_errno = 0; \ 667 info.si_errno = 0; \
691 info.si_code = sicode; \ 668 info.si_code = sicode; \
692 info.si_addr = (void __user *)siaddr; \ 669 info.si_addr = (void __user *)siaddr; \
693 trace_hardirqs_fixup(); \ 670 trace_hardirqs_fixup(); \
694 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 671 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
695 == NOTIFY_STOP) \ 672 == NOTIFY_STOP) \
696 return; \ 673 return; \
697 conditional_sti(regs); \ 674 conditional_sti(regs); \
698 do_trap(trapnr, signr, str, regs, error_code, &info); \ 675 do_trap(trapnr, signr, str, regs, error_code, &info); \
699} 676}
700 677
701DO_ERROR_INFO( 0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip) 678DO_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip)
702DO_ERROR( 4, SIGSEGV, "overflow", overflow) 679DO_ERROR(4, SIGSEGV, "overflow", overflow)
703DO_ERROR( 5, SIGSEGV, "bounds", bounds) 680DO_ERROR(5, SIGSEGV, "bounds", bounds)
704DO_ERROR_INFO( 6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip) 681DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip)
705DO_ERROR( 7, SIGSEGV, "device not available", device_not_available) 682DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
706DO_ERROR( 9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
707DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS) 683DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS)
708DO_ERROR(11, SIGBUS, "segment not present", segment_not_present) 684DO_ERROR(11, SIGBUS, "segment not present", segment_not_present)
709DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0) 685DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0)
710DO_ERROR(18, SIGSEGV, "reserved", reserved)
711 686
712/* Runs on IST stack */ 687/* Runs on IST stack */
713asmlinkage void do_stack_segment(struct pt_regs *regs, long error_code) 688asmlinkage void do_stack_segment(struct pt_regs *regs, long error_code)
@@ -737,31 +712,34 @@ asmlinkage void do_double_fault(struct pt_regs * regs, long error_code)
737 die(str, regs, error_code); 712 die(str, regs, error_code);
738} 713}
739 714
740asmlinkage void __kprobes do_general_protection(struct pt_regs * regs, 715asmlinkage void __kprobes
741 long error_code) 716do_general_protection(struct pt_regs *regs, long error_code)
742{ 717{
743 struct task_struct *tsk = current; 718 struct task_struct *tsk;
744 719
745 conditional_sti(regs); 720 conditional_sti(regs);
746 721
747 if (user_mode(regs)) { 722 tsk = current;
748 tsk->thread.error_code = error_code; 723 if (!user_mode(regs))
749 tsk->thread.trap_no = 13; 724 goto gp_in_kernel;
750
751 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
752 printk_ratelimit()) {
753 printk(KERN_INFO
754 "%s[%d] general protection ip:%lx sp:%lx error:%lx",
755 tsk->comm, tsk->pid,
756 regs->ip, regs->sp, error_code);
757 print_vma_addr(" in ", regs->ip);
758 printk("\n");
759 }
760 725
761 force_sig(SIGSEGV, tsk); 726 tsk->thread.error_code = error_code;
762 return; 727 tsk->thread.trap_no = 13;
763 } 728
729 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
730 printk_ratelimit()) {
731 printk(KERN_INFO
732 "%s[%d] general protection ip:%lx sp:%lx error:%lx",
733 tsk->comm, tsk->pid,
734 regs->ip, regs->sp, error_code);
735 print_vma_addr(" in ", regs->ip);
736 printk("\n");
737 }
764 738
739 force_sig(SIGSEGV, tsk);
740 return;
741
742gp_in_kernel:
765 if (fixup_exception(regs)) 743 if (fixup_exception(regs))
766 return; 744 return;
767 745
@@ -774,14 +752,14 @@ asmlinkage void __kprobes do_general_protection(struct pt_regs * regs,
774} 752}
775 753
776static notrace __kprobes void 754static notrace __kprobes void
777mem_parity_error(unsigned char reason, struct pt_regs * regs) 755mem_parity_error(unsigned char reason, struct pt_regs *regs)
778{ 756{
779 printk(KERN_EMERG "Uhhuh. NMI received for unknown reason %02x.\n", 757 printk(KERN_EMERG "Uhhuh. NMI received for unknown reason %02x.\n",
780 reason); 758 reason);
781 printk(KERN_EMERG "You have some hardware problem, likely on the PCI bus.\n"); 759 printk(KERN_EMERG "You have some hardware problem, likely on the PCI bus.\n");
782 760
783#if defined(CONFIG_EDAC) 761#if defined(CONFIG_EDAC)
784 if(edac_handler_set()) { 762 if (edac_handler_set()) {
785 edac_atomic_assert_error(); 763 edac_atomic_assert_error();
786 return; 764 return;
787 } 765 }
@@ -798,7 +776,7 @@ mem_parity_error(unsigned char reason, struct pt_regs * regs)
798} 776}
799 777
800static notrace __kprobes void 778static notrace __kprobes void
801io_check_error(unsigned char reason, struct pt_regs * regs) 779io_check_error(unsigned char reason, struct pt_regs *regs)
802{ 780{
803 printk("NMI: IOCK error (debug interrupt?)\n"); 781 printk("NMI: IOCK error (debug interrupt?)\n");
804 show_registers(regs); 782 show_registers(regs);
@@ -828,14 +806,14 @@ unknown_nmi_error(unsigned char reason, struct pt_regs * regs)
828 806
829/* Runs on IST stack. This code must keep interrupts off all the time. 807/* Runs on IST stack. This code must keep interrupts off all the time.
830 Nested NMIs are prevented by the CPU. */ 808 Nested NMIs are prevented by the CPU. */
831asmlinkage notrace __kprobes void default_do_nmi(struct pt_regs *regs) 809asmlinkage notrace __kprobes void default_do_nmi(struct pt_regs *regs)
832{ 810{
833 unsigned char reason = 0; 811 unsigned char reason = 0;
834 int cpu; 812 int cpu;
835 813
836 cpu = smp_processor_id(); 814 cpu = smp_processor_id();
837 815
838 /* Only the BSP gets external NMIs from the system. */ 816 /* Only the BSP gets external NMIs from the system. */
839 if (!cpu) 817 if (!cpu)
840 reason = get_nmi_reason(); 818 reason = get_nmi_reason();
841 819
@@ -847,32 +825,57 @@ asmlinkage notrace __kprobes void default_do_nmi(struct pt_regs *regs)
847 * Ok, so this is none of the documented NMI sources, 825 * Ok, so this is none of the documented NMI sources,
848 * so it must be the NMI watchdog. 826 * so it must be the NMI watchdog.
849 */ 827 */
850 if (nmi_watchdog_tick(regs,reason)) 828 if (nmi_watchdog_tick(regs, reason))
851 return; 829 return;
852 if (!do_nmi_callback(regs,cpu)) 830 if (!do_nmi_callback(regs, cpu))
853 unknown_nmi_error(reason, regs); 831 unknown_nmi_error(reason, regs);
854 832
855 return; 833 return;
856 } 834 }
857 if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) 835 if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP)
858 return; 836 return;
859 837
860 /* AK: following checks seem to be broken on modern chipsets. FIXME */ 838 /* AK: following checks seem to be broken on modern chipsets. FIXME */
861
862 if (reason & 0x80) 839 if (reason & 0x80)
863 mem_parity_error(reason, regs); 840 mem_parity_error(reason, regs);
864 if (reason & 0x40) 841 if (reason & 0x40)
865 io_check_error(reason, regs); 842 io_check_error(reason, regs);
866} 843}
867 844
845asmlinkage notrace __kprobes void
846do_nmi(struct pt_regs *regs, long error_code)
847{
848 nmi_enter();
849
850 add_pda(__nmi_count, 1);
851
852 if (!ignore_nmis)
853 default_do_nmi(regs);
854
855 nmi_exit();
856}
857
858void stop_nmi(void)
859{
860 acpi_nmi_disable();
861 ignore_nmis++;
862}
863
864void restart_nmi(void)
865{
866 ignore_nmis--;
867 acpi_nmi_enable();
868}
869
868/* runs on IST stack. */ 870/* runs on IST stack. */
869asmlinkage void __kprobes do_int3(struct pt_regs * regs, long error_code) 871asmlinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
870{ 872{
871 trace_hardirqs_fixup(); 873 trace_hardirqs_fixup();
872 874
873 if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP) == NOTIFY_STOP) { 875 if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
876 == NOTIFY_STOP)
874 return; 877 return;
875 } 878
876 preempt_conditional_sti(regs); 879 preempt_conditional_sti(regs);
877 do_trap(3, SIGTRAP, "int3", regs, error_code, NULL); 880 do_trap(3, SIGTRAP, "int3", regs, error_code, NULL);
878 preempt_conditional_cli(regs); 881 preempt_conditional_cli(regs);
@@ -903,8 +906,8 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
903asmlinkage void __kprobes do_debug(struct pt_regs * regs, 906asmlinkage void __kprobes do_debug(struct pt_regs * regs,
904 unsigned long error_code) 907 unsigned long error_code)
905{ 908{
906 unsigned long condition;
907 struct task_struct *tsk = current; 909 struct task_struct *tsk = current;
910 unsigned long condition;
908 siginfo_t info; 911 siginfo_t info;
909 912
910 trace_hardirqs_fixup(); 913 trace_hardirqs_fixup();
@@ -925,21 +928,19 @@ asmlinkage void __kprobes do_debug(struct pt_regs * regs,
925 928
926 /* Mask out spurious debug traps due to lazy DR7 setting */ 929 /* Mask out spurious debug traps due to lazy DR7 setting */
927 if (condition & (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3)) { 930 if (condition & (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3)) {
928 if (!tsk->thread.debugreg7) { 931 if (!tsk->thread.debugreg7)
929 goto clear_dr7; 932 goto clear_dr7;
930 }
931 } 933 }
932 934
933 tsk->thread.debugreg6 = condition; 935 tsk->thread.debugreg6 = condition;
934 936
935
936 /* 937 /*
937 * Single-stepping through TF: make sure we ignore any events in 938 * Single-stepping through TF: make sure we ignore any events in
938 * kernel space (but re-enable TF when returning to user mode). 939 * kernel space (but re-enable TF when returning to user mode).
939 */ 940 */
940 if (condition & DR_STEP) { 941 if (condition & DR_STEP) {
941 if (!user_mode(regs)) 942 if (!user_mode(regs))
942 goto clear_TF_reenable; 943 goto clear_TF_reenable;
943 } 944 }
944 945
945 /* Ok, finally something we can handle */ 946 /* Ok, finally something we can handle */
@@ -952,7 +953,7 @@ asmlinkage void __kprobes do_debug(struct pt_regs * regs,
952 force_sig_info(SIGTRAP, &info, tsk); 953 force_sig_info(SIGTRAP, &info, tsk);
953 954
954clear_dr7: 955clear_dr7:
955 set_debugreg(0UL, 7); 956 set_debugreg(0, 7);
956 preempt_conditional_cli(regs); 957 preempt_conditional_cli(regs);
957 return; 958 return;
958 959
@@ -960,6 +961,7 @@ clear_TF_reenable:
960 set_tsk_thread_flag(tsk, TIF_SINGLESTEP); 961 set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
961 regs->flags &= ~X86_EFLAGS_TF; 962 regs->flags &= ~X86_EFLAGS_TF;
962 preempt_conditional_cli(regs); 963 preempt_conditional_cli(regs);
964 return;
963} 965}
964 966
965static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr) 967static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr)
@@ -982,7 +984,7 @@ static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr)
982asmlinkage void do_coprocessor_error(struct pt_regs *regs) 984asmlinkage void do_coprocessor_error(struct pt_regs *regs)
983{ 985{
984 void __user *ip = (void __user *)(regs->ip); 986 void __user *ip = (void __user *)(regs->ip);
985 struct task_struct * task; 987 struct task_struct *task;
986 siginfo_t info; 988 siginfo_t info;
987 unsigned short cwd, swd; 989 unsigned short cwd, swd;
988 990
@@ -1015,30 +1017,30 @@ asmlinkage void do_coprocessor_error(struct pt_regs *regs)
1015 cwd = get_fpu_cwd(task); 1017 cwd = get_fpu_cwd(task);
1016 swd = get_fpu_swd(task); 1018 swd = get_fpu_swd(task);
1017 switch (swd & ~cwd & 0x3f) { 1019 switch (swd & ~cwd & 0x3f) {
1018 case 0x000: 1020 case 0x000: /* No unmasked exception */
1019 default: 1021 default: /* Multiple exceptions */
1020 break; 1022 break;
1021 case 0x001: /* Invalid Op */ 1023 case 0x001: /* Invalid Op */
1022 /* 1024 /*
1023 * swd & 0x240 == 0x040: Stack Underflow 1025 * swd & 0x240 == 0x040: Stack Underflow
1024 * swd & 0x240 == 0x240: Stack Overflow 1026 * swd & 0x240 == 0x240: Stack Overflow
1025 * User must clear the SF bit (0x40) if set 1027 * User must clear the SF bit (0x40) if set
1026 */ 1028 */
1027 info.si_code = FPE_FLTINV; 1029 info.si_code = FPE_FLTINV;
1028 break; 1030 break;
1029 case 0x002: /* Denormalize */ 1031 case 0x002: /* Denormalize */
1030 case 0x010: /* Underflow */ 1032 case 0x010: /* Underflow */
1031 info.si_code = FPE_FLTUND; 1033 info.si_code = FPE_FLTUND;
1032 break; 1034 break;
1033 case 0x004: /* Zero Divide */ 1035 case 0x004: /* Zero Divide */
1034 info.si_code = FPE_FLTDIV; 1036 info.si_code = FPE_FLTDIV;
1035 break; 1037 break;
1036 case 0x008: /* Overflow */ 1038 case 0x008: /* Overflow */
1037 info.si_code = FPE_FLTOVF; 1039 info.si_code = FPE_FLTOVF;
1038 break; 1040 break;
1039 case 0x020: /* Precision */ 1041 case 0x020: /* Precision */
1040 info.si_code = FPE_FLTRES; 1042 info.si_code = FPE_FLTRES;
1041 break; 1043 break;
1042 } 1044 }
1043 force_sig_info(SIGFPE, &info, task); 1045 force_sig_info(SIGFPE, &info, task);
1044} 1046}
@@ -1051,7 +1053,7 @@ asmlinkage void bad_intr(void)
1051asmlinkage void do_simd_coprocessor_error(struct pt_regs *regs) 1053asmlinkage void do_simd_coprocessor_error(struct pt_regs *regs)
1052{ 1054{
1053 void __user *ip = (void __user *)(regs->ip); 1055 void __user *ip = (void __user *)(regs->ip);
1054 struct task_struct * task; 1056 struct task_struct *task;
1055 siginfo_t info; 1057 siginfo_t info;
1056 unsigned short mxcsr; 1058 unsigned short mxcsr;
1057 1059
@@ -1079,25 +1081,25 @@ asmlinkage void do_simd_coprocessor_error(struct pt_regs *regs)
1079 */ 1081 */
1080 mxcsr = get_fpu_mxcsr(task); 1082 mxcsr = get_fpu_mxcsr(task);
1081 switch (~((mxcsr & 0x1f80) >> 7) & (mxcsr & 0x3f)) { 1083 switch (~((mxcsr & 0x1f80) >> 7) & (mxcsr & 0x3f)) {
1082 case 0x000: 1084 case 0x000:
1083 default: 1085 default:
1084 break; 1086 break;
1085 case 0x001: /* Invalid Op */ 1087 case 0x001: /* Invalid Op */
1086 info.si_code = FPE_FLTINV; 1088 info.si_code = FPE_FLTINV;
1087 break; 1089 break;
1088 case 0x002: /* Denormalize */ 1090 case 0x002: /* Denormalize */
1089 case 0x010: /* Underflow */ 1091 case 0x010: /* Underflow */
1090 info.si_code = FPE_FLTUND; 1092 info.si_code = FPE_FLTUND;
1091 break; 1093 break;
1092 case 0x004: /* Zero Divide */ 1094 case 0x004: /* Zero Divide */
1093 info.si_code = FPE_FLTDIV; 1095 info.si_code = FPE_FLTDIV;
1094 break; 1096 break;
1095 case 0x008: /* Overflow */ 1097 case 0x008: /* Overflow */
1096 info.si_code = FPE_FLTOVF; 1098 info.si_code = FPE_FLTOVF;
1097 break; 1099 break;
1098 case 0x020: /* Precision */ 1100 case 0x020: /* Precision */
1099 info.si_code = FPE_FLTRES; 1101 info.si_code = FPE_FLTRES;
1100 break; 1102 break;
1101 } 1103 }
1102 force_sig_info(SIGFPE, &info, task); 1104 force_sig_info(SIGFPE, &info, task);
1103} 1105}
@@ -1115,7 +1117,7 @@ asmlinkage void __attribute__((weak)) mce_threshold_interrupt(void)
1115} 1117}
1116 1118
1117/* 1119/*
1118 * 'math_state_restore()' saves the current math information in the 1120 * 'math_state_restore()' saves the current math information in the
1119 * old math state array, and gets the new ones from the current task 1121 * old math state array, and gets the new ones from the current task
1120 * 1122 *
1121 * Careful.. There are problems with IBM-designed IRQ13 behaviour. 1123 * Careful.. There are problems with IBM-designed IRQ13 behaviour.
@@ -1140,7 +1142,7 @@ asmlinkage void math_state_restore(void)
1140 local_irq_disable(); 1142 local_irq_disable();
1141 } 1143 }
1142 1144
1143 clts(); /* Allow maths ops (or we recurse) */ 1145 clts(); /* Allow maths ops (or we recurse) */
1144 restore_fpu_checking(&me->thread.xstate->fxsave); 1146 restore_fpu_checking(&me->thread.xstate->fxsave);
1145 task_thread_info(me)->status |= TS_USEDFPU; 1147 task_thread_info(me)->status |= TS_USEDFPU;
1146 me->fpu_counter++; 1148 me->fpu_counter++;
@@ -1149,64 +1151,61 @@ EXPORT_SYMBOL_GPL(math_state_restore);
1149 1151
1150void __init trap_init(void) 1152void __init trap_init(void)
1151{ 1153{
1152 set_intr_gate(0,&divide_error); 1154 set_intr_gate(0, &divide_error);
1153 set_intr_gate_ist(1,&debug,DEBUG_STACK); 1155 set_intr_gate_ist(1, &debug, DEBUG_STACK);
1154 set_intr_gate_ist(2,&nmi,NMI_STACK); 1156 set_intr_gate_ist(2, &nmi, NMI_STACK);
1155 set_system_gate_ist(3,&int3,DEBUG_STACK); /* int3 can be called from all */ 1157 set_system_gate_ist(3, &int3, DEBUG_STACK); /* int3 can be called from all */
1156 set_system_gate(4,&overflow); /* int4 can be called from all */ 1158 set_system_gate(4, &overflow); /* int4 can be called from all */
1157 set_intr_gate(5,&bounds); 1159 set_intr_gate(5, &bounds);
1158 set_intr_gate(6,&invalid_op); 1160 set_intr_gate(6, &invalid_op);
1159 set_intr_gate(7,&device_not_available); 1161 set_intr_gate(7, &device_not_available);
1160 set_intr_gate_ist(8,&double_fault, DOUBLEFAULT_STACK); 1162 set_intr_gate_ist(8, &double_fault, DOUBLEFAULT_STACK);
1161 set_intr_gate(9,&coprocessor_segment_overrun); 1163 set_intr_gate(9, &coprocessor_segment_overrun);
1162 set_intr_gate(10,&invalid_TSS); 1164 set_intr_gate(10, &invalid_TSS);
1163 set_intr_gate(11,&segment_not_present); 1165 set_intr_gate(11, &segment_not_present);
1164 set_intr_gate_ist(12,&stack_segment,STACKFAULT_STACK); 1166 set_intr_gate_ist(12, &stack_segment, STACKFAULT_STACK);
1165 set_intr_gate(13,&general_protection); 1167 set_intr_gate(13, &general_protection);
1166 set_intr_gate(14,&page_fault); 1168 set_intr_gate(14, &page_fault);
1167 set_intr_gate(15,&spurious_interrupt_bug); 1169 set_intr_gate(15, &spurious_interrupt_bug);
1168 set_intr_gate(16,&coprocessor_error); 1170 set_intr_gate(16, &coprocessor_error);
1169 set_intr_gate(17,&alignment_check); 1171 set_intr_gate(17, &alignment_check);
1170#ifdef CONFIG_X86_MCE 1172#ifdef CONFIG_X86_MCE
1171 set_intr_gate_ist(18,&machine_check, MCE_STACK); 1173 set_intr_gate_ist(18, &machine_check, MCE_STACK);
1172#endif 1174#endif
1173 set_intr_gate(19,&simd_coprocessor_error); 1175 set_intr_gate(19, &simd_coprocessor_error);
1174 1176
1175#ifdef CONFIG_IA32_EMULATION 1177#ifdef CONFIG_IA32_EMULATION
1176 set_system_gate(IA32_SYSCALL_VECTOR, ia32_syscall); 1178 set_system_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
1177#endif 1179#endif
1178
1179 /* 1180 /*
1180 * initialize the per thread extended state: 1181 * initialize the per thread extended state:
1181 */ 1182 */
1182 init_thread_xstate(); 1183 init_thread_xstate();
1183 /* 1184 /*
1184 * Should be a barrier for any external CPU state. 1185 * Should be a barrier for any external CPU state:
1185 */ 1186 */
1186 cpu_init(); 1187 cpu_init();
1187} 1188}
1188 1189
1189
1190static int __init oops_setup(char *s) 1190static int __init oops_setup(char *s)
1191{ 1191{
1192 if (!s) 1192 if (!s)
1193 return -EINVAL; 1193 return -EINVAL;
1194 if (!strcmp(s, "panic")) 1194 if (!strcmp(s, "panic"))
1195 panic_on_oops = 1; 1195 panic_on_oops = 1;
1196 return 0; 1196 return 0;
1197} 1197}
1198early_param("oops", oops_setup); 1198early_param("oops", oops_setup);
1199 1199
1200static int __init kstack_setup(char *s) 1200static int __init kstack_setup(char *s)
1201{ 1201{
1202 if (!s) 1202 if (!s)
1203 return -EINVAL; 1203 return -EINVAL;
1204 kstack_depth_to_print = simple_strtoul(s,NULL,0); 1204 kstack_depth_to_print = simple_strtoul(s, NULL, 0);
1205 return 0; 1205 return 0;
1206} 1206}
1207early_param("kstack", kstack_setup); 1207early_param("kstack", kstack_setup);
1208 1208
1209
1210static int __init code_bytes_setup(char *s) 1209static int __init code_bytes_setup(char *s)
1211{ 1210{
1212 code_bytes = simple_strtoul(s, NULL, 0); 1211 code_bytes = simple_strtoul(s, NULL, 0);
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
new file mode 100644
index 000000000000..7603c0553909
--- /dev/null
+++ b/arch/x86/kernel/tsc.c
@@ -0,0 +1,535 @@
1#include <linux/kernel.h>
2#include <linux/sched.h>
3#include <linux/init.h>
4#include <linux/module.h>
5#include <linux/timer.h>
6#include <linux/acpi_pmtmr.h>
7#include <linux/cpufreq.h>
8#include <linux/dmi.h>
9#include <linux/delay.h>
10#include <linux/clocksource.h>
11#include <linux/percpu.h>
12
13#include <asm/hpet.h>
14#include <asm/timer.h>
15#include <asm/vgtod.h>
16#include <asm/time.h>
17#include <asm/delay.h>
18
19unsigned int cpu_khz; /* TSC clocks / usec, not used here */
20EXPORT_SYMBOL(cpu_khz);
21unsigned int tsc_khz;
22EXPORT_SYMBOL(tsc_khz);
23
24/*
25 * TSC can be unstable due to cpufreq or due to unsynced TSCs
26 */
27static int tsc_unstable;
28
29/* native_sched_clock() is called before tsc_init(), so
30 we must start with the TSC soft disabled to prevent
31 erroneous rdtsc usage on !cpu_has_tsc processors */
32static int tsc_disabled = -1;
33
34/*
35 * Scheduler clock - returns current time in nanosec units.
36 */
37u64 native_sched_clock(void)
38{
39 u64 this_offset;
40
41 /*
42 * Fall back to jiffies if there's no TSC available:
43 * ( But note that we still use it if the TSC is marked
44 * unstable. We do this because unlike Time Of Day,
45 * the scheduler clock tolerates small errors and it's
46 * very important for it to be as fast as the platform
47 * can achive it. )
48 */
49 if (unlikely(tsc_disabled)) {
50 /* No locking but a rare wrong value is not a big deal: */
51 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
52 }
53
54 /* read the Time Stamp Counter: */
55 rdtscll(this_offset);
56
57 /* return the value in ns */
58 return cycles_2_ns(this_offset);
59}
60
61/* We need to define a real function for sched_clock, to override the
62 weak default version */
63#ifdef CONFIG_PARAVIRT
64unsigned long long sched_clock(void)
65{
66 return paravirt_sched_clock();
67}
68#else
69unsigned long long
70sched_clock(void) __attribute__((alias("native_sched_clock")));
71#endif
72
73int check_tsc_unstable(void)
74{
75 return tsc_unstable;
76}
77EXPORT_SYMBOL_GPL(check_tsc_unstable);
78
79#ifdef CONFIG_X86_TSC
80int __init notsc_setup(char *str)
81{
82 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
83 "cannot disable TSC completely.\n");
84 tsc_disabled = 1;
85 return 1;
86}
87#else
88/*
89 * disable flag for tsc. Takes effect by clearing the TSC cpu flag
90 * in cpu/common.c
91 */
92int __init notsc_setup(char *str)
93{
94 setup_clear_cpu_cap(X86_FEATURE_TSC);
95 return 1;
96}
97#endif
98
99__setup("notsc", notsc_setup);
100
101#define MAX_RETRIES 5
102#define SMI_TRESHOLD 50000
103
104/*
105 * Read TSC and the reference counters. Take care of SMI disturbance
106 */
107static u64 __init tsc_read_refs(u64 *pm, u64 *hpet)
108{
109 u64 t1, t2;
110 int i;
111
112 for (i = 0; i < MAX_RETRIES; i++) {
113 t1 = get_cycles();
114 if (hpet)
115 *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
116 else
117 *pm = acpi_pm_read_early();
118 t2 = get_cycles();
119 if ((t2 - t1) < SMI_TRESHOLD)
120 return t2;
121 }
122 return ULLONG_MAX;
123}
124
125/**
126 * native_calibrate_tsc - calibrate the tsc on boot
127 */
128unsigned long native_calibrate_tsc(void)
129{
130 unsigned long flags;
131 u64 tsc1, tsc2, tr1, tr2, delta, pm1, pm2, hpet1, hpet2;
132 int hpet = is_hpet_enabled();
133 unsigned int tsc_khz_val = 0;
134
135 local_irq_save(flags);
136
137 tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL);
138
139 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
140
141 outb(0xb0, 0x43);
142 outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42);
143 outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42);
144 tr1 = get_cycles();
145 while ((inb(0x61) & 0x20) == 0);
146 tr2 = get_cycles();
147
148 tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL);
149
150 local_irq_restore(flags);
151
152 /*
153 * Preset the result with the raw and inaccurate PIT
154 * calibration value
155 */
156 delta = (tr2 - tr1);
157 do_div(delta, 50);
158 tsc_khz_val = delta;
159
160 /* hpet or pmtimer available ? */
161 if (!hpet && !pm1 && !pm2) {
162 printk(KERN_INFO "TSC calibrated against PIT\n");
163 goto out;
164 }
165
166 /* Check, whether the sampling was disturbed by an SMI */
167 if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX) {
168 printk(KERN_WARNING "TSC calibration disturbed by SMI, "
169 "using PIT calibration result\n");
170 goto out;
171 }
172
173 tsc2 = (tsc2 - tsc1) * 1000000LL;
174
175 if (hpet) {
176 printk(KERN_INFO "TSC calibrated against HPET\n");
177 if (hpet2 < hpet1)
178 hpet2 += 0x100000000ULL;
179 hpet2 -= hpet1;
180 tsc1 = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
181 do_div(tsc1, 1000000);
182 } else {
183 printk(KERN_INFO "TSC calibrated against PM_TIMER\n");
184 if (pm2 < pm1)
185 pm2 += (u64)ACPI_PM_OVRRUN;
186 pm2 -= pm1;
187 tsc1 = pm2 * 1000000000LL;
188 do_div(tsc1, PMTMR_TICKS_PER_SEC);
189 }
190
191 do_div(tsc2, tsc1);
192 tsc_khz_val = tsc2;
193
194out:
195 return tsc_khz_val;
196}
197
198
199#ifdef CONFIG_X86_32
200/* Only called from the Powernow K7 cpu freq driver */
201int recalibrate_cpu_khz(void)
202{
203#ifndef CONFIG_SMP
204 unsigned long cpu_khz_old = cpu_khz;
205
206 if (cpu_has_tsc) {
207 tsc_khz = calibrate_tsc();
208 cpu_khz = tsc_khz;
209 cpu_data(0).loops_per_jiffy =
210 cpufreq_scale(cpu_data(0).loops_per_jiffy,
211 cpu_khz_old, cpu_khz);
212 return 0;
213 } else
214 return -ENODEV;
215#else
216 return -ENODEV;
217#endif
218}
219
220EXPORT_SYMBOL(recalibrate_cpu_khz);
221
222#endif /* CONFIG_X86_32 */
223
224/* Accelerators for sched_clock()
225 * convert from cycles(64bits) => nanoseconds (64bits)
226 * basic equation:
227 * ns = cycles / (freq / ns_per_sec)
228 * ns = cycles * (ns_per_sec / freq)
229 * ns = cycles * (10^9 / (cpu_khz * 10^3))
230 * ns = cycles * (10^6 / cpu_khz)
231 *
232 * Then we use scaling math (suggested by george@mvista.com) to get:
233 * ns = cycles * (10^6 * SC / cpu_khz) / SC
234 * ns = cycles * cyc2ns_scale / SC
235 *
236 * And since SC is a constant power of two, we can convert the div
237 * into a shift.
238 *
239 * We can use khz divisor instead of mhz to keep a better precision, since
240 * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
241 * (mathieu.desnoyers@polymtl.ca)
242 *
243 * -johnstul@us.ibm.com "math is hard, lets go shopping!"
244 */
245
246DEFINE_PER_CPU(unsigned long, cyc2ns);
247
248static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
249{
250 unsigned long long tsc_now, ns_now;
251 unsigned long flags, *scale;
252
253 local_irq_save(flags);
254 sched_clock_idle_sleep_event();
255
256 scale = &per_cpu(cyc2ns, cpu);
257
258 rdtscll(tsc_now);
259 ns_now = __cycles_2_ns(tsc_now);
260
261 if (cpu_khz)
262 *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
263
264 sched_clock_idle_wakeup_event(0);
265 local_irq_restore(flags);
266}
267
268#ifdef CONFIG_CPU_FREQ
269
270/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency
271 * changes.
272 *
273 * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's
274 * not that important because current Opteron setups do not support
275 * scaling on SMP anyroads.
276 *
277 * Should fix up last_tsc too. Currently gettimeofday in the
278 * first tick after the change will be slightly wrong.
279 */
280
281static unsigned int ref_freq;
282static unsigned long loops_per_jiffy_ref;
283static unsigned long tsc_khz_ref;
284
285static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
286 void *data)
287{
288 struct cpufreq_freqs *freq = data;
289 unsigned long *lpj, dummy;
290
291 if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC))
292 return 0;
293
294 lpj = &dummy;
295 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
296#ifdef CONFIG_SMP
297 lpj = &cpu_data(freq->cpu).loops_per_jiffy;
298#else
299 lpj = &boot_cpu_data.loops_per_jiffy;
300#endif
301
302 if (!ref_freq) {
303 ref_freq = freq->old;
304 loops_per_jiffy_ref = *lpj;
305 tsc_khz_ref = tsc_khz;
306 }
307 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
308 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
309 (val == CPUFREQ_RESUMECHANGE)) {
310 *lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
311
312 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
313 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
314 mark_tsc_unstable("cpufreq changes");
315 }
316
317 set_cyc2ns_scale(tsc_khz_ref, freq->cpu);
318
319 return 0;
320}
321
322static struct notifier_block time_cpufreq_notifier_block = {
323 .notifier_call = time_cpufreq_notifier
324};
325
326static int __init cpufreq_tsc(void)
327{
328 cpufreq_register_notifier(&time_cpufreq_notifier_block,
329 CPUFREQ_TRANSITION_NOTIFIER);
330 return 0;
331}
332
333core_initcall(cpufreq_tsc);
334
335#endif /* CONFIG_CPU_FREQ */
336
337/* clocksource code */
338
339static struct clocksource clocksource_tsc;
340
341/*
342 * We compare the TSC to the cycle_last value in the clocksource
343 * structure to avoid a nasty time-warp. This can be observed in a
344 * very small window right after one CPU updated cycle_last under
345 * xtime/vsyscall_gtod lock and the other CPU reads a TSC value which
346 * is smaller than the cycle_last reference value due to a TSC which
347 * is slighty behind. This delta is nowhere else observable, but in
348 * that case it results in a forward time jump in the range of hours
349 * due to the unsigned delta calculation of the time keeping core
350 * code, which is necessary to support wrapping clocksources like pm
351 * timer.
352 */
353static cycle_t read_tsc(void)
354{
355 cycle_t ret = (cycle_t)get_cycles();
356
357 return ret >= clocksource_tsc.cycle_last ?
358 ret : clocksource_tsc.cycle_last;
359}
360
361#ifdef CONFIG_X86_64
362static cycle_t __vsyscall_fn vread_tsc(void)
363{
364 cycle_t ret = (cycle_t)vget_cycles();
365
366 return ret >= __vsyscall_gtod_data.clock.cycle_last ?
367 ret : __vsyscall_gtod_data.clock.cycle_last;
368}
369#endif
370
371static struct clocksource clocksource_tsc = {
372 .name = "tsc",
373 .rating = 300,
374 .read = read_tsc,
375 .mask = CLOCKSOURCE_MASK(64),
376 .shift = 22,
377 .flags = CLOCK_SOURCE_IS_CONTINUOUS |
378 CLOCK_SOURCE_MUST_VERIFY,
379#ifdef CONFIG_X86_64
380 .vread = vread_tsc,
381#endif
382};
383
384void mark_tsc_unstable(char *reason)
385{
386 if (!tsc_unstable) {
387 tsc_unstable = 1;
388 printk("Marking TSC unstable due to %s\n", reason);
389 /* Change only the rating, when not registered */
390 if (clocksource_tsc.mult)
391 clocksource_change_rating(&clocksource_tsc, 0);
392 else
393 clocksource_tsc.rating = 0;
394 }
395}
396
397EXPORT_SYMBOL_GPL(mark_tsc_unstable);
398
399static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
400{
401 printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
402 d->ident);
403 tsc_unstable = 1;
404 return 0;
405}
406
407/* List of systems that have known TSC problems */
408static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
409 {
410 .callback = dmi_mark_tsc_unstable,
411 .ident = "IBM Thinkpad 380XD",
412 .matches = {
413 DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
414 DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
415 },
416 },
417 {}
418};
419
420/*
421 * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
422 */
423#ifdef CONFIG_MGEODE_LX
424/* RTSC counts during suspend */
425#define RTSC_SUSP 0x100
426
427static void __init check_geode_tsc_reliable(void)
428{
429 unsigned long res_low, res_high;
430
431 rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
432 if (res_low & RTSC_SUSP)
433 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
434}
435#else
436static inline void check_geode_tsc_reliable(void) { }
437#endif
438
439/*
440 * Make an educated guess if the TSC is trustworthy and synchronized
441 * over all CPUs.
442 */
443__cpuinit int unsynchronized_tsc(void)
444{
445 if (!cpu_has_tsc || tsc_unstable)
446 return 1;
447
448#ifdef CONFIG_SMP
449 if (apic_is_clustered_box())
450 return 1;
451#endif
452
453 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
454 return 0;
455 /*
456 * Intel systems are normally all synchronized.
457 * Exceptions must mark TSC as unstable:
458 */
459 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
460 /* assume multi socket systems are not synchronized: */
461 if (num_possible_cpus() > 1)
462 tsc_unstable = 1;
463 }
464
465 return tsc_unstable;
466}
467
468static void __init init_tsc_clocksource(void)
469{
470 clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
471 clocksource_tsc.shift);
472 /* lower the rating if we already know its unstable: */
473 if (check_tsc_unstable()) {
474 clocksource_tsc.rating = 0;
475 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
476 }
477 clocksource_register(&clocksource_tsc);
478}
479
480void __init tsc_init(void)
481{
482 u64 lpj;
483 int cpu;
484
485 if (!cpu_has_tsc)
486 return;
487
488 tsc_khz = calibrate_tsc();
489 cpu_khz = tsc_khz;
490
491 if (!tsc_khz) {
492 mark_tsc_unstable("could not calculate TSC khz");
493 return;
494 }
495
496#ifdef CONFIG_X86_64
497 if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) &&
498 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD))
499 cpu_khz = calibrate_cpu();
500#endif
501
502 lpj = ((u64)tsc_khz * 1000);
503 do_div(lpj, HZ);
504 lpj_fine = lpj;
505
506 printk("Detected %lu.%03lu MHz processor.\n",
507 (unsigned long)cpu_khz / 1000,
508 (unsigned long)cpu_khz % 1000);
509
510 /*
511 * Secondary CPUs do not run through tsc_init(), so set up
512 * all the scale factors for all CPUs, assuming the same
513 * speed as the bootup CPU. (cpufreq notifiers will fix this
514 * up if their speed diverges)
515 */
516 for_each_possible_cpu(cpu)
517 set_cyc2ns_scale(cpu_khz, cpu);
518
519 if (tsc_disabled > 0)
520 return;
521
522 /* now allow native_sched_clock() to use rdtsc */
523 tsc_disabled = 0;
524
525 use_tsc_delay();
526 /* Check and install the TSC clocksource */
527 dmi_check_system(bad_tsc_dmi_table);
528
529 if (unsynchronized_tsc())
530 mark_tsc_unstable("TSCs unsynchronized");
531
532 check_geode_tsc_reliable();
533 init_tsc_clocksource();
534}
535
diff --git a/arch/x86/kernel/tsc_32.c b/arch/x86/kernel/tsc_32.c
deleted file mode 100644
index 65b70637ad97..000000000000
--- a/arch/x86/kernel/tsc_32.c
+++ /dev/null
@@ -1,451 +0,0 @@
1#include <linux/sched.h>
2#include <linux/clocksource.h>
3#include <linux/workqueue.h>
4#include <linux/cpufreq.h>
5#include <linux/jiffies.h>
6#include <linux/init.h>
7#include <linux/dmi.h>
8#include <linux/percpu.h>
9
10#include <asm/delay.h>
11#include <asm/tsc.h>
12#include <asm/io.h>
13#include <asm/timer.h>
14
15#include "mach_timer.h"
16
17/* native_sched_clock() is called before tsc_init(), so
18 we must start with the TSC soft disabled to prevent
19 erroneous rdtsc usage on !cpu_has_tsc processors */
20static int tsc_disabled = -1;
21
22/*
23 * On some systems the TSC frequency does not
24 * change with the cpu frequency. So we need
25 * an extra value to store the TSC freq
26 */
27unsigned int tsc_khz;
28EXPORT_SYMBOL_GPL(tsc_khz);
29
30#ifdef CONFIG_X86_TSC
31static int __init tsc_setup(char *str)
32{
33 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
34 "cannot disable TSC completely.\n");
35 tsc_disabled = 1;
36 return 1;
37}
38#else
39/*
40 * disable flag for tsc. Takes effect by clearing the TSC cpu flag
41 * in cpu/common.c
42 */
43static int __init tsc_setup(char *str)
44{
45 setup_clear_cpu_cap(X86_FEATURE_TSC);
46 return 1;
47}
48#endif
49
50__setup("notsc", tsc_setup);
51
52/*
53 * code to mark and check if the TSC is unstable
54 * due to cpufreq or due to unsynced TSCs
55 */
56static int tsc_unstable;
57
58int check_tsc_unstable(void)
59{
60 return tsc_unstable;
61}
62EXPORT_SYMBOL_GPL(check_tsc_unstable);
63
64/* Accelerators for sched_clock()
65 * convert from cycles(64bits) => nanoseconds (64bits)
66 * basic equation:
67 * ns = cycles / (freq / ns_per_sec)
68 * ns = cycles * (ns_per_sec / freq)
69 * ns = cycles * (10^9 / (cpu_khz * 10^3))
70 * ns = cycles * (10^6 / cpu_khz)
71 *
72 * Then we use scaling math (suggested by george@mvista.com) to get:
73 * ns = cycles * (10^6 * SC / cpu_khz) / SC
74 * ns = cycles * cyc2ns_scale / SC
75 *
76 * And since SC is a constant power of two, we can convert the div
77 * into a shift.
78 *
79 * We can use khz divisor instead of mhz to keep a better precision, since
80 * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
81 * (mathieu.desnoyers@polymtl.ca)
82 *
83 * -johnstul@us.ibm.com "math is hard, lets go shopping!"
84 */
85
86DEFINE_PER_CPU(unsigned long, cyc2ns);
87
88static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
89{
90 unsigned long long tsc_now, ns_now;
91 unsigned long flags, *scale;
92
93 local_irq_save(flags);
94 sched_clock_idle_sleep_event();
95
96 scale = &per_cpu(cyc2ns, cpu);
97
98 rdtscll(tsc_now);
99 ns_now = __cycles_2_ns(tsc_now);
100
101 if (cpu_khz)
102 *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
103
104 /*
105 * Start smoothly with the new frequency:
106 */
107 sched_clock_idle_wakeup_event(0);
108 local_irq_restore(flags);
109}
110
111/*
112 * Scheduler clock - returns current time in nanosec units.
113 */
114unsigned long long native_sched_clock(void)
115{
116 unsigned long long this_offset;
117
118 /*
119 * Fall back to jiffies if there's no TSC available:
120 * ( But note that we still use it if the TSC is marked
121 * unstable. We do this because unlike Time Of Day,
122 * the scheduler clock tolerates small errors and it's
123 * very important for it to be as fast as the platform
124 * can achive it. )
125 */
126 if (unlikely(tsc_disabled))
127 /* No locking but a rare wrong value is not a big deal: */
128 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
129
130 /* read the Time Stamp Counter: */
131 rdtscll(this_offset);
132
133 /* return the value in ns */
134 return cycles_2_ns(this_offset);
135}
136
137/* We need to define a real function for sched_clock, to override the
138 weak default version */
139#ifdef CONFIG_PARAVIRT
140unsigned long long sched_clock(void)
141{
142 return paravirt_sched_clock();
143}
144#else
145unsigned long long sched_clock(void)
146 __attribute__((alias("native_sched_clock")));
147#endif
148
149unsigned long native_calculate_cpu_khz(void)
150{
151 unsigned long long start, end;
152 unsigned long count;
153 u64 delta64 = (u64)ULLONG_MAX;
154 int i;
155 unsigned long flags;
156
157 local_irq_save(flags);
158
159 /* run 3 times to ensure the cache is warm and to get an accurate reading */
160 for (i = 0; i < 3; i++) {
161 mach_prepare_counter();
162 rdtscll(start);
163 mach_countup(&count);
164 rdtscll(end);
165
166 /*
167 * Error: ECTCNEVERSET
168 * The CTC wasn't reliable: we got a hit on the very first read,
169 * or the CPU was so fast/slow that the quotient wouldn't fit in
170 * 32 bits..
171 */
172 if (count <= 1)
173 continue;
174
175 /* cpu freq too slow: */
176 if ((end - start) <= CALIBRATE_TIME_MSEC)
177 continue;
178
179 /*
180 * We want the minimum time of all runs in case one of them
181 * is inaccurate due to SMI or other delay
182 */
183 delta64 = min(delta64, (end - start));
184 }
185
186 /* cpu freq too fast (or every run was bad): */
187 if (delta64 > (1ULL<<32))
188 goto err;
189
190 delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
191 do_div(delta64,CALIBRATE_TIME_MSEC);
192
193 local_irq_restore(flags);
194 return (unsigned long)delta64;
195err:
196 local_irq_restore(flags);
197 return 0;
198}
199
200int recalibrate_cpu_khz(void)
201{
202#ifndef CONFIG_SMP
203 unsigned long cpu_khz_old = cpu_khz;
204
205 if (cpu_has_tsc) {
206 cpu_khz = calculate_cpu_khz();
207 tsc_khz = cpu_khz;
208 cpu_data(0).loops_per_jiffy =
209 cpufreq_scale(cpu_data(0).loops_per_jiffy,
210 cpu_khz_old, cpu_khz);
211 return 0;
212 } else
213 return -ENODEV;
214#else
215 return -ENODEV;
216#endif
217}
218
219EXPORT_SYMBOL(recalibrate_cpu_khz);
220
221#ifdef CONFIG_CPU_FREQ
222
223/*
224 * if the CPU frequency is scaled, TSC-based delays will need a different
225 * loops_per_jiffy value to function properly.
226 */
227static unsigned int ref_freq;
228static unsigned long loops_per_jiffy_ref;
229static unsigned long cpu_khz_ref;
230
231static int
232time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
233{
234 struct cpufreq_freqs *freq = data;
235
236 if (!ref_freq) {
237 if (!freq->old){
238 ref_freq = freq->new;
239 return 0;
240 }
241 ref_freq = freq->old;
242 loops_per_jiffy_ref = cpu_data(freq->cpu).loops_per_jiffy;
243 cpu_khz_ref = cpu_khz;
244 }
245
246 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
247 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
248 (val == CPUFREQ_RESUMECHANGE)) {
249 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
250 cpu_data(freq->cpu).loops_per_jiffy =
251 cpufreq_scale(loops_per_jiffy_ref,
252 ref_freq, freq->new);
253
254 if (cpu_khz) {
255
256 if (num_online_cpus() == 1)
257 cpu_khz = cpufreq_scale(cpu_khz_ref,
258 ref_freq, freq->new);
259 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
260 tsc_khz = cpu_khz;
261 set_cyc2ns_scale(cpu_khz, freq->cpu);
262 /*
263 * TSC based sched_clock turns
264 * to junk w/ cpufreq
265 */
266 mark_tsc_unstable("cpufreq changes");
267 }
268 }
269 }
270
271 return 0;
272}
273
274static struct notifier_block time_cpufreq_notifier_block = {
275 .notifier_call = time_cpufreq_notifier
276};
277
278static int __init cpufreq_tsc(void)
279{
280 return cpufreq_register_notifier(&time_cpufreq_notifier_block,
281 CPUFREQ_TRANSITION_NOTIFIER);
282}
283core_initcall(cpufreq_tsc);
284
285#endif
286
287/* clock source code */
288
289static unsigned long current_tsc_khz;
290static struct clocksource clocksource_tsc;
291
292/*
293 * We compare the TSC to the cycle_last value in the clocksource
294 * structure to avoid a nasty time-warp issue. This can be observed in
295 * a very small window right after one CPU updated cycle_last under
296 * xtime lock and the other CPU reads a TSC value which is smaller
297 * than the cycle_last reference value due to a TSC which is slighty
298 * behind. This delta is nowhere else observable, but in that case it
299 * results in a forward time jump in the range of hours due to the
300 * unsigned delta calculation of the time keeping core code, which is
301 * necessary to support wrapping clocksources like pm timer.
302 */
303static cycle_t read_tsc(void)
304{
305 cycle_t ret;
306
307 rdtscll(ret);
308
309 return ret >= clocksource_tsc.cycle_last ?
310 ret : clocksource_tsc.cycle_last;
311}
312
313static struct clocksource clocksource_tsc = {
314 .name = "tsc",
315 .rating = 300,
316 .read = read_tsc,
317 .mask = CLOCKSOURCE_MASK(64),
318 .mult = 0, /* to be set */
319 .shift = 22,
320 .flags = CLOCK_SOURCE_IS_CONTINUOUS |
321 CLOCK_SOURCE_MUST_VERIFY,
322};
323
324void mark_tsc_unstable(char *reason)
325{
326 if (!tsc_unstable) {
327 tsc_unstable = 1;
328 printk("Marking TSC unstable due to: %s.\n", reason);
329 /* Can be called before registration */
330 if (clocksource_tsc.mult)
331 clocksource_change_rating(&clocksource_tsc, 0);
332 else
333 clocksource_tsc.rating = 0;
334 }
335}
336EXPORT_SYMBOL_GPL(mark_tsc_unstable);
337
338static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
339{
340 printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
341 d->ident);
342 tsc_unstable = 1;
343 return 0;
344}
345
346/* List of systems that have known TSC problems */
347static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
348 {
349 .callback = dmi_mark_tsc_unstable,
350 .ident = "IBM Thinkpad 380XD",
351 .matches = {
352 DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
353 DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
354 },
355 },
356 {}
357};
358
359/*
360 * Make an educated guess if the TSC is trustworthy and synchronized
361 * over all CPUs.
362 */
363__cpuinit int unsynchronized_tsc(void)
364{
365 if (!cpu_has_tsc || tsc_unstable)
366 return 1;
367
368 /* Anything with constant TSC should be synchronized */
369 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
370 return 0;
371
372 /*
373 * Intel systems are normally all synchronized.
374 * Exceptions must mark TSC as unstable:
375 */
376 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
377 /* assume multi socket systems are not synchronized: */
378 if (num_possible_cpus() > 1)
379 tsc_unstable = 1;
380 }
381 return tsc_unstable;
382}
383
384/*
385 * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
386 */
387#ifdef CONFIG_MGEODE_LX
388/* RTSC counts during suspend */
389#define RTSC_SUSP 0x100
390
391static void __init check_geode_tsc_reliable(void)
392{
393 unsigned long res_low, res_high;
394
395 rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
396 if (res_low & RTSC_SUSP)
397 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
398}
399#else
400static inline void check_geode_tsc_reliable(void) { }
401#endif
402
403
404void __init tsc_init(void)
405{
406 int cpu;
407
408 if (!cpu_has_tsc || tsc_disabled > 0)
409 return;
410
411 cpu_khz = calculate_cpu_khz();
412 tsc_khz = cpu_khz;
413
414 if (!cpu_khz) {
415 mark_tsc_unstable("could not calculate TSC khz");
416 return;
417 }
418
419 /* now allow native_sched_clock() to use rdtsc */
420 tsc_disabled = 0;
421
422 printk("Detected %lu.%03lu MHz processor.\n",
423 (unsigned long)cpu_khz / 1000,
424 (unsigned long)cpu_khz % 1000);
425
426 /*
427 * Secondary CPUs do not run through tsc_init(), so set up
428 * all the scale factors for all CPUs, assuming the same
429 * speed as the bootup CPU. (cpufreq notifiers will fix this
430 * up if their speed diverges)
431 */
432 for_each_possible_cpu(cpu)
433 set_cyc2ns_scale(cpu_khz, cpu);
434
435 use_tsc_delay();
436
437 /* Check and install the TSC clocksource */
438 dmi_check_system(bad_tsc_dmi_table);
439
440 unsynchronized_tsc();
441 check_geode_tsc_reliable();
442 current_tsc_khz = tsc_khz;
443 clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
444 clocksource_tsc.shift);
445 /* lower the rating if we already know its unstable: */
446 if (check_tsc_unstable()) {
447 clocksource_tsc.rating = 0;
448 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
449 }
450 clocksource_register(&clocksource_tsc);
451}
diff --git a/arch/x86/kernel/tsc_64.c b/arch/x86/kernel/tsc_64.c
deleted file mode 100644
index 1784b8077a12..000000000000
--- a/arch/x86/kernel/tsc_64.c
+++ /dev/null
@@ -1,357 +0,0 @@
1#include <linux/kernel.h>
2#include <linux/sched.h>
3#include <linux/interrupt.h>
4#include <linux/init.h>
5#include <linux/clocksource.h>
6#include <linux/time.h>
7#include <linux/acpi.h>
8#include <linux/cpufreq.h>
9#include <linux/acpi_pmtmr.h>
10
11#include <asm/hpet.h>
12#include <asm/timex.h>
13#include <asm/timer.h>
14#include <asm/vgtod.h>
15
16static int notsc __initdata = 0;
17
18unsigned int cpu_khz; /* TSC clocks / usec, not used here */
19EXPORT_SYMBOL(cpu_khz);
20unsigned int tsc_khz;
21EXPORT_SYMBOL(tsc_khz);
22
23/* Accelerators for sched_clock()
24 * convert from cycles(64bits) => nanoseconds (64bits)
25 * basic equation:
26 * ns = cycles / (freq / ns_per_sec)
27 * ns = cycles * (ns_per_sec / freq)
28 * ns = cycles * (10^9 / (cpu_khz * 10^3))
29 * ns = cycles * (10^6 / cpu_khz)
30 *
31 * Then we use scaling math (suggested by george@mvista.com) to get:
32 * ns = cycles * (10^6 * SC / cpu_khz) / SC
33 * ns = cycles * cyc2ns_scale / SC
34 *
35 * And since SC is a constant power of two, we can convert the div
36 * into a shift.
37 *
38 * We can use khz divisor instead of mhz to keep a better precision, since
39 * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
40 * (mathieu.desnoyers@polymtl.ca)
41 *
42 * -johnstul@us.ibm.com "math is hard, lets go shopping!"
43 */
44DEFINE_PER_CPU(unsigned long, cyc2ns);
45
46static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
47{
48 unsigned long long tsc_now, ns_now;
49 unsigned long flags, *scale;
50
51 local_irq_save(flags);
52 sched_clock_idle_sleep_event();
53
54 scale = &per_cpu(cyc2ns, cpu);
55
56 rdtscll(tsc_now);
57 ns_now = __cycles_2_ns(tsc_now);
58
59 if (cpu_khz)
60 *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
61
62 sched_clock_idle_wakeup_event(0);
63 local_irq_restore(flags);
64}
65
66unsigned long long native_sched_clock(void)
67{
68 unsigned long a = 0;
69
70 /* Could do CPU core sync here. Opteron can execute rdtsc speculatively,
71 * which means it is not completely exact and may not be monotonous
72 * between CPUs. But the errors should be too small to matter for
73 * scheduling purposes.
74 */
75
76 rdtscll(a);
77 return cycles_2_ns(a);
78}
79
80/* We need to define a real function for sched_clock, to override the
81 weak default version */
82#ifdef CONFIG_PARAVIRT
83unsigned long long sched_clock(void)
84{
85 return paravirt_sched_clock();
86}
87#else
88unsigned long long
89sched_clock(void) __attribute__((alias("native_sched_clock")));
90#endif
91
92
93static int tsc_unstable;
94
95int check_tsc_unstable(void)
96{
97 return tsc_unstable;
98}
99EXPORT_SYMBOL_GPL(check_tsc_unstable);
100
101#ifdef CONFIG_CPU_FREQ
102
103/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency
104 * changes.
105 *
106 * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's
107 * not that important because current Opteron setups do not support
108 * scaling on SMP anyroads.
109 *
110 * Should fix up last_tsc too. Currently gettimeofday in the
111 * first tick after the change will be slightly wrong.
112 */
113
114static unsigned int ref_freq;
115static unsigned long loops_per_jiffy_ref;
116static unsigned long tsc_khz_ref;
117
118static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
119 void *data)
120{
121 struct cpufreq_freqs *freq = data;
122 unsigned long *lpj, dummy;
123
124 if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC))
125 return 0;
126
127 lpj = &dummy;
128 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
129#ifdef CONFIG_SMP
130 lpj = &cpu_data(freq->cpu).loops_per_jiffy;
131#else
132 lpj = &boot_cpu_data.loops_per_jiffy;
133#endif
134
135 if (!ref_freq) {
136 ref_freq = freq->old;
137 loops_per_jiffy_ref = *lpj;
138 tsc_khz_ref = tsc_khz;
139 }
140 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
141 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
142 (val == CPUFREQ_RESUMECHANGE)) {
143 *lpj =
144 cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
145
146 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
147 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
148 mark_tsc_unstable("cpufreq changes");
149 }
150
151 set_cyc2ns_scale(tsc_khz_ref, freq->cpu);
152
153 return 0;
154}
155
156static struct notifier_block time_cpufreq_notifier_block = {
157 .notifier_call = time_cpufreq_notifier
158};
159
160static int __init cpufreq_tsc(void)
161{
162 cpufreq_register_notifier(&time_cpufreq_notifier_block,
163 CPUFREQ_TRANSITION_NOTIFIER);
164 return 0;
165}
166
167core_initcall(cpufreq_tsc);
168
169#endif
170
171#define MAX_RETRIES 5
172#define SMI_TRESHOLD 50000
173
174/*
175 * Read TSC and the reference counters. Take care of SMI disturbance
176 */
177static unsigned long __init tsc_read_refs(unsigned long *pm,
178 unsigned long *hpet)
179{
180 unsigned long t1, t2;
181 int i;
182
183 for (i = 0; i < MAX_RETRIES; i++) {
184 t1 = get_cycles();
185 if (hpet)
186 *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
187 else
188 *pm = acpi_pm_read_early();
189 t2 = get_cycles();
190 if ((t2 - t1) < SMI_TRESHOLD)
191 return t2;
192 }
193 return ULONG_MAX;
194}
195
196/**
197 * tsc_calibrate - calibrate the tsc on boot
198 */
199void __init tsc_calibrate(void)
200{
201 unsigned long flags, tsc1, tsc2, tr1, tr2, pm1, pm2, hpet1, hpet2;
202 int hpet = is_hpet_enabled(), cpu;
203
204 local_irq_save(flags);
205
206 tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL);
207
208 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
209
210 outb(0xb0, 0x43);
211 outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42);
212 outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42);
213 tr1 = get_cycles();
214 while ((inb(0x61) & 0x20) == 0);
215 tr2 = get_cycles();
216
217 tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL);
218
219 local_irq_restore(flags);
220
221 /*
222 * Preset the result with the raw and inaccurate PIT
223 * calibration value
224 */
225 tsc_khz = (tr2 - tr1) / 50;
226
227 /* hpet or pmtimer available ? */
228 if (!hpet && !pm1 && !pm2) {
229 printk(KERN_INFO "TSC calibrated against PIT\n");
230 goto out;
231 }
232
233 /* Check, whether the sampling was disturbed by an SMI */
234 if (tsc1 == ULONG_MAX || tsc2 == ULONG_MAX) {
235 printk(KERN_WARNING "TSC calibration disturbed by SMI, "
236 "using PIT calibration result\n");
237 goto out;
238 }
239
240 tsc2 = (tsc2 - tsc1) * 1000000L;
241
242 if (hpet) {
243 printk(KERN_INFO "TSC calibrated against HPET\n");
244 if (hpet2 < hpet1)
245 hpet2 += 0x100000000;
246 hpet2 -= hpet1;
247 tsc1 = (hpet2 * hpet_readl(HPET_PERIOD)) / 1000000;
248 } else {
249 printk(KERN_INFO "TSC calibrated against PM_TIMER\n");
250 if (pm2 < pm1)
251 pm2 += ACPI_PM_OVRRUN;
252 pm2 -= pm1;
253 tsc1 = (pm2 * 1000000000) / PMTMR_TICKS_PER_SEC;
254 }
255
256 tsc_khz = tsc2 / tsc1;
257
258out:
259 for_each_possible_cpu(cpu)
260 set_cyc2ns_scale(tsc_khz, cpu);
261}
262
263/*
264 * Make an educated guess if the TSC is trustworthy and synchronized
265 * over all CPUs.
266 */
267__cpuinit int unsynchronized_tsc(void)
268{
269 if (tsc_unstable)
270 return 1;
271
272#ifdef CONFIG_SMP
273 if (apic_is_clustered_box())
274 return 1;
275#endif
276
277 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
278 return 0;
279
280 /* Assume multi socket systems are not synchronized */
281 return num_present_cpus() > 1;
282}
283
284int __init notsc_setup(char *s)
285{
286 notsc = 1;
287 return 1;
288}
289
290__setup("notsc", notsc_setup);
291
292static struct clocksource clocksource_tsc;
293
294/*
295 * We compare the TSC to the cycle_last value in the clocksource
296 * structure to avoid a nasty time-warp. This can be observed in a
297 * very small window right after one CPU updated cycle_last under
298 * xtime/vsyscall_gtod lock and the other CPU reads a TSC value which
299 * is smaller than the cycle_last reference value due to a TSC which
300 * is slighty behind. This delta is nowhere else observable, but in
301 * that case it results in a forward time jump in the range of hours
302 * due to the unsigned delta calculation of the time keeping core
303 * code, which is necessary to support wrapping clocksources like pm
304 * timer.
305 */
306static cycle_t read_tsc(void)
307{
308 cycle_t ret = (cycle_t)get_cycles();
309
310 return ret >= clocksource_tsc.cycle_last ?
311 ret : clocksource_tsc.cycle_last;
312}
313
314static cycle_t __vsyscall_fn vread_tsc(void)
315{
316 cycle_t ret = (cycle_t)vget_cycles();
317
318 return ret >= __vsyscall_gtod_data.clock.cycle_last ?
319 ret : __vsyscall_gtod_data.clock.cycle_last;
320}
321
322static struct clocksource clocksource_tsc = {
323 .name = "tsc",
324 .rating = 300,
325 .read = read_tsc,
326 .mask = CLOCKSOURCE_MASK(64),
327 .shift = 22,
328 .flags = CLOCK_SOURCE_IS_CONTINUOUS |
329 CLOCK_SOURCE_MUST_VERIFY,
330 .vread = vread_tsc,
331};
332
333void mark_tsc_unstable(char *reason)
334{
335 if (!tsc_unstable) {
336 tsc_unstable = 1;
337 printk("Marking TSC unstable due to %s\n", reason);
338 /* Change only the rating, when not registered */
339 if (clocksource_tsc.mult)
340 clocksource_change_rating(&clocksource_tsc, 0);
341 else
342 clocksource_tsc.rating = 0;
343 }
344}
345EXPORT_SYMBOL_GPL(mark_tsc_unstable);
346
347void __init init_tsc_clocksource(void)
348{
349 if (!notsc) {
350 clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
351 clocksource_tsc.shift);
352 if (check_tsc_unstable())
353 clocksource_tsc.rating = 0;
354
355 clocksource_register(&clocksource_tsc);
356 }
357}
diff --git a/arch/x86/kernel/visws_quirks.c b/arch/x86/kernel/visws_quirks.c
new file mode 100644
index 000000000000..e94bdb6add1d
--- /dev/null
+++ b/arch/x86/kernel/visws_quirks.c
@@ -0,0 +1,709 @@
1/*
2 * SGI Visual Workstation support and quirks, unmaintained.
3 *
4 * Split out from setup.c by davej@suse.de
5 *
6 * Copyright (C) 1999 Bent Hagemark, Ingo Molnar
7 *
8 * SGI Visual Workstation interrupt controller
9 *
10 * The Cobalt system ASIC in the Visual Workstation contains a "Cobalt" APIC
11 * which serves as the main interrupt controller in the system. Non-legacy
12 * hardware in the system uses this controller directly. Legacy devices
13 * are connected to the PIIX4 which in turn has its 8259(s) connected to
14 * a of the Cobalt APIC entry.
15 *
16 * 09/02/2000 - Updated for 2.4 by jbarnes@sgi.com
17 *
18 * 25/11/2002 - Updated for 2.5 by Andrey Panin <pazke@orbita1.ru>
19 */
20#include <linux/interrupt.h>
21#include <linux/module.h>
22#include <linux/init.h>
23#include <linux/smp.h>
24
25#include <asm/visws/cobalt.h>
26#include <asm/visws/piix4.h>
27#include <asm/arch_hooks.h>
28#include <asm/fixmap.h>
29#include <asm/reboot.h>
30#include <asm/setup.h>
31#include <asm/e820.h>
32#include <asm/smp.h>
33#include <asm/io.h>
34
35#include <mach_ipi.h>
36
37#include "mach_apic.h"
38
39#include <linux/init.h>
40#include <linux/smp.h>
41
42#include <linux/kernel_stat.h>
43#include <linux/interrupt.h>
44#include <linux/init.h>
45
46#include <asm/io.h>
47#include <asm/apic.h>
48#include <asm/i8259.h>
49#include <asm/irq_vectors.h>
50#include <asm/visws/cobalt.h>
51#include <asm/visws/lithium.h>
52#include <asm/visws/piix4.h>
53
54#include <linux/sched.h>
55#include <linux/kernel.h>
56#include <linux/init.h>
57#include <linux/pci.h>
58#include <linux/pci_ids.h>
59
60extern int no_broadcast;
61
62#include <asm/io.h>
63#include <asm/apic.h>
64#include <asm/arch_hooks.h>
65#include <asm/visws/cobalt.h>
66#include <asm/visws/lithium.h>
67
68char visws_board_type = -1;
69char visws_board_rev = -1;
70
71int is_visws_box(void)
72{
73 return visws_board_type >= 0;
74}
75
76static int __init visws_time_init_quirk(void)
77{
78 printk(KERN_INFO "Starting Cobalt Timer system clock\n");
79
80 /* Set the countdown value */
81 co_cpu_write(CO_CPU_TIMEVAL, CO_TIME_HZ/HZ);
82
83 /* Start the timer */
84 co_cpu_write(CO_CPU_CTRL, co_cpu_read(CO_CPU_CTRL) | CO_CTRL_TIMERUN);
85
86 /* Enable (unmask) the timer interrupt */
87 co_cpu_write(CO_CPU_CTRL, co_cpu_read(CO_CPU_CTRL) & ~CO_CTRL_TIMEMASK);
88
89 /*
90 * Zero return means the generic timer setup code will set up
91 * the standard vector:
92 */
93 return 0;
94}
95
96static int __init visws_pre_intr_init_quirk(void)
97{
98 init_VISWS_APIC_irqs();
99
100 /*
101 * We dont want ISA irqs to be set up by the generic code:
102 */
103 return 1;
104}
105
106/* Quirk for machine specific memory setup. */
107
108#define MB (1024 * 1024)
109
110unsigned long sgivwfb_mem_phys;
111unsigned long sgivwfb_mem_size;
112EXPORT_SYMBOL(sgivwfb_mem_phys);
113EXPORT_SYMBOL(sgivwfb_mem_size);
114
115long long mem_size __initdata = 0;
116
117static char * __init visws_memory_setup_quirk(void)
118{
119 long long gfx_mem_size = 8 * MB;
120
121 mem_size = boot_params.alt_mem_k;
122
123 if (!mem_size) {
124 printk(KERN_WARNING "Bootloader didn't set memory size, upgrade it !\n");
125 mem_size = 128 * MB;
126 }
127
128 /*
129 * this hardcodes the graphics memory to 8 MB
130 * it really should be sized dynamically (or at least
131 * set as a boot param)
132 */
133 if (!sgivwfb_mem_size) {
134 printk(KERN_WARNING "Defaulting to 8 MB framebuffer size\n");
135 sgivwfb_mem_size = 8 * MB;
136 }
137
138 /*
139 * Trim to nearest MB
140 */
141 sgivwfb_mem_size &= ~((1 << 20) - 1);
142 sgivwfb_mem_phys = mem_size - gfx_mem_size;
143
144 e820_add_region(0, LOWMEMSIZE(), E820_RAM);
145 e820_add_region(HIGH_MEMORY, mem_size - sgivwfb_mem_size - HIGH_MEMORY, E820_RAM);
146 e820_add_region(sgivwfb_mem_phys, sgivwfb_mem_size, E820_RESERVED);
147
148 return "PROM";
149}
150
151static void visws_machine_emergency_restart(void)
152{
153 /*
154 * Visual Workstations restart after this
155 * register is poked on the PIIX4
156 */
157 outb(PIIX4_RESET_VAL, PIIX4_RESET_PORT);
158}
159
160static void visws_machine_power_off(void)
161{
162 unsigned short pm_status;
163/* extern unsigned int pci_bus0; */
164
165 while ((pm_status = inw(PMSTS_PORT)) & 0x100)
166 outw(pm_status, PMSTS_PORT);
167
168 outw(PM_SUSPEND_ENABLE, PMCNTRL_PORT);
169
170 mdelay(10);
171
172#define PCI_CONF1_ADDRESS(bus, devfn, reg) \
173 (0x80000000 | (bus << 16) | (devfn << 8) | (reg & ~3))
174
175/* outl(PCI_CONF1_ADDRESS(pci_bus0, SPECIAL_DEV, SPECIAL_REG), 0xCF8); */
176 outl(PIIX_SPECIAL_STOP, 0xCFC);
177}
178
179static int __init visws_get_smp_config_quirk(unsigned int early)
180{
181 /*
182 * Prevent MP-table parsing by the generic code:
183 */
184 return 1;
185}
186
187extern unsigned int __cpuinitdata maxcpus;
188
189/*
190 * The Visual Workstation is Intel MP compliant in the hardware
191 * sense, but it doesn't have a BIOS(-configuration table).
192 * No problem for Linux.
193 */
194
195static void __init MP_processor_info (struct mpc_config_processor *m)
196{
197 int ver, logical_apicid;
198 physid_mask_t apic_cpus;
199
200 if (!(m->mpc_cpuflag & CPU_ENABLED))
201 return;
202
203 logical_apicid = m->mpc_apicid;
204 printk(KERN_INFO "%sCPU #%d %u:%u APIC version %d\n",
205 m->mpc_cpuflag & CPU_BOOTPROCESSOR ? "Bootup " : "",
206 m->mpc_apicid,
207 (m->mpc_cpufeature & CPU_FAMILY_MASK) >> 8,
208 (m->mpc_cpufeature & CPU_MODEL_MASK) >> 4,
209 m->mpc_apicver);
210
211 if (m->mpc_cpuflag & CPU_BOOTPROCESSOR)
212 boot_cpu_physical_apicid = m->mpc_apicid;
213
214 ver = m->mpc_apicver;
215 if ((ver >= 0x14 && m->mpc_apicid >= 0xff) || m->mpc_apicid >= 0xf) {
216 printk(KERN_ERR "Processor #%d INVALID. (Max ID: %d).\n",
217 m->mpc_apicid, MAX_APICS);
218 return;
219 }
220
221 apic_cpus = apicid_to_cpu_present(m->mpc_apicid);
222 physids_or(phys_cpu_present_map, phys_cpu_present_map, apic_cpus);
223 /*
224 * Validate version
225 */
226 if (ver == 0x0) {
227 printk(KERN_ERR "BIOS bug, APIC version is 0 for CPU#%d! "
228 "fixing up to 0x10. (tell your hw vendor)\n",
229 m->mpc_apicid);
230 ver = 0x10;
231 }
232 apic_version[m->mpc_apicid] = ver;
233}
234
235int __init visws_find_smp_config_quirk(unsigned int reserve)
236{
237 struct mpc_config_processor *mp = phys_to_virt(CO_CPU_TAB_PHYS);
238 unsigned short ncpus = readw(phys_to_virt(CO_CPU_NUM_PHYS));
239
240 if (ncpus > CO_CPU_MAX) {
241 printk(KERN_WARNING "find_visws_smp: got cpu count of %d at %p\n",
242 ncpus, mp);
243
244 ncpus = CO_CPU_MAX;
245 }
246
247 if (ncpus > maxcpus)
248 ncpus = maxcpus;
249
250#ifdef CONFIG_X86_LOCAL_APIC
251 smp_found_config = 1;
252#endif
253 while (ncpus--)
254 MP_processor_info(mp++);
255
256 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
257
258 return 1;
259}
260
261extern int visws_trap_init_quirk(void);
262
263void __init visws_early_detect(void)
264{
265 int raw;
266
267 visws_board_type = (char)(inb_p(PIIX_GPI_BD_REG) & PIIX_GPI_BD_REG)
268 >> PIIX_GPI_BD_SHIFT;
269
270 if (visws_board_type < 0)
271 return;
272
273 /*
274 * Install special quirks for timer, interrupt and memory setup:
275 */
276 arch_time_init_quirk = visws_time_init_quirk;
277 arch_pre_intr_init_quirk = visws_pre_intr_init_quirk;
278 arch_memory_setup_quirk = visws_memory_setup_quirk;
279
280 /*
281 * Fall back to generic behavior for traps:
282 */
283 arch_intr_init_quirk = NULL;
284 arch_trap_init_quirk = visws_trap_init_quirk;
285
286 /*
287 * Install reboot quirks:
288 */
289 pm_power_off = visws_machine_power_off;
290 machine_ops.emergency_restart = visws_machine_emergency_restart;
291
292 /*
293 * Do not use broadcast IPIs:
294 */
295 no_broadcast = 0;
296
297 /*
298 * Override generic MP-table parsing:
299 */
300 mach_get_smp_config_quirk = visws_get_smp_config_quirk;
301 mach_find_smp_config_quirk = visws_find_smp_config_quirk;
302
303#ifdef CONFIG_X86_IO_APIC
304 /*
305 * Turn off IO-APIC detection and initialization:
306 */
307 skip_ioapic_setup = 1;
308#endif
309
310 /*
311 * Get Board rev.
312 * First, we have to initialize the 307 part to allow us access
313 * to the GPIO registers. Let's map them at 0x0fc0 which is right
314 * after the PIIX4 PM section.
315 */
316 outb_p(SIO_DEV_SEL, SIO_INDEX);
317 outb_p(SIO_GP_DEV, SIO_DATA); /* Talk to GPIO regs. */
318
319 outb_p(SIO_DEV_MSB, SIO_INDEX);
320 outb_p(SIO_GP_MSB, SIO_DATA); /* MSB of GPIO base address */
321
322 outb_p(SIO_DEV_LSB, SIO_INDEX);
323 outb_p(SIO_GP_LSB, SIO_DATA); /* LSB of GPIO base address */
324
325 outb_p(SIO_DEV_ENB, SIO_INDEX);
326 outb_p(1, SIO_DATA); /* Enable GPIO registers. */
327
328 /*
329 * Now, we have to map the power management section to write
330 * a bit which enables access to the GPIO registers.
331 * What lunatic came up with this shit?
332 */
333 outb_p(SIO_DEV_SEL, SIO_INDEX);
334 outb_p(SIO_PM_DEV, SIO_DATA); /* Talk to GPIO regs. */
335
336 outb_p(SIO_DEV_MSB, SIO_INDEX);
337 outb_p(SIO_PM_MSB, SIO_DATA); /* MSB of PM base address */
338
339 outb_p(SIO_DEV_LSB, SIO_INDEX);
340 outb_p(SIO_PM_LSB, SIO_DATA); /* LSB of PM base address */
341
342 outb_p(SIO_DEV_ENB, SIO_INDEX);
343 outb_p(1, SIO_DATA); /* Enable PM registers. */
344
345 /*
346 * Now, write the PM register which enables the GPIO registers.
347 */
348 outb_p(SIO_PM_FER2, SIO_PM_INDEX);
349 outb_p(SIO_PM_GP_EN, SIO_PM_DATA);
350
351 /*
352 * Now, initialize the GPIO registers.
353 * We want them all to be inputs which is the
354 * power on default, so let's leave them alone.
355 * So, let's just read the board rev!
356 */
357 raw = inb_p(SIO_GP_DATA1);
358 raw &= 0x7f; /* 7 bits of valid board revision ID. */
359
360 if (visws_board_type == VISWS_320) {
361 if (raw < 0x6) {
362 visws_board_rev = 4;
363 } else if (raw < 0xc) {
364 visws_board_rev = 5;
365 } else {
366 visws_board_rev = 6;
367 }
368 } else if (visws_board_type == VISWS_540) {
369 visws_board_rev = 2;
370 } else {
371 visws_board_rev = raw;
372 }
373
374 printk(KERN_INFO "Silicon Graphics Visual Workstation %s (rev %d) detected\n",
375 (visws_board_type == VISWS_320 ? "320" :
376 (visws_board_type == VISWS_540 ? "540" :
377 "unknown")), visws_board_rev);
378}
379
380#define A01234 (LI_INTA_0 | LI_INTA_1 | LI_INTA_2 | LI_INTA_3 | LI_INTA_4)
381#define BCD (LI_INTB | LI_INTC | LI_INTD)
382#define ALLDEVS (A01234 | BCD)
383
384static __init void lithium_init(void)
385{
386 set_fixmap(FIX_LI_PCIA, LI_PCI_A_PHYS);
387 set_fixmap(FIX_LI_PCIB, LI_PCI_B_PHYS);
388
389 if ((li_pcia_read16(PCI_VENDOR_ID) != PCI_VENDOR_ID_SGI) ||
390 (li_pcia_read16(PCI_DEVICE_ID) != PCI_DEVICE_ID_SGI_LITHIUM)) {
391 printk(KERN_EMERG "Lithium hostbridge %c not found\n", 'A');
392/* panic("This machine is not SGI Visual Workstation 320/540"); */
393 }
394
395 if ((li_pcib_read16(PCI_VENDOR_ID) != PCI_VENDOR_ID_SGI) ||
396 (li_pcib_read16(PCI_DEVICE_ID) != PCI_DEVICE_ID_SGI_LITHIUM)) {
397 printk(KERN_EMERG "Lithium hostbridge %c not found\n", 'B');
398/* panic("This machine is not SGI Visual Workstation 320/540"); */
399 }
400
401 li_pcia_write16(LI_PCI_INTEN, ALLDEVS);
402 li_pcib_write16(LI_PCI_INTEN, ALLDEVS);
403}
404
405static __init void cobalt_init(void)
406{
407 /*
408 * On normal SMP PC this is used only with SMP, but we have to
409 * use it and set it up here to start the Cobalt clock
410 */
411 set_fixmap(FIX_APIC_BASE, APIC_DEFAULT_PHYS_BASE);
412 setup_local_APIC();
413 printk(KERN_INFO "Local APIC Version %#x, ID %#x\n",
414 (unsigned int)apic_read(APIC_LVR),
415 (unsigned int)apic_read(APIC_ID));
416
417 set_fixmap(FIX_CO_CPU, CO_CPU_PHYS);
418 set_fixmap(FIX_CO_APIC, CO_APIC_PHYS);
419 printk(KERN_INFO "Cobalt Revision %#lx, APIC ID %#lx\n",
420 co_cpu_read(CO_CPU_REV), co_apic_read(CO_APIC_ID));
421
422 /* Enable Cobalt APIC being careful to NOT change the ID! */
423 co_apic_write(CO_APIC_ID, co_apic_read(CO_APIC_ID) | CO_APIC_ENABLE);
424
425 printk(KERN_INFO "Cobalt APIC enabled: ID reg %#lx\n",
426 co_apic_read(CO_APIC_ID));
427}
428
429int __init visws_trap_init_quirk(void)
430{
431 lithium_init();
432 cobalt_init();
433
434 return 1;
435}
436
437/*
438 * IRQ controller / APIC support:
439 */
440
441static DEFINE_SPINLOCK(cobalt_lock);
442
443/*
444 * Set the given Cobalt APIC Redirection Table entry to point
445 * to the given IDT vector/index.
446 */
447static inline void co_apic_set(int entry, int irq)
448{
449 co_apic_write(CO_APIC_LO(entry), CO_APIC_LEVEL | (irq + FIRST_EXTERNAL_VECTOR));
450 co_apic_write(CO_APIC_HI(entry), 0);
451}
452
453/*
454 * Cobalt (IO)-APIC functions to handle PCI devices.
455 */
456static inline int co_apic_ide0_hack(void)
457{
458 extern char visws_board_type;
459 extern char visws_board_rev;
460
461 if (visws_board_type == VISWS_320 && visws_board_rev == 5)
462 return 5;
463 return CO_APIC_IDE0;
464}
465
466static int is_co_apic(unsigned int irq)
467{
468 if (IS_CO_APIC(irq))
469 return CO_APIC(irq);
470
471 switch (irq) {
472 case 0: return CO_APIC_CPU;
473 case CO_IRQ_IDE0: return co_apic_ide0_hack();
474 case CO_IRQ_IDE1: return CO_APIC_IDE1;
475 default: return -1;
476 }
477}
478
479
480/*
481 * This is the SGI Cobalt (IO-)APIC:
482 */
483
484static void enable_cobalt_irq(unsigned int irq)
485{
486 co_apic_set(is_co_apic(irq), irq);
487}
488
489static void disable_cobalt_irq(unsigned int irq)
490{
491 int entry = is_co_apic(irq);
492
493 co_apic_write(CO_APIC_LO(entry), CO_APIC_MASK);
494 co_apic_read(CO_APIC_LO(entry));
495}
496
497/*
498 * "irq" really just serves to identify the device. Here is where we
499 * map this to the Cobalt APIC entry where it's physically wired.
500 * This is called via request_irq -> setup_irq -> irq_desc->startup()
501 */
502static unsigned int startup_cobalt_irq(unsigned int irq)
503{
504 unsigned long flags;
505
506 spin_lock_irqsave(&cobalt_lock, flags);
507 if ((irq_desc[irq].status & (IRQ_DISABLED | IRQ_INPROGRESS | IRQ_WAITING)))
508 irq_desc[irq].status &= ~(IRQ_DISABLED | IRQ_INPROGRESS | IRQ_WAITING);
509 enable_cobalt_irq(irq);
510 spin_unlock_irqrestore(&cobalt_lock, flags);
511 return 0;
512}
513
514static void ack_cobalt_irq(unsigned int irq)
515{
516 unsigned long flags;
517
518 spin_lock_irqsave(&cobalt_lock, flags);
519 disable_cobalt_irq(irq);
520 apic_write(APIC_EOI, APIC_EIO_ACK);
521 spin_unlock_irqrestore(&cobalt_lock, flags);
522}
523
524static void end_cobalt_irq(unsigned int irq)
525{
526 unsigned long flags;
527
528 spin_lock_irqsave(&cobalt_lock, flags);
529 if (!(irq_desc[irq].status & (IRQ_DISABLED | IRQ_INPROGRESS)))
530 enable_cobalt_irq(irq);
531 spin_unlock_irqrestore(&cobalt_lock, flags);
532}
533
534static struct irq_chip cobalt_irq_type = {
535 .typename = "Cobalt-APIC",
536 .startup = startup_cobalt_irq,
537 .shutdown = disable_cobalt_irq,
538 .enable = enable_cobalt_irq,
539 .disable = disable_cobalt_irq,
540 .ack = ack_cobalt_irq,
541 .end = end_cobalt_irq,
542};
543
544
545/*
546 * This is the PIIX4-based 8259 that is wired up indirectly to Cobalt
547 * -- not the manner expected by the code in i8259.c.
548 *
549 * there is a 'master' physical interrupt source that gets sent to
550 * the CPU. But in the chipset there are various 'virtual' interrupts
551 * waiting to be handled. We represent this to Linux through a 'master'
552 * interrupt controller type, and through a special virtual interrupt-
553 * controller. Device drivers only see the virtual interrupt sources.
554 */
555static unsigned int startup_piix4_master_irq(unsigned int irq)
556{
557 init_8259A(0);
558
559 return startup_cobalt_irq(irq);
560}
561
562static void end_piix4_master_irq(unsigned int irq)
563{
564 unsigned long flags;
565
566 spin_lock_irqsave(&cobalt_lock, flags);
567 enable_cobalt_irq(irq);
568 spin_unlock_irqrestore(&cobalt_lock, flags);
569}
570
571static struct irq_chip piix4_master_irq_type = {
572 .typename = "PIIX4-master",
573 .startup = startup_piix4_master_irq,
574 .ack = ack_cobalt_irq,
575 .end = end_piix4_master_irq,
576};
577
578
579static struct irq_chip piix4_virtual_irq_type = {
580 .typename = "PIIX4-virtual",
581 .shutdown = disable_8259A_irq,
582 .enable = enable_8259A_irq,
583 .disable = disable_8259A_irq,
584};
585
586
587/*
588 * PIIX4-8259 master/virtual functions to handle interrupt requests
589 * from legacy devices: floppy, parallel, serial, rtc.
590 *
591 * None of these get Cobalt APIC entries, neither do they have IDT
592 * entries. These interrupts are purely virtual and distributed from
593 * the 'master' interrupt source: CO_IRQ_8259.
594 *
595 * When the 8259 interrupts its handler figures out which of these
596 * devices is interrupting and dispatches to its handler.
597 *
598 * CAREFUL: devices see the 'virtual' interrupt only. Thus disable/
599 * enable_irq gets the right irq. This 'master' irq is never directly
600 * manipulated by any driver.
601 */
602static irqreturn_t piix4_master_intr(int irq, void *dev_id)
603{
604 int realirq;
605 irq_desc_t *desc;
606 unsigned long flags;
607
608 spin_lock_irqsave(&i8259A_lock, flags);
609
610 /* Find out what's interrupting in the PIIX4 master 8259 */
611 outb(0x0c, 0x20); /* OCW3 Poll command */
612 realirq = inb(0x20);
613
614 /*
615 * Bit 7 == 0 means invalid/spurious
616 */
617 if (unlikely(!(realirq & 0x80)))
618 goto out_unlock;
619
620 realirq &= 7;
621
622 if (unlikely(realirq == 2)) {
623 outb(0x0c, 0xa0);
624 realirq = inb(0xa0);
625
626 if (unlikely(!(realirq & 0x80)))
627 goto out_unlock;
628
629 realirq = (realirq & 7) + 8;
630 }
631
632 /* mask and ack interrupt */
633 cached_irq_mask |= 1 << realirq;
634 if (unlikely(realirq > 7)) {
635 inb(0xa1);
636 outb(cached_slave_mask, 0xa1);
637 outb(0x60 + (realirq & 7), 0xa0);
638 outb(0x60 + 2, 0x20);
639 } else {
640 inb(0x21);
641 outb(cached_master_mask, 0x21);
642 outb(0x60 + realirq, 0x20);
643 }
644
645 spin_unlock_irqrestore(&i8259A_lock, flags);
646
647 desc = irq_desc + realirq;
648
649 /*
650 * handle this 'virtual interrupt' as a Cobalt one now.
651 */
652 kstat_cpu(smp_processor_id()).irqs[realirq]++;
653
654 if (likely(desc->action != NULL))
655 handle_IRQ_event(realirq, desc->action);
656
657 if (!(desc->status & IRQ_DISABLED))
658 enable_8259A_irq(realirq);
659
660 return IRQ_HANDLED;
661
662out_unlock:
663 spin_unlock_irqrestore(&i8259A_lock, flags);
664 return IRQ_NONE;
665}
666
667static struct irqaction master_action = {
668 .handler = piix4_master_intr,
669 .name = "PIIX4-8259",
670};
671
672static struct irqaction cascade_action = {
673 .handler = no_action,
674 .name = "cascade",
675};
676
677
678void init_VISWS_APIC_irqs(void)
679{
680 int i;
681
682 for (i = 0; i < CO_IRQ_APIC0 + CO_APIC_LAST + 1; i++) {
683 irq_desc[i].status = IRQ_DISABLED;
684 irq_desc[i].action = 0;
685 irq_desc[i].depth = 1;
686
687 if (i == 0) {
688 irq_desc[i].chip = &cobalt_irq_type;
689 }
690 else if (i == CO_IRQ_IDE0) {
691 irq_desc[i].chip = &cobalt_irq_type;
692 }
693 else if (i == CO_IRQ_IDE1) {
694 irq_desc[i].chip = &cobalt_irq_type;
695 }
696 else if (i == CO_IRQ_8259) {
697 irq_desc[i].chip = &piix4_master_irq_type;
698 }
699 else if (i < CO_IRQ_APIC0) {
700 irq_desc[i].chip = &piix4_virtual_irq_type;
701 }
702 else if (IS_CO_APIC(i)) {
703 irq_desc[i].chip = &cobalt_irq_type;
704 }
705 }
706
707 setup_irq(CO_IRQ_8259, &master_action);
708 setup_irq(2, &cascade_action);
709}
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
index 956f38927aa7..b15346092b7b 100644
--- a/arch/x86/kernel/vmi_32.c
+++ b/arch/x86/kernel/vmi_32.c
@@ -151,7 +151,7 @@ static unsigned vmi_patch(u8 type, u16 clobbers, void *insns,
151 insns, ip); 151 insns, ip);
152 case PARAVIRT_PATCH(pv_cpu_ops.iret): 152 case PARAVIRT_PATCH(pv_cpu_ops.iret):
153 return patch_internal(VMI_CALL_IRET, len, insns, ip); 153 return patch_internal(VMI_CALL_IRET, len, insns, ip);
154 case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret): 154 case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit):
155 return patch_internal(VMI_CALL_SYSEXIT, len, insns, ip); 155 return patch_internal(VMI_CALL_SYSEXIT, len, insns, ip);
156 default: 156 default:
157 break; 157 break;
@@ -896,7 +896,7 @@ static inline int __init activate_vmi(void)
896 * the backend. They are performance critical anyway, so requiring 896 * the backend. They are performance critical anyway, so requiring
897 * a patch is not a big problem. 897 * a patch is not a big problem.
898 */ 898 */
899 pv_cpu_ops.irq_enable_syscall_ret = (void *)0xfeedbab0; 899 pv_cpu_ops.irq_enable_sysexit = (void *)0xfeedbab0;
900 pv_cpu_ops.iret = (void *)0xbadbab0; 900 pv_cpu_ops.iret = (void *)0xbadbab0;
901 901
902#ifdef CONFIG_SMP 902#ifdef CONFIG_SMP
@@ -932,7 +932,7 @@ static inline int __init activate_vmi(void)
932 pv_apic_ops.setup_secondary_clock = vmi_time_ap_init; 932 pv_apic_ops.setup_secondary_clock = vmi_time_ap_init;
933#endif 933#endif
934 pv_time_ops.sched_clock = vmi_sched_clock; 934 pv_time_ops.sched_clock = vmi_sched_clock;
935 pv_time_ops.get_cpu_khz = vmi_cpu_khz; 935 pv_time_ops.get_tsc_khz = vmi_tsc_khz;
936 936
937 /* We have true wallclock functions; disable CMOS clock sync */ 937 /* We have true wallclock functions; disable CMOS clock sync */
938 no_sync_cmos_clock = 1; 938 no_sync_cmos_clock = 1;
diff --git a/arch/x86/kernel/vmiclock_32.c b/arch/x86/kernel/vmiclock_32.c
index a2b030780aa9..6953859fe289 100644
--- a/arch/x86/kernel/vmiclock_32.c
+++ b/arch/x86/kernel/vmiclock_32.c
@@ -33,8 +33,7 @@
33#include <asm/apic.h> 33#include <asm/apic.h>
34#include <asm/timer.h> 34#include <asm/timer.h>
35#include <asm/i8253.h> 35#include <asm/i8253.h>
36 36#include <asm/irq_vectors.h>
37#include <irq_vectors.h>
38 37
39#define VMI_ONESHOT (VMI_ALARM_IS_ONESHOT | VMI_CYCLES_REAL | vmi_get_alarm_wiring()) 38#define VMI_ONESHOT (VMI_ALARM_IS_ONESHOT | VMI_CYCLES_REAL | vmi_get_alarm_wiring())
40#define VMI_PERIODIC (VMI_ALARM_IS_PERIODIC | VMI_CYCLES_REAL | vmi_get_alarm_wiring()) 39#define VMI_PERIODIC (VMI_ALARM_IS_PERIODIC | VMI_CYCLES_REAL | vmi_get_alarm_wiring())
@@ -70,8 +69,8 @@ unsigned long long vmi_sched_clock(void)
70 return cycles_2_ns(vmi_timer_ops.get_cycle_counter(VMI_CYCLES_AVAILABLE)); 69 return cycles_2_ns(vmi_timer_ops.get_cycle_counter(VMI_CYCLES_AVAILABLE));
71} 70}
72 71
73/* paravirt_ops.get_cpu_khz = vmi_cpu_khz */ 72/* paravirt_ops.get_tsc_khz = vmi_tsc_khz */
74unsigned long vmi_cpu_khz(void) 73unsigned long vmi_tsc_khz(void)
75{ 74{
76 unsigned long long khz; 75 unsigned long long khz;
77 khz = vmi_timer_ops.get_cycle_frequency(); 76 khz = vmi_timer_ops.get_cycle_frequency();
diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S
index ce5ed083a1e9..cdb2363697d2 100644
--- a/arch/x86/kernel/vmlinux_32.lds.S
+++ b/arch/x86/kernel/vmlinux_32.lds.S
@@ -49,23 +49,14 @@ SECTIONS
49 _etext = .; /* End of text section */ 49 _etext = .; /* End of text section */
50 } :text = 0x9090 50 } :text = 0x9090
51 51
52 NOTES :text :note
53
52 . = ALIGN(16); /* Exception table */ 54 . = ALIGN(16); /* Exception table */
53 __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { 55 __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
54 __start___ex_table = .; 56 __start___ex_table = .;
55 *(__ex_table) 57 *(__ex_table)
56 __stop___ex_table = .; 58 __stop___ex_table = .;
57 } 59 } :text = 0x9090
58
59 NOTES :text :note
60
61 BUG_TABLE :text
62
63 . = ALIGN(4);
64 .tracedata : AT(ADDR(.tracedata) - LOAD_OFFSET) {
65 __tracedata_start = .;
66 *(.tracedata)
67 __tracedata_end = .;
68 }
69 60
70 RODATA 61 RODATA
71 62
diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S
index fad3674b06a5..63e5c1a22e88 100644
--- a/arch/x86/kernel/vmlinux_64.lds.S
+++ b/arch/x86/kernel/vmlinux_64.lds.S
@@ -19,7 +19,7 @@ PHDRS {
19 data PT_LOAD FLAGS(7); /* RWE */ 19 data PT_LOAD FLAGS(7); /* RWE */
20 user PT_LOAD FLAGS(7); /* RWE */ 20 user PT_LOAD FLAGS(7); /* RWE */
21 data.init PT_LOAD FLAGS(7); /* RWE */ 21 data.init PT_LOAD FLAGS(7); /* RWE */
22 note PT_NOTE FLAGS(4); /* R__ */ 22 note PT_NOTE FLAGS(0); /* ___ */
23} 23}
24SECTIONS 24SECTIONS
25{ 25{
@@ -40,26 +40,17 @@ SECTIONS
40 _etext = .; /* End of text section */ 40 _etext = .; /* End of text section */
41 } :text = 0x9090 41 } :text = 0x9090
42 42
43 NOTES :text :note
44
43 . = ALIGN(16); /* Exception table */ 45 . = ALIGN(16); /* Exception table */
44 __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { 46 __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
45 __start___ex_table = .; 47 __start___ex_table = .;
46 *(__ex_table) 48 *(__ex_table)
47 __stop___ex_table = .; 49 __stop___ex_table = .;
48 } 50 } :text = 0x9090
49
50 NOTES :text :note
51
52 BUG_TABLE :text
53 51
54 RODATA 52 RODATA
55 53
56 . = ALIGN(4);
57 .tracedata : AT(ADDR(.tracedata) - LOAD_OFFSET) {
58 __tracedata_start = .;
59 *(.tracedata)
60 __tracedata_end = .;
61 }
62
63 . = ALIGN(PAGE_SIZE); /* Align data segment to page size boundary */ 54 . = ALIGN(PAGE_SIZE); /* Align data segment to page size boundary */
64 /* Data */ 55 /* Data */
65 .data : AT(ADDR(.data) - LOAD_OFFSET) { 56 .data : AT(ADDR(.data) - LOAD_OFFSET) {
@@ -177,6 +168,7 @@ SECTIONS
177 *(.con_initcall.init) 168 *(.con_initcall.init)
178 } 169 }
179 __con_initcall_end = .; 170 __con_initcall_end = .;
171 . = ALIGN(16);
180 __x86cpuvendor_start = .; 172 __x86cpuvendor_start = .;
181 .x86cpuvendor.init : AT(ADDR(.x86cpuvendor.init) - LOAD_OFFSET) { 173 .x86cpuvendor.init : AT(ADDR(.x86cpuvendor.init) - LOAD_OFFSET) {
182 *(.x86cpuvendor.init) 174 *(.x86cpuvendor.init)
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index ba8c0b75ab0a..0c029e8959c7 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,9 +15,12 @@
15#include <linux/init.h> 15#include <linux/init.h>
16#include <linux/pci_ids.h> 16#include <linux/pci_ids.h>
17#include <linux/pci_regs.h> 17#include <linux/pci_regs.h>
18
19#include <asm/apic.h>
18#include <asm/pci-direct.h> 20#include <asm/pci-direct.h>
19#include <asm/io.h> 21#include <asm/io.h>
20#include <asm/paravirt.h> 22#include <asm/paravirt.h>
23#include <asm/setup.h>
21 24
22#if defined CONFIG_PCI && defined CONFIG_PARAVIRT 25#if defined CONFIG_PCI && defined CONFIG_PARAVIRT
23/* 26/*
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 61efa2f7d564..0b8b6690a86d 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -42,7 +42,8 @@
42#include <asm/topology.h> 42#include <asm/topology.h>
43#include <asm/vgtod.h> 43#include <asm/vgtod.h>
44 44
45#define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr))) 45#define __vsyscall(nr) \
46 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
46#define __syscall_clobber "r11","cx","memory" 47#define __syscall_clobber "r11","cx","memory"
47 48
48/* 49/*
@@ -249,7 +250,7 @@ static ctl_table kernel_root_table2[] = {
249 doesn't violate that. We'll find out if it does. */ 250 doesn't violate that. We'll find out if it does. */
250static void __cpuinit vsyscall_set_cpu(int cpu) 251static void __cpuinit vsyscall_set_cpu(int cpu)
251{ 252{
252 unsigned long *d; 253 unsigned long d;
253 unsigned long node = 0; 254 unsigned long node = 0;
254#ifdef CONFIG_NUMA 255#ifdef CONFIG_NUMA
255 node = cpu_to_node(cpu); 256 node = cpu_to_node(cpu);
@@ -260,11 +261,11 @@ static void __cpuinit vsyscall_set_cpu(int cpu)
260 /* Store cpu number in limit so that it can be loaded quickly 261 /* Store cpu number in limit so that it can be loaded quickly
261 in user space in vgetcpu. 262 in user space in vgetcpu.
262 12 bits for the CPU and 8 bits for the node. */ 263 12 bits for the CPU and 8 bits for the node. */
263 d = (unsigned long *)(get_cpu_gdt_table(cpu) + GDT_ENTRY_PER_CPU); 264 d = 0x0f40000000000ULL;
264 *d = 0x0f40000000000ULL; 265 d |= cpu;
265 *d |= cpu; 266 d |= (node & 0xf) << 12;
266 *d |= (node & 0xf) << 12; 267 d |= (node >> 4) << 48;
267 *d |= (node >> 4) << 48; 268 write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
268} 269}
269 270
270static void __cpuinit cpu_vsyscall_init(void *arg) 271static void __cpuinit cpu_vsyscall_init(void *arg)
@@ -278,7 +279,7 @@ cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
278{ 279{
279 long cpu = (long)arg; 280 long cpu = (long)arg;
280 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) 281 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
281 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1); 282 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
282 return NOTIFY_DONE; 283 return NOTIFY_DONE;
283} 284}
284 285
@@ -301,7 +302,7 @@ static int __init vsyscall_init(void)
301#ifdef CONFIG_SYSCTL 302#ifdef CONFIG_SYSCTL
302 register_sysctl_table(kernel_root_table2); 303 register_sysctl_table(kernel_root_table2);
303#endif 304#endif
304 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1); 305 on_each_cpu(cpu_vsyscall_init, NULL, 1);
305 hotcpu_notifier(cpu_vsyscall_notifier, 0); 306 hotcpu_notifier(cpu_vsyscall_notifier, 0);
306 return 0; 307 return 0;
307} 308}
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index f6c05d0410fb..b545f371b5f5 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -2,13 +2,20 @@
2 All C exports should go in the respective C files. */ 2 All C exports should go in the respective C files. */
3 3
4#include <linux/module.h> 4#include <linux/module.h>
5#include <net/checksum.h>
6#include <linux/smp.h> 5#include <linux/smp.h>
7 6
7#include <net/checksum.h>
8
8#include <asm/processor.h> 9#include <asm/processor.h>
9#include <asm/uaccess.h>
10#include <asm/pgtable.h> 10#include <asm/pgtable.h>
11#include <asm/uaccess.h>
11#include <asm/desc.h> 12#include <asm/desc.h>
13#include <asm/ftrace.h>
14
15#ifdef CONFIG_FTRACE
16/* mcount is defined in assembly */
17EXPORT_SYMBOL(mcount);
18#endif
12 19
13EXPORT_SYMBOL(kernel_thread); 20EXPORT_SYMBOL(kernel_thread);
14 21
@@ -53,8 +60,3 @@ EXPORT_SYMBOL(init_level4_pgt);
53EXPORT_SYMBOL(load_gs_index); 60EXPORT_SYMBOL(load_gs_index);
54 61
55EXPORT_SYMBOL(_proxy_pda); 62EXPORT_SYMBOL(_proxy_pda);
56
57#ifdef CONFIG_PARAVIRT
58/* Virtualized guests may want to use it */
59EXPORT_SYMBOL_GPL(cpu_gdt_descr);
60#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-15 04:11:00 -0500