From 874c4fe389d1358f82c96dc9b5092fc5c7690604 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] i386: Allow to use GENERICARCH for UP kernels There are some machines around (large xSeries or Unisys ES7000) that need physical IO-APIC destination mode to access all of their IO devices. This currently doesn't work in UP kernels as used in distribution installers. This patch allows to compile even UP kernels as GENERICARCH which allows to use physical or clustered APIC mode. Signed-off-by: Andi Kleen --- include/asm-i386/genapic.h | 69 +++++++++++++++++++------------- include/asm-i386/mach-es7000/mach_apic.h | 4 ++ include/asm-i386/mach-summit/mach_apic.h | 11 ++++- include/asm-i386/smp.h | 19 +++++---- 4 files changed, 66 insertions(+), 37 deletions(-) (limited to 'include') diff --git a/include/asm-i386/genapic.h b/include/asm-i386/genapic.h index b3783a32abee..8ffbb0f07457 100644 --- a/include/asm-i386/genapic.h +++ b/include/asm-i386/genapic.h @@ -1,6 +1,8 @@ #ifndef _ASM_GENAPIC_H #define _ASM_GENAPIC_H 1 +#include + /* * Generic APIC driver interface. * @@ -63,14 +65,25 @@ struct genapic { unsigned (*get_apic_id)(unsigned long x); unsigned long apic_id_mask; unsigned int (*cpu_mask_to_apicid)(cpumask_t cpumask); - + +#ifdef CONFIG_SMP /* ipi */ void (*send_IPI_mask)(cpumask_t mask, int vector); void (*send_IPI_allbutself)(int vector); void (*send_IPI_all)(int vector); +#endif }; -#define APICFUNC(x) .x = x +#define APICFUNC(x) .x = x, + +/* More functions could be probably marked IPIFUNC and save some space + in UP GENERICARCH kernels, but I don't have the nerve right now + to untangle this mess. -AK */ +#ifdef CONFIG_SMP +#define IPIFUNC(x) APICFUNC(x) +#else +#define IPIFUNC(x) +#endif #define APIC_INIT(aname, aprobe) { \ .name = aname, \ @@ -80,33 +93,33 @@ struct genapic { .no_balance_irq = NO_BALANCE_IRQ, \ .ESR_DISABLE = esr_disable, \ .apic_destination_logical = APIC_DEST_LOGICAL, \ - APICFUNC(apic_id_registered), \ - APICFUNC(target_cpus), \ - APICFUNC(check_apicid_used), \ - APICFUNC(check_apicid_present), \ - APICFUNC(init_apic_ldr), \ - APICFUNC(ioapic_phys_id_map), \ - APICFUNC(clustered_apic_check), \ - APICFUNC(multi_timer_check), \ - APICFUNC(apicid_to_node), \ - APICFUNC(cpu_to_logical_apicid), \ - APICFUNC(cpu_present_to_apicid), \ - APICFUNC(apicid_to_cpu_present), \ - APICFUNC(mpc_apic_id), \ - APICFUNC(setup_portio_remap), \ - APICFUNC(check_phys_apicid_present), \ - APICFUNC(mpc_oem_bus_info), \ - APICFUNC(mpc_oem_pci_bus), \ - APICFUNC(mps_oem_check), \ - APICFUNC(get_apic_id), \ + APICFUNC(apic_id_registered) \ + APICFUNC(target_cpus) \ + APICFUNC(check_apicid_used) \ + APICFUNC(check_apicid_present) \ + APICFUNC(init_apic_ldr) \ + APICFUNC(ioapic_phys_id_map) \ + APICFUNC(clustered_apic_check) \ + APICFUNC(multi_timer_check) \ + APICFUNC(apicid_to_node) \ + APICFUNC(cpu_to_logical_apicid) \ + APICFUNC(cpu_present_to_apicid) \ + APICFUNC(apicid_to_cpu_present) \ + APICFUNC(mpc_apic_id) \ + APICFUNC(setup_portio_remap) \ + APICFUNC(check_phys_apicid_present) \ + APICFUNC(mpc_oem_bus_info) \ + APICFUNC(mpc_oem_pci_bus) \ + APICFUNC(mps_oem_check) \ + APICFUNC(get_apic_id) \ .apic_id_mask = APIC_ID_MASK, \ - APICFUNC(cpu_mask_to_apicid), \ - APICFUNC(acpi_madt_oem_check), \ - APICFUNC(send_IPI_mask), \ - APICFUNC(send_IPI_allbutself), \ - APICFUNC(send_IPI_all), \ - APICFUNC(enable_apic_mode), \ - APICFUNC(phys_pkg_id), \ + APICFUNC(cpu_mask_to_apicid) \ + APICFUNC(acpi_madt_oem_check) \ + IPIFUNC(send_IPI_mask) \ + IPIFUNC(send_IPI_allbutself) \ + IPIFUNC(send_IPI_all) \ + APICFUNC(enable_apic_mode) \ + APICFUNC(phys_pkg_id) \ } extern struct genapic *genapic; diff --git a/include/asm-i386/mach-es7000/mach_apic.h b/include/asm-i386/mach-es7000/mach_apic.h index b5f3f0d0b2bc..26333685a7fb 100644 --- a/include/asm-i386/mach-es7000/mach_apic.h +++ b/include/asm-i386/mach-es7000/mach_apic.h @@ -123,9 +123,13 @@ extern u8 cpu_2_logical_apicid[]; /* Mapping from cpu number to logical apicid */ static inline int cpu_to_logical_apicid(int cpu) { +#ifdef CONFIG_SMP if (cpu >= NR_CPUS) return BAD_APICID; return (int)cpu_2_logical_apicid[cpu]; +#else + return logical_smp_processor_id(); +#endif } static inline int mpc_apic_id(struct mpc_config_processor *m, struct mpc_config_translation *unused) diff --git a/include/asm-i386/mach-summit/mach_apic.h b/include/asm-i386/mach-summit/mach_apic.h index 9fd073286289..a81b05961595 100644 --- a/include/asm-i386/mach-summit/mach_apic.h +++ b/include/asm-i386/mach-summit/mach_apic.h @@ -46,10 +46,12 @@ extern u8 cpu_2_logical_apicid[]; static inline void init_apic_ldr(void) { unsigned long val, id; - int i, count; - u8 lid; + int count = 0; u8 my_id = (u8)hard_smp_processor_id(); u8 my_cluster = (u8)apicid_cluster(my_id); +#ifdef CONFIG_SMP + u8 lid; + int i; /* Create logical APIC IDs by counting CPUs already in cluster. */ for (count = 0, i = NR_CPUS; --i >= 0; ) { @@ -57,6 +59,7 @@ static inline void init_apic_ldr(void) if (lid != BAD_APICID && apicid_cluster(lid) == my_cluster) ++count; } +#endif /* We only have a 4 wide bitmap in cluster mode. If a deranged * BIOS puts 5 CPUs in one APIC cluster, we're hosed. */ BUG_ON(count >= XAPIC_DEST_CPUS_SHIFT); @@ -91,9 +94,13 @@ static inline int apicid_to_node(int logical_apicid) /* Mapping from cpu number to logical apicid */ static inline int cpu_to_logical_apicid(int cpu) { +#ifdef CONFIG_SMP if (cpu >= NR_CPUS) return BAD_APICID; return (int)cpu_2_logical_apicid[cpu]; +#else + return logical_smp_processor_id(); +#endif } static inline int cpu_present_to_apicid(int mps_cpu) diff --git a/include/asm-i386/smp.h b/include/asm-i386/smp.h index 142d10e34ade..f87826039a51 100644 --- a/include/asm-i386/smp.h +++ b/include/asm-i386/smp.h @@ -80,17 +80,11 @@ static inline int hard_smp_processor_id(void) return GET_APIC_ID(*(unsigned long *)(APIC_BASE+APIC_ID)); } #endif - -static __inline int logical_smp_processor_id(void) -{ - /* we don't want to mark this access volatile - bad code generation */ - return GET_APIC_LOGICAL_ID(*(unsigned long *)(APIC_BASE+APIC_LDR)); -} - #endif extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); + #endif /* !__ASSEMBLY__ */ #else /* CONFIG_SMP */ @@ -100,4 +94,15 @@ extern void __cpu_die(unsigned int cpu); #define NO_PROC_ID 0xFF /* No processor magic marker */ #endif + +#ifndef __ASSEMBLY__ +#ifdef CONFIG_X86_LOCAL_APIC +static __inline int logical_smp_processor_id(void) +{ + /* we don't want to mark this access volatile - bad code generation */ + return GET_APIC_LOGICAL_ID(*(unsigned long *)(APIC_BASE+APIC_LDR)); +} +#endif +#endif + #endif -- cgit v1.2.2 From b07f8915cda3fcd73b8b68075ba1e6cd0673365d Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] x86: Temporarily revert parts of the Core 2 nmi nmi watchdog support This makes merging easier. They are readded a few patches later. Signed-off-by: Andi Kleen --- include/asm-i386/intel_arch_perfmon.h | 19 ------------------- include/asm-x86_64/intel_arch_perfmon.h | 19 ------------------- 2 files changed, 38 deletions(-) delete mode 100644 include/asm-i386/intel_arch_perfmon.h delete mode 100644 include/asm-x86_64/intel_arch_perfmon.h (limited to 'include') diff --git a/include/asm-i386/intel_arch_perfmon.h b/include/asm-i386/intel_arch_perfmon.h deleted file mode 100644 index 134ea9cc5283..000000000000 --- a/include/asm-i386/intel_arch_perfmon.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef X86_INTEL_ARCH_PERFMON_H -#define X86_INTEL_ARCH_PERFMON_H 1 - -#define MSR_ARCH_PERFMON_PERFCTR0 0xc1 -#define MSR_ARCH_PERFMON_PERFCTR1 0xc2 - -#define MSR_ARCH_PERFMON_EVENTSEL0 0x186 -#define MSR_ARCH_PERFMON_EVENTSEL1 0x187 - -#define ARCH_PERFMON_EVENTSEL0_ENABLE (1 << 22) -#define ARCH_PERFMON_EVENTSEL_INT (1 << 20) -#define ARCH_PERFMON_EVENTSEL_OS (1 << 17) -#define ARCH_PERFMON_EVENTSEL_USR (1 << 16) - -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL (0x3c) -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT (1 << 0) - -#endif /* X86_INTEL_ARCH_PERFMON_H */ diff --git a/include/asm-x86_64/intel_arch_perfmon.h b/include/asm-x86_64/intel_arch_perfmon.h deleted file mode 100644 index 59c396431569..000000000000 --- a/include/asm-x86_64/intel_arch_perfmon.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef X86_64_INTEL_ARCH_PERFMON_H -#define X86_64_INTEL_ARCH_PERFMON_H 1 - -#define MSR_ARCH_PERFMON_PERFCTR0 0xc1 -#define MSR_ARCH_PERFMON_PERFCTR1 0xc2 - -#define MSR_ARCH_PERFMON_EVENTSEL0 0x186 -#define MSR_ARCH_PERFMON_EVENTSEL1 0x187 - -#define ARCH_PERFMON_EVENTSEL0_ENABLE (1 << 22) -#define ARCH_PERFMON_EVENTSEL_INT (1 << 20) -#define ARCH_PERFMON_EVENTSEL_OS (1 << 17) -#define ARCH_PERFMON_EVENTSEL_USR (1 << 16) - -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL (0x3c) -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT (1 << 0) - -#endif /* X86_64_INTEL_ARCH_PERFMON_H */ -- cgit v1.2.2 From 828f0afda123a96ff4e8078f057a302f4b4232ae Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] x86: Add performance counter reservation framework for UP kernels Adds basic infrastructure to allow subsystems to reserve performance counters on the x86 chips. Only UP kernels are supported in this patch to make reviewing easier. The SMP portion makes a lot more changes. Think of this as a locking mechanism where each bit represents a different counter. In addition, each subsystem should also reserve an appropriate event selection register that will correspond to the performance counter it will be using (this is mainly neccessary for the Pentium 4 chips as they break the 1:1 relationship to performance counters). This will help prevent subsystems like oprofile from interfering with the nmi watchdog. Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/asm-i386/nmi.h | 7 +++++++ include/asm-x86_64/nmi.h | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-i386/nmi.h b/include/asm-i386/nmi.h index 67d994799999..27fc9e6f630e 100644 --- a/include/asm-i386/nmi.h +++ b/include/asm-i386/nmi.h @@ -25,6 +25,13 @@ void set_nmi_callback(nmi_callback_t callback); */ void unset_nmi_callback(void); +extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); +extern int avail_to_resrv_perfctr_nmi(unsigned int); +extern int reserve_perfctr_nmi(unsigned int); +extern void release_perfctr_nmi(unsigned int); +extern int reserve_evntsel_nmi(unsigned int); +extern void release_evntsel_nmi(unsigned int); + extern void setup_apic_nmi_watchdog (void); extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index efb45c894d76..62a784cb8f0c 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -56,7 +56,13 @@ extern int panic_on_timeout; extern int unknown_nmi_panic; extern int check_nmi_watchdog(void); - +extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); +extern int avail_to_resrv_perfctr_nmi(unsigned int); +extern int reserve_perfctr_nmi(unsigned int); +extern void release_perfctr_nmi(unsigned int); +extern int reserve_evntsel_nmi(unsigned int); +extern void release_evntsel_nmi(unsigned int); + extern void setup_apic_nmi_watchdog (void); extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); -- cgit v1.2.2 From f2802e7f571c05f9a901b1f5bd144aa730ccc88e Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] Add SMP support on x86_64 to reservation framework This patch includes the changes to make the nmi watchdog on x86_64 SMP aware. A bunch of code was moved around to make it simpler to read. In addition, it is now possible to determine if a particular NMI was the result of the watchdog or not. This feature allows the kernel to filter out unknown NMIs easier. Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/asm-x86_64/nmi.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index 62a784cb8f0c..5918136fd853 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -63,7 +63,7 @@ extern void release_perfctr_nmi(unsigned int); extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); -extern void setup_apic_nmi_watchdog (void); +extern void setup_apic_nmi_watchdog (void *); extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); @@ -73,6 +73,7 @@ extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); extern void nmi_watchdog_default(void); extern int setup_nmi_watchdog(char *); +extern atomic_t nmi_active; extern unsigned int nmi_watchdog; #define NMI_DEFAULT -1 #define NMI_NONE 0 -- cgit v1.2.2 From b7471c6da94d30d3deadc55986cc38d1ff57f9ca Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] i386: Add SMP support on i386 to reservation framework This patch includes the changes to make the nmi watchdog on i386 SMP aware. A bunch of code was moved around to make it simpler to read. In addition, it is now possible to determine if a particular NMI was the result of the watchdog or not. This feature allows the kernel to filter out unknown NMIs easier. Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/asm-i386/nmi.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-i386/nmi.h b/include/asm-i386/nmi.h index 27fc9e6f630e..4cda6801ecb8 100644 --- a/include/asm-i386/nmi.h +++ b/include/asm-i386/nmi.h @@ -32,13 +32,14 @@ extern void release_perfctr_nmi(unsigned int); extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); -extern void setup_apic_nmi_watchdog (void); +extern void setup_apic_nmi_watchdog (void *); extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs); +extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); +extern atomic_t nmi_active; extern unsigned int nmi_watchdog; #define NMI_DEFAULT -1 #define NMI_NONE 0 -- cgit v1.2.2 From 3adbbcce9a49b900d4cc118cdccfdefa78bf1afb Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] x86: Cleanup NMI interrupt path This patch cleans up the NMI interrupt path. Instead of being gated by if the 'nmi callback' is set, the interrupt handler now calls everyone who is registered on the die_chain and additionally checks the nmi watchdog, reseting it if enabled. This allows more subsystems to hook into the NMI if they need to (without being block by set_nmi_callback). Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/asm-i386/nmi.h | 2 +- include/asm-x86_64/nmi.h | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-i386/nmi.h b/include/asm-i386/nmi.h index 4cda6801ecb8..da0e0b4e9139 100644 --- a/include/asm-i386/nmi.h +++ b/include/asm-i386/nmi.h @@ -37,7 +37,7 @@ extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); +extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); extern atomic_t nmi_active; extern unsigned int nmi_watchdog; diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index 5918136fd853..8f02a2a416e6 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -26,6 +26,14 @@ void set_nmi_callback(nmi_callback_t callback); */ void unset_nmi_callback(void); +/** + * do_nmi_callback + * + * Check to see if a callback exists and execute it. Return 1 + * if the handler exists and was handled successfully. + */ +int do_nmi_callback(struct pt_regs *regs, int cpu); + #ifdef CONFIG_PM /** Replace the PM callback routine for NMI. */ @@ -68,7 +76,7 @@ extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); +extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); extern void nmi_watchdog_default(void); extern int setup_nmi_watchdog(char *); -- cgit v1.2.2 From 1d001df19d5323e642ba8ac821c713675ebccd82 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:26 +0200 Subject: [PATCH] Add TIF_RESTORE_SIGMASK We need TIF_RESTORE_SIGMASK in order to support ppoll() and pselect() system calls. This patch originally came from Andi, and was based heavily on David Howells' implementation of same on i386. I fixed a typo which was causing do_signal() to use the wrong signal mask. Signed-off-by: David Woodhouse Signed-off-by: Andi Kleen --- include/asm-x86_64/signal.h | 4 ---- include/asm-x86_64/thread_info.h | 2 ++ include/asm-x86_64/unistd.h | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/signal.h b/include/asm-x86_64/signal.h index 3ede2a61973a..4581f978b299 100644 --- a/include/asm-x86_64/signal.h +++ b/include/asm-x86_64/signal.h @@ -24,10 +24,6 @@ typedef struct { } sigset_t; -struct pt_regs; -asmlinkage int do_signal(struct pt_regs *regs, sigset_t *oldset); - - #else /* Here we must cater to libcs that poke about in kernel headers. */ diff --git a/include/asm-x86_64/thread_info.h b/include/asm-x86_64/thread_info.h index 2029b00351f3..790c512a4369 100644 --- a/include/asm-x86_64/thread_info.h +++ b/include/asm-x86_64/thread_info.h @@ -114,6 +114,7 @@ static inline struct thread_info *stack_thread_info(void) #define TIF_IRET 5 /* force IRET */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SECCOMP 8 /* secure computing */ +#define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal */ /* 16 free */ #define TIF_IA32 17 /* 32bit process */ #define TIF_FORK 18 /* ret_from_fork */ @@ -128,6 +129,7 @@ static inline struct thread_info *stack_thread_info(void) #define _TIF_IRET (1< Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] Add ppoll/pselect syscalls Needed TIF_RESTORE_SIGMASK first Signed-off-by: Andi Kleen --- include/asm-x86_64/unistd.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/unistd.h b/include/asm-x86_64/unistd.h index f266de294003..eeb98c168e98 100644 --- a/include/asm-x86_64/unistd.h +++ b/include/asm-x86_64/unistd.h @@ -600,9 +600,9 @@ __SYSCALL(__NR_fchmodat, sys_fchmodat) #define __NR_faccessat 269 __SYSCALL(__NR_faccessat, sys_faccessat) #define __NR_pselect6 270 -__SYSCALL(__NR_pselect6, sys_ni_syscall) /* for now */ +__SYSCALL(__NR_pselect6, sys_pselect6) #define __NR_ppoll 271 -__SYSCALL(__NR_ppoll, sys_ni_syscall) /* for now */ +__SYSCALL(__NR_ppoll, sys_ppoll) #define __NR_unshare 272 __SYSCALL(__NR_unshare, sys_unshare) #define __NR_set_robust_list 273 -- cgit v1.2.2 From 2fbe7b25c8edaf2d10e6c1a4cc9f8afe714c4764 Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] i386/x86-64: Remove un/set_nmi_callback and reserve/release_lapic_nmi functions Removes the un/set_nmi_callback and reserve/release_lapic_nmi functions as they are no longer needed. The various subsystems are modified to register with the die_notifier instead. Also includes compile fixes by Andrew Morton. Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/asm-i386/nmi.h | 21 ++++----------------- include/asm-x86_64/nmi.h | 21 --------------------- 2 files changed, 4 insertions(+), 38 deletions(-) (limited to 'include') diff --git a/include/asm-i386/nmi.h b/include/asm-i386/nmi.h index da0e0b4e9139..34d6bf063b6e 100644 --- a/include/asm-i386/nmi.h +++ b/include/asm-i386/nmi.h @@ -6,24 +6,13 @@ #include -struct pt_regs; - -typedef int (*nmi_callback_t)(struct pt_regs * regs, int cpu); - -/** - * set_nmi_callback - * - * Set a handler for an NMI. Only one handler may be - * set. Return 1 if the NMI was handled. - */ -void set_nmi_callback(nmi_callback_t callback); - /** - * unset_nmi_callback + * do_nmi_callback * - * Remove the handler previously set. + * Check to see if a callback exists and execute it. Return 1 + * if the handler exists and was handled successfully. */ -void unset_nmi_callback(void); +int do_nmi_callback(struct pt_regs *regs, int cpu); extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); extern int avail_to_resrv_perfctr_nmi(unsigned int); @@ -33,8 +22,6 @@ extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); extern void setup_apic_nmi_watchdog (void *); -extern int reserve_lapic_nmi(void); -extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index 8f02a2a416e6..8818c39d34e0 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -7,25 +7,6 @@ #include #include -struct pt_regs; - -typedef int (*nmi_callback_t)(struct pt_regs * regs, int cpu); - -/** - * set_nmi_callback - * - * Set a handler for an NMI. Only one handler may be - * set. Return 1 if the NMI was handled. - */ -void set_nmi_callback(nmi_callback_t callback); - -/** - * unset_nmi_callback - * - * Remove the handler previously set. - */ -void unset_nmi_callback(void); - /** * do_nmi_callback * @@ -72,8 +53,6 @@ extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); extern void setup_apic_nmi_watchdog (void *); -extern int reserve_lapic_nmi(void); -extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); -- cgit v1.2.2 From 407984f1af259b31957c7c05075a454a751bb801 Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] x86: Add abilty to enable/disable nmi watchdog with sysctl Adds a new /proc/sys/kernel/nmi call that will enable/disable the nmi watchdog. Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/asm-i386/nmi.h | 1 + include/asm-x86_64/nmi.h | 1 + include/linux/sysctl.h | 1 + 3 files changed, 3 insertions(+) (limited to 'include') diff --git a/include/asm-i386/nmi.h b/include/asm-i386/nmi.h index 34d6bf063b6e..13b5d8311bf7 100644 --- a/include/asm-i386/nmi.h +++ b/include/asm-i386/nmi.h @@ -14,6 +14,7 @@ */ int do_nmi_callback(struct pt_regs *regs, int cpu); +extern int nmi_watchdog_enabled; extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); extern int avail_to_resrv_perfctr_nmi(unsigned int); extern int reserve_perfctr_nmi(unsigned int); diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index 8818c39d34e0..2c23b0df87d2 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -43,6 +43,7 @@ extern void die_nmi(char *str, struct pt_regs *regs); extern int panic_on_timeout; extern int unknown_nmi_panic; +extern int nmi_watchdog_enabled; extern int check_nmi_watchdog(void); extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 736ed917a4f8..ecb79ba52ae1 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -150,6 +150,7 @@ enum KERN_IA64_UNALIGNED=72, /* int: ia64 unaligned userland trap enable */ KERN_COMPAT_LOG=73, /* int: print compat layer messages */ KERN_MAX_LOCK_DEPTH=74, + KERN_NMI_WATCHDOG=75, /* int: enable/disable nmi watchdog */ }; -- cgit v1.2.2 From 8da5adda91df3d2fcc5300e68da491694c9af019 Mon Sep 17 00:00:00 2001 From: Don Zickus Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] x86: Allow users to force a panic on NMI To quote Alan Cox: The default Linux behaviour on an NMI of either memory or unknown is to continue operation. For many environments such as scientific computing it is preferable that the box is taken out and the error dealt with than an uncorrected parity/ECC error get propogated. A small number of systems do generate NMI's for bizarre random reasons such as power management so the default is unchanged. In other respects the new proc/sys entry works like the existing panic controls already in that directory. This is separate to the edac support - EDAC allows supported chipsets to handle ECC errors well, this change allows unsupported cases to at least panic rather than cause problems further down the line. Signed-off-by: Don Zickus Signed-off-by: Andi Kleen --- include/linux/kernel.h | 1 + include/linux/sysctl.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 2b2ae4fdce8b..1ff9609300b4 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -186,6 +186,7 @@ extern void bust_spinlocks(int yes); extern int oops_in_progress; /* If set, an oops, panic(), BUG() or die() is in progress */ extern int panic_timeout; extern int panic_on_oops; +extern int panic_on_unrecovered_nmi; extern int tainted; extern const char *print_tainted(void); extern void add_taint(unsigned); diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index ecb79ba52ae1..432778446ad2 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -151,6 +151,7 @@ enum KERN_COMPAT_LOG=73, /* int: print compat layer messages */ KERN_MAX_LOCK_DEPTH=74, KERN_NMI_WATCHDOG=75, /* int: enable/disable nmi watchdog */ + KERN_PANIC_ON_NMI=76, /* int: whether we will panic on an unrecovered */ }; -- cgit v1.2.2 From 4038f901cf102a40715b900984ed7540a9fa637f Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] i386/x86-64: Fix NMI watchdog suspend/resume Making NMI suspend/resume work with SMP. We use CPU hotplug to offline APs in SMP suspend/resume. Only BSP executes sysdev's .suspend/.resume method. APs should follow CPU hotplug code path. And: +From: Don Zickus Makes the start/stop paths of nmi watchdog more robust to handle the suspend/resume cases more gracefully. AK: I merged the two patches together Signed-off-by: Shaohua Li Signed-off-by: Andi Kleen Cc: Don Zickus Cc: Andi Kleen Signed-off-by: Andrew Morton --- include/asm-i386/nmi.h | 1 + include/asm-x86_64/nmi.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include') diff --git a/include/asm-i386/nmi.h b/include/asm-i386/nmi.h index 13b5d8311bf7..303bcd4592bb 100644 --- a/include/asm-i386/nmi.h +++ b/include/asm-i386/nmi.h @@ -23,6 +23,7 @@ extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); extern void setup_apic_nmi_watchdog (void *); +extern void stop_apic_nmi_watchdog (void *); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index 2c23b0df87d2..578596494275 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -54,6 +54,7 @@ extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); extern void setup_apic_nmi_watchdog (void *); +extern void stop_apic_nmi_watchdog (void *); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); -- cgit v1.2.2 From fac58550e80c307bf17cfa0dd544fca4eff120a5 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] Fix up panic messages for different NMI panics When a unknown NMI happened the panic would claim a NMI watchdog timeout. Also it would check the variable set by nmi_watchdog=panic and panic then. Fix up the panic message to be generic Unconditionally panic on unknown NMI when panic on unknown nmi is enabled. Noticed by Jan Beulich Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-x86_64/nmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index 578596494275..cbf2669bca71 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -37,7 +37,7 @@ static inline void unset_nmi_pm_callback(struct pm_dev * dev) #endif /* CONFIG_PM */ extern void default_do_nmi(struct pt_regs *); -extern void die_nmi(char *str, struct pt_regs *regs); +extern void die_nmi(char *str, struct pt_regs *regs, int do_panic); #define get_nmi_reason() inb(0x61) -- cgit v1.2.2 From 248dcb2ffffe8f3e4a369556a68988788c208111 Mon Sep 17 00:00:00 2001 From: Venkatesh Pallipadi Date: Tue, 26 Sep 2006 10:52:27 +0200 Subject: [PATCH] x86: i386/x86-64 Add nmi watchdog support for new Intel CPUs AK: This redoes the changes I temporarily reverted. Intel now has support for Architectural Performance Monitoring Counters ( Refer to IA-32 Intel Architecture Software Developer's Manual http://www.intel.com/design/pentium4/manuals/253669.htm ). This feature is present starting from Intel Core Duo and Intel Core Solo processors. What this means is, the performance monitoring counters and some performance monitoring events are now defined in an architectural way (using cpuid). And there will be no need to check for family/model etc for these architectural events. Below is the patch to use this performance counters in nmi watchdog driver. Patch handles both i386 and x86-64 kernels. Signed-off-by: Venkatesh Pallipadi Signed-off-by: Andi Kleen --- include/asm-i386/intel_arch_perfmon.h | 31 +++++++++++++++++++++++++++++++ include/asm-x86_64/intel_arch_perfmon.h | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 include/asm-i386/intel_arch_perfmon.h create mode 100644 include/asm-x86_64/intel_arch_perfmon.h (limited to 'include') diff --git a/include/asm-i386/intel_arch_perfmon.h b/include/asm-i386/intel_arch_perfmon.h new file mode 100644 index 000000000000..b52cd60a075b --- /dev/null +++ b/include/asm-i386/intel_arch_perfmon.h @@ -0,0 +1,31 @@ +#ifndef X86_INTEL_ARCH_PERFMON_H +#define X86_INTEL_ARCH_PERFMON_H 1 + +#define MSR_ARCH_PERFMON_PERFCTR0 0xc1 +#define MSR_ARCH_PERFMON_PERFCTR1 0xc2 + +#define MSR_ARCH_PERFMON_EVENTSEL0 0x186 +#define MSR_ARCH_PERFMON_EVENTSEL1 0x187 + +#define ARCH_PERFMON_EVENTSEL0_ENABLE (1 << 22) +#define ARCH_PERFMON_EVENTSEL_INT (1 << 20) +#define ARCH_PERFMON_EVENTSEL_OS (1 << 17) +#define ARCH_PERFMON_EVENTSEL_USR (1 << 16) + +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL (0x3c) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX (0) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \ + (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX)) + +union cpuid10_eax { + struct { + unsigned int version_id:8; + unsigned int num_counters:8; + unsigned int bit_width:8; + unsigned int mask_length:8; + } split; + unsigned int full; +}; + +#endif /* X86_INTEL_ARCH_PERFMON_H */ diff --git a/include/asm-x86_64/intel_arch_perfmon.h b/include/asm-x86_64/intel_arch_perfmon.h new file mode 100644 index 000000000000..8633331420ec --- /dev/null +++ b/include/asm-x86_64/intel_arch_perfmon.h @@ -0,0 +1,31 @@ +#ifndef X86_64_INTEL_ARCH_PERFMON_H +#define X86_64_INTEL_ARCH_PERFMON_H 1 + +#define MSR_ARCH_PERFMON_PERFCTR0 0xc1 +#define MSR_ARCH_PERFMON_PERFCTR1 0xc2 + +#define MSR_ARCH_PERFMON_EVENTSEL0 0x186 +#define MSR_ARCH_PERFMON_EVENTSEL1 0x187 + +#define ARCH_PERFMON_EVENTSEL0_ENABLE (1 << 22) +#define ARCH_PERFMON_EVENTSEL_INT (1 << 20) +#define ARCH_PERFMON_EVENTSEL_OS (1 << 17) +#define ARCH_PERFMON_EVENTSEL_USR (1 << 16) + +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL (0x3c) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX (0) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \ + (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX)) + +union cpuid10_eax { + struct { + unsigned int version_id:8; + unsigned int num_counters:8; + unsigned int bit_width:8; + unsigned int mask_length:8; + } split; + unsigned int full; +}; + +#endif /* X86_64_INTEL_ARCH_PERFMON_H */ -- cgit v1.2.2 From 81af4449af9c9b686a4eeeb00112614621655704 Mon Sep 17 00:00:00 2001 From: Vojtech Pavlik Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] Add macros for rdtscp This patch adds macros for reading tsc via the RDTSCP instruction, as well as writing the auxilliary MSR read by RDTSCP to msr.h [AK: changed rdtscp definition for old binutils] Signed-off-by: Vojtech Pavlik Signed-off-by: Andi Kleen --- include/asm-x86_64/msr.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/msr.h b/include/asm-x86_64/msr.h index 10f8b51cec8b..37e194169fac 100644 --- a/include/asm-x86_64/msr.h +++ b/include/asm-x86_64/msr.h @@ -66,14 +66,25 @@ #define rdtscl(low) \ __asm__ __volatile__ ("rdtsc" : "=a" (low) : : "edx") +#define rdtscp(low,high,aux) \ + asm volatile (".byte 0x0f,0x01,0xf9" : "=a" (low), "=d" (high), "=c" (aux)) + #define rdtscll(val) do { \ unsigned int __a,__d; \ asm volatile("rdtsc" : "=a" (__a), "=d" (__d)); \ (val) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \ } while(0) +#define rdtscpll(val, aux) do { \ + unsigned long __a, __d; \ + asm volatile (".byte 0x0f,0x01,0xf9" : "=a" (__a), "=d" (__d), "=c" (aux)); \ + (val) = (__d << 32) | __a; \ +} while (0) + #define write_tsc(val1,val2) wrmsr(0x10, val1, val2) +#define write_rdtscp_aux(val) wrmsr(0xc0000103, val, 0) + #define rdpmc(counter,low,high) \ __asm__ __volatile__("rdpmc" \ : "=a" (low), "=d" (high) \ -- cgit v1.2.2 From a670fad0adb1cc6202a607d250f10bd380593905 Mon Sep 17 00:00:00 2001 From: Vojtech Pavlik Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] Add initalization of the RDTSCP auxilliary values This patch adds initalization of the RDTSCP auxilliary values to CPU numbers to time.c. If RDTSCP is available, the MSRs are written with the respective values. It can be later used to initalize per-cpu timekeeping variables. AK: Some cleanups. Move externs into headers and fix CPU hotplug. Signed-off-by: Vojtech Pavlik Signed-off-by: Andi Kleen --- include/asm-x86_64/proto.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/asm-x86_64/proto.h b/include/asm-x86_64/proto.h index 038fe1f47e6f..3b1c60247902 100644 --- a/include/asm-x86_64/proto.h +++ b/include/asm-x86_64/proto.h @@ -51,6 +51,7 @@ extern unsigned long long monotonic_base; extern int sysctl_vsyscall; extern int nohpet; extern unsigned long vxtime_hz; +extern void time_init_gtod(void); extern int numa_setup(char *opt); -- cgit v1.2.2 From c08c820508233b424deab3302bc404bbecc6493a Mon Sep 17 00:00:00 2001 From: Vojtech Pavlik Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] Add the vgetcpu vsyscall This patch adds a vgetcpu vsyscall, which depending on the CPU RDTSCP capability uses either the RDTSCP or CPUID to obtain a CPU and node numbers and pass them to the program. AK: Lots of changes over Vojtech's original code: Better prototype for vgetcpu() It's better to pass the cpu / node numbers as separate arguments to avoid mistakes when going from SMP to NUMA. Also add a fast time stamp based cache using a user supplied argument to speed things more up. Use fast method from Chuck Ebbert to retrieve node/cpu from GDT limit instead of CPUID Made sure RDTSCP init is always executed after node is known. Drop printk Signed-off-by: Vojtech Pavlik Signed-off-by: Andi Kleen --- include/asm-x86_64/segment.h | 5 +++-- include/asm-x86_64/smp.h | 12 +++++++++--- include/asm-x86_64/vsyscall.h | 9 +++++++++ include/linux/getcpu.h | 16 ++++++++++++++++ 4 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 include/linux/getcpu.h (limited to 'include') diff --git a/include/asm-x86_64/segment.h b/include/asm-x86_64/segment.h index d4bed33fb32c..334ddcdd8f92 100644 --- a/include/asm-x86_64/segment.h +++ b/include/asm-x86_64/segment.h @@ -20,15 +20,16 @@ #define __USER_CS 0x33 /* 6*8+3 */ #define __USER32_DS __USER_DS -#define GDT_ENTRY_TLS 1 #define GDT_ENTRY_TSS 8 /* needs two entries */ #define GDT_ENTRY_LDT 10 /* needs two entries */ #define GDT_ENTRY_TLS_MIN 12 #define GDT_ENTRY_TLS_MAX 14 -/* 15 free */ #define GDT_ENTRY_TLS_ENTRIES 3 +#define GDT_ENTRY_PER_CPU 15 /* Abused to load per CPU data from limit */ +#define __PER_CPU_SEG (GDT_ENTRY_PER_CPU * 8 + 3) + /* TLS indexes for 64bit - hardcoded in arch_prctl */ #define FS_TLS 0 #define GS_TLS 1 diff --git a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h index 6805e1feb300..d61547fd833b 100644 --- a/include/asm-x86_64/smp.h +++ b/include/asm-x86_64/smp.h @@ -133,13 +133,19 @@ static __inline int logical_smp_processor_id(void) /* we don't want to mark this access volatile - bad code generation */ return GET_APIC_LOGICAL_ID(*(unsigned long *)(APIC_BASE+APIC_LDR)); } -#endif #ifdef CONFIG_SMP #define cpu_physical_id(cpu) x86_cpu_to_apicid[cpu] #else #define cpu_physical_id(cpu) boot_cpu_id -#endif - +static inline int smp_call_function_single(int cpuid, void (*func) (void *info), + void *info, int retry, int wait) +{ + /* Disable interrupts here? */ + func(info); + return 0; +} +#endif /* !CONFIG_SMP */ +#endif /* !__ASSEMBLY */ #endif diff --git a/include/asm-x86_64/vsyscall.h b/include/asm-x86_64/vsyscall.h index 146b24402a5f..2281e9399b96 100644 --- a/include/asm-x86_64/vsyscall.h +++ b/include/asm-x86_64/vsyscall.h @@ -4,6 +4,7 @@ enum vsyscall_num { __NR_vgettimeofday, __NR_vtime, + __NR_vgetcpu, }; #define VSYSCALL_START (-10UL << 20) @@ -15,6 +16,7 @@ enum vsyscall_num { #include #define __section_vxtime __attribute__ ((unused, __section__ (".vxtime"), aligned(16))) +#define __section_vgetcpu_mode __attribute__ ((unused, __section__ (".vgetcpu_mode"), aligned(16))) #define __section_wall_jiffies __attribute__ ((unused, __section__ (".wall_jiffies"), aligned(16))) #define __section_jiffies __attribute__ ((unused, __section__ (".jiffies"), aligned(16))) #define __section_sys_tz __attribute__ ((unused, __section__ (".sys_tz"), aligned(16))) @@ -26,6 +28,9 @@ enum vsyscall_num { #define VXTIME_HPET 2 #define VXTIME_PMTMR 3 +#define VGETCPU_RDTSCP 1 +#define VGETCPU_LSL 2 + struct vxtime_data { long hpet_address; /* HPET base address */ int last; @@ -40,6 +45,7 @@ struct vxtime_data { /* vsyscall space (readonly) */ extern struct vxtime_data __vxtime; +extern int __vgetcpu_mode; extern struct timespec __xtime; extern volatile unsigned long __jiffies; extern unsigned long __wall_jiffies; @@ -48,6 +54,7 @@ extern seqlock_t __xtime_lock; /* kernel space (writeable) */ extern struct vxtime_data vxtime; +extern int vgetcpu_mode; extern unsigned long wall_jiffies; extern struct timezone sys_tz; extern int sysctl_vsyscall; @@ -55,6 +62,8 @@ extern seqlock_t xtime_lock; extern int sysctl_vsyscall; +extern void vsyscall_set_cpu(int cpu); + #define ARCH_HAVE_XTIME_LOCK 1 #endif /* __KERNEL__ */ diff --git a/include/linux/getcpu.h b/include/linux/getcpu.h new file mode 100644 index 000000000000..031ed3780e45 --- /dev/null +++ b/include/linux/getcpu.h @@ -0,0 +1,16 @@ +#ifndef _LINUX_GETCPU_H +#define _LINUX_GETCPU_H 1 + +/* Cache for getcpu() to speed it up. Results might be upto a jiffie + out of date, but will be faster. + User programs should not refer to the contents of this structure. + It is only a cache for vgetcpu(). It might change in future kernels. + The user program must store this information per thread (__thread) + If you want 100% accurate information pass NULL instead. */ +struct getcpu_cache { + unsigned long t0; + unsigned long t1; + unsigned long res[4]; +}; + +#endif -- cgit v1.2.2 From 3cfc348bf90ffaa777c188652aa297f04eb94de8 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] x86: Add portable getcpu call For NUMA optimization and some other algorithms it is useful to have a fast to get the current CPU and node numbers in user space. x86-64 added a fast way to do this in a vsyscall. This adds a generic syscall for other architectures to make it a generic portable facility. I expect some of them will also implement it as a faster vsyscall. The cache is an optimization for the x86-64 vsyscall optimization. Since what the syscall returns is an approximation anyways and user space often wants very fast results it can be cached for some time. The norma methods to get this information in user space are relatively slow The vsyscall is in a better position to manage the cache because it has direct access to a fast time stamp (jiffies). For the generic syscall optimization it doesn't help much, but enforce a valid argument to keep programs portable I only added an i386 syscall entry for now. Other architectures can follow as needed. AK: Also added some cleanups from Andrew Morton Signed-off-by: Andi Kleen --- include/asm-i386/unistd.h | 3 ++- include/linux/syscalls.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-i386/unistd.h b/include/asm-i386/unistd.h index fc1c8ddae149..565d0897b205 100644 --- a/include/asm-i386/unistd.h +++ b/include/asm-i386/unistd.h @@ -323,10 +323,11 @@ #define __NR_tee 315 #define __NR_vmsplice 316 #define __NR_move_pages 317 +#define __NR_getcpu 318 #ifdef __KERNEL__ -#define NR_syscalls 318 +#define NR_syscalls 319 /* * user-visible error numbers are in the range -1 - -128: see diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 008f04c56737..3f0f716225ec 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -53,6 +53,7 @@ struct mq_attr; struct compat_stat; struct compat_timeval; struct robust_list_head; +struct getcpu_cache; #include #include @@ -596,5 +597,6 @@ asmlinkage long sys_get_robust_list(int pid, size_t __user *len_ptr); asmlinkage long sys_set_robust_list(struct robust_list_head __user *head, size_t len); +asmlinkage long sys_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *cache); #endif -- cgit v1.2.2 From 2f766d16062d0147edff91be15de4a950667ca42 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] Clean up asm/smp.h includes No need to include it from entry.S Drop all the #ifdef __ASSEMBLY__ Signed-off-by: Andi Kleen --- include/asm-x86_64/smp.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h index d61547fd833b..612d208961a2 100644 --- a/include/asm-x86_64/smp.h +++ b/include/asm-x86_64/smp.h @@ -4,15 +4,12 @@ /* * We need the APIC definitions automatically as part of 'smp.h' */ -#ifndef __ASSEMBLY__ #include #include #include extern int disable_apic; -#endif #ifdef CONFIG_X86_LOCAL_APIC -#ifndef __ASSEMBLY__ #include #include #ifdef CONFIG_X86_IO_APIC @@ -21,10 +18,8 @@ extern int disable_apic; #include #include #endif -#endif #ifdef CONFIG_SMP -#ifndef ASSEMBLY #include @@ -83,13 +78,10 @@ extern void prefill_possible_map(void); extern unsigned num_processors; extern unsigned disabled_cpus; -#endif /* !ASSEMBLY */ - #define NO_PROC_ID 0xFF /* No processor magic marker */ #endif -#ifndef ASSEMBLY /* * Some lowlevel functions might want to know about * the real APIC ID <-> CPU # mapping. @@ -111,8 +103,6 @@ static inline int cpu_present_to_apicid(int mps_cpu) return BAD_APICID; } -#endif /* !ASSEMBLY */ - #ifndef CONFIG_SMP #define stack_smp_processor_id() 0 #define safe_smp_processor_id() 0 @@ -127,7 +117,6 @@ static inline int cpu_present_to_apicid(int mps_cpu) }) #endif -#ifndef __ASSEMBLY__ static __inline int logical_smp_processor_id(void) { /* we don't want to mark this access volatile - bad code generation */ @@ -146,6 +135,5 @@ static inline int smp_call_function_single(int cpuid, void (*func) (void *info), return 0; } #endif /* !CONFIG_SMP */ -#endif /* !__ASSEMBLY */ #endif -- cgit v1.2.2 From d3a4f48d4866b8623ca9adde8ce4e5fde979c132 Mon Sep 17 00:00:00 2001 From: Stephane Eranian Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] x86-64 TIF flags for debug regs and io bitmap in ctxsw Hello, Following my discussion with Andi. Here is a patch that introduces two new TIF flags to simplify the context switch code in __switch_to(). The idea is to minimize the number of cache lines accessed in the common case, i.e., when neither the debug registers nor the I/O bitmap are used. This patch covers the x86-64 modifications. A patch for i386 follows. Changelog: - add TIF_DEBUG to track when debug registers are active - add TIF_IO_BITMAP to track when I/O bitmap is used - modify __switch_to() to use the new TIF flags : eranian@hpl.hp.com Signed-off-by: Andi Kleen --- include/asm-x86_64/thread_info.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/thread_info.h b/include/asm-x86_64/thread_info.h index 790c512a4369..787a08114b48 100644 --- a/include/asm-x86_64/thread_info.h +++ b/include/asm-x86_64/thread_info.h @@ -120,6 +120,8 @@ static inline struct thread_info *stack_thread_info(void) #define TIF_FORK 18 /* ret_from_fork */ #define TIF_ABI_PENDING 19 #define TIF_MEMDIE 20 +#define TIF_DEBUG 21 /* uses debug registers */ +#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ #define _TIF_SYSCALL_TRACE (1< Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] i386: Account spinlocks to the caller during profiling for !FP kernels This ports the algorithm from x86-64 (with improvements) to i386. Previously this only worked for frame pointer enabled kernels. But spinlocks have a very simple stack frame that can be manually analyzed. Do this. Signed-off-by: Andi Kleen --- include/asm-i386/ptrace.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include') diff --git a/include/asm-i386/ptrace.h b/include/asm-i386/ptrace.h index f324c53b6f9a..30a442ec2059 100644 --- a/include/asm-i386/ptrace.h +++ b/include/asm-i386/ptrace.h @@ -80,11 +80,7 @@ static inline int user_mode_vm(struct pt_regs *regs) return ((regs->xcs & 3) | (regs->eflags & VM_MASK)) != 0; } #define instruction_pointer(regs) ((regs)->eip) -#if defined(CONFIG_SMP) && defined(CONFIG_FRAME_POINTER) extern unsigned long profile_pc(struct pt_regs *regs); -#else -#define profile_pc(regs) instruction_pointer(regs) -#endif #endif /* __KERNEL__ */ #endif -- cgit v1.2.2 From b4062b16094038334d9bbadac0397a3fc9e981b0 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] Support patchable lock prefix for pure assembly files Signed-off-by: Andi Kleen --- include/asm-x86_64/alternative-asm.i | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 include/asm-x86_64/alternative-asm.i (limited to 'include') diff --git a/include/asm-x86_64/alternative-asm.i b/include/asm-x86_64/alternative-asm.i new file mode 100644 index 000000000000..e4041f4fa4dc --- /dev/null +++ b/include/asm-x86_64/alternative-asm.i @@ -0,0 +1,14 @@ +#include + +#ifdef CONFIG_SMP + .macro LOCK_PREFIX +1: lock + .section .smp_locks,"a" + .align 8 + .quad 1b + .previous + .endm +#else + .macro LOCK_PREFIX + .endm +#endif -- cgit v1.2.2 From 9a0b26e6bc4ae1979d9bcc6194e57a71b2b5cac6 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:28 +0200 Subject: [PATCH] Clean up read write lock assembly - Move the slow path fallbacks to their own assembly files This makes them much easier to read and is needed for the next change. - Add CFI annotations for unwinding (XXX need review) - Remove constant case which can never happen with out of line spinlocks - Use patchable LOCK prefixes - Don't use lock sections anymore for inline code because they can't be expressed by the unwinder (this adds one taken jump to the lock fast path) Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-x86_64/rwlock.h | 68 +++++++------------------------------------ include/asm-x86_64/spinlock.h | 11 ++----- 2 files changed, 12 insertions(+), 67 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/rwlock.h b/include/asm-x86_64/rwlock.h index dea0e9459264..28a080d23119 100644 --- a/include/asm-x86_64/rwlock.h +++ b/include/asm-x86_64/rwlock.h @@ -18,69 +18,21 @@ #ifndef _ASM_X86_64_RWLOCK_H #define _ASM_X86_64_RWLOCK_H -#include - #define RW_LOCK_BIAS 0x01000000 -#define RW_LOCK_BIAS_STR "0x01000000" +#define RW_LOCK_BIAS_STR "0x01000000" -#define __build_read_lock_ptr(rw, helper) \ +#define __build_read_lock(rw) \ asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \ - "js 2f\n" \ - "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tcall " helper "\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - ::"a" (rw) : "memory") - -#define __build_read_lock_const(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $1,%0\n\t" \ - "js 2f\n" \ + "jns 1f\n" \ + "call __read_lock_failed\n" \ "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tpushq %%rax\n\t" \ - "leaq %0,%%rax\n\t" \ - "call " helper "\n\t" \ - "popq %%rax\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - :"=m" (*((volatile int *)rw))::"memory") - -#define __build_read_lock(rw, helper) do { \ - if (__builtin_constant_p(rw)) \ - __build_read_lock_const(rw, helper); \ - else \ - __build_read_lock_ptr(rw, helper); \ - } while (0) + ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory") -#define __build_write_lock_ptr(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ - "jnz 2f\n" \ +#define __build_write_lock(rw) \ + asm volatile(LOCK_PREFIX "subl %1,(%0)\n\t" \ + "jz 1f\n" \ + "\tcall __write_lock_failed\n\t" \ "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tcall " helper "\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - ::"a" (rw) : "memory") - -#define __build_write_lock_const(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ - "jnz 2f\n" \ - "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tpushq %%rax\n\t" \ - "leaq %0,%%rax\n\t" \ - "call " helper "\n\t" \ - "popq %%rax\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - :"=m" (*((volatile long *)rw))::"memory") - -#define __build_write_lock(rw, helper) do { \ - if (__builtin_constant_p(rw)) \ - __build_write_lock_const(rw, helper); \ - else \ - __build_write_lock_ptr(rw, helper); \ - } while (0) + ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory") #endif diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h index 248a79f0eaff..a8e3d89f591f 100644 --- a/include/asm-x86_64/spinlock.h +++ b/include/asm-x86_64/spinlock.h @@ -79,13 +79,6 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) * * On x86, we implement read-write locks as a 32-bit counter * with the high bit (sign) being the "contended" bit. - * - * The inline assembly is non-obvious. Think about it. - * - * Changed to use the same technique as rw semaphores. See - * semaphore.h for details. -ben - * - * the helpers are in arch/i386/kernel/semaphore.c */ #define __raw_read_can_lock(x) ((int)(x)->lock > 0) @@ -93,12 +86,12 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) static inline void __raw_read_lock(raw_rwlock_t *rw) { - __build_read_lock(rw, "__read_lock_failed"); + __build_read_lock(rw); } static inline void __raw_write_lock(raw_rwlock_t *rw) { - __build_write_lock(rw, "__write_lock_failed"); + __build_write_lock(rw); } static inline int __raw_read_trylock(raw_rwlock_t *lock) -- cgit v1.2.2 From 1a015b5644ec6df0a2c4cbeff1f8a3d24ba0478e Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] i386: Remove const case for rwlocks rwlocks are now out of line, so it near never triggers. Also it was incompatible with the new dwarf2 unwinder because it had unannotiatable push/pops. Signed-off-by: Andi Kleen --- include/asm-i386/rwlock.h | 38 ++------------------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) (limited to 'include') diff --git a/include/asm-i386/rwlock.h b/include/asm-i386/rwlock.h index 87c069ccba08..f40ccbd8cb7f 100644 --- a/include/asm-i386/rwlock.h +++ b/include/asm-i386/rwlock.h @@ -20,52 +20,18 @@ #define RW_LOCK_BIAS 0x01000000 #define RW_LOCK_BIAS_STR "0x01000000" -#define __build_read_lock_ptr(rw, helper) \ +#define __build_read_lock(rw, helper) \ asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" \ "jns 1f\n" \ "call " helper "\n\t" \ "1:\n" \ ::"a" (rw) : "memory") -#define __build_read_lock_const(rw, helper) \ - asm volatile(LOCK_PREFIX " subl $1,%0\n\t" \ - "jns 1f\n" \ - "pushl %%eax\n\t" \ - "leal %0,%%eax\n\t" \ - "call " helper "\n\t" \ - "popl %%eax\n\t" \ - "1:\n" \ - :"+m" (*(volatile int *)rw) : : "memory") - -#define __build_read_lock(rw, helper) do { \ - if (__builtin_constant_p(rw)) \ - __build_read_lock_const(rw, helper); \ - else \ - __build_read_lock_ptr(rw, helper); \ - } while (0) - -#define __build_write_lock_ptr(rw, helper) \ +#define __build_write_lock(rw, helper) \ asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ "jz 1f\n" \ "call " helper "\n\t" \ "1:\n" \ ::"a" (rw) : "memory") -#define __build_write_lock_const(rw, helper) \ - asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ - "jz 1f\n" \ - "pushl %%eax\n\t" \ - "leal %0,%%eax\n\t" \ - "call " helper "\n\t" \ - "popl %%eax\n\t" \ - "1:\n" \ - :"+m" (*(volatile int *)rw) : : "memory") - -#define __build_write_lock(rw, helper) do { \ - if (__builtin_constant_p(rw)) \ - __build_write_lock_const(rw, helper); \ - else \ - __build_write_lock_ptr(rw, helper); \ - } while (0) - #endif -- cgit v1.2.2 From b06babac45e1546dfb504f1f25eb0495632bfc41 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] Add proper alignment to ENTRY Previously it didn't align. Use the same one as the C compiler in blended mode, which is good for K8 and Core2 and doesn't hurt on P4. Signed-off-by: Andi Kleen --- include/asm-x86_64/linkage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/linkage.h b/include/asm-x86_64/linkage.h index 291c2d01c44f..b5f39d0189ce 100644 --- a/include/asm-x86_64/linkage.h +++ b/include/asm-x86_64/linkage.h @@ -1,6 +1,6 @@ #ifndef __ASM_LINKAGE_H #define __ASM_LINKAGE_H -/* Nothing to see here... */ +#define __ALIGN .p2align 4,,15 #endif -- cgit v1.2.2 From 07c9819b31eda7954feddc83f2fae035f31c11e1 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] i386: add alternative-asm.h to allow LOCK_PREFIX replacement in .S files LOCK_PREFIX is replaced by nops on UP systems, so it has to be a special macro. Previously this was only possible from C. Allow it for pure assembly files too. Similar to earlier x86-64 patch. Signed-off-by: Andi Kleen --- include/asm-i386/alternative-asm.i | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 include/asm-i386/alternative-asm.i (limited to 'include') diff --git a/include/asm-i386/alternative-asm.i b/include/asm-i386/alternative-asm.i new file mode 100644 index 000000000000..6c47e3b9484b --- /dev/null +++ b/include/asm-i386/alternative-asm.i @@ -0,0 +1,14 @@ +#include + +#ifdef CONFIG_SMP + .macro LOCK_PREFIX +1: lock + .section .smp_locks,"a" + .align 4 + .long 1b + .previous + .endm +#else + .macro LOCK_PREFIX + .endm +#endif -- cgit v1.2.2 From ecaf45ee5ce60afe7cc46e91d82c1b0cbda09387 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] i386: Redo semaphore and rwlock assembly helpers - Move them to a pure assembly file. Previously they were in a C file that only consisted of inline assembly. Doing it in pure assembler is much nicer. - Add a frame.i include with FRAME/ENDFRAME macros to easily add frame pointers to assembly functions - Add dwarf2 annotation to them so that the new dwarf2 unwinder doesn't get stuck on them - Random cleanups Includes feedback from Jan Beulich and a UML build fix from Andrew Morton. Cc: jbeulich@novell.com Cc: jdike@addtoit.com Signed-off-by: Andi Kleen --- include/asm-i386/frame.i | 24 ++++++++++++++++++++++++ include/asm-um/alternative-asm.i | 6 ++++++ include/asm-um/frame.i | 6 ++++++ 3 files changed, 36 insertions(+) create mode 100644 include/asm-i386/frame.i create mode 100644 include/asm-um/alternative-asm.i create mode 100644 include/asm-um/frame.i (limited to 'include') diff --git a/include/asm-i386/frame.i b/include/asm-i386/frame.i new file mode 100644 index 000000000000..4d68ddce18b6 --- /dev/null +++ b/include/asm-i386/frame.i @@ -0,0 +1,24 @@ +#include +#include + +/* The annotation hides the frame from the unwinder and makes it look + like a ordinary ebp save/restore. This avoids some special cases for + frame pointer later */ +#ifdef CONFIG_FRAME_POINTER + .macro FRAME + pushl %ebp + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ebp,0 + movl %esp,%ebp + .endm + .macro ENDFRAME + popl %ebp + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE ebp + .endm +#else + .macro FRAME + .endm + .macro ENDFRAME + .endm +#endif diff --git a/include/asm-um/alternative-asm.i b/include/asm-um/alternative-asm.i new file mode 100644 index 000000000000..cae9faca132f --- /dev/null +++ b/include/asm-um/alternative-asm.i @@ -0,0 +1,6 @@ +#ifndef __UM_ALTERNATIVE_ASM_I +#define __UM_ALTERNATIVE_ASM_I + +#include "asm/arch/alternative-asm.i" + +#endif diff --git a/include/asm-um/frame.i b/include/asm-um/frame.i new file mode 100644 index 000000000000..09d5dca5d928 --- /dev/null +++ b/include/asm-um/frame.i @@ -0,0 +1,6 @@ +#ifndef __UM_FRAME_I +#define __UM_FRAME_I + +#include "asm/arch/frame.i" + +#endif -- cgit v1.2.2 From 7f11d8a5efd625ffa41cde1d8472f85e885478ec Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] Remove all ifdefs for local/io apic IO-APIC or local APIC can only be disabled at runtime anyways and Kconfig has forced these options on for a long time now. The Kconfigs are kept only now for the benefit of the shared acpi boot.c code. Signed-off-by: Andi Kleen --- include/asm-x86_64/apic.h | 4 ---- include/asm-x86_64/fixmap.h | 4 ---- include/asm-x86_64/io_apic.h | 6 ------ include/asm-x86_64/irq.h | 2 -- include/asm-x86_64/mpspec.h | 2 -- include/asm-x86_64/smp.h | 4 ---- 6 files changed, 22 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/apic.h b/include/asm-x86_64/apic.h index 9c96a0a8d1bd..8ed0f4d67b8d 100644 --- a/include/asm-x86_64/apic.h +++ b/include/asm-x86_64/apic.h @@ -29,8 +29,6 @@ extern int apic_runs_main_timer; printk(s, ##a); \ } while (0) -#ifdef CONFIG_X86_LOCAL_APIC - struct pt_regs; /* @@ -104,8 +102,6 @@ void switch_ipi_to_APIC_timer(void *cpumask); #define ARCH_APICTIMER_STOPS_ON_C3 1 -#endif /* CONFIG_X86_LOCAL_APIC */ - extern unsigned boot_cpu_id; #endif /* __ASM_APIC_H */ diff --git a/include/asm-x86_64/fixmap.h b/include/asm-x86_64/fixmap.h index 0b4ffbd1a125..1b620db5b9e3 100644 --- a/include/asm-x86_64/fixmap.h +++ b/include/asm-x86_64/fixmap.h @@ -37,13 +37,9 @@ enum fixed_addresses { VSYSCALL_FIRST_PAGE = VSYSCALL_LAST_PAGE + ((VSYSCALL_END-VSYSCALL_START) >> PAGE_SHIFT) - 1, VSYSCALL_HPET, FIX_HPET_BASE, -#ifdef CONFIG_X86_LOCAL_APIC FIX_APIC_BASE, /* local (CPU) APIC) -- required for SMP or not */ -#endif -#ifdef CONFIG_X86_IO_APIC FIX_IO_APIC_BASE_0, FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS-1, -#endif __end_of_fixed_addresses }; diff --git a/include/asm-x86_64/io_apic.h b/include/asm-x86_64/io_apic.h index fb7a0909a174..5d1b5c68e36e 100644 --- a/include/asm-x86_64/io_apic.h +++ b/include/asm-x86_64/io_apic.h @@ -10,8 +10,6 @@ * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar */ -#ifdef CONFIG_X86_IO_APIC - #ifdef CONFIG_PCI_MSI static inline int use_pci_vector(void) {return 1;} static inline void disable_edge_ioapic_vector(unsigned int vector) { } @@ -209,10 +207,6 @@ extern int timer_uses_ioapic_pin_0; extern int sis_apic_bug; /* dummy */ -#else /* !CONFIG_X86_IO_APIC */ -#define io_apic_assign_pci_irqs 0 -#endif - extern int assign_irq_vector(int irq); void enable_NMI_through_LVT0 (void * dummy); diff --git a/include/asm-x86_64/irq.h b/include/asm-x86_64/irq.h index 9db5a1b4f7b1..43469d8ab71a 100644 --- a/include/asm-x86_64/irq.h +++ b/include/asm-x86_64/irq.h @@ -44,9 +44,7 @@ static __inline__ int irq_canonicalize(int irq) return ((irq == 2) ? 9 : irq); } -#ifdef CONFIG_X86_LOCAL_APIC #define ARCH_HAS_NMI_WATCHDOG /* See include/linux/nmi.h */ -#endif #ifdef CONFIG_HOTPLUG_CPU #include diff --git a/include/asm-x86_64/mpspec.h b/include/asm-x86_64/mpspec.h index 14fc3ddd9031..3c7aae9fd157 100644 --- a/include/asm-x86_64/mpspec.h +++ b/include/asm-x86_64/mpspec.h @@ -184,12 +184,10 @@ extern int pic_mode; extern void mp_register_lapic (u8 id, u8 enabled); extern void mp_register_lapic_address (u64 address); -#ifdef CONFIG_X86_IO_APIC extern void mp_register_ioapic (u8 id, u32 address, u32 gsi_base); extern void mp_override_legacy_irq (u8 bus_irq, u8 polarity, u8 trigger, u32 gsi); extern void mp_config_acpi_legacy_irqs (void); extern int mp_register_gsi (u32 gsi, int triggering, int polarity); -#endif /*CONFIG_X86_IO_APIC*/ #endif extern int using_apic_timer; diff --git a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h index 612d208961a2..6353fa41bebe 100644 --- a/include/asm-x86_64/smp.h +++ b/include/asm-x86_64/smp.h @@ -9,15 +9,11 @@ #include extern int disable_apic; -#ifdef CONFIG_X86_LOCAL_APIC #include #include -#ifdef CONFIG_X86_IO_APIC #include -#endif #include #include -#endif #ifdef CONFIG_SMP -- cgit v1.2.2 From b1c78c0fcc29097567e1afc39701012e6d89adb7 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] Clean up and minor fixes to TLB flush - Convert CR* accesses to dedicated inline functions and rewrite the rest as C inlines - Don't do a double flush for global flushes (pointed out by Zach Amsden) This was a bug workaround for old CPUs that don't do 64bit and is obsolete. - Add a proper memory clobber to invlpg - Remove an unused extern Signed-off-by: Andi Kleen --- include/asm-x86_64/pgtable.h | 2 -- include/asm-x86_64/tlbflush.h | 70 +++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 37 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h index a31ab4e68a9b..0c1e2422400a 100644 --- a/include/asm-x86_64/pgtable.h +++ b/include/asm-x86_64/pgtable.h @@ -25,8 +25,6 @@ extern int nonx_setup(char *str); extern void paging_init(void); extern void clear_kernel_mapping(unsigned long addr, unsigned long size); -extern unsigned long pgkern_mask; - /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. diff --git a/include/asm-x86_64/tlbflush.h b/include/asm-x86_64/tlbflush.h index d16d5b60f419..983bd296c81a 100644 --- a/include/asm-x86_64/tlbflush.h +++ b/include/asm-x86_64/tlbflush.h @@ -4,44 +4,44 @@ #include #include -#define __flush_tlb() \ - do { \ - unsigned long tmpreg; \ - \ - __asm__ __volatile__( \ - "movq %%cr3, %0; # flush TLB \n" \ - "movq %0, %%cr3; \n" \ - : "=r" (tmpreg) \ - :: "memory"); \ - } while (0) +static inline unsigned long get_cr3(void) +{ + unsigned long cr3; + asm volatile("mov %%cr3,%0" : "=r" (cr3)); + return cr3; +} -/* - * Global pages have to be flushed a bit differently. Not a real - * performance problem because this does not happen often. - */ -#define __flush_tlb_global() \ - do { \ - unsigned long tmpreg, cr4, cr4_orig; \ - \ - __asm__ __volatile__( \ - "movq %%cr4, %2; # turn off PGE \n" \ - "movq %2, %1; \n" \ - "andq %3, %1; \n" \ - "movq %1, %%cr4; \n" \ - "movq %%cr3, %0; # flush TLB \n" \ - "movq %0, %%cr3; \n" \ - "movq %2, %%cr4; # turn PGE back on \n" \ - : "=&r" (tmpreg), "=&r" (cr4), "=&r" (cr4_orig) \ - : "i" (~X86_CR4_PGE) \ - : "memory"); \ - } while (0) - -extern unsigned long pgkern_mask; - -#define __flush_tlb_all() __flush_tlb_global() +static inline void set_cr3(unsigned long cr3) +{ + asm volatile("mov %0,%%cr3" :: "r" (cr3) : "memory"); +} + +static inline void __flush_tlb(void) +{ + set_cr3(get_cr3()); +} + +static inline unsigned long get_cr4(void) +{ + unsigned long cr4; + asm volatile("mov %%cr4,%0" : "=r" (cr4)); + return cr4; +} + +static inline void set_cr4(unsigned long cr4) +{ + asm volatile("mov %0,%%cr4" :: "r" (cr4) : "memory"); +} + +static inline void __flush_tlb_all(void) +{ + unsigned long cr4 = get_cr4(); + set_cr4(cr4 & ~X86_CR4_PGE); /* clear PGE */ + set_cr4(cr4); /* write old PGE again and flush TLBs */ +} #define __flush_tlb_one(addr) \ - __asm__ __volatile__("invlpg %0": :"m" (*(char *) addr)) + __asm__ __volatile__("invlpg (%0)" :: "r" (addr) : "memory") /* -- cgit v1.2.2 From 107878bb14b1fb6bddc646f4d5e72e8beaa2f6a2 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] i386: Minor fixes & cleanup to tlb flush (based on x86-64 changes) - Add a proper memory clobber to invlpg - Remove an unused extern Signed-off-by: Andi Kleen --- include/asm-i386/tlbflush.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-i386/tlbflush.h b/include/asm-i386/tlbflush.h index d57ca5c540b6..360648b0f2b3 100644 --- a/include/asm-i386/tlbflush.h +++ b/include/asm-i386/tlbflush.h @@ -36,8 +36,6 @@ : "memory"); \ } while (0) -extern unsigned long pgkern_mask; - # define __flush_tlb_all() \ do { \ if (cpu_has_pge) \ @@ -49,7 +47,7 @@ extern unsigned long pgkern_mask; #define cpu_has_invlpg (boot_cpu_data.x86 > 3) #define __flush_tlb_single(addr) \ - __asm__ __volatile__("invlpg %0": :"m" (*(char *) addr)) + __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory") #ifdef CONFIG_X86_INVLPG # define __flush_tlb_one(addr) __flush_tlb_single(addr) -- cgit v1.2.2 From a8fcf1a24a16e1c735c795d99773b5dcefb71518 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:30 +0200 Subject: [PATCH] Remove obsolete PIC mode PIC mode is an outdated way to drive the APICs that was used on some early MP boards. It is not supported in the ACPI model. It is unlikely to be ever configured by any x86-64 system Remove it thus. Signed-off-by: Andi Kleen --- include/asm-x86_64/mpspec.h | 1 - include/asm-x86_64/smp.h | 1 - 2 files changed, 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/mpspec.h b/include/asm-x86_64/mpspec.h index 3c7aae9fd157..1dc83436cc91 100644 --- a/include/asm-x86_64/mpspec.h +++ b/include/asm-x86_64/mpspec.h @@ -178,7 +178,6 @@ extern int mp_irq_entries; extern struct mpc_config_intsrc mp_irqs [MAX_IRQ_SOURCES]; extern int mpc_default_type; extern unsigned long mp_lapic_addr; -extern int pic_mode; #ifdef CONFIG_ACPI extern void mp_register_lapic (u8 id, u8 enabled); diff --git a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h index 6353fa41bebe..498fbc1fc179 100644 --- a/include/asm-x86_64/smp.h +++ b/include/asm-x86_64/smp.h @@ -33,7 +33,6 @@ extern cpumask_t cpu_initialized; extern void smp_alloc_memory(void); extern volatile unsigned long smp_invalidate_needed; -extern int pic_mode; extern void lock_ipi_call_lock(void); extern void unlock_ipi_call_lock(void); extern int smp_num_siblings; -- cgit v1.2.2 From dfa4698c50bf85b7927214b0e4a3dc4bc3b3c4a9 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:30 +0200 Subject: [PATCH] Move early chipset quirks out to new file They did not really belong into io_apic.c. Move them into a new file and clean it up a bit. Also remove outdated ATI quirk that was obsolete, Signed-off-by: Andi Kleen --- include/asm-x86_64/proto.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/proto.h b/include/asm-x86_64/proto.h index 3b1c60247902..58fec91318e4 100644 --- a/include/asm-x86_64/proto.h +++ b/include/asm-x86_64/proto.h @@ -92,7 +92,7 @@ extern void syscall32_cpu_init(void); extern void setup_node_bootmem(int nodeid, unsigned long start, unsigned long end); -extern void check_ioapic(void); +extern void early_quirks(void); extern void check_efer(void); extern int unhandled_signal(struct task_struct *tsk, int sig); -- cgit v1.2.2 From 55f05ffaa788e039df2f1ebe0d7bfbcb6f39d0b4 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:30 +0200 Subject: [PATCH] Replace mp bus array with bitmap for bus not pci Since we only support PCI and ISA legacy busses now there is no need to have an full array with checking. Signed-off-by: Andi Kleen --- include/asm-x86_64/mpspec.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/mpspec.h b/include/asm-x86_64/mpspec.h index 1dc83436cc91..017fddb61dc5 100644 --- a/include/asm-x86_64/mpspec.h +++ b/include/asm-x86_64/mpspec.h @@ -159,13 +159,7 @@ struct mpc_config_lintsrc #define MAX_MP_BUSSES 256 /* Each PCI slot may be a combo card with its own bus. 4 IRQ pins per slot. */ #define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4) -enum mp_bustype { - MP_BUS_ISA = 1, - MP_BUS_EISA, - MP_BUS_PCI, - MP_BUS_MCA -}; -extern unsigned char mp_bus_id_to_type [MAX_MP_BUSSES]; +extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); extern int mp_bus_id_to_pci_bus [MAX_MP_BUSSES]; extern unsigned int boot_cpu_physical_apicid; -- cgit v1.2.2 From e2414910f212c52d9d7c64c99a22863488ac5b48 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:30 +0200 Subject: [PATCH] x86: Detect CFI support in the assembler at runtime ... instead of using a CONFIG option. The config option still controls if the resulting executable actually has unwind information. This is useful to prevent compilation errors when users select CONFIG_STACK_UNWIND on old binutils and also allows to use CFI in the future for non kernel debugging applications. Cc: jbeulich@novell.com Cc: sam@ravnborg.org Signed-off-by: Andi Kleen --- include/asm-x86_64/dwarf2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/dwarf2.h b/include/asm-x86_64/dwarf2.h index 0744db777676..2b9368365fad 100644 --- a/include/asm-x86_64/dwarf2.h +++ b/include/asm-x86_64/dwarf2.h @@ -13,7 +13,7 @@ away for older version. */ -#ifdef CONFIG_UNWIND_INFO +#ifdef CONFIG_AS_CFI #define CFI_STARTPROC .cfi_startproc #define CFI_ENDPROC .cfi_endproc -- cgit v1.2.2 From 3b94355c47e2b025a7ececa0a14180e8fce6b0f1 Mon Sep 17 00:00:00 2001 From: Jan Beulich Date: Tue, 26 Sep 2006 10:52:31 +0200 Subject: [PATCH] remove int_delivery_dest The genapic field and the accessor macro weren't used anywhere. Signed-off-by: Jan Beulich Signed-off-by: Andi Kleen --- include/asm-x86_64/genapic.h | 1 - include/asm-x86_64/mach_apic.h | 1 - 2 files changed, 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/genapic.h b/include/asm-x86_64/genapic.h index 50b38e7c58e4..81e714665344 100644 --- a/include/asm-x86_64/genapic.h +++ b/include/asm-x86_64/genapic.h @@ -16,7 +16,6 @@ struct genapic { char *name; u32 int_delivery_mode; u32 int_dest_mode; - u32 int_delivery_dest; /* for quick IPIs */ int (*apic_id_registered)(void); cpumask_t (*target_cpus)(void); void (*init_apic_ldr)(void); diff --git a/include/asm-x86_64/mach_apic.h b/include/asm-x86_64/mach_apic.h index 0acea44c9377..d33422450c00 100644 --- a/include/asm-x86_64/mach_apic.h +++ b/include/asm-x86_64/mach_apic.h @@ -16,7 +16,6 @@ #define INT_DELIVERY_MODE (genapic->int_delivery_mode) #define INT_DEST_MODE (genapic->int_dest_mode) -#define INT_DELIVERY_DEST (genapic->int_delivery_dest) #define TARGET_CPUS (genapic->target_cpus()) #define apic_id_registered (genapic->apic_id_registered) #define init_apic_ldr (genapic->init_apic_ldr) -- cgit v1.2.2 From 5f4a7a93886ce1a4327f6028cc05d423f39eebf0 Mon Sep 17 00:00:00 2001 From: Muli Ben-Yehuda Date: Tue, 26 Sep 2006 10:52:31 +0200 Subject: [PATCH] Calgary IOMMU: rearrange 'struct iommu_table' members Rearrange struct members loosely based on size for improved alignment and to save a few bytes. Signed-off-by: Muli Ben-Yehuda Signed-off-by: Jon Mason Signed-off-by: Andi Kleen --- include/asm-x86_64/calgary.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/calgary.h b/include/asm-x86_64/calgary.h index 4e3919524240..0a03bda94d03 100644 --- a/include/asm-x86_64/calgary.h +++ b/include/asm-x86_64/calgary.h @@ -34,12 +34,12 @@ struct iommu_table { unsigned long it_base; /* mapped address of tce table */ unsigned long it_hint; /* Hint for next alloc */ unsigned long *it_map; /* A simple allocation bitmap for now */ + void __iomem *bbar; /* Bridge BAR */ + u64 tar_val; /* Table Address Register */ + struct timer_list watchdog_timer; spinlock_t it_lock; /* Protects it_map */ unsigned int it_size; /* Size of iommu table in entries */ unsigned char it_busno; /* Bus number this table belongs to */ - void __iomem *bbar; - u64 tar_val; - struct timer_list watchdog_timer; }; #define TCE_TABLE_SIZE_UNSPECIFIED ~0 -- cgit v1.2.2 From f38db651d5da5e10235fd7dd31095969fb7ef6fb Mon Sep 17 00:00:00 2001 From: Muli Ben-Yehuda Date: Tue, 26 Sep 2006 10:52:31 +0200 Subject: [PATCH] Calgary IOMMU: consolidate per bus data structures Move the tce_table_kva array, disabled bitmap and bus_to_phb array into a new per bus 'struct calgary_bus_info'. Also slightly reorganize build_tce_table and tce_table_setparms to avoid exporting bus_info to tce.c. Signed-off-by: Muli Ben-Yehuda Signed-off-by: Jon Mason Signed-off-by: Andi Kleen --- include/asm-x86_64/tce.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/tce.h b/include/asm-x86_64/tce.h index 53e9a68b3336..dbb047febc5e 100644 --- a/include/asm-x86_64/tce.h +++ b/include/asm-x86_64/tce.h @@ -24,7 +24,6 @@ #ifndef _ASM_X86_64_TCE_H #define _ASM_X86_64_TCE_H -extern void* tce_table_kva[]; extern unsigned int specified_table_size; struct iommu_table; -- cgit v1.2.2 From 01215ad8d83e18321d99e9b5750a6f21cac243a2 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:31 +0200 Subject: [PATCH] i386: Remove lock section support in mutex.h Lock sections don't work the new dwarf2 unwinder This generates slightly smaller code. It adds one more taken jump to the fast path. Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-i386/mutex.h | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'include') diff --git a/include/asm-i386/mutex.h b/include/asm-i386/mutex.h index 05a538531229..7a17d9e58ad6 100644 --- a/include/asm-i386/mutex.h +++ b/include/asm-i386/mutex.h @@ -30,14 +30,10 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " decl (%%eax) \n" \ - " js 2f \n" \ + " jns 1f \n" \ + " call "#fail_fn" \n" \ "1: \n" \ \ - LOCK_SECTION_START("") \ - "2: call "#fail_fn" \n" \ - " jmp 1b \n" \ - LOCK_SECTION_END \ - \ :"=a" (dummy) \ : "a" (count) \ : "memory", "ecx", "edx"); \ @@ -86,14 +82,10 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " incl (%%eax) \n" \ - " jle 2f \n" \ + " jg 1f \n" \ + " call "#fail_fn" \n" \ "1: \n" \ \ - LOCK_SECTION_START("") \ - "2: call "#fail_fn" \n" \ - " jmp 1b \n" \ - LOCK_SECTION_END \ - \ :"=a" (dummy) \ : "a" (count) \ : "memory", "ecx", "edx"); \ -- cgit v1.2.2 From add659bf8aa92f8b3f01a8c0220557c959507fb1 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:31 +0200 Subject: [PATCH] i386: Remove lock section support in rwsem.h Lock sections don't work the new dwarf2 unwinder This generates slightly smaller code. It adds one more taken jump to the fast path. Also move the trampolines into semaphore.S and add proper CFI annotations. Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-i386/rwsem.h | 62 ++++++++++-------------------------------------- 1 file changed, 12 insertions(+), 50 deletions(-) (limited to 'include') diff --git a/include/asm-i386/rwsem.h b/include/asm-i386/rwsem.h index 43113f5608eb..bc598d6388e3 100644 --- a/include/asm-i386/rwsem.h +++ b/include/asm-i386/rwsem.h @@ -99,17 +99,9 @@ static inline void __down_read(struct rw_semaphore *sem) __asm__ __volatile__( "# beginning down_read\n\t" LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */ - " js 2f\n\t" /* jump if we weren't granted the lock */ + " jns 1f\n" + " call call_rwsem_down_read_failed\n" "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " pushl %%edx\n\t" - " call rwsem_down_read_failed\n\t" - " popl %%edx\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END "# ending down_read\n\t" : "+m" (sem->count) : "a" (sem) @@ -151,15 +143,9 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) "# beginning down_write\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */ " testl %%edx,%%edx\n\t" /* was the count 0 before? */ - " jnz 2f\n\t" /* jump if we weren't granted the lock */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " call rwsem_down_write_failed\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END + " jz 1f\n" + " call call_rwsem_down_write_failed\n" + "1:\n" "# ending down_write" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (tmp) @@ -193,17 +179,9 @@ static inline void __up_read(struct rw_semaphore *sem) __asm__ __volatile__( "# beginning __up_read\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ - " js 2f\n\t" /* jump if the lock is being waited upon */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " decw %%dx\n\t" /* do nothing if still outstanding active readers */ - " jnz 1b\n\t" - " pushl %%ecx\n\t" - " call rwsem_wake\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END + " jns 1f\n\t" + " call call_rwsem_wake\n" + "1:\n" "# ending __up_read\n" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (tmp) @@ -219,17 +197,9 @@ static inline void __up_write(struct rw_semaphore *sem) "# beginning __up_write\n\t" " movl %2,%%edx\n\t" LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ - " jnz 2f\n\t" /* jump if the lock is being waited upon */ + " jz 1f\n" + " call call_rwsem_wake\n" "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " decw %%dx\n\t" /* did the active count reduce to 0? */ - " jnz 1b\n\t" /* jump back if not */ - " pushl %%ecx\n\t" - " call rwsem_wake\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END "# ending __up_write\n" : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) @@ -244,17 +214,9 @@ static inline void __downgrade_write(struct rw_semaphore *sem) __asm__ __volatile__( "# beginning __downgrade_write\n\t" LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ - " js 2f\n\t" /* jump if the lock is being waited upon */ + " jns 1f\n\t" + " call call_rwsem_downgrade_wake\n" "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " pushl %%edx\n\t" - " call rwsem_downgrade_wake\n\t" - " popl %%edx\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END "# ending __downgrade_write\n" : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_WAITING_BIAS) -- cgit v1.2.2 From 7ca2b49b06a6d26e89e3535653889f1d7892b085 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:32 +0200 Subject: [PATCH] i386: Remove lock section support in semaphore.h Lock sections don't work the new dwarf2 unwinder This generates slightly smaller code. It adds one more taken jump to the fast path. Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-i386/semaphore.h | 49 +++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) (limited to 'include') diff --git a/include/asm-i386/semaphore.h b/include/asm-i386/semaphore.h index d51e800acf29..e63b6a68f04c 100644 --- a/include/asm-i386/semaphore.h +++ b/include/asm-i386/semaphore.h @@ -100,13 +100,10 @@ static inline void down(struct semaphore * sem) __asm__ __volatile__( "# atomic down operation\n\t" LOCK_PREFIX "decl %0\n\t" /* --sem->count */ - "js 2f\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tlea %0,%%eax\n\t" - "call __down_failed\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 2f\n" + "\tlea %0,%%eax\n\t" + "call __down_failed\n" + "2:" :"+m" (sem->count) : :"memory","ax"); @@ -123,15 +120,12 @@ static inline int down_interruptible(struct semaphore * sem) might_sleep(); __asm__ __volatile__( "# atomic interruptible down operation\n\t" + "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "js 2f\n\t" - "xorl %0,%0\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tlea %1,%%eax\n\t" - "call __down_failed_interruptible\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 2f\n\t" + "lea %1,%%eax\n\t" + "call __down_failed_interruptible\n" + "2:" :"=a" (result), "+m" (sem->count) : :"memory"); @@ -148,15 +142,12 @@ static inline int down_trylock(struct semaphore * sem) __asm__ __volatile__( "# atomic interruptible down operation\n\t" + "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "js 2f\n\t" - "xorl %0,%0\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tlea %1,%%eax\n\t" + "jns 2f\n\t" + "lea %1,%%eax\n\t" "call __down_failed_trylock\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "2:\n" :"=a" (result), "+m" (sem->count) : :"memory"); @@ -166,22 +157,16 @@ static inline int down_trylock(struct semaphore * sem) /* * Note! This is subtle. We jump to wake people up only if * the semaphore was negative (== somebody was waiting on it). - * The default case (no contention) will result in NO - * jumps for both down() and up(). */ static inline void up(struct semaphore * sem) { __asm__ __volatile__( "# atomic up operation\n\t" LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ - "jle 2f\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tlea %0,%%eax\n\t" - "call __up_wakeup\n\t" - "jmp 1b\n" - LOCK_SECTION_END - ".subsection 0\n" + "jg 1f\n\t" + "lea %0,%%eax\n\t" + "call __up_wakeup\n" + "1:" :"+m" (sem->count) : :"memory","ax"); -- cgit v1.2.2 From 8578bdf835d6d6ece6718858d351456367b8a5cf Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:32 +0200 Subject: [PATCH] Don't use lock section for mutexes and semaphores Lock sections cannot be handled by the dwarf2 unwinder. Disadvantage is a taken branch in the hot path. Signed-off-by: Andi Kleen --- include/asm-x86_64/mutex.h | 20 ++++++-------------- include/asm-x86_64/semaphore.h | 40 ++++++++++++++-------------------------- include/asm-x86_64/spinlock.h | 9 ++++----- 3 files changed, 24 insertions(+), 45 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/mutex.h b/include/asm-x86_64/mutex.h index 06fab6de2a88..16396b1de3e4 100644 --- a/include/asm-x86_64/mutex.h +++ b/include/asm-x86_64/mutex.h @@ -25,13 +25,9 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " decl (%%rdi) \n" \ - " js 2f \n" \ - "1: \n" \ - \ - LOCK_SECTION_START("") \ - "2: call "#fail_fn" \n" \ - " jmp 1b \n" \ - LOCK_SECTION_END \ + " jns 1f \n" \ + " call "#fail_fn" \n" \ + "1:" \ \ :"=D" (dummy) \ : "D" (v) \ @@ -75,13 +71,9 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " incl (%%rdi) \n" \ - " jle 2f \n" \ - "1: \n" \ - \ - LOCK_SECTION_START("") \ - "2: call "#fail_fn" \n" \ - " jmp 1b \n" \ - LOCK_SECTION_END \ + " jg 1f \n" \ + " call "#fail_fn" \n" \ + "1: " \ \ :"=D" (dummy) \ : "D" (v) \ diff --git a/include/asm-x86_64/semaphore.h b/include/asm-x86_64/semaphore.h index 064df08b9a0f..107bd90429e8 100644 --- a/include/asm-x86_64/semaphore.h +++ b/include/asm-x86_64/semaphore.h @@ -107,12 +107,9 @@ static inline void down(struct semaphore * sem) __asm__ __volatile__( "# atomic down operation\n\t" LOCK_PREFIX "decl %0\n\t" /* --sem->count */ - "js 2f\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __down_failed\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 1f\n\t" + "call __down_failed\n" + "1:" :"=m" (sem->count) :"D" (sem) :"memory"); @@ -130,14 +127,11 @@ static inline int down_interruptible(struct semaphore * sem) __asm__ __volatile__( "# atomic interruptible down operation\n\t" + "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "js 2f\n\t" - "xorl %0,%0\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __down_failed_interruptible\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 2f\n\t" + "call __down_failed_interruptible\n" + "2:\n" :"=a" (result), "=m" (sem->count) :"D" (sem) :"memory"); @@ -154,14 +148,11 @@ static inline int down_trylock(struct semaphore * sem) __asm__ __volatile__( "# atomic interruptible down operation\n\t" + "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "js 2f\n\t" - "xorl %0,%0\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __down_failed_trylock\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 2f\n\t" + "call __down_failed_trylock\n\t" + "2:\n" :"=a" (result), "=m" (sem->count) :"D" (sem) :"memory","cc"); @@ -179,12 +170,9 @@ static inline void up(struct semaphore * sem) __asm__ __volatile__( "# atomic up operation\n\t" LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ - "jle 2f\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __up_wakeup\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jg 1f\n\t" + "call __up_wakeup\n" + "1:" :"=m" (sem->count) :"D" (sem) :"memory"); diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h index a8e3d89f591f..1d0733b38a4b 100644 --- a/include/asm-x86_64/spinlock.h +++ b/include/asm-x86_64/spinlock.h @@ -22,14 +22,13 @@ #define __raw_spin_lock_string \ "\n1:\t" \ LOCK_PREFIX " ; decl %0\n\t" \ - "js 2f\n" \ - LOCK_SECTION_START("") \ - "2:\t" \ + "jns 2f\n" \ + "3:\n" \ "rep;nop\n\t" \ "cmpl $0,%0\n\t" \ - "jle 2b\n\t" \ + "jle 3b\n\t" \ "jmp 1b\n" \ - LOCK_SECTION_END + "2:\t" \ #define __raw_spin_lock_string_up \ "\n\tdecl %0" -- cgit v1.2.2 From 8b059d2373c16b6d32787a49daf8ccf72dc61b71 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:32 +0200 Subject: [PATCH] Clean up spin/rwlocks - Inline spinlock strings into their inline functions - Convert macros to typesafe inlines - Replace some leftover __asm__ __volatile__s with asm volatile Signed-off-by: Andi Kleen --- include/asm-x86_64/rwlock.h | 14 +-------- include/asm-x86_64/spinlock.h | 71 ++++++++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 44 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/rwlock.h b/include/asm-x86_64/rwlock.h index 28a080d23119..72aeebed920b 100644 --- a/include/asm-x86_64/rwlock.h +++ b/include/asm-x86_64/rwlock.h @@ -21,18 +21,6 @@ #define RW_LOCK_BIAS 0x01000000 #define RW_LOCK_BIAS_STR "0x01000000" -#define __build_read_lock(rw) \ - asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \ - "jns 1f\n" \ - "call __read_lock_failed\n" \ - "1:\n" \ - ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory") - -#define __build_write_lock(rw) \ - asm volatile(LOCK_PREFIX "subl %1,(%0)\n\t" \ - "jz 1f\n" \ - "\tcall __write_lock_failed\n\t" \ - "1:\n" \ - ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory") +/* Actual code is in asm/spinlock.h or in arch/x86_64/lib/rwlock.S */ #endif diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h index 1d0733b38a4b..be7a9e629fb2 100644 --- a/include/asm-x86_64/spinlock.h +++ b/include/asm-x86_64/spinlock.h @@ -16,30 +16,23 @@ * (the type definitions are in asm/spinlock_types.h) */ -#define __raw_spin_is_locked(x) \ - (*(volatile signed int *)(&(x)->slock) <= 0) - -#define __raw_spin_lock_string \ - "\n1:\t" \ - LOCK_PREFIX " ; decl %0\n\t" \ - "jns 2f\n" \ - "3:\n" \ - "rep;nop\n\t" \ - "cmpl $0,%0\n\t" \ - "jle 3b\n\t" \ - "jmp 1b\n" \ - "2:\t" \ - -#define __raw_spin_lock_string_up \ - "\n\tdecl %0" - -#define __raw_spin_unlock_string \ - "movl $1,%0" \ - :"=m" (lock->slock) : : "memory" +static inline int __raw_spin_is_locked(raw_spinlock_t *lock) +{ + return *(volatile signed int *)(&(lock)->slock) <= 0; +} static inline void __raw_spin_lock(raw_spinlock_t *lock) { - asm volatile(__raw_spin_lock_string : "=m" (lock->slock) : : "memory"); + asm volatile( + "\n1:\t" + LOCK_PREFIX " ; decl %0\n\t" + "jns 2f\n" + "3:\n" + "rep;nop\n\t" + "cmpl $0,%0\n\t" + "jle 3b\n\t" + "jmp 1b\n" + "2:\t" : "=m" (lock->slock) : : "memory"); } #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) @@ -48,7 +41,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) { int oldval; - __asm__ __volatile__( + asm volatile( "xchgl %0,%1" :"=q" (oldval), "=m" (lock->slock) :"0" (0) : "memory"); @@ -58,13 +51,14 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) static inline void __raw_spin_unlock(raw_spinlock_t *lock) { - __asm__ __volatile__( - __raw_spin_unlock_string - ); + asm volatile("movl $1,%0" :"=m" (lock->slock) :: "memory"); } -#define __raw_spin_unlock_wait(lock) \ - do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) +{ + while (__raw_spin_is_locked(lock)) + cpu_relax(); +} /* * Read-write spinlocks, allowing multiple readers @@ -80,17 +74,32 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) * with the high bit (sign) being the "contended" bit. */ -#define __raw_read_can_lock(x) ((int)(x)->lock > 0) -#define __raw_write_can_lock(x) ((x)->lock == RW_LOCK_BIAS) +static inline int __raw_read_can_lock(raw_rwlock_t *lock) +{ + return (int)(lock)->lock > 0; +} + +static inline int __raw_write_can_lock(raw_rwlock_t *lock) +{ + return (lock)->lock == RW_LOCK_BIAS; +} static inline void __raw_read_lock(raw_rwlock_t *rw) { - __build_read_lock(rw); + asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" + "jns 1f\n" + "call __read_lock_failed\n" + "1:\n" + ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory"); } static inline void __raw_write_lock(raw_rwlock_t *rw) { - __build_write_lock(rw); + asm volatile(LOCK_PREFIX "subl %1,(%0)\n\t" + "jz 1f\n" + "\tcall __write_lock_failed\n\t" + "1:\n" + ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory"); } static inline int __raw_read_trylock(raw_rwlock_t *lock) -- cgit v1.2.2 From fb2e28485679418e459583605f9b19807a72ceca Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:32 +0200 Subject: [PATCH] i386: Clean up spin/rwlocks - Inline spinlock strings into their inline functions - Convert macros to typesafe inlines - Replace some leftover __asm__ __volatile__s with asm volatile Signed-off-by: Andi Kleen --- include/asm-i386/rwlock.h | 14 +---- include/asm-i386/spinlock.h | 131 ++++++++++++++++++++++---------------------- 2 files changed, 68 insertions(+), 77 deletions(-) (limited to 'include') diff --git a/include/asm-i386/rwlock.h b/include/asm-i386/rwlock.h index f40ccbd8cb7f..c3e5db32fa48 100644 --- a/include/asm-i386/rwlock.h +++ b/include/asm-i386/rwlock.h @@ -20,18 +20,6 @@ #define RW_LOCK_BIAS 0x01000000 #define RW_LOCK_BIAS_STR "0x01000000" -#define __build_read_lock(rw, helper) \ - asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" \ - "jns 1f\n" \ - "call " helper "\n\t" \ - "1:\n" \ - ::"a" (rw) : "memory") - -#define __build_write_lock(rw, helper) \ - asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ - "jz 1f\n" \ - "call " helper "\n\t" \ - "1:\n" \ - ::"a" (rw) : "memory") +/* Code is in asm-i386/spinlock.h */ #endif diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h index d1020363c41a..324329313af8 100644 --- a/include/asm-i386/spinlock.h +++ b/include/asm-i386/spinlock.h @@ -4,6 +4,7 @@ #include #include #include +#include #include /* @@ -17,67 +18,64 @@ * (the type definitions are in asm/spinlock_types.h) */ -#define __raw_spin_is_locked(x) \ - (*(volatile signed char *)(&(x)->slock) <= 0) - -#define __raw_spin_lock_string \ - "\n1:\t" \ - LOCK_PREFIX " ; decb %0\n\t" \ - "jns 3f\n" \ - "2:\t" \ - "rep;nop\n\t" \ - "cmpb $0,%0\n\t" \ - "jle 2b\n\t" \ - "jmp 1b\n" \ - "3:\n\t" - -/* - * NOTE: there's an irqs-on section here, which normally would have to be - * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use - * __raw_spin_lock_string_flags(). - */ -#define __raw_spin_lock_string_flags \ - "\n1:\t" \ - LOCK_PREFIX " ; decb %0\n\t" \ - "jns 5f\n" \ - "2:\t" \ - "testl $0x200, %1\n\t" \ - "jz 4f\n\t" \ - "sti\n" \ - "3:\t" \ - "rep;nop\n\t" \ - "cmpb $0, %0\n\t" \ - "jle 3b\n\t" \ - "cli\n\t" \ - "jmp 1b\n" \ - "4:\t" \ - "rep;nop\n\t" \ - "cmpb $0, %0\n\t" \ - "jg 1b\n\t" \ - "jmp 4b\n" \ - "5:\n\t" +static inline int __raw_spin_is_locked(raw_spinlock_t *x) +{ + return *(volatile signed char *)(&(x)->slock) <= 0; +} static inline void __raw_spin_lock(raw_spinlock_t *lock) { - asm(__raw_spin_lock_string : "+m" (lock->slock) : : "memory"); + asm volatile("\n1:\t" + LOCK_PREFIX " ; decb %0\n\t" + "jns 3f\n" + "2:\t" + "rep;nop\n\t" + "cmpb $0,%0\n\t" + "jle 2b\n\t" + "jmp 1b\n" + "3:\n\t" + : "+m" (lock->slock) : : "memory"); } /* * It is easier for the lock validator if interrupts are not re-enabled * in the middle of a lock-acquire. This is a performance feature anyway * so we turn it off: + * + * NOTE: there's an irqs-on section here, which normally would have to be + * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant. */ #ifndef CONFIG_PROVE_LOCKING static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) { - asm(__raw_spin_lock_string_flags : "+m" (lock->slock) : "r" (flags) : "memory"); + asm volatile( + "\n1:\t" + LOCK_PREFIX " ; decb %0\n\t" + "jns 5f\n" + "2:\t" + "testl $0x200, %1\n\t" + "jz 4f\n\t" + "sti\n" + "3:\t" + "rep;nop\n\t" + "cmpb $0, %0\n\t" + "jle 3b\n\t" + "cli\n\t" + "jmp 1b\n" + "4:\t" + "rep;nop\n\t" + "cmpb $0, %0\n\t" + "jg 1b\n\t" + "jmp 4b\n" + "5:\n\t" + : "+m" (lock->slock) : "r" (flags) : "memory"); } #endif static inline int __raw_spin_trylock(raw_spinlock_t *lock) { char oldval; - __asm__ __volatile__( + asm volatile( "xchgb %b0,%1" :"=q" (oldval), "+m" (lock->slock) :"0" (0) : "memory"); @@ -93,38 +91,29 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) #if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) -#define __raw_spin_unlock_string \ - "movb $1,%0" \ - :"+m" (lock->slock) : : "memory" - - static inline void __raw_spin_unlock(raw_spinlock_t *lock) { - __asm__ __volatile__( - __raw_spin_unlock_string - ); + asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory"); } #else -#define __raw_spin_unlock_string \ - "xchgb %b0, %1" \ - :"=q" (oldval), "+m" (lock->slock) \ - :"0" (oldval) : "memory" - static inline void __raw_spin_unlock(raw_spinlock_t *lock) { char oldval = 1; - __asm__ __volatile__( - __raw_spin_unlock_string - ); + asm volatile("xchgb %b0, %1" + : "=q" (oldval), "+m" (lock->slock) + : "0" (oldval) : "memory"); } #endif -#define __raw_spin_unlock_wait(lock) \ - do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) +{ + while (__raw_spin_is_locked(lock)) + cpu_relax(); +} /* * Read-write spinlocks, allowing multiple readers @@ -151,22 +140,36 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) * read_can_lock - would read_trylock() succeed? * @lock: the rwlock in question. */ -#define __raw_read_can_lock(x) ((int)(x)->lock > 0) +static inline int __raw_read_can_lock(raw_rwlock_t *x) +{ + return (int)(x)->lock > 0; +} /** * write_can_lock - would write_trylock() succeed? * @lock: the rwlock in question. */ -#define __raw_write_can_lock(x) ((x)->lock == RW_LOCK_BIAS) +static inline int __raw_write_can_lock(raw_rwlock_t *x) +{ + return (x)->lock == RW_LOCK_BIAS; +} static inline void __raw_read_lock(raw_rwlock_t *rw) { - __build_read_lock(rw, "__read_lock_failed"); + asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" + "jns 1f\n" + "call __read_lock_failed\n\t" + "1:\n" + ::"a" (rw) : "memory"); } static inline void __raw_write_lock(raw_rwlock_t *rw) { - __build_write_lock(rw, "__write_lock_failed"); + asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" + "jz 1f\n" + "call __write_lock_failed\n\t" + "1:\n" + ::"a" (rw) : "memory"); } static inline int __raw_read_trylock(raw_rwlock_t *lock) -- cgit v1.2.2 From 1a3f239ddf9208f2e52d36fef1c1c4518cbbbabe Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 26 Sep 2006 10:52:32 +0200 Subject: [PATCH] i386: Replace i386 open-coded cmdline parsing with This patch replaces the open-coded early commandline parsing throughout the i386 boot code with the generic mechanism (already used by ppc, powerpc, ia64 and s390). The code was inconsistent with whether it deletes the option from the cmdline or not, meaning some of these will get passed through the environment into init. This transformation is mainly mechanical, but there are some notable parts: 1) Grammar: s/linux never set's it up/linux never sets it up/ 2) Remove hacked-in earlyprintk= option scanning. When someone actually implements CONFIG_EARLY_PRINTK, then they can use early_param(). [AK: actually it is implemented, but I'm adding the early_param it in the next x86-64 patch] 3) Move declaration of generic_apic_probe() from setup.c into asm/apic.h 4) Various parameters now moved into their appropriate files (thanks Andi). 5) All parse functions which examine arg need to check for NULL, except one where it has subtle humor value. AK: readded acpi_sci handling which was completely dropped AK: moved some more variables into acpi/boot.c Cc: len.brown@intel.com Signed-off-by: Rusty Russell Signed-off-by: Andi Kleen --- include/asm-i386/acpi.h | 14 -------------- include/asm-i386/apic.h | 4 ++-- include/asm-i386/io_apic.h | 11 +++++++++++ include/asm-i386/pgtable.h | 2 -- 4 files changed, 13 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/include/asm-i386/acpi.h b/include/asm-i386/acpi.h index 20f523954218..6016632d032f 100644 --- a/include/asm-i386/acpi.h +++ b/include/asm-i386/acpi.h @@ -131,21 +131,7 @@ static inline void disable_acpi(void) extern int acpi_gsi_to_irq(u32 gsi, unsigned int *irq); #ifdef CONFIG_X86_IO_APIC -extern int skip_ioapic_setup; extern int acpi_skip_timer_override; - -static inline void disable_ioapic_setup(void) -{ - skip_ioapic_setup = 1; -} - -static inline int ioapic_setup_disabled(void) -{ - return skip_ioapic_setup; -} - -#else -static inline void disable_ioapic_setup(void) { } #endif static inline void acpi_noirq_set(void) { acpi_noirq = 1; } diff --git a/include/asm-i386/apic.h b/include/asm-i386/apic.h index 2c1e371cebb6..8c5331ceca7b 100644 --- a/include/asm-i386/apic.h +++ b/include/asm-i386/apic.h @@ -42,6 +42,8 @@ static inline void lapic_enable(void) } while (0) +extern void generic_apic_probe(void); + #ifdef CONFIG_X86_LOCAL_APIC /* @@ -117,8 +119,6 @@ extern void enable_APIC_timer(void); extern void enable_NMI_through_LVT0 (void * dummy); -extern int disable_timer_pin_1; - void smp_send_timer_broadcast_ipi(struct pt_regs *regs); void switch_APIC_timer_to_ipi(void *cpumask); void switch_ipi_to_APIC_timer(void *cpumask); diff --git a/include/asm-i386/io_apic.h b/include/asm-i386/io_apic.h index 5092e819b8a2..5d309275a1dc 100644 --- a/include/asm-i386/io_apic.h +++ b/include/asm-i386/io_apic.h @@ -188,6 +188,16 @@ static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned /* 1 if "noapic" boot option passed */ extern int skip_ioapic_setup; +static inline void disable_ioapic_setup(void) +{ + skip_ioapic_setup = 1; +} + +static inline int ioapic_setup_disabled(void) +{ + return skip_ioapic_setup; +} + /* * If we use the IO-APIC for IRQ routing, disable automatic * assignment of PCI IRQ's. @@ -206,6 +216,7 @@ extern int (*ioapic_renumber_irq)(int ioapic, int irq); #else /* !CONFIG_X86_IO_APIC */ #define io_apic_assign_pci_irqs 0 +static inline void disable_ioapic_setup(void) { } #endif extern int assign_irq_vector(int irq); diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h index 09697fec3d2b..64140f2f1b95 100644 --- a/include/asm-i386/pgtable.h +++ b/include/asm-i386/pgtable.h @@ -391,8 +391,6 @@ extern pte_t *lookup_address(unsigned long address); static inline int set_kernel_exec(unsigned long vaddr, int enable) { return 0;} #endif -extern void noexec_setup(const char *str); - #if defined(CONFIG_HIGHPTE) #define pte_offset_map(dir, address) \ ((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE0) + pte_index(address)) -- cgit v1.2.2 From 2c8c0e6b8d7700a990da8d24eff767f9ca223b96 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:32 +0200 Subject: [PATCH] Convert x86-64 to early param Instead of hackish manual parsing Requires earlier i386 patchkit, but also fixes i386 early_printk again. I removed some obsolete really early parameters which didn't do anything useful. Also made a few parameters that needed it early (mostly oops printing setup) Also removed one panic check that wasn't visible without early console anyways (the early console is now initialized after that panic) This cleans up a lot of code. Signed-off-by: Andi Kleen --- include/asm-x86_64/apic.h | 4 +--- include/asm-x86_64/e820.h | 3 +-- include/asm-x86_64/pgtable.h | 1 - include/asm-x86_64/proto.h | 10 ---------- 4 files changed, 2 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/apic.h b/include/asm-x86_64/apic.h index 8ed0f4d67b8d..29ee735278f2 100644 --- a/include/asm-x86_64/apic.h +++ b/include/asm-x86_64/apic.h @@ -17,6 +17,7 @@ extern int apic_verbosity; extern int apic_runs_main_timer; +extern int ioapic_force; /* * Define the default level of output to be very little @@ -93,9 +94,6 @@ extern void setup_APIC_extened_lvt(unsigned char lvt_off, unsigned char vector, #define K8_APIC_EXT_INT_MSG_EXT 0x7 #define K8_APIC_EXT_LVT_ENTRY_THRESHOLD 0 -extern int disable_timer_pin_1; - - void smp_send_timer_broadcast_ipi(void); void switch_APIC_timer_to_ipi(void *cpumask); void switch_ipi_to_APIC_timer(void *cpumask); diff --git a/include/asm-x86_64/e820.h b/include/asm-x86_64/e820.h index 670a3388e70a..dba012151856 100644 --- a/include/asm-x86_64/e820.h +++ b/include/asm-x86_64/e820.h @@ -55,8 +55,7 @@ extern void e820_setup_gap(void); extern unsigned long e820_hole_size(unsigned long start_pfn, unsigned long end_pfn); -extern void __init parse_memopt(char *p, char **end); -extern void __init parse_memmapopt(char *p, char **end); +extern void finish_e820_parsing(void); extern struct e820map e820; diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h index 0c1e2422400a..d79e7441b513 100644 --- a/include/asm-x86_64/pgtable.h +++ b/include/asm-x86_64/pgtable.h @@ -21,7 +21,6 @@ extern unsigned long __supported_pte_mask; #define swapper_pg_dir init_level4_pgt -extern int nonx_setup(char *str); extern void paging_init(void); extern void clear_kernel_mapping(unsigned long addr, unsigned long size); diff --git a/include/asm-x86_64/proto.h b/include/asm-x86_64/proto.h index 58fec91318e4..151826f8ce69 100644 --- a/include/asm-x86_64/proto.h +++ b/include/asm-x86_64/proto.h @@ -53,9 +53,6 @@ extern int nohpet; extern unsigned long vxtime_hz; extern void time_init_gtod(void); -extern int numa_setup(char *opt); - -extern int setup_early_printk(char *); extern void early_printk(const char *fmt, ...) __attribute__((format(printf,1,2))); extern void early_identify_cpu(struct cpuinfo_x86 *c); @@ -104,13 +101,7 @@ extern void select_idle_routine(const struct cpuinfo_x86 *c); extern unsigned long table_start, table_end; extern int exception_trace; -extern int using_apic_timer; -extern int disable_apic; extern unsigned cpu_khz; -extern int ioapic_force; -extern int skip_ioapic_setup; -extern int acpi_ht; -extern int acpi_disabled; extern void no_iommu_init(void); extern int force_iommu, no_iommu; @@ -132,7 +123,6 @@ extern int fix_aperture; extern int reboot_force; extern int notsc_setup(char *); -extern int setup_additional_cpus(char *); extern void smp_local_timer_interrupt(struct pt_regs * regs); -- cgit v1.2.2 From 91cd444e56ebe0c2acd9576a045d77490b26f607 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:33 +0200 Subject: [PATCH] x86: Remove unneeded externs in acpi/boot.c And move one into proto.h Cc: len.brown@intel.com Signed-off-by: Andi Kleen --- include/asm-x86_64/proto.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/proto.h b/include/asm-x86_64/proto.h index 151826f8ce69..b73d0c76613c 100644 --- a/include/asm-x86_64/proto.h +++ b/include/asm-x86_64/proto.h @@ -124,6 +124,8 @@ extern int fix_aperture; extern int reboot_force; extern int notsc_setup(char *); +extern int gsi_irq_sharing(int gsi); + extern void smp_local_timer_interrupt(struct pt_regs * regs); long do_arch_prctl(struct task_struct *task, int code, unsigned long addr); -- cgit v1.2.2 From 5a1b3999d6cb7ab87f1f3b1700bc91839fd6fa29 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:34 +0200 Subject: [PATCH] x86: Some preparationary cleanup for stack trace - Remove unused all_contexts parameter No caller used it - Move skip argument into the structure (needed for followon patches) Cc: mingo@elte.hu Signed-off-by: Andi Kleen --- include/linux/stacktrace.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h index 9cc81e572224..50e2b01e517c 100644 --- a/include/linux/stacktrace.h +++ b/include/linux/stacktrace.h @@ -5,15 +5,16 @@ struct stack_trace { unsigned int nr_entries, max_entries; unsigned long *entries; + int skip; /* input argument: How many entries to skip */ + int all_contexts; /* input argument: if true do than one stack */ }; extern void save_stack_trace(struct stack_trace *trace, - struct task_struct *task, int all_contexts, - unsigned int skip); + struct task_struct *task); extern void print_stack_trace(struct stack_trace *trace, int spaces); #else -# define save_stack_trace(trace, task, all, skip) do { } while (0) +# define save_stack_trace(trace, task) do { } while (0) # define print_stack_trace(trace) do { } while (0) #endif -- cgit v1.2.2 From b7f5e3c7742d5332b78b831131f43fc3630e6322 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:34 +0200 Subject: [PATCH] Don't access the APIC in safe_smp_processor_id when it is not mapped yet Lockdep can call the dwarf2 unwinder early, and the dwarf2 code uses safe_smp_processor_id which tries to access the local APIC page. But that doesn't work before the APIC code has set up its fixmap. Check for this case and always return boot cpu then. Cc: jbeulich@novell.com Cc: mingo@elte.hu Signed-off-by: Andi Kleen --- include/asm-x86_64/apic.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/asm-x86_64/apic.h b/include/asm-x86_64/apic.h index 29ee735278f2..9e66d32330c9 100644 --- a/include/asm-x86_64/apic.h +++ b/include/asm-x86_64/apic.h @@ -18,6 +18,7 @@ extern int apic_verbosity; extern int apic_runs_main_timer; extern int ioapic_force; +extern int apic_mapped; /* * Define the default level of output to be very little -- cgit v1.2.2 From c0b766f13d8e1189ce4d00e54700c9d96b543b9a Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:34 +0200 Subject: [PATCH] Merge stacktrace and show_trace This unifies the standard backtracer and the new stacktrace in memory backtracer. The standard one is converted to use callbacks and then reimplement stacktrace using new callbacks. The main advantage is that stacktrace can now use the new dwarf2 unwinder and avoid false positives in many cases. I kept it simple to make sure the standard backtracer stays reliable. Cc: mingo@elte.hu Signed-off-by: Andi Kleen --- include/asm-x86_64/stacktrace.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 include/asm-x86_64/stacktrace.h (limited to 'include') diff --git a/include/asm-x86_64/stacktrace.h b/include/asm-x86_64/stacktrace.h new file mode 100644 index 000000000000..5eb9799bef76 --- /dev/null +++ b/include/asm-x86_64/stacktrace.h @@ -0,0 +1,18 @@ +#ifndef _ASM_STACKTRACE_H +#define _ASM_STACKTRACE_H 1 + +/* Generic stack tracer with callbacks */ + +struct stacktrace_ops { + void (*warning)(void *data, char *msg); + /* msg must contain %s for the symbol */ + void (*warning_symbol)(void *data, char *msg, unsigned long symbol); + void (*address)(void *data, unsigned long address); + /* On negative return stop dumping */ + int (*stack)(void *data, char *name); +}; + +void dump_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long *stack, + struct stacktrace_ops *ops, void *data); + +#endif -- cgit v1.2.2 From 2b14a78cd07a52001b8c3865ed615d8b9b905b78 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:34 +0200 Subject: [PATCH] i386: Do stacktracer conversion too Following x86-64 patches. Reuses code from them in fact. Convert the standard backtracer to do all output using callbacks. Use the x86-64 stack tracer implementation that uses these callbacks to implement the stacktrace interface. This allows to use the new dwarf2 unwinder for stacktrace and get better backtraces. Cc: mingo@elte.hu Signed-off-by: Andi Kleen --- include/asm-i386/stacktrace.h | 1 + 1 file changed, 1 insertion(+) create mode 100644 include/asm-i386/stacktrace.h (limited to 'include') diff --git a/include/asm-i386/stacktrace.h b/include/asm-i386/stacktrace.h new file mode 100644 index 000000000000..7d1f6a5cbfca --- /dev/null +++ b/include/asm-i386/stacktrace.h @@ -0,0 +1 @@ +#include -- cgit v1.2.2 From a32cf3975bed3b84491f8ffeb24abe8c45d86ab0 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:34 +0200 Subject: [PATCH] i386: Get ebp from unwinder state when continuing fallback backtrace Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-i386/unwind.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/asm-i386/unwind.h b/include/asm-i386/unwind.h index 4c1a0b968569..f0ac399bae3c 100644 --- a/include/asm-i386/unwind.h +++ b/include/asm-i386/unwind.h @@ -28,6 +28,8 @@ struct unwind_frame_info #define FRAME_LINK_OFFSET 0 #define STACK_BOTTOM(tsk) STACK_LIMIT((tsk)->thread.esp0) #define STACK_TOP(tsk) ((tsk)->thread.esp0) +#else +#define UNW_FP(frame) ((void)(frame), 0) #endif #define STACK_LIMIT(ptr) (((ptr) - 1) & ~(THREAD_SIZE - 1)) @@ -88,6 +90,7 @@ static inline int arch_unw_user_mode(const struct unwind_frame_info *info) #define UNW_PC(frame) ((void)(frame), 0) #define UNW_SP(frame) ((void)(frame), 0) +#define UNW_FP(frame) ((void)(frame), 0) static inline int arch_unw_user_mode(const void *info) { -- cgit v1.2.2 From d28c4393a7bf558538e9def269c1caeab6ec056f Mon Sep 17 00:00:00 2001 From: "Prasanna S.P" Date: Tue, 26 Sep 2006 10:52:34 +0200 Subject: [PATCH] x86: error_code is not safe for kprobes This patch moves the entry.S:error_entry to .kprobes.text section, since code marked unsafe for kprobes jumps directly to entry.S::error_entry, that must be marked unsafe as well. This patch also moves all the ".previous.text" asm directives to ".previous" for kprobes section. AK: Following a similar i386 patch from Chuck Ebbert AK: Also merged Jeremy's fix in. +From: Jeremy Fitzhardinge KPROBE_ENTRY does a .section .kprobes.text, and expects its users to do a .previous at the end of the function. Unfortunately, if any code within the function switches sections, for example .fixup, then the .previous ends up putting all subsequent code into .fixup. Worse, any subsequent .fixup code gets intermingled with the code its supposed to be fixing (which is also in .fixup). It's surprising this didn't cause more havok. The fix is to use .pushsection/.popsection, so this stuff nests properly. A further cleanup would be to get rid of all .section/.previous pairs, since they're inherently fragile. +From: Chuck Ebbert <76306.1226@compuserve.com> Because code marked unsafe for kprobes jumps directly to entry.S::error_code, that must be marked unsafe as well. The easiest way to do that is to move the page fault entry point to just before error_code and let it inherit the same section. Also moved all the ".previous" asm directives for kprobes sections to column 1 and removed ".text" from them. Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com> Signed-off-by: Andi Kleen --- include/linux/linkage.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/linkage.h b/include/linux/linkage.h index 932021f872d5..6c9873f88287 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -35,9 +35,13 @@ #endif #define KPROBE_ENTRY(name) \ - .section .kprobes.text, "ax"; \ + .pushsection .kprobes.text, "ax"; \ ENTRY(name) +#define KPROBE_END(name) \ + END(name); \ + .popsection + #ifndef END #define END(name) \ .size name, .-name -- cgit v1.2.2 From 3d08a256da8aed5300bd0752200ece426f49b050 Mon Sep 17 00:00:00 2001 From: Adrian Bunk Date: Tue, 26 Sep 2006 10:52:35 +0200 Subject: [PATCH] i386: Make enable_local_apic static enable_local_apic can now become static. Cc: len.brown@intel.com Signed-off-by: Adrian Bunk Signed-off-by: Andi Kleen --- include/asm-i386/apic.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include') diff --git a/include/asm-i386/apic.h b/include/asm-i386/apic.h index 8c5331ceca7b..3a42b7d6fc92 100644 --- a/include/asm-i386/apic.h +++ b/include/asm-i386/apic.h @@ -16,20 +16,8 @@ #define APIC_VERBOSE 1 #define APIC_DEBUG 2 -extern int enable_local_apic; extern int apic_verbosity; -static inline void lapic_disable(void) -{ - enable_local_apic = -1; - clear_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability); -} - -static inline void lapic_enable(void) -{ - enable_local_apic = 1; -} - /* * Define the default level of output to be very little * This can be turned up by using apic=verbose for more -- cgit v1.2.2 From 522e93e3fcdbf00ba85c72fde6df28cfc0486a65 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 26 Sep 2006 10:52:35 +0200 Subject: [PATCH] i386: Descriptor and trap table cleanups. The implementation comes from Zach's [RFC, PATCH 10/24] i386 Vmi descriptor changes: Descriptor and trap table cleanups. Add cleanly written accessors for IDT and GDT gates so the subarch may override them. Note that this allows the hypervisor to transparently tweak the DPL of the descriptors as well as the RPL of segments in those descriptors, with no unnecessary kernel code modification. It also allows the hypervisor implementation of the VMI to tweak the gates, allowing for custom exception frames or extra layers of indirection above the guest fault / IRQ handlers. Signed-off-by: Zachary Amsden Signed-off-by: Rusty Russell Signed-off-by: Andi Kleen --- include/asm-i386/desc.h | 121 ++++++++++++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 45 deletions(-) (limited to 'include') diff --git a/include/asm-i386/desc.h b/include/asm-i386/desc.h index 89b8b82c82b3..5db9e96e8dc1 100644 --- a/include/asm-i386/desc.h +++ b/include/asm-i386/desc.h @@ -33,50 +33,99 @@ static inline struct desc_struct *get_cpu_gdt_table(unsigned int cpu) return (struct desc_struct *)per_cpu(cpu_gdt_descr, cpu).address; } +/* + * This is the ldt that every process will get unless we need + * something other than this. + */ +extern struct desc_struct default_ldt[]; +extern struct desc_struct idt_table[]; +extern void set_intr_gate(unsigned int irq, void * addr); + +static inline void pack_descriptor(__u32 *a, __u32 *b, + unsigned long base, unsigned long limit, unsigned char type, unsigned char flags) +{ + *a = ((base & 0xffff) << 16) | (limit & 0xffff); + *b = (base & 0xff000000) | ((base & 0xff0000) >> 16) | + ((type & 0xff) << 8) | ((flags & 0xf) << 12); +} + +static inline void pack_gate(__u32 *a, __u32 *b, + unsigned long base, unsigned short seg, unsigned char type, unsigned char flags) +{ + *a = (seg << 16) | (base & 0xffff); + *b = (base & 0xffff0000) | ((type & 0xff) << 8) | (flags & 0xff); +} + +#define DESCTYPE_LDT 0x82 /* present, system, DPL-0, LDT */ +#define DESCTYPE_TSS 0x89 /* present, system, DPL-0, 32-bit TSS */ +#define DESCTYPE_TASK 0x85 /* present, system, DPL-0, task gate */ +#define DESCTYPE_INT 0x8e /* present, system, DPL-0, interrupt gate */ +#define DESCTYPE_TRAP 0x8f /* present, system, DPL-0, trap gate */ +#define DESCTYPE_DPL3 0x60 /* DPL-3 */ +#define DESCTYPE_S 0x10 /* !system */ + #define load_TR_desc() __asm__ __volatile__("ltr %w0"::"q" (GDT_ENTRY_TSS*8)) #define load_LDT_desc() __asm__ __volatile__("lldt %w0"::"q" (GDT_ENTRY_LDT*8)) #define load_gdt(dtr) __asm__ __volatile("lgdt %0"::"m" (*dtr)) #define load_idt(dtr) __asm__ __volatile("lidt %0"::"m" (*dtr)) -#define load_tr(tr) __asm__ __volatile("ltr %0"::"mr" (tr)) -#define load_ldt(ldt) __asm__ __volatile("lldt %0"::"mr" (ldt)) +#define load_tr(tr) __asm__ __volatile("ltr %0"::"m" (tr)) +#define load_ldt(ldt) __asm__ __volatile("lldt %0"::"m" (ldt)) #define store_gdt(dtr) __asm__ ("sgdt %0":"=m" (*dtr)) #define store_idt(dtr) __asm__ ("sidt %0":"=m" (*dtr)) -#define store_tr(tr) __asm__ ("str %0":"=mr" (tr)) -#define store_ldt(ldt) __asm__ ("sldt %0":"=mr" (ldt)) +#define store_tr(tr) __asm__ ("str %0":"=m" (tr)) +#define store_ldt(ldt) __asm__ ("sldt %0":"=m" (ldt)) -/* - * This is the ldt that every process will get unless we need - * something other than this. - */ -extern struct desc_struct default_ldt[]; -extern void set_intr_gate(unsigned int irq, void * addr); +#if TLS_SIZE != 24 +# error update this code. +#endif -#define _set_tssldt_desc(n,addr,limit,type) \ -__asm__ __volatile__ ("movw %w3,0(%2)\n\t" \ - "movw %w1,2(%2)\n\t" \ - "rorl $16,%1\n\t" \ - "movb %b1,4(%2)\n\t" \ - "movb %4,5(%2)\n\t" \ - "movb $0,6(%2)\n\t" \ - "movb %h1,7(%2)\n\t" \ - "rorl $16,%1" \ - : "=m"(*(n)) : "q" (addr), "r"(n), "ir"(limit), "i"(type)) - -static inline void __set_tss_desc(unsigned int cpu, unsigned int entry, void *addr) +static inline void load_TLS(struct thread_struct *t, unsigned int cpu) { - _set_tssldt_desc(&get_cpu_gdt_table(cpu)[entry], (int)addr, - offsetof(struct tss_struct, __cacheline_filler) - 1, 0x89); +#define C(i) get_cpu_gdt_table(cpu)[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i] + C(0); C(1); C(2); +#undef C } -#define set_tss_desc(cpu,addr) __set_tss_desc(cpu, GDT_ENTRY_TSS, addr) +static inline void write_dt_entry(void *dt, int entry, __u32 entry_a, __u32 entry_b) +{ + __u32 *lp = (__u32 *)((char *)dt + entry*8); + *lp = entry_a; + *(lp+1) = entry_b; +} + +#define write_ldt_entry(dt, entry, a, b) write_dt_entry(dt, entry, a, b) +#define write_gdt_entry(dt, entry, a, b) write_dt_entry(dt, entry, a, b) +#define write_idt_entry(dt, entry, a, b) write_dt_entry(dt, entry, a, b) + +static inline void _set_gate(int gate, unsigned int type, void *addr, unsigned short seg) +{ + __u32 a, b; + pack_gate(&a, &b, (unsigned long)addr, seg, type, 0); + write_idt_entry(idt_table, gate, a, b); +} -static inline void set_ldt_desc(unsigned int cpu, void *addr, unsigned int size) +static inline void __set_tss_desc(unsigned int cpu, unsigned int entry, const void *addr) { - _set_tssldt_desc(&get_cpu_gdt_table(cpu)[GDT_ENTRY_LDT], (int)addr, ((size << 3)-1), 0x82); + __u32 a, b; + pack_descriptor(&a, &b, (unsigned long)addr, + offsetof(struct tss_struct, __cacheline_filler) - 1, + DESCTYPE_TSS, 0); + write_gdt_entry(get_cpu_gdt_table(cpu), entry, a, b); } +static inline void set_ldt_desc(unsigned int cpu, void *addr, unsigned int entries) +{ + __u32 a, b; + pack_descriptor(&a, &b, (unsigned long)addr, + entries * sizeof(struct desc_struct) - 1, + DESCTYPE_LDT, 0); + write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_LDT, a, b); +} + +#define set_tss_desc(cpu,addr) __set_tss_desc(cpu, GDT_ENTRY_TSS, addr) + #define LDT_entry_a(info) \ ((((info)->base_addr & 0x0000ffff) << 16) | ((info)->limit & 0x0ffff)) @@ -102,24 +151,6 @@ static inline void set_ldt_desc(unsigned int cpu, void *addr, unsigned int size) (info)->seg_not_present == 1 && \ (info)->useable == 0 ) -static inline void write_ldt_entry(void *ldt, int entry, __u32 entry_a, __u32 entry_b) -{ - __u32 *lp = (__u32 *)((char *)ldt + entry*8); - *lp = entry_a; - *(lp+1) = entry_b; -} - -#if TLS_SIZE != 24 -# error update this code. -#endif - -static inline void load_TLS(struct thread_struct *t, unsigned int cpu) -{ -#define C(i) get_cpu_gdt_table(cpu)[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i] - C(0); C(1); C(2); -#undef C -} - static inline void clear_LDT(void) { int cpu = get_cpu(); -- cgit v1.2.2 From ba4d40bb5c465f0a4dcc30d02dab80c2cb7e1ff3 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Tue, 26 Sep 2006 10:52:35 +0200 Subject: [PATCH] Auto size the per cpu area. Now for a completely different but trivial approach. I just boot tested it with 255 CPUS and everything worked. Currently everything (except module data) we place in the per cpu area we know about at compile time. So instead of allocating a fixed size for the per_cpu area allocate the number of bytes we need plus a fixed constant for to be used for modules. It isn't perfect but it is much less of a pain to work with than what we are doing now. AK: fixed warning Signed-off-by: Eric W. Biederman Signed-off-by: Andi Kleen --- include/asm-x86_64/percpu.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/percpu.h b/include/asm-x86_64/percpu.h index 08dd9f9dda81..39d2bab9b520 100644 --- a/include/asm-x86_64/percpu.h +++ b/include/asm-x86_64/percpu.h @@ -11,6 +11,16 @@ #include +#ifdef CONFIG_MODULES +# define PERCPU_MODULE_RESERVE 8192 +#else +# define PERCPU_MODULE_RESERVE 0 +#endif + +#define PERCPU_ENOUGH_ROOM \ + (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \ + PERCPU_MODULE_RESERVE) + #define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset) #define __my_cpu_offset() read_pda(data_offset) -- cgit v1.2.2 From 73fea175303926055440c06bc8894f0c5c58afc8 Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Tue, 26 Sep 2006 10:52:35 +0200 Subject: [PATCH] i386: Support physical cpu hotplug for x86_64 This patch enables ACPI based physical CPU hotplug support for x86_64. Implements acpi_map_lsapic() and acpi_unmap_lsapic() to support physical cpu hotplug. Signed-off-by: Ashok Raj Signed-off-by: Andi Kleen Cc: Andi Kleen Cc: "Brown, Len" Signed-off-by: Andrew Morton --- include/asm-i386/smp.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/asm-i386/smp.h b/include/asm-i386/smp.h index f87826039a51..32ac8c91d5c5 100644 --- a/include/asm-i386/smp.h +++ b/include/asm-i386/smp.h @@ -84,6 +84,7 @@ static inline int hard_smp_processor_id(void) extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); +extern unsigned int num_processors; #endif /* !__ASSEMBLY__ */ -- cgit v1.2.2 From e07e23e1fd3000289fc7ccc6c71879070d3b19e0 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Tue, 26 Sep 2006 10:52:36 +0200 Subject: [PATCH] non lazy "sleazy" fpu implementation Right now the kernel on x86-64 has a 100% lazy fpu behavior: after *every* context switch a trap is taken for the first FPU use to restore the FPU context lazily. This is of course great for applications that have very sporadic or no FPU use (since then you avoid doing the expensive save/restore all the time). However for very frequent FPU users... you take an extra trap every context switch. The patch below adds a simple heuristic to this code: After 5 consecutive context switches of FPU use, the lazy behavior is disabled and the context gets restored every context switch. If the app indeed uses the FPU, the trap is avoided. (the chance of the 6th time slice using FPU after the previous 5 having done so are quite high obviously). After 256 switches, this is reset and lazy behavior is returned (until there are 5 consecutive ones again). The reason for this is to give apps that do longer bursts of FPU use still the lazy behavior back after some time. [akpm@osdl.org: place new task_struct field next to jit_keyring to save space] Signed-off-by: Arjan van de Ven Signed-off-by: Andi Kleen Cc: Andi Kleen Signed-off-by: Andrew Morton --- include/asm-x86_64/i387.h | 5 ++++- include/linux/sched.h | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/i387.h b/include/asm-x86_64/i387.h index cba8a3b0cded..60c0f4853fdb 100644 --- a/include/asm-x86_64/i387.h +++ b/include/asm-x86_64/i387.h @@ -24,6 +24,7 @@ extern unsigned int mxcsr_feature_mask; extern void mxcsr_feature_mask_init(void); extern void init_fpu(struct task_struct *child); extern int save_i387(struct _fpstate __user *buf); +extern asmlinkage void math_state_restore(void); /* * FPU lazy state save handling... @@ -31,7 +32,9 @@ extern int save_i387(struct _fpstate __user *buf); #define unlazy_fpu(tsk) do { \ if (task_thread_info(tsk)->status & TS_USEDFPU) \ - save_init_fpu(tsk); \ + save_init_fpu(tsk); \ + else \ + tsk->fpu_counter = 0; \ } while (0) /* Ignore delayed exceptions from user space */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 34ed0d99b1bd..807556c5bcd2 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -865,6 +865,15 @@ struct task_struct { struct key *thread_keyring; /* keyring private to this thread */ unsigned char jit_keyring; /* default keyring to attach requested keys to */ #endif + /* + * fpu_counter contains the number of consecutive context switches + * that the FPU is used. If this is over a threshold, the lazy fpu + * saving becomes unlazy to save the trap. This is an unsigned char + * so that after 256 times the counter wraps and the behavior turns + * lazy again; this to deal with bursty apps that only use FPU for + * a short time + */ + unsigned char fpu_counter; int oomkilladj; /* OOM kill score adjustment (bit shift). */ char comm[TASK_COMM_LEN]; /* executable name excluding path - access with [gs]et_task_comm (which lock -- cgit v1.2.2 From f704cb935006580db0495e54d3c82631f6e2a984 Mon Sep 17 00:00:00 2001 From: Dave Jones Date: Tue, 26 Sep 2006 10:52:36 +0200 Subject: [PATCH] x86: remove config.h includes from asm-i386 & asm-x86_64 This is now automatically included by kbuild. Signed-off-by: Dave Jones Signed-off-by: Andi Kleen --- include/asm-i386/dwarf2.h | 2 -- include/asm-i386/tsc.h | 1 - include/asm-x86_64/calgary.h | 1 - 3 files changed, 4 deletions(-) (limited to 'include') diff --git a/include/asm-i386/dwarf2.h b/include/asm-i386/dwarf2.h index 2280f6272f80..fe2392f361ee 100644 --- a/include/asm-i386/dwarf2.h +++ b/include/asm-i386/dwarf2.h @@ -1,8 +1,6 @@ #ifndef _DWARF2_H #define _DWARF2_H -#include - #ifndef __ASSEMBLY__ #warning "asm/dwarf2.h should be only included in pure assembly files" #endif diff --git a/include/asm-i386/tsc.h b/include/asm-i386/tsc.h index 97b828ce31e0..c13933185c1c 100644 --- a/include/asm-i386/tsc.h +++ b/include/asm-i386/tsc.h @@ -6,7 +6,6 @@ #ifndef _ASM_i386_TSC_H #define _ASM_i386_TSC_H -#include #include /* diff --git a/include/asm-x86_64/calgary.h b/include/asm-x86_64/calgary.h index 0a03bda94d03..6b93f5a3a5c8 100644 --- a/include/asm-x86_64/calgary.h +++ b/include/asm-x86_64/calgary.h @@ -24,7 +24,6 @@ #ifndef _ASM_X86_64_CALGARY_H #define _ASM_X86_64_CALGARY_H -#include #include #include #include -- cgit v1.2.2 From 1bb4996bcebca1cde49d964b4e012699ce180e61 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] Move compiler check for modules to ia64 only Apparently IA64 needs it, but i386/x86-64 don't anymore since gcc 2.95 support was dropped. Nobody else on linux-arch requested keeping it generically Cc: tony.luck@intel.com Cc: kaos@sgi.com Signed-off-by: Andi Kleen --- include/asm-ia64/module.h | 3 ++- include/linux/vermagic.h | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-ia64/module.h b/include/asm-ia64/module.h index 85c82bd819f2..d2da61e4c49b 100644 --- a/include/asm-ia64/module.h +++ b/include/asm-ia64/module.h @@ -28,7 +28,8 @@ struct mod_arch_specific { #define Elf_Ehdr Elf64_Ehdr #define MODULE_PROC_FAMILY "ia64" -#define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY +#define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY \ + "gcc-" __stringify(__GNUC__) "." __stringify(__GNUC_MINOR__) #define ARCH_SHF_SMALL SHF_IA_64_SHORT diff --git a/include/linux/vermagic.h b/include/linux/vermagic.h index 46919f9f5eb3..4d0909e53595 100644 --- a/include/linux/vermagic.h +++ b/include/linux/vermagic.h @@ -24,5 +24,5 @@ #define VERMAGIC_STRING \ UTS_RELEASE " " \ MODULE_VERMAGIC_SMP MODULE_VERMAGIC_PREEMPT \ - MODULE_VERMAGIC_MODULE_UNLOAD MODULE_ARCH_VERMAGIC \ - "gcc-" __stringify(__GNUC__) "." __stringify(__GNUC_MINOR__) + MODULE_VERMAGIC_MODULE_UNLOAD MODULE_ARCH_VERMAGIC + -- cgit v1.2.2 From a549b86dd0f3cbffcd5f9343f4ae7fcd59f7e756 Mon Sep 17 00:00:00 2001 From: Chuck Ebbert <76306.1226@compuserve.com> Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] i386: annotate FIX_STACK() and the rest of nmi() In i386's entry.S, FIX_STACK() needs annotation because it replaces the stack pointer. And the rest of nmi() needs annotation in order to compile with these new annotations. Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com> Signed-off-by: Andi Kleen --- include/asm-i386/dwarf2.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/asm-i386/dwarf2.h b/include/asm-i386/dwarf2.h index fe2392f361ee..5d1a8db5a9b0 100644 --- a/include/asm-i386/dwarf2.h +++ b/include/asm-i386/dwarf2.h @@ -26,6 +26,7 @@ #define CFI_RESTORE .cfi_restore #define CFI_REMEMBER_STATE .cfi_remember_state #define CFI_RESTORE_STATE .cfi_restore_state +#define CFI_UNDEFINED .cfi_undefined #else @@ -46,6 +47,7 @@ #define CFI_RESTORE ignore #define CFI_REMEMBER_STATE ignore #define CFI_RESTORE_STATE ignore +#define CFI_UNDEFINED ignore #endif -- cgit v1.2.2 From 151f8cc1169f9052095b2be36183ab132d75c6c2 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] Remove safe_smp_processor_id() And replace all users with ordinary smp_processor_id. The function was originally added to get some basic oops information out even if the GS register was corrupted. However that didn't work for some anymore because printk is needed to print the oops and it uses smp_processor_id() already. Also GS register corruptions are not particularly common anymore. This also helps the Xen port which would otherwise need to do this in a special way because it can't access the local APIC. Cc: Chris Wright Signed-off-by: Andi Kleen --- include/asm-x86_64/smp.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h index 498fbc1fc179..58b5d6149a42 100644 --- a/include/asm-x86_64/smp.h +++ b/include/asm-x86_64/smp.h @@ -66,7 +66,6 @@ static inline int hard_smp_processor_id(void) return GET_APIC_ID(*(unsigned int *)(APIC_BASE+APIC_ID)); } -extern int safe_smp_processor_id(void); extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); extern void prefill_possible_map(void); @@ -100,7 +99,6 @@ static inline int cpu_present_to_apicid(int mps_cpu) #ifndef CONFIG_SMP #define stack_smp_processor_id() 0 -#define safe_smp_processor_id() 0 #define cpu_logical_map(x) (x) #else #include -- cgit v1.2.2 From df992848f5aa803fcacd2c5e7d67034bb89e3fa3 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] Fix pte_exec/mkexec and use it in change_page_attr() Fix the pte_exec/mkexec page table accessor functions to really use the NX bit. Previously they only checked the USER bit, but weren't actually used for anything. Then use them in change_page_attr() to manipulate the NX bit properly. Signed-off-by: Andi Kleen --- include/asm-x86_64/pgtable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h index d79e7441b513..f7614670c655 100644 --- a/include/asm-x86_64/pgtable.h +++ b/include/asm-x86_64/pgtable.h @@ -265,7 +265,7 @@ static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) #define __LARGE_PTE (_PAGE_PSE|_PAGE_PRESENT) static inline int pte_user(pte_t pte) { return pte_val(pte) & _PAGE_USER; } static inline int pte_read(pte_t pte) { return pte_val(pte) & _PAGE_USER; } -static inline int pte_exec(pte_t pte) { return pte_val(pte) & _PAGE_USER; } +static inline int pte_exec(pte_t pte) { return !(pte_val(pte) & _PAGE_NX); } static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } @@ -278,7 +278,7 @@ static inline pte_t pte_mkclean(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & static inline pte_t pte_mkold(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_ACCESSED)); return pte; } static inline pte_t pte_wrprotect(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_RW)); return pte; } static inline pte_t pte_mkread(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_USER)); return pte; } -static inline pte_t pte_mkexec(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_USER)); return pte; } +static inline pte_t pte_mkexec(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_NX)); return pte; } static inline pte_t pte_mkdirty(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_DIRTY)); return pte; } static inline pte_t pte_mkyoung(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_ACCESSED)); return pte; } static inline pte_t pte_mkwrite(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_RW)); return pte; } -- cgit v1.2.2 From 5e6b0bfe5b452957b7be4b6ef181cd41880f8359 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] Use proper accessors to change PSE bits in change_page_attr() Use normal pte accessors in change_page_attr() to access the PSE bits. Signed-off-by: Andi Kleen --- include/asm-x86_64/pgtable.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h index f7614670c655..b34f43acdef1 100644 --- a/include/asm-x86_64/pgtable.h +++ b/include/asm-x86_64/pgtable.h @@ -283,6 +283,7 @@ static inline pte_t pte_mkdirty(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | static inline pte_t pte_mkyoung(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_ACCESSED)); return pte; } static inline pte_t pte_mkwrite(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_RW)); return pte; } static inline pte_t pte_mkhuge(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_PSE)); return pte; } +static inline pte_t pte_clrhuge(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_PSE)); return pte; } struct vm_area_struct; -- cgit v1.2.2 From e4251e130deef9de5226cc36faa70a1c6671d3c5 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] Remove some cruft in apic id checking during processor setup - Remove a define that was used only once - Remove the too large APIC ID check because we always support the full 8bit range of APICs. - Restructure code a bit to be simpler. Cc: len.brown@intel.com Signed-off-by: Andi Kleen --- include/asm-x86_64/acpi.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/acpi.h b/include/asm-x86_64/acpi.h index 2c95a319c056..ed59aa4c6ff9 100644 --- a/include/asm-x86_64/acpi.h +++ b/include/asm-x86_64/acpi.h @@ -155,8 +155,6 @@ extern void acpi_reserve_bootmem(void); #endif /*CONFIG_ACPI_SLEEP*/ -#define boot_cpu_physical_apicid boot_cpu_id - extern int acpi_disabled; extern int acpi_pci_disabled; -- cgit v1.2.2 From 8380aabb99719af583447133f19a4d8074b5c337 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:37 +0200 Subject: [PATCH] Remove non e820 fallbacks in high level code Drop support for non e820 BIOS calls to get the memory map. The boot assembler code still has some support, but not the C code now. Signed-off-by: Andi Kleen --- include/asm-x86_64/e820.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/e820.h b/include/asm-x86_64/e820.h index dba012151856..8dac397bf85e 100644 --- a/include/asm-x86_64/e820.h +++ b/include/asm-x86_64/e820.h @@ -22,10 +22,6 @@ #define E820_ACPI 3 /* usable as RAM once ACPI tables have been read */ #define E820_NVS 4 -#define HIGH_MEMORY (1024*1024) - -#define LOWMEMSIZE() (0x9f000) - #ifndef __ASSEMBLY__ struct e820entry { u64 addr; /* start of memory segment */ -- cgit v1.2.2 From 0136611c62e8650e354b95c76dff6d2ce6030eff Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] optimize hweight64 for x86_64 Based on patch from David Rientjes , but changed by AK. Optimizes the 64-bit hamming weight for x86_64 processors assuming they have fast multiplication. Uses five fewer bitops than the generic hweight64. Benchmark on one EMT64 showed ~25% speedup with 2^24 consecutive calls. Define a new ARCH_HAS_FAST_MULTIPLIER that can be set by other architectures that can also multiply fast. Signed-off-by: Andi Kleen --- include/asm-x86_64/bitops.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/bitops.h b/include/asm-x86_64/bitops.h index f7ba57b1cc08..5b535eaf5309 100644 --- a/include/asm-x86_64/bitops.h +++ b/include/asm-x86_64/bitops.h @@ -399,6 +399,8 @@ static __inline__ int fls(int x) return r+1; } +#define ARCH_HAS_FAST_MULTIPLIER 1 + #include #endif /* __KERNEL__ */ -- cgit v1.2.2 From 80d2679cbc8e170011c9649fb8fb684ffd7e5c8f Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] x86: Remove incorrect comment about ACPI e820 entries They cannot be actually freed because the FACS table has a shared-with-the-BIOS lock. Signed-off-by: Andi Kleen --- include/asm-i386/e820.h | 2 +- include/asm-x86_64/e820.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-i386/e820.h b/include/asm-i386/e820.h index ca82acb8cb1f..f7514fb6e8e4 100644 --- a/include/asm-i386/e820.h +++ b/include/asm-i386/e820.h @@ -18,7 +18,7 @@ #define E820_RAM 1 #define E820_RESERVED 2 -#define E820_ACPI 3 /* usable as RAM once ACPI tables have been read */ +#define E820_ACPI 3 #define E820_NVS 4 #define HIGH_MEMORY (1024*1024) diff --git a/include/asm-x86_64/e820.h b/include/asm-x86_64/e820.h index 8dac397bf85e..fd4198b49637 100644 --- a/include/asm-x86_64/e820.h +++ b/include/asm-x86_64/e820.h @@ -19,7 +19,7 @@ #define E820_RAM 1 #define E820_RESERVED 2 -#define E820_ACPI 3 /* usable as RAM once ACPI tables have been read */ +#define E820_ACPI 3 #define E820_NVS 4 #ifndef __ASSEMBLY__ -- cgit v1.2.2 From 53ee11ae0d73f28029a5f0d991bc4dcd7c817e7a Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] Optimize PDA accesses slightly Based on a idea by Jeremy Fitzhardinge: Replace the volatiles and memory clobbers in the PDA access with telling gcc about access to a proxy PDA structure that doesn't actually exist. But the dummy accesses give a defined ordering for read/write accesses. Also add some memory barriers to the early GS initialization to make sure no PDA access is moved before it. Advantage is some .text savings (probably most from better code for accessing "current"): text data bss dec hex filename 4845647 1223688 615864 6685199 66020f vmlinux 4837780 1223688 615864 6677332 65e354 vmlinux-pda 1.2% smaller code Cc: Jeremy Fitzhardinge Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index b47c3df9ed1d..55e21da96e7a 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -36,40 +36,43 @@ extern struct x8664_pda boot_cpu_pda[]; * There is no fast way to get the base address of the PDA, all the accesses * have to mention %fs/%gs. So it needs to be done this Torvaldian way. */ -#define sizeof_field(type,field) (sizeof(((type *)0)->field)) -#define typeof_field(type,field) typeof(((type *)0)->field) - extern void __bad_pda_field(void); +/* proxy_pda doesn't actually exist, but tell gcc it is accessed + for all PDA accesses so it gets read/write dependencies right. */ +extern struct x8664_pda _proxy_pda; + #define pda_offset(field) offsetof(struct x8664_pda, field) #define pda_to_op(op,field,val) do { \ - typedef typeof_field(struct x8664_pda, field) T__; \ - switch (sizeof_field(struct x8664_pda, field)) { \ + typedef typeof(_proxy_pda.field) T__; \ + switch (sizeof(_proxy_pda.field)) { \ case 2: \ -asm volatile(op "w %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \ +asm(op "w %1,%%gs:%P2" : "+m" (_proxy_pda.field) : \ + "ri" ((T__)val),"i"(pda_offset(field))); break; \ case 4: \ -asm volatile(op "l %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \ +asm(op "l %1,%%gs:%P2" : "+m" (_proxy_pda.field) : \ + "ri" ((T__)val),"i"(pda_offset(field))); break; \ case 8: \ -asm volatile(op "q %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \ - default: __bad_pda_field(); \ +asm(op "q %1,%%gs:%P2": "+m" (_proxy_pda.field) : \ + "ri" ((T__)val),"i"(pda_offset(field))); break; \ +default: __bad_pda_field(); \ } \ } while (0) -/* - * AK: PDA read accesses should be neither volatile nor have an memory clobber. - * Unfortunately removing them causes all hell to break lose currently. - */ #define pda_from_op(op,field) ({ \ - typeof_field(struct x8664_pda, field) ret__; \ - switch (sizeof_field(struct x8664_pda, field)) { \ + typeof(_proxy_pda.field) ret__; \ + switch (sizeof(_proxy_pda.field)) { \ case 2: \ -asm volatile(op "w %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\ +asm(op "w %%gs:%P1,%0":"=r" (ret__):\ + "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ case 4: \ -asm volatile(op "l %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\ +asm(op "l %%gs:%P1,%0":"=r" (ret__):\ + "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ case 8: \ -asm volatile(op "q %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\ - default: __bad_pda_field(); \ +asm(op "q %%gs:%P1,%0":"=r" (ret__):\ + "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ +default: __bad_pda_field(); \ } \ ret__; }) -- cgit v1.2.2 From 575400d1b483fbe9e03c68758059bfaf4e4768d1 Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] i386: Fix the EDD code misparsing the command line The EDD code would scan the command line as a fixed array, without taking account of either whitespace, null-termination, the old command-line protocol, late overrides early, or the fact that the command line may not be reachable from INITSEG. This should fix those problems, and enable us to use a longer command line. Signed-off-by: H. Peter Anvin Signed-off-by: Andi Kleen --- include/linux/edd.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/edd.h b/include/linux/edd.h index 162512b886f7..b2b3e68aa512 100644 --- a/include/linux/edd.h +++ b/include/linux/edd.h @@ -52,6 +52,7 @@ #define EDD_CL_EQUALS 0x3d646465 /* "edd=" */ #define EDD_CL_OFF 0x666f /* "of" for off */ #define EDD_CL_SKIP 0x6b73 /* "sk" for skipmbr */ +#define EDD_CL_ON 0x6e6f /* "on" for on */ #ifndef __ASSEMBLY__ -- cgit v1.2.2 From f574164491d00d28b727d713685fb5edc9138200 Mon Sep 17 00:00:00 2001 From: Keith Owens Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] Remove most of the special cases for the debug IST stack Remove most of the special cases for the debug IST stack. This is a follow on clean up patch, it requires the bug fix patch that adds orig_ist. Signed-off-by: Keith Owens Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index 55e21da96e7a..e2b23e337b94 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -13,9 +13,6 @@ struct x8664_pda { unsigned long data_offset; /* Per cpu data offset from linker address */ unsigned long kernelstack; /* top of kernel stack for current */ unsigned long oldrsp; /* user rsp for system call */ -#if DEBUG_STKSZ > EXCEPTION_STKSZ - unsigned long debugstack; /* #DB/#BP stack. */ -#endif int irqcount; /* Irq nesting counter. Starts with -1 */ int cpunumber; /* Logical CPU number */ char *irqstackptr; /* top of irqstack */ -- cgit v1.2.2 From 4bfaaef01a1badb9e8ffb0c0a37cd2379008d21f Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] Avoid overwriting the current pgd (V4, x86_64) kexec: Avoid overwriting the current pgd (V4, x86_64) This patch upgrades the x86_64-specific kexec code to avoid overwriting the current pgd. Overwriting the current pgd is bad when CONFIG_CRASH_DUMP is used to start a secondary kernel that dumps the memory of the previous kernel. The code introduces a new set of page tables. These tables are used to provide an executable identity mapping without overwriting the current pgd. Signed-off-by: Magnus Damm Signed-off-by: Andi Kleen --- include/asm-x86_64/kexec.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/kexec.h b/include/asm-x86_64/kexec.h index c564bae03433..5fab957e1091 100644 --- a/include/asm-x86_64/kexec.h +++ b/include/asm-x86_64/kexec.h @@ -1,6 +1,27 @@ #ifndef _X86_64_KEXEC_H #define _X86_64_KEXEC_H +#define PA_CONTROL_PAGE 0 +#define VA_CONTROL_PAGE 1 +#define PA_PGD 2 +#define VA_PGD 3 +#define PA_PUD_0 4 +#define VA_PUD_0 5 +#define PA_PMD_0 6 +#define VA_PMD_0 7 +#define PA_PTE_0 8 +#define VA_PTE_0 9 +#define PA_PUD_1 10 +#define VA_PUD_1 11 +#define PA_PMD_1 12 +#define VA_PMD_1 13 +#define PA_PTE_1 14 +#define VA_PTE_1 15 +#define PA_TABLE_PAGE 16 +#define PAGES_NR 17 + +#ifndef __ASSEMBLY__ + #include #include @@ -64,4 +85,12 @@ static inline void crash_setup_regs(struct pt_regs *newregs, newregs->rip = (unsigned long)current_text_addr(); } } + +NORET_TYPE void +relocate_kernel(unsigned long indirection_page, + unsigned long page_list, + unsigned long start_address) ATTRIB_NORET; + +#endif /* __ASSEMBLY__ */ + #endif /* _X86_64_KEXEC_H */ -- cgit v1.2.2 From 3566561bfadffcb5dbc85d576be80c0dbf2cccc9 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] i386: Avoid overwriting the current pgd (V4, i386) kexec: Avoid overwriting the current pgd (V4, i386) This patch upgrades the i386-specific kexec code to avoid overwriting the current pgd. Overwriting the current pgd is bad when CONFIG_CRASH_DUMP is used to start a secondary kernel that dumps the memory of the previous kernel. The code introduces a new set of page tables. These tables are used to provide an executable identity mapping without overwriting the current pgd. Signed-off-by: Magnus Damm Signed-off-by: Andi Kleen --- include/asm-i386/kexec.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'include') diff --git a/include/asm-i386/kexec.h b/include/asm-i386/kexec.h index 53f0e06672dc..4dfc9f5ed031 100644 --- a/include/asm-i386/kexec.h +++ b/include/asm-i386/kexec.h @@ -1,6 +1,26 @@ #ifndef _I386_KEXEC_H #define _I386_KEXEC_H +#define PA_CONTROL_PAGE 0 +#define VA_CONTROL_PAGE 1 +#define PA_PGD 2 +#define VA_PGD 3 +#define PA_PTE_0 4 +#define VA_PTE_0 5 +#define PA_PTE_1 6 +#define VA_PTE_1 7 +#ifdef CONFIG_X86_PAE +#define PA_PMD_0 8 +#define VA_PMD_0 9 +#define PA_PMD_1 10 +#define VA_PMD_1 11 +#define PAGES_NR 12 +#else +#define PAGES_NR 8 +#endif + +#ifndef __ASSEMBLY__ + #include #include #include @@ -72,5 +92,12 @@ static inline void crash_setup_regs(struct pt_regs *newregs, newregs->eip = (unsigned long)current_text_addr(); } } +asmlinkage NORET_TYPE void +relocate_kernel(unsigned long indirection_page, + unsigned long control_page, + unsigned long start_address, + unsigned int has_pae) ATTRIB_NORET; + +#endif /* __ASSEMBLY__ */ #endif /* _I386_KEXEC_H */ -- cgit v1.2.2 From 29a9af60e2120f874d0c600bf9e27617254a0488 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] Add comments to the PDA structure to annotate offsets Change the comments in the pda structure to make the first fields to have their offset documented and to have the comments aligned. The stack protector series needs a field at offset 40 (gcc ABI); annotate upto 40 for that reason. Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Andi Kleen CC: Andi Kleen --- include/asm-x86_64/pda.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index e2b23e337b94..6794ffaae433 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -9,13 +9,14 @@ /* Per processor datastructure. %gs points to it while the kernel runs */ struct x8664_pda { - struct task_struct *pcurrent; /* Current process */ - unsigned long data_offset; /* Per cpu data offset from linker address */ - unsigned long kernelstack; /* top of kernel stack for current */ - unsigned long oldrsp; /* user rsp for system call */ - int irqcount; /* Irq nesting counter. Starts with -1 */ - int cpunumber; /* Logical CPU number */ - char *irqstackptr; /* top of irqstack */ + struct task_struct *pcurrent; /* 0 Current process */ + unsigned long data_offset; /* 8 Per cpu data offset from linker + address */ + unsigned long kernelstack; /* 16 top of kernel stack for current */ + unsigned long oldrsp; /* 24 user rsp for system call */ + int irqcount; /* 32 Irq nesting counter. Starts with -1 */ + int cpunumber; /* 36 Logical CPU number */ + char *irqstackptr; /* 40 top of irqstack */ int nodenumber; /* number of current node */ unsigned int __softirq_pending; unsigned int __nmi_count; /* number of NMI on this CPUs */ -- cgit v1.2.2 From 0a4254058037eb172758961d0a5b94f4320a1425 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Tue, 26 Sep 2006 10:52:38 +0200 Subject: [PATCH] Add the canary field to the PDA area and the task struct This patch adds the per thread cookie field to the task struct and the PDA. Also it makes sure that the PDA value gets the new cookie value at context switch, and that a new task gets a new cookie at task creation time. Signed-off-by: Arjan van Ven Signed-off-by: Ingo Molnar Signed-off-by: Andi Kleen CC: Andi Kleen --- include/asm-x86_64/pda.h | 7 ++++++- include/linux/sched.h | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index 6794ffaae433..e7773e0af865 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -16,7 +16,12 @@ struct x8664_pda { unsigned long oldrsp; /* 24 user rsp for system call */ int irqcount; /* 32 Irq nesting counter. Starts with -1 */ int cpunumber; /* 36 Logical CPU number */ - char *irqstackptr; /* 40 top of irqstack */ +#ifdef CONFIG_CC_STACKPROTECTOR + unsigned long stack_canary; /* 40 stack canary value */ + /* gcc-ABI: this canary MUST be at + offset 40!!! */ +#endif + char *irqstackptr; int nodenumber; /* number of current node */ unsigned int __softirq_pending; unsigned int __nmi_count; /* number of NMI on this CPUs */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 807556c5bcd2..9d4aa7f95bc8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -819,6 +819,11 @@ struct task_struct { unsigned did_exec:1; pid_t pid; pid_t tgid; + +#ifdef CONFIG_CC_STACKPROTECTOR + /* Canary value for the -fstack-protector gcc feature */ + unsigned long stack_canary; +#endif /* * pointers to (original) parent process, youngest child, younger sibling, * older sibling, respectively. (p->father can be replaced with -- cgit v1.2.2 From baf5695dd1a49bb48a3daf08726d7f243f42e97e Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:39 +0200 Subject: [PATCH] Use %c instead of %P modifier in pda access Apparently that is the more official way to get numbers without $ in inline assembly Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index e7773e0af865..c2aac96ac323 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -51,13 +51,13 @@ extern struct x8664_pda _proxy_pda; typedef typeof(_proxy_pda.field) T__; \ switch (sizeof(_proxy_pda.field)) { \ case 2: \ -asm(op "w %1,%%gs:%P2" : "+m" (_proxy_pda.field) : \ +asm(op "w %1,%%gs:%c2" : "+m" (_proxy_pda.field) : \ "ri" ((T__)val),"i"(pda_offset(field))); break; \ case 4: \ -asm(op "l %1,%%gs:%P2" : "+m" (_proxy_pda.field) : \ +asm(op "l %1,%%gs:%c2" : "+m" (_proxy_pda.field) : \ "ri" ((T__)val),"i"(pda_offset(field))); break; \ case 8: \ -asm(op "q %1,%%gs:%P2": "+m" (_proxy_pda.field) : \ +asm(op "q %1,%%gs:%c2": "+m" (_proxy_pda.field) : \ "ri" ((T__)val),"i"(pda_offset(field))); break; \ default: __bad_pda_field(); \ } \ @@ -67,13 +67,13 @@ default: __bad_pda_field(); \ typeof(_proxy_pda.field) ret__; \ switch (sizeof(_proxy_pda.field)) { \ case 2: \ -asm(op "w %%gs:%P1,%0":"=r" (ret__):\ +asm(op "w %%gs:%c1,%0":"=r" (ret__):\ "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ case 4: \ -asm(op "l %%gs:%P1,%0":"=r" (ret__):\ +asm(op "l %%gs:%c1,%0":"=r" (ret__):\ "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ case 8: \ -asm(op "q %%gs:%P1,%0":"=r" (ret__):\ +asm(op "q %%gs:%c1,%0":"=r" (ret__):\ "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ default: __bad_pda_field(); \ } \ -- cgit v1.2.2 From 85691f135db78f3548107a0abe383dfab3bc38fa Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Tue, 26 Sep 2006 10:52:39 +0200 Subject: [PATCH] Type checking for write_pda() I just added type checking for assignments the PDA in the i386 PDA code. Here's the x86-64 equivalent. (Obviously this doesn't contain the latest x86-64 PDA change.) Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index c2aac96ac323..5dadb201f769 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -49,7 +49,8 @@ extern struct x8664_pda _proxy_pda; #define pda_to_op(op,field,val) do { \ typedef typeof(_proxy_pda.field) T__; \ - switch (sizeof(_proxy_pda.field)) { \ + if (0) { T__ tmp__; tmp__ = (val); } \ + switch (sizeof(_proxy_pda.field)) { \ case 2: \ asm(op "w %1,%%gs:%c2" : "+m" (_proxy_pda.field) : \ "ri" ((T__)val),"i"(pda_offset(field))); break; \ -- cgit v1.2.2 From 0da5db313317e3195482d3e660a1074857374a89 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 26 Sep 2006 10:52:39 +0200 Subject: [PATCH] i386: Abstract sensitive instructions Abstract sensitive instructions in assembler code, replacing them with macros (which currently are #defined to the native versions). We use long names: assembler is case-insensitive, so if something goes wrong and macros do not expand, it would assemble anyway. Resulting object files are exactly the same as before. Signed-off-by: Rusty Russell Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Andrew Morton Signed-off-by: Andi Kleen --- include/asm-i386/spinlock.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h index 324329313af8..b0b3043f05e1 100644 --- a/include/asm-i386/spinlock.h +++ b/include/asm-i386/spinlock.h @@ -7,6 +7,9 @@ #include #include +#define CLI_STRING "cli" +#define STI_STRING "sti" + /* * Your basic SMP spinlocks, allowing only a single CPU anywhere * @@ -55,12 +58,12 @@ static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long fla "2:\t" "testl $0x200, %1\n\t" "jz 4f\n\t" - "sti\n" + STI_STRING "\n" "3:\t" "rep;nop\n\t" "cmpb $0, %0\n\t" "jle 3b\n\t" - "cli\n\t" + CLI_STRING "\n\t" "jmp 1b\n" "4:\t" "rep;nop\n\t" -- cgit v1.2.2 From 78be3706b21a232310590fe00258b224177ac05f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 26 Sep 2006 10:52:39 +0200 Subject: [PATCH] i386: Allow a kernel not to be in ring 0 We allow for the fact that the guest kernel may not run in ring 0. This requires some abstraction in a few places when setting %cs or checking privilege level (user vs kernel). This is Chris' [RFC PATCH 15/33] move segment checks to subarch, except rather than using #define USER_MODE_MASK which depends on a config option, we use Zach's more flexible approach of assuming ring 3 == userspace. I also used "get_kernel_rpl()" over "get_kernel_cs()" because I think it reads better in the code... 1) Remove the hardcoded 3 and introduce #define SEGMENT_RPL_MASK 3 2) Add a get_kernel_rpl() macro, and don't assume it's zero. And: Clean up of patch for letting kernel run other than ring 0: a. Add some comments about the SEGMENT_IS_*_CODE() macros. b. Add a USER_RPL macro. (Code was comparing a value to a mask in some places and to the magic number 3 in other places.) c. Add macros for table indicator field and use them. d. Change the entry.S tests for LDT stack segment to use the macros Signed-off-by: Rusty Russell Signed-off-by: Zachary Amsden Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Andrew Morton Signed-off-by: Andi Kleen --- include/asm-i386/ptrace.h | 5 +++-- include/asm-i386/segment.h | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-i386/ptrace.h b/include/asm-i386/ptrace.h index 30a442ec2059..21bb91679c82 100644 --- a/include/asm-i386/ptrace.h +++ b/include/asm-i386/ptrace.h @@ -60,6 +60,7 @@ struct pt_regs { #ifdef __KERNEL__ #include +#include struct task_struct; extern void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs, int error_code); @@ -73,11 +74,11 @@ extern void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs, int erro */ static inline int user_mode(struct pt_regs *regs) { - return (regs->xcs & 3) != 0; + return (regs->xcs & SEGMENT_RPL_MASK) == USER_RPL; } static inline int user_mode_vm(struct pt_regs *regs) { - return ((regs->xcs & 3) | (regs->eflags & VM_MASK)) != 0; + return ((regs->xcs & SEGMENT_RPL_MASK) | (regs->eflags & VM_MASK)) >= USER_RPL; } #define instruction_pointer(regs) ((regs)->eip) extern unsigned long profile_pc(struct pt_regs *regs); diff --git a/include/asm-i386/segment.h b/include/asm-i386/segment.h index faf995307b9e..b7ab59685ba7 100644 --- a/include/asm-i386/segment.h +++ b/include/asm-i386/segment.h @@ -83,6 +83,11 @@ #define GDT_SIZE (GDT_ENTRIES * 8) +/* Matches __KERNEL_CS and __USER_CS (they must be 2 entries apart) */ +#define SEGMENT_IS_FLAT_CODE(x) (((x) & 0xec) == GDT_ENTRY_KERNEL_CS * 8) +/* Matches PNP_CS32 and PNP_CS16 (they must be consecutive) */ +#define SEGMENT_IS_PNP_CODE(x) (((x) & 0xf4) == GDT_ENTRY_PNPBIOS_BASE * 8) + /* Simple and small GDT entries for booting only */ #define GDT_ENTRY_BOOT_CS 2 @@ -112,4 +117,16 @@ */ #define IDT_ENTRIES 256 +/* Bottom two bits of selector give the ring privilege level */ +#define SEGMENT_RPL_MASK 0x3 +/* Bit 2 is table indicator (LDT/GDT) */ +#define SEGMENT_TI_MASK 0x4 + +/* User mode is privilege level 3 */ +#define USER_RPL 0x3 +/* LDT segment has TI set, GDT has it cleared */ +#define SEGMENT_LDT 0x4 +#define SEGMENT_GDT 0x0 + +#define get_kernel_rpl() 0 #endif -- cgit v1.2.2 From 3022d734a54cbd2b65eea9a024564821101b4a9a Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:39 +0200 Subject: [PATCH] Fix zeroing on exception in copy_*_user - Don't zero for __copy_from_user_inatomic following i386. This will prevent spurious zeros for parallel file system writers when one does a exception - The string instruction version didn't zero the output on exception. Oops. Also I cleaned up the code a bit while I was at it and added a minor optimization to the string instruction path. Signed-off-by: Andi Kleen --- include/asm-x86_64/uaccess.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/uaccess.h b/include/asm-x86_64/uaccess.h index 1e1fa003daa3..bc6812009bd7 100644 --- a/include/asm-x86_64/uaccess.h +++ b/include/asm-x86_64/uaccess.h @@ -238,6 +238,7 @@ do { \ /* Handles exceptions in both to and from, but doesn't do access_ok */ extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); +extern unsigned long copy_user_generic_dontzero(void *to, const void *from, unsigned len); extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); @@ -303,7 +304,6 @@ static __always_inline int __copy_to_user(void __user *dst, const void *src, uns } } - static __always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { int ret = 0; @@ -352,7 +352,7 @@ long strlen_user(const char __user *str); unsigned long clear_user(void __user *mem, unsigned long len); unsigned long __clear_user(void __user *mem, unsigned long len); -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +extern long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); +#define __copy_to_user_inatomic copy_user_generic #endif /* __X86_64_UACCESS_H */ -- cgit v1.2.2 From 95912008ba1fb9d0677c1ce5930aeb0e85ba5710 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:39 +0200 Subject: [PATCH] Add __must_check to copy_*_user Following i386. And also fix the two occurrences that caused warnings in arch/x86_64/* Signed-off-by: Andi Kleen --- include/asm-x86_64/i387.h | 4 ++-- include/asm-x86_64/uaccess.h | 46 ++++++++++++++++++++++++++------------------ 2 files changed, 29 insertions(+), 21 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/i387.h b/include/asm-x86_64/i387.h index 60c0f4853fdb..0217b74cc9fc 100644 --- a/include/asm-x86_64/i387.h +++ b/include/asm-x86_64/i387.h @@ -137,8 +137,8 @@ static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) #else : [fx] "cdaSDb" (fx), "0" (0)); #endif - if (unlikely(err)) - __clear_user(fx, sizeof(struct i387_fxsave_struct)); + if (unlikely(err) && __clear_user(fx, sizeof(struct i387_fxsave_struct))) + err = -EFAULT; /* No need to clear here because the caller clears USED_MATH */ return err; } diff --git a/include/asm-x86_64/uaccess.h b/include/asm-x86_64/uaccess.h index bc6812009bd7..802a4a068ef6 100644 --- a/include/asm-x86_64/uaccess.h +++ b/include/asm-x86_64/uaccess.h @@ -237,14 +237,18 @@ do { \ */ /* Handles exceptions in both to and from, but doesn't do access_ok */ -extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); -extern unsigned long copy_user_generic_dontzero(void *to, const void *from, unsigned len); - -extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); -extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); -extern unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len); - -static __always_inline int __copy_from_user(void *dst, const void __user *src, unsigned size) +__must_check unsigned long +copy_user_generic(void *to, const void *from, unsigned len); + +__must_check unsigned long +copy_to_user(void __user *to, const void *from, unsigned len); +__must_check unsigned long +copy_from_user(void *to, const void __user *from, unsigned len); +__must_check unsigned long +copy_in_user(void __user *to, const void __user *from, unsigned len); + +static __always_inline __must_check +int __copy_from_user(void *dst, const void __user *src, unsigned size) { int ret = 0; if (!__builtin_constant_p(size)) @@ -273,7 +277,8 @@ static __always_inline int __copy_from_user(void *dst, const void __user *src, u } } -static __always_inline int __copy_to_user(void __user *dst, const void *src, unsigned size) +static __always_inline __must_check +int __copy_to_user(void __user *dst, const void *src, unsigned size) { int ret = 0; if (!__builtin_constant_p(size)) @@ -304,7 +309,8 @@ static __always_inline int __copy_to_user(void __user *dst, const void *src, uns } } -static __always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size) +static __always_inline __must_check +int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { int ret = 0; if (!__builtin_constant_p(size)) @@ -344,15 +350,17 @@ static __always_inline int __copy_in_user(void __user *dst, const void __user *s } } -long strncpy_from_user(char *dst, const char __user *src, long count); -long __strncpy_from_user(char *dst, const char __user *src, long count); -long strnlen_user(const char __user *str, long n); -long __strnlen_user(const char __user *str, long n); -long strlen_user(const char __user *str); -unsigned long clear_user(void __user *mem, unsigned long len); -unsigned long __clear_user(void __user *mem, unsigned long len); - -extern long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); +__must_check long +strncpy_from_user(char *dst, const char __user *src, long count); +__must_check long +__strncpy_from_user(char *dst, const char __user *src, long count); +__must_check long strnlen_user(const char __user *str, long n); +__must_check long __strnlen_user(const char __user *str, long n); +__must_check long strlen_user(const char __user *str); +__must_check unsigned long clear_user(void __user *mem, unsigned long len); +__must_check unsigned long __clear_user(void __user *mem, unsigned long len); + +__must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); #define __copy_to_user_inatomic copy_user_generic #endif /* __X86_64_UACCESS_H */ -- cgit v1.2.2 From 383d079bfdfcfccd6a720405a510fe325b3e6576 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:40 +0200 Subject: [PATCH] Fix some stylistic issues in uaccess.h - Replace some broken white space. - Replace __ keywords with standard names No functional changes. Signed-off-by: Andi Kleen --- include/asm-x86_64/uaccess.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/uaccess.h b/include/asm-x86_64/uaccess.h index 802a4a068ef6..e856570c0689 100644 --- a/include/asm-x86_64/uaccess.h +++ b/include/asm-x86_64/uaccess.h @@ -84,7 +84,7 @@ struct exception_table_entry */ #define __get_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __get_user_" #size \ + asm volatile("call __get_user_" #size \ :"=a" (ret),"=d" (x) \ :"c" (ptr) \ :"r8") @@ -101,7 +101,7 @@ struct exception_table_entry case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ default: __get_user_bad(); break; \ } \ - (x) = (__typeof__(*(ptr)))__val_gu; \ + (x) = (typeof(*(ptr)))__val_gu; \ __ret_gu; \ }) @@ -112,7 +112,7 @@ extern void __put_user_8(void); extern void __put_user_bad(void); #define __put_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __put_user_" #size \ + asm volatile("call __put_user_" #size \ :"=a" (ret) \ :"c" (ptr),"d" (x) \ :"r8") @@ -139,7 +139,7 @@ extern void __put_user_bad(void); #define __put_user_check(x,ptr,size) \ ({ \ int __pu_err; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + typeof(*(ptr)) __user *__pu_addr = (ptr); \ switch (size) { \ case 1: __put_user_x(1,__pu_err,x,__pu_addr); break; \ case 2: __put_user_x(2,__pu_err,x,__pu_addr); break; \ @@ -173,7 +173,7 @@ struct __large_struct { unsigned long buf[100]; }; * aliasing issues. */ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - __asm__ __volatile__( \ + asm volatile( \ "1: mov"itype" %"rtype"1,%2\n" \ "2:\n" \ ".section .fixup,\"ax\"\n" \ @@ -193,7 +193,7 @@ struct __large_struct { unsigned long buf[100]; }; int __gu_err; \ unsigned long __gu_val; \ __get_user_size(__gu_val,(ptr),(size),__gu_err); \ - (x) = (__typeof__(*(ptr)))__gu_val; \ + (x) = (typeof(*(ptr)))__gu_val; \ __gu_err; \ }) @@ -217,7 +217,7 @@ do { \ } while (0) #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - __asm__ __volatile__( \ + asm volatile( \ "1: mov"itype" %2,%"rtype"1\n" \ "2:\n" \ ".section .fixup,\"ax\"\n" \ @@ -250,7 +250,7 @@ copy_in_user(void __user *to, const void __user *from, unsigned len); static __always_inline __must_check int __copy_from_user(void *dst, const void __user *src, unsigned size) { - int ret = 0; + int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic(dst,(__force void *)src,size); switch (size) { @@ -280,7 +280,7 @@ int __copy_from_user(void *dst, const void __user *src, unsigned size) static __always_inline __must_check int __copy_to_user(void __user *dst, const void *src, unsigned size) { - int ret = 0; + int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst,src,size); switch (size) { @@ -312,7 +312,7 @@ int __copy_to_user(void __user *dst, const void *src, unsigned size) static __always_inline __must_check int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { - int ret = 0; + int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst,(__force void *)src,size); switch (size) { -- cgit v1.2.2 From c1a9d41f4f103bfef2ed0bea1e95b3190e39e448 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:40 +0200 Subject: [PATCH] Reindent macros in pda.h Reindent the macros in x86-64 pda.h, making them much more readable. Follows Jeremy's i386 version of this. No functional changes Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 85 ++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 32 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index 5dadb201f769..9e3aaf74475d 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -41,46 +41,67 @@ extern struct x8664_pda boot_cpu_pda[]; */ extern void __bad_pda_field(void); -/* proxy_pda doesn't actually exist, but tell gcc it is accessed - for all PDA accesses so it gets read/write dependencies right. */ +/* + * proxy_pda doesn't actually exist, but tell gcc it is accessed for + * all PDA accesses so it gets read/write dependencies right. + */ extern struct x8664_pda _proxy_pda; #define pda_offset(field) offsetof(struct x8664_pda, field) -#define pda_to_op(op,field,val) do { \ - typedef typeof(_proxy_pda.field) T__; \ - if (0) { T__ tmp__; tmp__ = (val); } \ - switch (sizeof(_proxy_pda.field)) { \ -case 2: \ -asm(op "w %1,%%gs:%c2" : "+m" (_proxy_pda.field) : \ - "ri" ((T__)val),"i"(pda_offset(field))); break; \ -case 4: \ -asm(op "l %1,%%gs:%c2" : "+m" (_proxy_pda.field) : \ - "ri" ((T__)val),"i"(pda_offset(field))); break; \ -case 8: \ -asm(op "q %1,%%gs:%c2": "+m" (_proxy_pda.field) : \ - "ri" ((T__)val),"i"(pda_offset(field))); break; \ -default: __bad_pda_field(); \ - } \ +#define pda_to_op(op,field,val) do { \ + typedef typeof(_proxy_pda.field) T__; \ + if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \ + switch (sizeof(_proxy_pda.field)) { \ + case 2: \ + asm(op "w %1,%%gs:%c2" : \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ + case 4: \ + asm(op "l %1,%%gs:%c2" : \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i" (pda_offset(field))); \ + break; \ + case 8: \ + asm(op "q %1,%%gs:%c2": \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ + default: \ + __bad_pda_field(); \ + } \ } while (0) -#define pda_from_op(op,field) ({ \ - typeof(_proxy_pda.field) ret__; \ - switch (sizeof(_proxy_pda.field)) { \ -case 2: \ -asm(op "w %%gs:%c1,%0":"=r" (ret__):\ - "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ -case 4: \ -asm(op "l %%gs:%c1,%0":"=r" (ret__):\ - "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ -case 8: \ -asm(op "q %%gs:%c1,%0":"=r" (ret__):\ - "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\ -default: __bad_pda_field(); \ - } \ +#define pda_from_op(op,field) ({ \ + typeof(_proxy_pda.field) ret__; \ + switch (sizeof(_proxy_pda.field)) { \ + case 2: \ + asm(op "w %%gs:%c1,%0" : \ + "=r" (ret__) : \ + "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ + case 4: \ + asm(op "l %%gs:%c1,%0": \ + "=r" (ret__): \ + "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ + case 8: \ + asm(op "q %%gs:%c1,%0": \ + "=r" (ret__) : \ + "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ + default: \ + __bad_pda_field(); \ + } \ ret__; }) - #define read_pda(field) pda_from_op("mov",field) #define write_pda(field,val) pda_to_op("mov",field,val) #define add_pda(field,val) pda_to_op("add",field,val) -- cgit v1.2.2 From fd167e42b237e0688005b3dec380eb5a6e5f3585 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:40 +0200 Subject: [PATCH] Define __bad_pda_field as noreturn This quietens so warnings about uninitialized use of the return value of the pda read operations. Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index 9e3aaf74475d..531f48a6c3af 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -39,7 +39,7 @@ extern struct x8664_pda boot_cpu_pda[]; * There is no fast way to get the base address of the PDA, all the accesses * have to mention %fs/%gs. So it needs to be done this Torvaldian way. */ -extern void __bad_pda_field(void); +extern void __bad_pda_field(void) __attribute__((noreturn)); /* * proxy_pda doesn't actually exist, but tell gcc it is accessed for -- cgit v1.2.2 From 73bb5117a448bdf0b56232ca28451fe4c534cb3a Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:40 +0200 Subject: [PATCH] Remove unused asm-x86_64/mmx.h Signed-off-by: Andi Kleen --- include/asm-x86_64/mmx.h | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 include/asm-x86_64/mmx.h (limited to 'include') diff --git a/include/asm-x86_64/mmx.h b/include/asm-x86_64/mmx.h deleted file mode 100644 index 46b71da99869..000000000000 --- a/include/asm-x86_64/mmx.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _ASM_MMX_H -#define _ASM_MMX_H - -/* - * MMX 3Dnow! helper operations - */ - -#include - -extern void *_mmx_memcpy(void *to, const void *from, size_t size); -extern void mmx_clear_page(void *page); -extern void mmx_copy_page(void *to, void *from); - -#endif -- cgit v1.2.2 From a15da49debaf7f09460a886b0ecd08588410715e Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:40 +0200 Subject: [PATCH] Fix idle notifiers Previously exit_idle would be called more often than enter_idle Now instead of using complicated tests just keep track of it using the per CPU variable as a flip flop. I moved the idle state into the PDA to make the access more efficient. Original bug report and an initial patch from Stephane Eranian, but redone by AK. Cc: Stephane Eranian Signed-off-by: Andi Kleen --- include/asm-x86_64/pda.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index 531f48a6c3af..14996d962bac 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -25,7 +25,8 @@ struct x8664_pda { int nodenumber; /* number of current node */ unsigned int __softirq_pending; unsigned int __nmi_count; /* number of NMI on this CPUs */ - int mmu_state; + short mmu_state; + short isidle; struct mm_struct *active_mm; unsigned apic_timer_irqs; } ____cacheline_aligned_in_smp; -- cgit v1.2.2 From 2817716ace8c397685bd702223cc5a8a42c7e997 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Tue, 26 Sep 2006 10:52:40 +0200 Subject: [PATCH] i386: Fix pack_descriptor() Fix pack_descriptor: 1. flags are bits 20-23 in the high word 2. limit's 4 msb are bits 16-19 in the high word These haven't mattered so far, because all users have had small limits and a flags setting of 0. Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Andi Kleen =================================================================== --- include/asm-i386/desc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-i386/desc.h b/include/asm-i386/desc.h index 5db9e96e8dc1..5874ef119ffd 100644 --- a/include/asm-i386/desc.h +++ b/include/asm-i386/desc.h @@ -46,7 +46,7 @@ static inline void pack_descriptor(__u32 *a, __u32 *b, { *a = ((base & 0xffff) << 16) | (limit & 0xffff); *b = (base & 0xff000000) | ((base & 0xff0000) >> 16) | - ((type & 0xff) << 8) | ((flags & 0xf) << 12); + (limit & 0x000f0000) | ((type & 0xff) << 8) | ((flags & 0xf) << 20); } static inline void pack_gate(__u32 *a, __u32 *b, -- cgit v1.2.2 From adf1423698f00d00b267f7dca8231340ce7d65ef Mon Sep 17 00:00:00 2001 From: Jan Beulich Date: Tue, 26 Sep 2006 10:52:41 +0200 Subject: [PATCH] i386/x86-64: Work around gcc bug with noreturn functions in unwinder Current gcc generates calls not jumps to noreturn functions. When that happens the return address can point to the next function, which confuses the unwinder. This patch works around it by marking asynchronous exception frames in contrast normal call frames in the unwind information. Then teach the unwinder to decode this. For normal call frames the unwinder now subtracts one from the address which avoids this problem. The standard libgcc unwinder uses the same trick. It doesn't include adjustment of the printed address (i.e. for the original example, it'd still be kernel_math_error+0 that gets displayed, but the unwinder wouldn't get confused anymore. This only works with binutils 2.6.17+ and some versions of H.J.Lu's 2.6.16 unfortunately because earlier binutils don't support .cfi_signal_frame [AK: added automatic detection of the new binutils and wrote description] Signed-off-by: Jan Beulich Signed-off-by: Andi Kleen --- include/asm-i386/dwarf2.h | 7 +++++++ include/asm-i386/unwind.h | 5 +++++ include/asm-x86_64/dwarf2.h | 6 ++++++ include/asm-x86_64/unwind.h | 5 +++++ 4 files changed, 23 insertions(+) (limited to 'include') diff --git a/include/asm-i386/dwarf2.h b/include/asm-i386/dwarf2.h index 5d1a8db5a9b0..6d66398a307d 100644 --- a/include/asm-i386/dwarf2.h +++ b/include/asm-i386/dwarf2.h @@ -28,6 +28,12 @@ #define CFI_RESTORE_STATE .cfi_restore_state #define CFI_UNDEFINED .cfi_undefined +#ifdef CONFIG_AS_CFI_SIGNAL_FRAME +#define CFI_SIGNAL_FRAME .cfi_signal_frame +#else +#define CFI_SIGNAL_FRAME +#endif + #else /* Due to the structure of pre-exisiting code, don't use assembler line @@ -48,6 +54,7 @@ #define CFI_REMEMBER_STATE ignore #define CFI_RESTORE_STATE ignore #define CFI_UNDEFINED ignore +#define CFI_SIGNAL_FRAME ignore #endif diff --git a/include/asm-i386/unwind.h b/include/asm-i386/unwind.h index f0ac399bae3c..5031d693b89d 100644 --- a/include/asm-i386/unwind.h +++ b/include/asm-i386/unwind.h @@ -18,6 +18,7 @@ struct unwind_frame_info { struct pt_regs regs; struct task_struct *task; + unsigned call_frame:1; }; #define UNW_PC(frame) (frame)->regs.eip @@ -44,6 +45,10 @@ struct unwind_frame_info PTREGS_INFO(edi), \ PTREGS_INFO(eip) +#define UNW_DEFAULT_RA(raItem, dataAlign) \ + ((raItem).where == Memory && \ + !((raItem).value * (dataAlign) + 4)) + static inline void arch_unw_init_frame_info(struct unwind_frame_info *info, /*const*/ struct pt_regs *regs) { diff --git a/include/asm-x86_64/dwarf2.h b/include/asm-x86_64/dwarf2.h index 2b9368365fad..eedc08526b0b 100644 --- a/include/asm-x86_64/dwarf2.h +++ b/include/asm-x86_64/dwarf2.h @@ -28,6 +28,11 @@ #define CFI_REMEMBER_STATE .cfi_remember_state #define CFI_RESTORE_STATE .cfi_restore_state #define CFI_UNDEFINED .cfi_undefined +#ifdef CONFIG_AS_CFI_SIGNAL_FRAME +#define CFI_SIGNAL_FRAME .cfi_signal_frame +#else +#define CFI_SIGNAL_FRAME +#endif #else @@ -45,6 +50,7 @@ #define CFI_REMEMBER_STATE # #define CFI_RESTORE_STATE # #define CFI_UNDEFINED # +#define CFI_SIGNAL_FRAME # #endif diff --git a/include/asm-x86_64/unwind.h b/include/asm-x86_64/unwind.h index 1f6e9bfb569e..b8fa5cb7ff88 100644 --- a/include/asm-x86_64/unwind.h +++ b/include/asm-x86_64/unwind.h @@ -18,6 +18,7 @@ struct unwind_frame_info { struct pt_regs regs; struct task_struct *task; + unsigned call_frame:1; }; #define UNW_PC(frame) (frame)->regs.rip @@ -57,6 +58,10 @@ struct unwind_frame_info PTREGS_INFO(r15), \ PTREGS_INFO(rip) +#define UNW_DEFAULT_RA(raItem, dataAlign) \ + ((raItem).where == Memory && \ + !((raItem).value * (dataAlign) + 8)) + static inline void arch_unw_init_frame_info(struct unwind_frame_info *info, /*const*/ struct pt_regs *regs) { -- cgit v1.2.2 From 658fdbef66e5e9be79b457edc2cbbb3add840aa9 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:41 +0200 Subject: [PATCH] Don't leak NT bit into next task SYSENTER can cause a NT to be set which might cause crashes on the IRET in the next task. Following similar i386 patch from Linus. Signed-off-by: Andi Kleen --- include/asm-x86_64/system.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/system.h b/include/asm-x86_64/system.h index 6bf170bceae1..bd376bc8c4ab 100644 --- a/include/asm-x86_64/system.h +++ b/include/asm-x86_64/system.h @@ -14,12 +14,13 @@ #define __RESTORE(reg,offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t" /* frame pointer must be last for get_wchan */ -#define SAVE_CONTEXT "pushq %%rbp ; movq %%rsi,%%rbp\n\t" -#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp\n\t" +#define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t" +#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t" #define __EXTRA_CLOBBER \ ,"rcx","rbx","rdx","r8","r9","r10","r11","r12","r13","r14","r15" +/* Save restore flags to clear handle leaking NT */ #define switch_to(prev,next,last) \ asm volatile(SAVE_CONTEXT \ "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \ -- cgit v1.2.2 From 8f60774a116ced9b73ae3913d511687889efe725 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:41 +0200 Subject: [PATCH] x86: Move direct PCI scanning functions out of line Saves about 200 bytes of code space. Signed-off-by: Andi Kleen --- include/asm-x86_64/pci-direct.h | 42 ++++------------------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/pci-direct.h b/include/asm-x86_64/pci-direct.h index 036b6ca5b53b..9d916cdaa18e 100644 --- a/include/asm-x86_64/pci-direct.h +++ b/include/asm-x86_64/pci-direct.h @@ -2,47 +2,13 @@ #define ASM_PCI_DIRECT_H 1 #include -#include /* Direct PCI access. This is used for PCI accesses in early boot before the PCI subsystem works. */ -#define PDprintk(x...) - -static inline u32 read_pci_config(u8 bus, u8 slot, u8 func, u8 offset) -{ - u32 v; - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - v = inl(0xcfc); - if (v != 0xffffffff) - PDprintk("%x reading 4 from %x: %x\n", slot, offset, v); - return v; -} - -static inline u8 read_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset) -{ - u8 v; - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - v = inb(0xcfc + (offset&3)); - PDprintk("%x reading 1 from %x: %x\n", slot, offset, v); - return v; -} - -static inline u16 read_pci_config_16(u8 bus, u8 slot, u8 func, u8 offset) -{ - u16 v; - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - v = inw(0xcfc + (offset&2)); - PDprintk("%x reading 2 from %x: %x\n", slot, offset, v); - return v; -} - -static inline void write_pci_config(u8 bus, u8 slot, u8 func, u8 offset, - u32 val) -{ - PDprintk("%x writing to %x: %x\n", slot, offset, val); - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - outl(val, 0xcfc); -} +extern u32 read_pci_config(u8 bus, u8 slot, u8 func, u8 offset); +extern u8 read_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset); +extern u16 read_pci_config_16(u8 bus, u8 slot, u8 func, u8 offset); +extern void write_pci_config(u8 bus, u8 slot, u8 func, u8 offset, u32 val); #endif -- cgit v1.2.2 From 0637a70a5db98182d9ad3d6ae1ee30acf20afde9 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:41 +0200 Subject: [PATCH] x86: Allow disabling early pci scans with pci=noearly or disallowing conf1 Some buggy systems can machine check when config space accesses happen for some non existent devices. i386/x86-64 do some early device scans that might trigger this. Allow pci=noearly to disable this. Also when type 1 is disabling also don't do any early accesses which are always type1. This moves the pci= configuration parsing to be a early parameter. I don't think this can break anything because it only changes a single global that is only used by PCI. Cc: gregkh@suse.de Cc: Trammell Hudson Signed-off-by: Andi Kleen --- include/asm-x86_64/pci-direct.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/asm-x86_64/pci-direct.h b/include/asm-x86_64/pci-direct.h index 9d916cdaa18e..eba9cb471df3 100644 --- a/include/asm-x86_64/pci-direct.h +++ b/include/asm-x86_64/pci-direct.h @@ -11,4 +11,6 @@ extern u8 read_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset); extern u16 read_pci_config_16(u8 bus, u8 slot, u8 func, u8 offset); extern void write_pci_config(u8 bus, u8 slot, u8 func, u8 offset, u32 val); +extern int early_pci_allowed(void); + #endif -- cgit v1.2.2 From b89ebd0b0a65d5371aa9ad98e873c4616056ca68 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:42 +0200 Subject: [PATCH] Fix unwinder warning in traps.c Fix linux/arch/x86_64/kernel/traps.c: In function 'dump_trace': linux/arch/x86_64/kernel/traps.c:275: warning: cast to pointer from integer of different size with allnoconfig Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- include/asm-x86_64/unwind.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-x86_64/unwind.h b/include/asm-x86_64/unwind.h index b8fa5cb7ff88..2e7ff10fd775 100644 --- a/include/asm-x86_64/unwind.h +++ b/include/asm-x86_64/unwind.h @@ -99,8 +99,8 @@ static inline int arch_unw_user_mode(const struct unwind_frame_info *info) #else -#define UNW_PC(frame) ((void)(frame), 0) -#define UNW_SP(frame) ((void)(frame), 0) +#define UNW_PC(frame) ((void)(frame), 0UL) +#define UNW_SP(frame) ((void)(frame), 0UL) static inline int arch_unw_user_mode(const void *info) { -- cgit v1.2.2 From 3b171672831b9633c2ed8fa94805255cd4d5af19 Mon Sep 17 00:00:00 2001 From: Dmitriy Zavin Date: Tue, 26 Sep 2006 10:52:42 +0200 Subject: [PATCH] Add 64bit jiffies compares (for use with get_jiffies_64) The current time_before/time_after macros will fail typechecks when passed u64 values (as returned by get_jiffies_64()). On 64bit systems, this will just result in a warning about mismatching types without explicit casts, but since unsigned long and u64 (unsigned long long) are of same size, it will still work. On 32bit systems, a long is 32bits, so the value from get_jiffies_64() will be truncated by the cast and thus lose all the precision gained by 64bit jiffies. Signed-off-by: Dmitriy Zavin Signed-off-by: Andi Kleen --- include/linux/jiffies.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index 329ebcffa106..c8d5f207c3d4 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -115,6 +115,21 @@ static inline u64 get_jiffies_64(void) ((long)(a) - (long)(b) >= 0)) #define time_before_eq(a,b) time_after_eq(b,a) +/* Same as above, but does so with platform independent 64bit types. + * These must be used when utilizing jiffies_64 (i.e. return value of + * get_jiffies_64() */ +#define time_after64(a,b) \ + (typecheck(__u64, a) && \ + typecheck(__u64, b) && \ + ((__s64)(b) - (__s64)(a) < 0)) +#define time_before64(a,b) time_after64(b,a) + +#define time_after_eq64(a,b) \ + (typecheck(__u64, a) && \ + typecheck(__u64, b) && \ + ((__s64)(a) - (__s64)(b) >= 0)) +#define time_before_eq64(a,b) time_after_eq64(b,a) + /* * Have the 32 bit jiffies value wrap 5 minutes after boot * so jiffies wrap bugs show up earlier. -- cgit v1.2.2 From 15d5f8398311f565682959daaca30e3ca7aea600 Mon Sep 17 00:00:00 2001 From: Dmitriy Zavin Date: Tue, 26 Sep 2006 10:52:42 +0200 Subject: [PATCH] x86: Refactor thermal throttle processing Refactor the event processing (syslog messaging and rate limiting) into separate file therm_throt.c. This allows consistent reporting of CPU thermal throttle events. After ACK'ing the interrupt, if the event is current, the user (p4.c/mce_intel.c) calls therm_throt_process to log (and rate limit) the event. If that function returns 1, the user has the option to log things further (such as to mce_log in x86_64). AK: minor cleanup Signed-off-by: Dmitriy Zavin Signed-off-by: Andi Kleen --- include/asm-i386/therm_throt.h | 6 ++++++ include/asm-x86_64/mce.h | 2 ++ include/asm-x86_64/therm_throt.h | 1 + 3 files changed, 9 insertions(+) create mode 100644 include/asm-i386/therm_throt.h create mode 100644 include/asm-x86_64/therm_throt.h (limited to 'include') diff --git a/include/asm-i386/therm_throt.h b/include/asm-i386/therm_throt.h new file mode 100644 index 000000000000..3c9c22cc10c9 --- /dev/null +++ b/include/asm-i386/therm_throt.h @@ -0,0 +1,6 @@ +#ifndef __ASM_I386_THERM_THROT_H__ +#define __ASM_I386_THERM_THROT_H__ 1 + +int therm_throt_process(int curr); + +#endif /* __ASM_I386_THERM_THROT_H__ */ diff --git a/include/asm-x86_64/mce.h b/include/asm-x86_64/mce.h index d13687dfd691..5a11146d6d9c 100644 --- a/include/asm-x86_64/mce.h +++ b/include/asm-x86_64/mce.h @@ -99,6 +99,8 @@ static inline void mce_amd_feature_init(struct cpuinfo_x86 *c) } #endif +void mce_log_therm_throt_event(unsigned int cpu, __u64 status); + extern atomic_t mce_entry; #endif diff --git a/include/asm-x86_64/therm_throt.h b/include/asm-x86_64/therm_throt.h new file mode 100644 index 000000000000..5aac059007ba --- /dev/null +++ b/include/asm-x86_64/therm_throt.h @@ -0,0 +1 @@ +#include -- cgit v1.2.2 From 3222b36f46c22f46697a0a53fa8804153a32669f Mon Sep 17 00:00:00 2001 From: Dmitriy Zavin Date: Tue, 26 Sep 2006 10:52:42 +0200 Subject: [PATCH] x86: Add a cumulative thermal throttle event counter. The counter is exported to /sys that keeps track of the number of thermal events, such that the user knows how bad the thermal problem might be (since the logging to syslog and mcelog is rate limited). AK: Fixed cpu hotplug locking Signed-off-by: Dmitriy Zavin Signed-off-by: Andi Kleen --- include/asm-i386/therm_throt.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/asm-i386/therm_throt.h b/include/asm-i386/therm_throt.h index 3c9c22cc10c9..399bf6026b16 100644 --- a/include/asm-i386/therm_throt.h +++ b/include/asm-i386/therm_throt.h @@ -1,6 +1,9 @@ #ifndef __ASM_I386_THERM_THROT_H__ #define __ASM_I386_THERM_THROT_H__ 1 +#include + +extern atomic_t therm_throt_en; int therm_throt_process(int curr); #endif /* __ASM_I386_THERM_THROT_H__ */ -- cgit v1.2.2