Merge branch 'core/percpu' into x86/paravirt

author: H. Peter Anvin <hpa@linux.intel.com> 2009-01-30 17:50:57 -0500
committer: H. Peter Anvin <hpa@linux.intel.com> 2009-01-30 17:50:57 -0500
commit: 9b7ed8faa034fc2d350e2eff5c68680eb5c43a07 (patch)
tree: 5c94c34ad30e312604c1ce4f08ab6631b64a94f5 /arch/x86
parent: 6522869c34664dd5f05a0a327e93915b1281c90d (diff)
parent: c43e0e46adf79c321ed3fbf0351e1005fb8a2413 (diff)
124 files changed, 2007 insertions, 2276 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 73f7fe8fd4d1..d6218e6c9824 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -133,7 +133,7 @@ config ARCH_HAS_CACHE_LINE_SIZE
        def_bool y
 config HAVE_SETUP_PER_CPU_AREA
-        def_bool X86_64_SMP || (X86_SMP && !X86_VOYAGER)
+        def_bool y
 config HAVE_CPUMASK_OF_CPU_MAP
        def_bool X86_64_SMP
@@ -391,6 +391,13 @@ config X86_RDC321X
          as R-8610-(G).
          If you don't have one of these chips, you should say N here.
+config X86_UV
+        bool "SGI Ultraviolet"
+        depends on X86_64
+        help
+          This option is needed in order to support SGI Ultraviolet systems.
+          If you don't have one of these, you should say N here.
 config SCHED_OMIT_FRAME_POINTER
        def_bool y
        prompt "Single-depth WCHAN output"
@@ -1340,13 +1347,17 @@ config SECCOMP
          If unsure, say Y. Only embedded should say N here.
+config CC_STACKPROTECTOR_ALL
+        bool
 config CC_STACKPROTECTOR
        bool "Enable -fstack-protector buffer overflow detection (EXPERIMENTAL)"
-        depends on X86_64 && EXPERIMENTAL && BROKEN
+        depends on X86_64
+        select CC_STACKPROTECTOR_ALL
        help
-         This option turns on the -fstack-protector GCC feature. This
+          This option turns on the -fstack-protector GCC feature. This
-          feature puts, at the beginning of critical functions, a canary
+          feature puts, at the beginning of functions, a canary value on
-          value on the stack just before the return address, and validates
+          the stack just before the return address, and validates
          the value just before actually returning.  Stack based buffer
          overflows (that need to overwrite this return address) now also
          overwrite the canary, which gets detected and the attack is then
@@ -1354,15 +1365,8 @@ config CC_STACKPROTECTOR
          This feature requires gcc version 4.2 or above, or a distribution
          gcc with the feature backported. Older versions are automatically
-          detected and for those versions, this configuration option is ignored.
+          detected and for those versions, this configuration option is
+          ignored. (and a warning is printed during bootup)
-config CC_STACKPROTECTOR_ALL
-        bool "Use stack-protector for all functions"
-        depends on CC_STACKPROTECTOR
-        help
-          Normally, GCC only inserts the canary value protection for
-          functions that use large-ish on-stack buffers. By enabling
-          this option, GCC will be asked to do this for ALL functions.
 source kernel/Kconfig.hz
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index 8078955845ae..8eb50ba9161e 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -292,25 +292,23 @@ config X86_CPU
 # Define implied options from the CPU selection here
 config X86_L1_CACHE_BYTES
        int
-        default "128" if GENERIC_CPU || MPSC
+        default "128" if MPSC
-        default "64" if MK8 || MCORE2
+        default "64" if GENERIC_CPU || MK8 || MCORE2 || X86_32
-        depends on X86_64
 config X86_INTERNODE_CACHE_BYTES
        int
        default "4096" if X86_VSMP
        default X86_L1_CACHE_BYTES if !X86_VSMP
-        depends on X86_64
 config X86_CMPXCHG
        def_bool X86_64 || (X86_32 && !M386)
 config X86_L1_CACHE_SHIFT
        int
-        default "7" if MPENTIUM4 || X86_GENERIC || GENERIC_CPU || MPSC
+        default "7" if MPENTIUM4 || MPSC
        default "4" if X86_ELAN || M486 || M386 || MGEODEGX1
        default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
-        default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MVIAC7
+        default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MVIAC7 || X86_GENERIC || GENERIC_CPU
 config X86_XADD
        def_bool y
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 10d6cc3fd052..28f111461ca8 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -117,6 +117,7 @@ config DEBUG_RODATA
 config DEBUG_RODATA_TEST
        bool "Testcase for the DEBUG_RODATA feature"
        depends on DEBUG_RODATA
+        default y
        help
          This option enables a testcase for the DEBUG_RODATA
          feature as well as for the change_page_attr() infrastructure.
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index d1a47adb5aec..cacee981d166 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -73,7 +73,7 @@ else
        stackp := $(CONFIG_SHELL) $(srctree)/scripts/gcc-x86_64-has-stack-protector.sh
        stackp-$(CONFIG_CC_STACKPROTECTOR) := $(shell $(stackp) \
-                "$(CC)" -fstack-protector )
+                "$(CC)" "-fstack-protector -DGCC_HAS_SP" )
        stackp-$(CONFIG_CC_STACKPROTECTOR_ALL) += $(shell $(stackp) \
                "$(CC)" -fstack-protector-all )
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index b30a08ed8eb4..edba00d98ac3 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -1331,8 +1331,8 @@ CONFIG_I2C_I801=y
 # Miscellaneous I2C Chip support
 #
 # CONFIG_DS1682 is not set
-# CONFIG_AT24 is not set
+# CONFIG_EEPROM_AT24 is not set
-# CONFIG_SENSORS_EEPROM is not set
+# CONFIG_EEPROM_LEGACY is not set
 # CONFIG_SENSORS_PCF8574 is not set
 # CONFIG_PCF8575 is not set
 # CONFIG_SENSORS_PCA9539 is not set
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index 0e7dbc0a3e46..322dd2748fc9 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -1311,8 +1311,8 @@ CONFIG_I2C_I801=y
 # Miscellaneous I2C Chip support
 #
 # CONFIG_DS1682 is not set
-# CONFIG_AT24 is not set
+# CONFIG_EEPROM_AT24 is not set
-# CONFIG_SENSORS_EEPROM is not set
+# CONFIG_EEPROM_LEGACY is not set
 # CONFIG_SENSORS_PCF8574 is not set
 # CONFIG_PCF8575 is not set
 # CONFIG_SENSORS_PCA9539 is not set
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 256b00b61892..9c79b2477008 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -112,8 +112,8 @@ ENTRY(ia32_sysenter_target)
        CFI_DEF_CFA     rsp,0
        CFI_REGISTER    rsp,rbp
        SWAPGS_UNSAFE_STACK
-        movq    %gs:pda_kernelstack, %rsp
+        movq    PER_CPU_VAR(kernel_stack), %rsp
-        addq    $(PDA_STACKOFFSET),%rsp 
+        addq    $(KERNEL_STACK_OFFSET),%rsp
        /*
         * No need to follow this irqs on/off section: the syscall
         * disabled irqs, here we enable it straight after entry:
@@ -273,13 +273,13 @@ ENDPROC(ia32_sysenter_target)
 ENTRY(ia32_cstar_target)
        CFI_STARTPROC32 simple
        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
+        CFI_DEF_CFA     rsp,KERNEL_STACK_OFFSET
        CFI_REGISTER    rip,rcx
        /*CFI_REGISTER  rflags,r11*/
        SWAPGS_UNSAFE_STACK
        movl    %esp,%r8d
        CFI_REGISTER    rsp,r8
-        movq    %gs:pda_kernelstack,%rsp
+        movq    PER_CPU_VAR(kernel_stack),%rsp
        /*
         * No need to follow this irqs on/off section: the syscall
         * disabled irqs and here we enable it straight after entry:
diff --git a/arch/x86/include/asm/apicnum.h b/arch/x86/include/asm/apicnum.h
new file mode 100644
index 000000000000..82f613c607ce
--- /dev/null
+++ b/arch/x86/include/asm/apicnum.h
@@ -0,0 +1,12 @@
+#ifndef _ASM_X86_APICNUM_H
+#define _ASM_X86_APICNUM_H
+/* define MAX_IO_APICS */
+#ifdef CONFIG_X86_32
+# define MAX_IO_APICS 64
+#else
+# define MAX_IO_APICS 128
+# define MAX_LOCAL_APIC 32768
+#endif
+#endif /* _ASM_X86_APICNUM_H */
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index e02a359d2aa5..02b47a603fc8 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -3,6 +3,9 @@
 /*
 * Copyright 1992, Linus Torvalds.
+ *
+ * Note: inlines with more than a single statement should be marked
+ * __always_inline to avoid problems with older gcc's inlining heuristics.
 */
 #ifndef _LINUX_BITOPS_H
@@ -53,7 +56,8 @@
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
-static inline void set_bit(unsigned int nr, volatile unsigned long *addr)
+static __always_inline void
+set_bit(unsigned int nr, volatile unsigned long *addr)
 {
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "orb %1,%0"
@@ -90,7 +94,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr)
 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
 * in order to ensure changes are visible on other processors.
 */
-static inline void clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+clear_bit(int nr, volatile unsigned long *addr)
 {
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "andb %1,%0"
@@ -204,7 +209,8 @@ static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
 *
 * This is the same as test_and_set_bit on x86.
 */
-static inline int test_and_set_bit_lock(int nr, volatile unsigned long *addr)
+static __always_inline int
+test_and_set_bit_lock(int nr, volatile unsigned long *addr)
 {
        return test_and_set_bit(nr, addr);
 }
@@ -300,7 +306,7 @@ static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
        return oldbit;
 }
-static inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
+static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
 {
        return ((1UL << (nr % BITS_PER_LONG)) &
                (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h
index bae482df6039..f03b23e32864 100644
--- a/arch/x86/include/asm/cpu.h
+++ b/arch/x86/include/asm/cpu.h
@@ -7,6 +7,20 @@
 #include <linux/nodemask.h>
 #include <linux/percpu.h>
+#ifdef CONFIG_SMP
+extern void prefill_possible_map(void);
+#else /* CONFIG_SMP */
+static inline void prefill_possible_map(void) {}
+#define cpu_physical_id(cpu)                    boot_cpu_physical_apicid
+#define safe_smp_processor_id()                 0
+#define stack_smp_processor_id()                0
+#endif /* CONFIG_SMP */
 struct x86_cpu {
        struct cpu cpu;
 };
@@ -17,4 +31,11 @@ extern void arch_unregister_cpu(int);
 #endif
 DECLARE_PER_CPU(int, cpu_state);
+#ifdef CONFIG_X86_HAS_BOOT_CPU_ID
+extern unsigned char boot_cpu_id;
+#else
+#define boot_cpu_id                             0
+#endif
 #endif /* _ASM_X86_CPU_H */
diff --git a/arch/x86/include/asm/cpumask.h b/arch/x86/include/asm/cpumask.h
new file mode 100644
index 000000000000..a7f3c75f8ad7
--- /dev/null
+++ b/arch/x86/include/asm/cpumask.h
@@ -0,0 +1,32 @@
+#ifndef _ASM_X86_CPUMASK_H
+#define _ASM_X86_CPUMASK_H
+#ifndef __ASSEMBLY__
+#include <linux/cpumask.h>
+#ifdef CONFIG_X86_64
+extern cpumask_var_t cpu_callin_mask;
+extern cpumask_var_t cpu_callout_mask;
+extern cpumask_var_t cpu_initialized_mask;
+extern cpumask_var_t cpu_sibling_setup_mask;
+extern void setup_cpu_local_masks(void);
+#else /* CONFIG_X86_32 */
+extern cpumask_t cpu_callin_map;
+extern cpumask_t cpu_callout_map;
+extern cpumask_t cpu_initialized;
+extern cpumask_t cpu_sibling_setup_map;
+#define cpu_callin_mask         ((struct cpumask *)&cpu_callin_map)
+#define cpu_callout_mask        ((struct cpumask *)&cpu_callout_map)
+#define cpu_initialized_mask    ((struct cpumask *)&cpu_initialized)
+#define cpu_sibling_setup_mask  ((struct cpumask *)&cpu_sibling_setup_map)
+static inline void setup_cpu_local_masks(void) { }
+#endif /* CONFIG_X86_32 */
+#endif /* __ASSEMBLY__ */
+#endif /* _ASM_X86_CPUMASK_H */
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 0930b4f8d672..c68c361697e1 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -1,39 +1,21 @@
 #ifndef _ASM_X86_CURRENT_H
 #define _ASM_X86_CURRENT_H
-#ifdef CONFIG_X86_32
 #include <linux/compiler.h>
 #include <asm/percpu.h>
+#ifndef __ASSEMBLY__
 struct task_struct;
 DECLARE_PER_CPU(struct task_struct *, current_task);
-static __always_inline struct task_struct *get_current(void)
-{
-        return x86_read_percpu(current_task);
-}
-#else /* X86_32 */
-#ifndef __ASSEMBLY__
-#include <asm/pda.h>
-struct task_struct;
 static __always_inline struct task_struct *get_current(void)
 {
-        return read_pda(pcurrent);
+        return percpu_read(current_task);
 }
-#else /* __ASSEMBLY__ */
+#define current get_current()
-#include <asm/asm-offsets.h>
-#define GET_CURRENT(reg) movq %gs:(pda_pcurrent),reg
 #endif /* __ASSEMBLY__ */
-#endif /* X86_32 */
-#define current get_current()
 #endif /* _ASM_X86_CURRENT_H */
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index 4035357f5b9d..132a134d12f2 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -2,8 +2,8 @@
 #define _ASM_X86_DMA_MAPPING_H
 /*
- * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
+ * IOMMU interface. See Documentation/PCI/PCI-DMA-mapping.txt and
- * documentation.
+ * Documentation/DMA-API.txt for documentation.
 */
 #include <linux/scatterlist.h>
diff --git a/arch/x86/include/asm/genapic_32.h b/arch/x86/include/asm/genapic_32.h
index 2c05b737ee22..4334502d3664 100644
--- a/arch/x86/include/asm/genapic_32.h
+++ b/arch/x86/include/asm/genapic_32.h
@@ -138,11 +138,4 @@ struct genapic {
 extern struct genapic *genapic;
 extern void es7000_update_genapic_to_cluster(void);
-enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
-#define get_uv_system_type()            UV_NONE
-#define is_uv_system()                  0
-#define uv_wakeup_secondary(a, b)       1
-#define uv_system_init()                do {} while (0)
 #endif /* _ASM_X86_GENAPIC_32_H */
diff --git a/arch/x86/include/asm/genapic_64.h b/arch/x86/include/asm/genapic_64.h
index adf32fb56aa6..7bb092c59055 100644
--- a/arch/x86/include/asm/genapic_64.h
+++ b/arch/x86/include/asm/genapic_64.h
@@ -51,15 +51,9 @@ extern struct genapic apic_x2apic_phys;
 extern int acpi_madt_oem_check(char *, char *);
 extern void apic_send_IPI_self(int vector);
-enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
-extern enum uv_system_type get_uv_system_type(void);
-extern int is_uv_system(void);
 extern struct genapic apic_x2apic_uv_x;
 DECLARE_PER_CPU(int, x2apic_extra_bits);
-extern void uv_cpu_init(void);
-extern void uv_system_init(void);
-extern int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip);
 extern void setup_apic_routing(void);
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 000787df66e6..176f058e7159 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -1,11 +1,52 @@
-#ifdef CONFIG_X86_32
+#ifndef _ASM_X86_HARDIRQ_H
-# include "hardirq_32.h"
+#define _ASM_X86_HARDIRQ_H
-#else
-# include "hardirq_64.h"
+#include <linux/threads.h>
+#include <linux/irq.h>
+typedef struct {
+        unsigned int __softirq_pending;
+        unsigned int __nmi_count;       /* arch dependent */
+        unsigned int irq0_irqs;
+#ifdef CONFIG_X86_LOCAL_APIC
+        unsigned int apic_timer_irqs;   /* arch dependent */
+        unsigned int irq_spurious_count;
+#endif
+#ifdef CONFIG_SMP
+        unsigned int irq_resched_count;
+        unsigned int irq_call_count;
+        unsigned int irq_tlb_count;
+#endif
+#ifdef CONFIG_X86_MCE
+        unsigned int irq_thermal_count;
+# ifdef CONFIG_X86_64
+        unsigned int irq_threshold_count;
+# endif
 #endif
+} ____cacheline_aligned irq_cpustat_t;
+DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
+/* We can have at most NR_VECTORS irqs routed to a cpu at a time */
+#define MAX_HARDIRQS_PER_CPU NR_VECTORS
+#define __ARCH_IRQ_STAT
+#define inc_irq_stat(member)    percpu_add(irq_stat.member, 1)
+#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define __ARCH_SET_SOFTIRQ_PENDING
+#define set_softirq_pending(x)  percpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x)   percpu_or(irq_stat.__softirq_pending, (x))
+extern void ack_bad_irq(unsigned int irq);
 extern u64 arch_irq_stat_cpu(unsigned int cpu);
 #define arch_irq_stat_cpu       arch_irq_stat_cpu
 extern u64 arch_irq_stat(void);
 #define arch_irq_stat           arch_irq_stat
+#endif /* _ASM_X86_HARDIRQ_H */
diff --git a/arch/x86/include/asm/hardirq_32.h b/arch/x86/include/asm/hardirq_32.h
deleted file mode 100644
index cf7954d1405f..000000000000
--- a/arch/x86/include/asm/hardirq_32.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef _ASM_X86_HARDIRQ_32_H
-#define _ASM_X86_HARDIRQ_32_H
-#include <linux/threads.h>
-#include <linux/irq.h>
-typedef struct {
-        unsigned int __softirq_pending;
-        unsigned long idle_timestamp;
-        unsigned int __nmi_count;       /* arch dependent */
-        unsigned int apic_timer_irqs;   /* arch dependent */
-        unsigned int irq0_irqs;
-        unsigned int irq_resched_count;
-        unsigned int irq_call_count;
-        unsigned int irq_tlb_count;
-        unsigned int irq_thermal_count;
-        unsigned int irq_spurious_count;
-} ____cacheline_aligned irq_cpustat_t;
-DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
-#define __ARCH_IRQ_STAT
-#define __IRQ_STAT(cpu, member) (per_cpu(irq_stat, cpu).member)
-#define inc_irq_stat(member)    (__get_cpu_var(irq_stat).member++)
-void ack_bad_irq(unsigned int irq);
-#include <linux/irq_cpustat.h>
-#endif /* _ASM_X86_HARDIRQ_32_H */
diff --git a/arch/x86/include/asm/hardirq_64.h b/arch/x86/include/asm/hardirq_64.h
deleted file mode 100644
index b5a6b5d56704..000000000000
--- a/arch/x86/include/asm/hardirq_64.h
+++ /dev/null
@@ -1,25 +0,0 @@
-#ifndef _ASM_X86_HARDIRQ_64_H
-#define _ASM_X86_HARDIRQ_64_H
-#include <linux/threads.h>
-#include <linux/irq.h>
-#include <asm/pda.h>
-#include <asm/apic.h>
-/* We can have at most NR_VECTORS irqs routed to a cpu at a time */
-#define MAX_HARDIRQS_PER_CPU NR_VECTORS
-#define __ARCH_IRQ_STAT 1
-#define inc_irq_stat(member)    add_pda(member, 1)
-#define local_softirq_pending() read_pda(__softirq_pending)
-#define __ARCH_SET_SOFTIRQ_PENDING 1
-#define set_softirq_pending(x) write_pda(__softirq_pending, (x))
-#define or_softirq_pending(x)  or_pda(__softirq_pending, (x))
-extern void ack_bad_irq(unsigned int irq);
-#endif /* _ASM_X86_HARDIRQ_64_H */
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 05cfed4485fa..1dbbdf4be9b4 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -99,7 +99,6 @@ extern void __iomem *ioremap_wc(unsigned long offset, unsigned long size);
 * A boot-time mapping is currently limited to at most 16 pages.
 */
 extern void early_ioremap_init(void);
-extern void early_ioremap_clear(void);
 extern void early_ioremap_reset(void);
 extern void __iomem *early_ioremap(unsigned long offset, unsigned long size);
 extern void __iomem *early_memremap(unsigned long offset, unsigned long size);
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 7a1f44ac1f17..08ec793aa043 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -114,38 +114,16 @@ struct IR_IO_APIC_route_entry {
 extern int nr_ioapics;
 extern int nr_ioapic_registers[MAX_IO_APICS];
-/*
- * MP-BIOS irq configuration table structures:
- */
 #define MP_MAX_IOAPIC_PIN 127
-struct mp_config_ioapic {
-        unsigned long mp_apicaddr;
-        unsigned int mp_apicid;
-        unsigned char mp_type;
-        unsigned char mp_apicver;
-        unsigned char mp_flags;
-};
-struct mp_config_intsrc {
-        unsigned int mp_dstapic;
-        unsigned char mp_type;
-        unsigned char mp_irqtype;
-        unsigned short mp_irqflag;
-        unsigned char mp_srcbus;
-        unsigned char mp_srcbusirq;
-        unsigned char mp_dstirq;
-};
 /* I/O APIC entries */
-extern struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
+extern struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
 /* # of MP IRQ source entries */
 extern int mp_irq_entries;
 /* MP IRQ source entries */
-extern struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
+extern struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
 /* non-0 if default (table-less) MP configuration */
 extern int mpc_default_type;
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 89c898ab298b..77843225b7ea 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -1,5 +1,31 @@
-#ifdef CONFIG_X86_32
+/*
-# include "irq_regs_32.h"
+ * Per-cpu current frame pointer - the location of the last exception frame on
-#else
+ * the stack, stored in the per-cpu area.
-# include "irq_regs_64.h"
+ *
-#endif
+ * Jeremy Fitzhardinge <jeremy@goop.org>
+ */
+#ifndef _ASM_X86_IRQ_REGS_H
+#define _ASM_X86_IRQ_REGS_H
+#include <asm/percpu.h>
+#define ARCH_HAS_OWN_IRQ_REGS
+DECLARE_PER_CPU(struct pt_regs *, irq_regs);
+static inline struct pt_regs *get_irq_regs(void)
+{
+        return percpu_read(irq_regs);
+}
+static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
+{
+        struct pt_regs *old_regs;
+        old_regs = get_irq_regs();
+        percpu_write(irq_regs, new_regs);
+        return old_regs;
+}
+#endif /* _ASM_X86_IRQ_REGS_32_H */
diff --git a/arch/x86/include/asm/irq_regs_32.h b/arch/x86/include/asm/irq_regs_32.h
deleted file mode 100644
index 86afd7473457..000000000000
--- a/arch/x86/include/asm/irq_regs_32.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Per-cpu current frame pointer - the location of the last exception frame on
- * the stack, stored in the per-cpu area.
- *
- * Jeremy Fitzhardinge <jeremy@goop.org>
- */
-#ifndef _ASM_X86_IRQ_REGS_32_H
-#define _ASM_X86_IRQ_REGS_32_H
-#include <asm/percpu.h>
-#define ARCH_HAS_OWN_IRQ_REGS
-DECLARE_PER_CPU(struct pt_regs *, irq_regs);
-static inline struct pt_regs *get_irq_regs(void)
-{
-        return x86_read_percpu(irq_regs);
-}
-static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
-{
-        struct pt_regs *old_regs;
-        old_regs = get_irq_regs();
-        x86_write_percpu(irq_regs, new_regs);
-        return old_regs;
-}
-#endif /* _ASM_X86_IRQ_REGS_32_H */
diff --git a/arch/x86/include/asm/irq_regs_64.h b/arch/x86/include/asm/irq_regs_64.h
deleted file mode 100644
index 3dd9c0b70270..000000000000
--- a/arch/x86/include/asm/irq_regs_64.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <asm-generic/irq_regs.h>
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index f7ff65032b9d..9a83a10a5d51 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -49,31 +49,33 @@
 *  some of the following vectors are 'rare', they are merged
 *  into a single vector (CALL_FUNCTION_VECTOR) to save vector space.
 *  TLB, reschedule and local APIC vectors are performance-critical.
- *
- *  Vectors 0xf0-0xfa are free (reserved for future Linux use).
 */
 #ifdef CONFIG_X86_32
 # define SPURIOUS_APIC_VECTOR           0xff
 # define ERROR_APIC_VECTOR              0xfe
-# define INVALIDATE_TLB_VECTOR          0xfd
+# define RESCHEDULE_VECTOR              0xfd
-# define RESCHEDULE_VECTOR              0xfc
+# define CALL_FUNCTION_VECTOR           0xfc
-# define CALL_FUNCTION_VECTOR           0xfb
+# define CALL_FUNCTION_SINGLE_VECTOR    0xfb
-# define CALL_FUNCTION_SINGLE_VECTOR    0xfa
+# define THERMAL_APIC_VECTOR            0xfa
-# define THERMAL_APIC_VECTOR            0xf0
+/* 0xf8 - 0xf9 : free */
+# define INVALIDATE_TLB_VECTOR_END      0xf7
+# define INVALIDATE_TLB_VECTOR_START    0xf0    /* f0-f7 used for TLB flush */
+# define NUM_INVALIDATE_TLB_VECTORS     8
 #else
-#define SPURIOUS_APIC_VECTOR            0xff
+# define SPURIOUS_APIC_VECTOR           0xff
-#define ERROR_APIC_VECTOR               0xfe
+# define ERROR_APIC_VECTOR              0xfe
-#define RESCHEDULE_VECTOR               0xfd
+# define RESCHEDULE_VECTOR              0xfd
-#define CALL_FUNCTION_VECTOR            0xfc
+# define CALL_FUNCTION_VECTOR           0xfc
-#define CALL_FUNCTION_SINGLE_VECTOR     0xfb
+# define CALL_FUNCTION_SINGLE_VECTOR    0xfb
-#define THERMAL_APIC_VECTOR             0xfa
+# define THERMAL_APIC_VECTOR            0xfa
-#define THRESHOLD_APIC_VECTOR           0xf9
+# define THRESHOLD_APIC_VECTOR          0xf9
-#define UV_BAU_MESSAGE                  0xf8
+# define UV_BAU_MESSAGE                 0xf8
-#define INVALIDATE_TLB_VECTOR_END       0xf7
+# define INVALIDATE_TLB_VECTOR_END      0xf7
-#define INVALIDATE_TLB_VECTOR_START     0xf0    /* f0-f7 used for TLB flush */
+# define INVALIDATE_TLB_VECTOR_START    0xf0    /* f0-f7 used for TLB flush */
 #define NUM_INVALIDATE_TLB_VECTORS      8
@@ -105,6 +107,8 @@
 #if defined(CONFIG_X86_IO_APIC) && !defined(CONFIG_X86_VOYAGER)
+#include <asm/apicnum.h>        /* need MAX_IO_APICS */
 #ifndef CONFIG_SPARSE_IRQ
 # if NR_CPUS < MAX_IO_APICS
 #  define NR_IRQS (NR_VECTORS + (32 * NR_CPUS))
@@ -112,11 +116,12 @@
 #  define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS))
 # endif
 #else
-# if (8 * NR_CPUS) > (32 * MAX_IO_APICS)
-#  define NR_IRQS (NR_VECTORS + (8 * NR_CPUS))
+# define NR_IRQS                                        \
-# else
+        ((8 * NR_CPUS) > (32 * MAX_IO_APICS) ?          \
-#  define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS))
+                (NR_VECTORS + (8 * NR_CPUS)) :          \
-# endif
+                (NR_VECTORS + (32 * MAX_IO_APICS)))     \
 #endif
 #elif defined(CONFIG_X86_VOYAGER)
diff --git a/arch/x86/include/asm/mach-default/entry_arch.h b/arch/x86/include/asm/mach-default/entry_arch.h
index 6b1add8e31dd..6fa399ad1de2 100644
--- a/arch/x86/include/asm/mach-default/entry_arch.h
+++ b/arch/x86/include/asm/mach-default/entry_arch.h
@@ -11,10 +11,26 @@
 */
 #ifdef CONFIG_X86_SMP
 BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR)
-BUILD_INTERRUPT(invalidate_interrupt,INVALIDATE_TLB_VECTOR)
 BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR)
 BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR)
 BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR)
+BUILD_INTERRUPT3(invalidate_interrupt0,INVALIDATE_TLB_VECTOR_START+0,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt1,INVALIDATE_TLB_VECTOR_START+1,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt2,INVALIDATE_TLB_VECTOR_START+2,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt3,INVALIDATE_TLB_VECTOR_START+3,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt4,INVALIDATE_TLB_VECTOR_START+4,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt5,INVALIDATE_TLB_VECTOR_START+5,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt6,INVALIDATE_TLB_VECTOR_START+6,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt7,INVALIDATE_TLB_VECTOR_START+7,
+                 smp_invalidate_interrupt)
 #endif
 /*
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 8aeeb3fd73db..52948df9cd1d 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -21,11 +21,54 @@ static inline void paravirt_activate_mm(struct mm_struct *prev,
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
 void destroy_context(struct mm_struct *mm);
-#ifdef CONFIG_X86_32
-# include "mmu_context_32.h"
+static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-#else
+{
-# include "mmu_context_64.h"
+#ifdef CONFIG_SMP
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+                percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+#endif
+}
+static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
+                             struct task_struct *tsk)
+{
+        unsigned cpu = smp_processor_id();
+        if (likely(prev != next)) {
+                /* stop flush ipis for the previous mm */
+                cpu_clear(cpu, prev->cpu_vm_mask);
+#ifdef CONFIG_SMP
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                percpu_write(cpu_tlbstate.active_mm, next);
 #endif
+                cpu_set(cpu, next->cpu_vm_mask);
+                /* Re-load page tables */
+                load_cr3(next->pgd);
+                /*
+                 * load the LDT, if the LDT is different:
+                 */
+                if (unlikely(prev->context.ldt != next->context.ldt))
+                        load_LDT_nolock(&next->context);
+        }
+#ifdef CONFIG_SMP
+        else {
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
+                        /* We were in lazy tlb mode and leave_mm disabled
+                         * tlb flush IPI delivery. We must reload CR3
+                         * to make sure to use no freed page tables.
+                         */
+                        load_cr3(next->pgd);
+                        load_LDT_nolock(&next->context);
+                }
+        }
+#endif
+}
 #define activate_mm(prev, next)                 \
 do {                                            \
@@ -33,5 +76,17 @@ do {						\
        switch_mm((prev), (next), NULL);        \
 } while (0);
+#ifdef CONFIG_X86_32
+#define deactivate_mm(tsk, mm)                  \
+do {                                            \
+        loadsegment(gs, 0);                     \
+} while (0)
+#else
+#define deactivate_mm(tsk, mm)                  \
+do {                                            \
+        load_gs_index(0);                       \
+        loadsegment(fs, 0);                     \
+} while (0)
+#endif
 #endif /* _ASM_X86_MMU_CONTEXT_H */
diff --git a/arch/x86/include/asm/mmu_context_32.h b/arch/x86/include/asm/mmu_context_32.h
deleted file mode 100644
index 7e98ce1d2c0e..000000000000
--- a/arch/x86/include/asm/mmu_context_32.h
+++ /dev/null
@@ -1,55 +0,0 @@
-#ifndef _ASM_X86_MMU_CONTEXT_32_H
-#define _ASM_X86_MMU_CONTEXT_32_H
-static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-{
-#ifdef CONFIG_SMP
-        if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK)
-                x86_write_percpu(cpu_tlbstate.state, TLBSTATE_LAZY);
-#endif
-}
-static inline void switch_mm(struct mm_struct *prev,
-                             struct mm_struct *next,
-                             struct task_struct *tsk)
-{
-        int cpu = smp_processor_id();
-        if (likely(prev != next)) {
-                /* stop flush ipis for the previous mm */
-                cpu_clear(cpu, prev->cpu_vm_mask);
-#ifdef CONFIG_SMP
-                x86_write_percpu(cpu_tlbstate.state, TLBSTATE_OK);
-                x86_write_percpu(cpu_tlbstate.active_mm, next);
-#endif
-                cpu_set(cpu, next->cpu_vm_mask);
-                /* Re-load page tables */
-                load_cr3(next->pgd);
-                /*
-                 * load the LDT, if the LDT is different:
-                 */
-                if (unlikely(prev->context.ldt != next->context.ldt))
-                        load_LDT_nolock(&next->context);
-        }
-#ifdef CONFIG_SMP
-        else {
-                x86_write_percpu(cpu_tlbstate.state, TLBSTATE_OK);
-                BUG_ON(x86_read_percpu(cpu_tlbstate.active_mm) != next);
-                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
-                        /* We were in lazy tlb mode and leave_mm disabled
-                         * tlb flush IPI delivery. We must reload %cr3.
-                         */
-                        load_cr3(next->pgd);
-                        load_LDT_nolock(&next->context);
-                }
-        }
-#endif
-}
-#define deactivate_mm(tsk, mm)                  \
-        asm("movl %0,%%gs": :"r" (0));
-#endif /* _ASM_X86_MMU_CONTEXT_32_H */
diff --git a/arch/x86/include/asm/mmu_context_64.h b/arch/x86/include/asm/mmu_context_64.h
deleted file mode 100644
index 677d36e9540a..000000000000
--- a/arch/x86/include/asm/mmu_context_64.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#ifndef _ASM_X86_MMU_CONTEXT_64_H
-#define _ASM_X86_MMU_CONTEXT_64_H
-#include <asm/pda.h>
-static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-{
-#ifdef CONFIG_SMP
-        if (read_pda(mmu_state) == TLBSTATE_OK)
-                write_pda(mmu_state, TLBSTATE_LAZY);
-#endif
-}
-static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
-                             struct task_struct *tsk)
-{
-        unsigned cpu = smp_processor_id();
-        if (likely(prev != next)) {
-                /* stop flush ipis for the previous mm */
-                cpu_clear(cpu, prev->cpu_vm_mask);
-#ifdef CONFIG_SMP
-                write_pda(mmu_state, TLBSTATE_OK);
-                write_pda(active_mm, next);
-#endif
-                cpu_set(cpu, next->cpu_vm_mask);
-                load_cr3(next->pgd);
-                if (unlikely(next->context.ldt != prev->context.ldt))
-                        load_LDT_nolock(&next->context);
-        }
-#ifdef CONFIG_SMP
-        else {
-                write_pda(mmu_state, TLBSTATE_OK);
-                if (read_pda(active_mm) != next)
-                        BUG();
-                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
-                        /* We were in lazy tlb mode and leave_mm disabled
-                         * tlb flush IPI delivery. We must reload CR3
-                         * to make sure to use no freed page tables.
-                         */
-                        load_cr3(next->pgd);
-                        load_LDT_nolock(&next->context);
-                }
-        }
-#endif
-}
-#define deactivate_mm(tsk, mm)                  \
-do {                                            \
-        load_gs_index(0);                       \
-        asm volatile("movl %0,%%fs"::"r"(0));   \
-} while (0)
-#endif /* _ASM_X86_MMU_CONTEXT_64_H */
diff --git a/arch/x86/include/asm/mpspec_def.h b/arch/x86/include/asm/mpspec_def.h
index 59568bc4767f..4a7f96d7c188 100644
--- a/arch/x86/include/asm/mpspec_def.h
+++ b/arch/x86/include/asm/mpspec_def.h
@@ -24,17 +24,18 @@
 # endif
 #endif
-struct intel_mp_floating {
+/* Intel MP Floating Pointer Structure */
-        char mpf_signature[4];          /* "_MP_"                       */
+struct mpf_intel {
-        unsigned int mpf_physptr;       /* Configuration table address  */
+        char signature[4];              /* "_MP_"                       */
-        unsigned char mpf_length;       /* Our length (paragraphs)      */
+        unsigned int physptr;           /* Configuration table address  */
-        unsigned char mpf_specification;/* Specification version        */
+        unsigned char length;           /* Our length (paragraphs)      */
-        unsigned char mpf_checksum;     /* Checksum (makes sum 0)       */
+        unsigned char specification;    /* Specification version        */
-        unsigned char mpf_feature1;     /* Standard or configuration ?  */
+        unsigned char checksum;         /* Checksum (makes sum 0)       */
-        unsigned char mpf_feature2;     /* Bit7 set for IMCR|PIC        */
+        unsigned char feature1;         /* Standard or configuration ?  */
-        unsigned char mpf_feature3;     /* Unused (0)                   */
+        unsigned char feature2;         /* Bit7 set for IMCR|PIC        */
-        unsigned char mpf_feature4;     /* Unused (0)                   */
+        unsigned char feature3;         /* Unused (0)                   */
-        unsigned char mpf_feature5;     /* Unused (0)                   */
+        unsigned char feature4;         /* Unused (0)                   */
+        unsigned char feature5;         /* Unused (0)                   */
 };
 #define MPC_SIGNATURE "PCMP"
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index cb58643947b9..358acc59ae04 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -202,6 +202,35 @@
 #define MSR_IA32_THERM_STATUS           0x0000019c
 #define MSR_IA32_MISC_ENABLE            0x000001a0
+/* MISC_ENABLE bits: architectural */
+#define MSR_IA32_MISC_ENABLE_FAST_STRING        (1ULL << 0)
+#define MSR_IA32_MISC_ENABLE_TCC                (1ULL << 1)
+#define MSR_IA32_MISC_ENABLE_EMON               (1ULL << 7)
+#define MSR_IA32_MISC_ENABLE_BTS_UNAVAIL        (1ULL << 11)
+#define MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL       (1ULL << 12)
+#define MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP (1ULL << 16)
+#define MSR_IA32_MISC_ENABLE_MWAIT              (1ULL << 18)
+#define MSR_IA32_MISC_ENABLE_LIMIT_CPUID        (1ULL << 22)
+#define MSR_IA32_MISC_ENABLE_XTPR_DISABLE       (1ULL << 23)
+#define MSR_IA32_MISC_ENABLE_XD_DISABLE         (1ULL << 34)
+/* MISC_ENABLE bits: model-specific, meaning may vary from core to core */
+#define MSR_IA32_MISC_ENABLE_X87_COMPAT         (1ULL << 2)
+#define MSR_IA32_MISC_ENABLE_TM1                (1ULL << 3)
+#define MSR_IA32_MISC_ENABLE_SPLIT_LOCK_DISABLE (1ULL << 4)
+#define MSR_IA32_MISC_ENABLE_L3CACHE_DISABLE    (1ULL << 6)
+#define MSR_IA32_MISC_ENABLE_SUPPRESS_LOCK      (1ULL << 8)
+#define MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE   (1ULL << 9)
+#define MSR_IA32_MISC_ENABLE_FERR               (1ULL << 10)
+#define MSR_IA32_MISC_ENABLE_FERR_MULTIPLEX     (1ULL << 10)
+#define MSR_IA32_MISC_ENABLE_TM2                (1ULL << 13)
+#define MSR_IA32_MISC_ENABLE_ADJ_PREF_DISABLE   (1ULL << 19)
+#define MSR_IA32_MISC_ENABLE_SPEEDSTEP_LOCK     (1ULL << 20)
+#define MSR_IA32_MISC_ENABLE_L1D_CONTEXT        (1ULL << 24)
+#define MSR_IA32_MISC_ENABLE_DCU_PREF_DISABLE   (1ULL << 37)
+#define MSR_IA32_MISC_ENABLE_TURBO_DISABLE      (1ULL << 38)
+#define MSR_IA32_MISC_ENABLE_IP_PREF_DISABLE    (1ULL << 39)
 /* Intel Model 6 */
 #define MSR_P6_EVNTSEL0                 0x00000186
 #define MSR_P6_EVNTSEL1                 0x00000187
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 5ebca29f44f0..e27fdbe5f9e4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -13,8 +13,8 @@
 #define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1)
 #define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER)
-#define IRQSTACK_ORDER 2
+#define IRQ_STACK_ORDER 2
-#define IRQSTACKSIZE (PAGE_SIZE << IRQSTACK_ORDER)
+#define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER)
 #define STACKFAULT_STACK 1
 #define DOUBLEFAULT_STACK 2
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index e25c410f3d8c..175778887090 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -244,7 +244,8 @@ struct pv_mmu_ops {
        void (*flush_tlb_user)(void);
        void (*flush_tlb_kernel)(void);
        void (*flush_tlb_single)(unsigned long addr);
-        void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm,
+        void (*flush_tlb_others)(const struct cpumask *cpus,
+                                 struct mm_struct *mm,
                                 unsigned long va);
        /* Hooks for allocating and freeing a pagetable top-level */
@@ -983,10 +984,11 @@ static inline void __flush_tlb_single(unsigned long addr)
        PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
 }
-static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
+static inline void flush_tlb_others(const struct cpumask *cpumask,
+                                    struct mm_struct *mm,
                                    unsigned long va)
 {
-        PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va);
+        PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
 }
 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
deleted file mode 100644
index 2fbfff88df37..000000000000
--- a/arch/x86/include/asm/pda.h
+++ /dev/null
@@ -1,137 +0,0 @@
-#ifndef _ASM_X86_PDA_H
-#define _ASM_X86_PDA_H
-#ifndef __ASSEMBLY__
-#include <linux/stddef.h>
-#include <linux/types.h>
-#include <linux/cache.h>
-#include <asm/page.h>
-/* Per processor datastructure. %gs points to it while the kernel runs */
-struct x8664_pda {
-        struct task_struct *pcurrent;   /* 0  Current process */
-        unsigned long data_offset;      /* 8 Per cpu data offset from linker
-                                           address */
-        unsigned long kernelstack;      /* 16 top of kernel stack for current */
-        unsigned long oldrsp;           /* 24 user rsp for system call */
-        int irqcount;                   /* 32 Irq nesting counter. Starts -1 */
-        unsigned int cpunumber;         /* 36 Logical CPU number */
-#ifdef CONFIG_CC_STACKPROTECTOR
-        unsigned long stack_canary;     /* 40 stack canary value */
-                                        /* gcc-ABI: this canary MUST be at
-                                           offset 40!!! */
-#endif
-        char *irqstackptr;
-        short nodenumber;               /* number of current node (32k max) */
-        short in_bootmem;               /* pda lives in bootmem */
-        unsigned int __softirq_pending;
-        unsigned int __nmi_count;       /* number of NMI on this CPUs */
-        short mmu_state;
-        short isidle;
-        struct mm_struct *active_mm;
-        unsigned apic_timer_irqs;
-        unsigned irq0_irqs;
-        unsigned irq_resched_count;
-        unsigned irq_call_count;
-        unsigned irq_tlb_count;
-        unsigned irq_thermal_count;
-        unsigned irq_threshold_count;
-        unsigned irq_spurious_count;
-} ____cacheline_aligned_in_smp;
-extern struct x8664_pda **_cpu_pda;
-extern void pda_init(int);
-#define cpu_pda(i) (_cpu_pda[i])
-/*
- * There is no fast way to get the base address of the PDA, all the accesses
- * have to mention %fs/%gs.  So it needs to be done this Torvaldian way.
- */
-extern void __bad_pda_field(void) __attribute__((noreturn));
-/*
- * proxy_pda doesn't actually exist, but tell gcc it is accessed for
- * all PDA accesses so it gets read/write dependencies right.
- */
-extern struct x8664_pda _proxy_pda;
-#define pda_offset(field) offsetof(struct x8664_pda, field)
-#define pda_to_op(op, field, val)                                       \
-do {                                                                    \
-        typedef typeof(_proxy_pda.field) T__;                           \
-        if (0) { T__ tmp__; tmp__ = (val); }    /* type checking */     \
-        switch (sizeof(_proxy_pda.field)) {                             \
-        case 2:                                                         \
-                asm(op "w %1,%%gs:%c2" :                                \
-                    "+m" (_proxy_pda.field) :                           \
-                    "ri" ((T__)val),                                    \
-                    "i"(pda_offset(field)));                            \
-                break;                                                  \
-        case 4:                                                         \
-                asm(op "l %1,%%gs:%c2" :                                \
-                    "+m" (_proxy_pda.field) :                           \
-                    "ri" ((T__)val),                                    \
-                    "i" (pda_offset(field)));                           \
-                break;                                                  \
-        case 8:                                                         \
-                asm(op "q %1,%%gs:%c2":                                 \
-                    "+m" (_proxy_pda.field) :                           \
-                    "ri" ((T__)val),                                    \
-                    "i"(pda_offset(field)));                            \
-                break;                                                  \
-        default:                                                        \
-                __bad_pda_field();                                      \
-        }                                                               \
-} while (0)
-#define pda_from_op(op, field)                  \
-({                                              \
-        typeof(_proxy_pda.field) ret__;         \
-        switch (sizeof(_proxy_pda.field)) {     \
-        case 2:                                 \
-                asm(op "w %%gs:%c1,%0" :        \
-                    "=r" (ret__) :              \
-                    "i" (pda_offset(field)),    \
-                    "m" (_proxy_pda.field));    \
-                break;                          \
-        case 4:                                 \
-                asm(op "l %%gs:%c1,%0":         \
-                    "=r" (ret__):               \
-                    "i" (pda_offset(field)),    \
-                    "m" (_proxy_pda.field));    \
-                break;                          \
-        case 8:                                 \
-                asm(op "q %%gs:%c1,%0":         \
-                    "=r" (ret__) :              \
-                    "i" (pda_offset(field)),    \
-                    "m" (_proxy_pda.field));    \
-                break;                          \
-        default:                                \
-                __bad_pda_field();              \
-        }                                       \
-        ret__;                                  \
-})
-#define read_pda(field)         pda_from_op("mov", field)
-#define write_pda(field, val)   pda_to_op("mov", field, val)
-#define add_pda(field, val)     pda_to_op("add", field, val)
-#define sub_pda(field, val)     pda_to_op("sub", field, val)
-#define or_pda(field, val)      pda_to_op("or", field, val)
-/* This is not atomic against other CPUs -- CPU preemption needs to be off */
-#define test_and_clear_bit_pda(bit, field)                              \
-({                                                                      \
-        int old__;                                                      \
-        asm volatile("btr %2,%%gs:%c3\n\tsbbl %0,%0"                    \
-                     : "=r" (old__), "+m" (_proxy_pda.field)            \
-                     : "dIr" (bit), "i" (pda_offset(field)) : "memory");\
-        old__;                                                          \
-})
-#endif
-#define PDA_STACKOFFSET (5*8)
-#endif /* _ASM_X86_PDA_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index ece72053ba63..0b64af4f13ac 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -2,53 +2,12 @@
 #define _ASM_X86_PERCPU_H
 #ifdef CONFIG_X86_64
-#include <linux/compiler.h>
+#define __percpu_seg            gs
+#define __percpu_mov_op         movq
-/* Same as asm-generic/percpu.h, except that we store the per cpu offset
+#else
-   in the PDA. Longer term the PDA and every per cpu variable
+#define __percpu_seg            fs
-   should be just put into a single section and referenced directly
+#define __percpu_mov_op         movl
-   from %gs */
-#ifdef CONFIG_SMP
-#include <asm/pda.h>
-#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
-#define __my_cpu_offset read_pda(data_offset)
-#define per_cpu_offset(x) (__per_cpu_offset(x))
 #endif
-#include <asm-generic/percpu.h>
-DECLARE_PER_CPU(struct x8664_pda, pda);
-/*
- * These are supposed to be implemented as a single instruction which
- * operates on the per-cpu data base segment.  x86-64 doesn't have
- * that yet, so this is a fairly inefficient workaround for the
- * meantime.  The single instruction is atomic with respect to
- * preemption and interrupts, so we need to explicitly disable
- * interrupts here to achieve the same effect.  However, because it
- * can be used from within interrupt-disable/enable, we can't actually
- * disable interrupts; disabling preemption is enough.
- */
-#define x86_read_percpu(var)                                            \
-        ({                                                              \
-                typeof(per_cpu_var(var)) __tmp;                         \
-                preempt_disable();                                      \
-                __tmp = __get_cpu_var(var);                             \
-                preempt_enable();                                       \
-                __tmp;                                                  \
-        })
-#define x86_write_percpu(var, val)                                      \
-        do {                                                            \
-                preempt_disable();                                      \
-                __get_cpu_var(var) = (val);                             \
-                preempt_enable();                                       \
-        } while(0)
-#else /* CONFIG_X86_64 */
 #ifdef __ASSEMBLY__
@@ -65,47 +24,26 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
 #ifdef CONFIG_SMP
-#define PER_CPU(var, reg)                               \
+#define PER_CPU(var, reg)                                               \
-        movl %fs:per_cpu__##this_cpu_off, reg;          \
+        __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg;       \
        lea per_cpu__##var(reg), reg
-#define PER_CPU_VAR(var)        %fs:per_cpu__##var
+#define PER_CPU_VAR(var)        %__percpu_seg:per_cpu__##var
 #else /* ! SMP */
-#define PER_CPU(var, reg)                       \
+#define PER_CPU(var, reg)                                               \
-        movl $per_cpu__##var, reg
+        __percpu_mov_op $per_cpu__##var, reg
 #define PER_CPU_VAR(var)        per_cpu__##var
 #endif  /* SMP */
 #else /* ...!ASSEMBLY */
-/*
+#include <linux/stringify.h>
- * PER_CPU finds an address of a per-cpu variable.
- *
- * Args:
- *    var - variable name
- *    cpu - 32bit register containing the current CPU number
- *
- * The resulting address is stored in the "cpu" argument.
- *
- * Example:
- *    PER_CPU(cpu_gdt_descr, %ebx)
- */
-#ifdef CONFIG_SMP
-#define __my_cpu_offset x86_read_percpu(this_cpu_off)
-/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
+#ifdef CONFIG_SMP
-#define __percpu_seg "%%fs:"
+#define __percpu_arg(x)         "%%"__stringify(__percpu_seg)":%P" #x
+#define __my_cpu_offset         percpu_read(this_cpu_off)
-#else  /* !SMP */
+#else
+#define __percpu_arg(x)         "%" #x
-#define __percpu_seg ""
+#endif
-#endif  /* SMP */
-#include <asm-generic/percpu.h>
-/* We can use this directly for local CPU (faster). */
-DECLARE_PER_CPU(unsigned long, this_cpu_off);
 /* For arch-specific code, we can use direct single-insn ops (they
 * don't give an lvalue though). */
@@ -120,20 +58,25 @@ do {							\
        }                                               \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b %1,"__percpu_seg"%0"          \
+                asm(op "b %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 2:                                         \
-                asm(op "w %1,"__percpu_seg"%0"          \
+                asm(op "w %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 4:                                         \
-                asm(op "l %1,"__percpu_seg"%0"          \
+                asm(op "l %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
+        case 8:                                         \
+                asm(op "q %1,"__percpu_arg(0)           \
+                    : "+m" (var)                        \
+                    : "re" ((T__)val));                 \
+                break;                                  \
        default: __bad_percpu_size();                   \
        }                                               \
 } while (0)
@@ -143,17 +86,22 @@ do {							\
        typeof(var) ret__;                              \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b "__percpu_seg"%1,%0"          \
+                asm(op "b "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 2:                                         \
-                asm(op "w "__percpu_seg"%1,%0"          \
+                asm(op "w "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 4:                                         \
-                asm(op "l "__percpu_seg"%1,%0"          \
+                asm(op "l "__percpu_arg(1)",%0"         \
+                    : "=r" (ret__)                      \
+                    : "m" (var));                       \
+                break;                                  \
+        case 8:                                         \
+                asm(op "q "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
@@ -162,13 +110,30 @@ do {							\
        ret__;                                          \
 })
-#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
+#define percpu_read(var)        percpu_from_op("mov", per_cpu__##var)
-#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val)
+#define percpu_write(var, val)  percpu_to_op("mov", per_cpu__##var, val)
-#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)
+#define percpu_add(var, val)    percpu_to_op("add", per_cpu__##var, val)
-#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)
+#define percpu_sub(var, val)    percpu_to_op("sub", per_cpu__##var, val)
-#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
+#define percpu_and(var, val)    percpu_to_op("and", per_cpu__##var, val)
+#define percpu_or(var, val)     percpu_to_op("or", per_cpu__##var, val)
+#define percpu_xor(var, val)    percpu_to_op("xor", per_cpu__##var, val)
+/* This is not atomic against other CPUs -- CPU preemption needs to be off */
+#define x86_test_and_clear_bit_percpu(bit, var)                         \
+({                                                                      \
+        int old__;                                                      \
+        asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"           \
+                     : "=r" (old__), "+m" (per_cpu__##var)              \
+                     : "dIr" (bit));                                    \
+        old__;                                                          \
+})
+#include <asm-generic/percpu.h>
+/* We can use this directly for local CPU (faster). */
+DECLARE_PER_CPU(unsigned long, this_cpu_off);
 #endif /* !__ASSEMBLY__ */
-#endif /* !CONFIG_X86_64 */
 #ifdef CONFIG_SMP
@@ -195,9 +160,9 @@ do {							\
 #define early_per_cpu_ptr(_name) (_name##_early_ptr)
 #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
 #define early_per_cpu(_name, _cpu)                              \
-        (early_per_cpu_ptr(_name) ?                             \
+        *(early_per_cpu_ptr(_name) ?                            \
-                early_per_cpu_ptr(_name)[_cpu] :                \
+                &early_per_cpu_ptr(_name)[_cpu] :               \
-                per_cpu(_name, _cpu))
+                &per_cpu(_name, _cpu))
 #else   /* !CONFIG_SMP */
 #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)          \
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index cb7c151a8bff..dd14c54ac718 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -42,6 +42,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 static inline void pte_free(struct mm_struct *mm, struct page *pte)
 {
+        pgtable_page_dtor(pte);
        __free_page(pte);
 }
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index ba09289accaa..1df9637dfda3 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -11,7 +11,6 @@
 #include <asm/processor.h>
 #include <linux/bitops.h>
 #include <linux/threads.h>
-#include <asm/pda.h>
 extern pud_t level3_kernel_pgt[512];
 extern pud_t level3_ident_pgt[512];
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 091cd8855f2e..befa20b4a68c 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -378,6 +378,22 @@ union thread_xstate {
 #ifdef CONFIG_X86_64
 DECLARE_PER_CPU(struct orig_ist, orig_ist);
+union irq_stack_union {
+        char irq_stack[IRQ_STACK_SIZE];
+        /*
+         * GCC hardcodes the stack canary as %gs:40.  Since the
+         * irq_stack is the object at %gs:0, we reserve the bottom
+         * 48 bytes of the irq stack for the canary.
+         */
+        struct {
+                char gs_base[40];
+                unsigned long stack_canary;
+        };
+};
+DECLARE_PER_CPU(union irq_stack_union, irq_stack_union);
+DECLARE_PER_CPU(char *, irq_stack_ptr);
 #endif
 extern void print_cpu_info(struct cpuinfo_x86 *);
@@ -754,7 +770,6 @@ extern struct desc_ptr		early_gdt_descr;
 extern void cpu_set_gdt(int);
 extern void switch_to_new_gdt(void);
 extern void cpu_init(void);
-extern void init_gdt(int cpu);
 static inline unsigned long get_debugctlmsr(void)
 {
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index ebe858cdc8a3..536949749bc2 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -100,7 +100,6 @@ extern unsigned long init_pg_tables_start;
 extern unsigned long init_pg_tables_end;
 #else
-void __init x86_64_init_pda(void);
 void __init x86_64_start_kernel(char *real_mode);
 void __init x86_64_start_reservations(char *real_mode_data);
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 19953df61c52..45ef8a1b9d7c 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -15,34 +15,8 @@
 #  include <asm/io_apic.h>
 # endif
 #endif
-#include <asm/pda.h>
 #include <asm/thread_info.h>
+#include <asm/cpumask.h>
-#ifdef CONFIG_X86_64
-extern cpumask_var_t cpu_callin_mask;
-extern cpumask_var_t cpu_callout_mask;
-extern cpumask_var_t cpu_initialized_mask;
-extern cpumask_var_t cpu_sibling_setup_mask;
-#else /* CONFIG_X86_32 */
-extern cpumask_t cpu_callin_map;
-extern cpumask_t cpu_callout_map;
-extern cpumask_t cpu_initialized;
-extern cpumask_t cpu_sibling_setup_map;
-#define cpu_callin_mask         ((struct cpumask *)&cpu_callin_map)
-#define cpu_callout_mask        ((struct cpumask *)&cpu_callout_map)
-#define cpu_initialized_mask    ((struct cpumask *)&cpu_initialized)
-#define cpu_sibling_setup_mask  ((struct cpumask *)&cpu_sibling_setup_map)
-#endif /* CONFIG_X86_32 */
-extern void (*mtrr_hook)(void);
-extern void zap_low_mappings(void);
-extern int __cpuinit get_local_pda(int cpu);
 extern int smp_num_siblings;
 extern unsigned int num_processors;
@@ -50,9 +24,7 @@ extern unsigned int num_processors;
 DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
 DECLARE_PER_CPU(cpumask_t, cpu_core_map);
 DECLARE_PER_CPU(u16, cpu_llc_id);
-#ifdef CONFIG_X86_32
 DECLARE_PER_CPU(int, cpu_number);
-#endif
 static inline struct cpumask *cpu_sibling_mask(int cpu)
 {
@@ -167,8 +139,6 @@ void play_dead_common(void);
 void native_send_call_func_ipi(const struct cpumask *mask);
 void native_send_call_func_single_ipi(int cpu);
-extern void prefill_possible_map(void);
 void smp_store_cpu_info(int id);
 #define cpu_physical_id(cpu)    per_cpu(x86_cpu_to_apicid, cpu)
@@ -177,10 +147,6 @@ static inline int num_booting_cpus(void)
 {
        return cpumask_weight(cpu_callout_mask);
 }
-#else
-static inline void prefill_possible_map(void)
-{
-}
 #endif /* CONFIG_SMP */
 extern unsigned disabled_cpus __cpuinitdata;
@@ -191,11 +157,11 @@ extern unsigned disabled_cpus __cpuinitdata;
 * from the initial startup. We map APIC_BASE very early in page_setup(),
 * so this is correct in the x86 case.
 */
-#define raw_smp_processor_id() (x86_read_percpu(cpu_number))
+#define raw_smp_processor_id() (percpu_read(cpu_number))
 extern int safe_smp_processor_id(void);
 #elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id()  read_pda(cpunumber)
+#define raw_smp_processor_id() (percpu_read(cpu_number))
 #define stack_smp_processor_id()                                        \
 ({                                                              \
@@ -205,10 +171,6 @@ extern int safe_smp_processor_id(void);
 })
 #define safe_smp_processor_id()         smp_processor_id()
-#else /* !CONFIG_X86_32_SMP && !CONFIG_X86_64_SMP */
-#define cpu_physical_id(cpu)            boot_cpu_physical_apicid
-#define safe_smp_processor_id()         0
-#define stack_smp_processor_id()        0
 #endif
 #ifdef CONFIG_X86_LOCAL_APIC
@@ -251,11 +213,5 @@ static inline int hard_smp_processor_id(void)
 #endif /* CONFIG_X86_LOCAL_APIC */
-#ifdef CONFIG_X86_HAS_BOOT_CPU_ID
-extern unsigned char boot_cpu_id;
-#else
-#define boot_cpu_id     0
-#endif
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_X86_SMP_H */
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
new file mode 100644
index 000000000000..36a700acaf2b
--- /dev/null
+++ b/arch/x86/include/asm/stackprotector.h
@@ -0,0 +1,38 @@
+#ifndef _ASM_STACKPROTECTOR_H
+#define _ASM_STACKPROTECTOR_H 1
+#include <asm/tsc.h>
+#include <asm/processor.h>
+/*
+ * Initialize the stackprotector canary value.
+ *
+ * NOTE: this must only be called from functions that never return,
+ * and it must always be inlined.
+ */
+static __always_inline void boot_init_stack_canary(void)
+{
+        u64 canary;
+        u64 tsc;
+        /*
+         * Build time only check to make sure the stack_canary is at
+         * offset 40 in the pda; this is a gcc ABI requirement
+         */
+        BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
+        /*
+         * We both use the random pool and the current TSC as a source
+         * of randomness. The TSC only matters for very early init,
+         * there it already has some randomness on most systems. Later
+         * on during the bootup the random pool has true entropy too.
+         */
+        get_random_bytes(&canary, sizeof(canary));
+        tsc = __native_read_tsc();
+        canary += tsc + (tsc << 32UL);
+        current->stack_canary = canary;
+        percpu_write(irq_stack_union.stack_canary, canary);
+}
+#endif
diff --git a/arch/x86/include/asm/syscalls.h b/arch/x86/include/asm/syscalls.h
index 9c6797c3e56c..c0b0bda754ee 100644
--- a/arch/x86/include/asm/syscalls.h
+++ b/arch/x86/include/asm/syscalls.h
@@ -40,7 +40,7 @@ asmlinkage int sys_sigaction(int, const struct old_sigaction __user *,
                             struct old_sigaction __user *);
 asmlinkage int sys_sigaltstack(unsigned long);
 asmlinkage unsigned long sys_sigreturn(unsigned long);
-asmlinkage int sys_rt_sigreturn(struct pt_regs);
+asmlinkage int sys_rt_sigreturn(unsigned long);
 /* kernel/ioport.c */
 asmlinkage long sys_iopl(unsigned long);
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 8e626ea33a1a..2fcc70bc85f3 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -86,27 +86,44 @@ do {									\
        , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
          "r12", "r13", "r14", "r15"
+#ifdef CONFIG_CC_STACKPROTECTOR
+#define __switch_canary                                                   \
+        "movq %P[task_canary](%%rsi),%%r8\n\t"                            \
+        "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
+#define __switch_canary_oparam                                            \
+        , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary))
+#define __switch_canary_iparam                                            \
+        , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
+#else   /* CC_STACKPROTECTOR */
+#define __switch_canary
+#define __switch_canary_oparam
+#define __switch_canary_iparam
+#endif  /* CC_STACKPROTECTOR */
 /* Save restore flags to clear handle leaking NT */
 #define switch_to(prev, next, last) \
-        asm volatile(SAVE_CONTEXT                                                   \
+        asm volatile(SAVE_CONTEXT                                         \
             "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */       \
             "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */    \
             "call __switch_to\n\t"                                       \
             ".globl thread_return\n"                                     \
             "thread_return:\n\t"                                         \
-             "movq %%gs:%P[pda_pcurrent],%%rsi\n\t"                       \
+             "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
+             __switch_canary                                              \
             "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
             LOCK_PREFIX "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"        \
             "movq %%rax,%%rdi\n\t"                                       \
             "jc   ret_from_fork\n\t"                                     \
             RESTORE_CONTEXT                                              \
             : "=a" (last)                                                \
+               __switch_canary_oparam                                     \
             : [next] "S" (next), [prev] "D" (prev),                      \
               [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
               [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
               [tif_fork] "i" (TIF_FORK),                                 \
               [thread_info] "i" (offsetof(struct task_struct, stack)),   \
-               [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent))  \
+               [current_task] "m" (per_cpu_var(current_task))             \
+               __switch_canary_iparam                                     \
             : "memory", "cc" __EXTRA_CLOBBER)
 #endif
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 98789647baa9..b46f8ca007b5 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -194,25 +194,21 @@ static inline struct thread_info *current_thread_info(void)
 #else /* X86_32 */
-#include <asm/pda.h>
+#include <asm/percpu.h>
+#define KERNEL_STACK_OFFSET (5*8)
 /*
 * macros/functions for gaining access to the thread information structure
 * preempt_count needs to be 1 initially, until the scheduler is functional.
 */
 #ifndef __ASSEMBLY__
-static inline struct thread_info *current_thread_info(void)
+DECLARE_PER_CPU(unsigned long, kernel_stack);
-{
-        struct thread_info *ti;
-        ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
-        return ti;
-}
-/* do not use in interrupt context */
+static inline struct thread_info *current_thread_info(void)
-static inline struct thread_info *stack_thread_info(void)
 {
        struct thread_info *ti;
-        asm("andq %%rsp,%0; " : "=r" (ti) : "0" (~(THREAD_SIZE - 1)));
+        ti = (void *)(percpu_read(kernel_stack) +
+                      KERNEL_STACK_OFFSET - THREAD_SIZE);
        return ti;
 }
@@ -220,8 +216,8 @@ static inline struct thread_info *stack_thread_info(void)
 /* how to get the thread information struct from ASM */
 #define GET_THREAD_INFO(reg) \
-        movq %gs:pda_kernelstack,reg ; \
+        movq PER_CPU_VAR(kernel_stack),reg ; \
-        subq $(THREAD_SIZE-PDA_STACKOFFSET),reg
+        subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
 #endif
diff --git a/arch/x86/include/asm/timex.h b/arch/x86/include/asm/timex.h
index 1287dc1347d6..b5c9d45c981f 100644
--- a/arch/x86/include/asm/timex.h
+++ b/arch/x86/include/asm/timex.h
@@ -1,18 +1,13 @@
-/* x86 architecture timex specifications */
 #ifndef _ASM_X86_TIMEX_H
 #define _ASM_X86_TIMEX_H
 #include <asm/processor.h>
 #include <asm/tsc.h>
-#ifdef CONFIG_X86_ELAN
+/* The PIT ticks at this frequency (in HZ): */
-#  define PIT_TICK_RATE 1189200 /* AMD Elan has different frequency! */
+#define PIT_TICK_RATE           1193182
-#elif defined(CONFIG_X86_RDC321X)
-#  define PIT_TICK_RATE 1041667 /* Underlying HZ for R8610 */
+#define CLOCK_TICK_RATE         PIT_TICK_RATE
-#else
-#  define PIT_TICK_RATE 1193182 /* Underlying HZ */
-#endif
-#define CLOCK_TICK_RATE PIT_TICK_RATE
 #define ARCH_HAS_READ_CURRENT_TIMER
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 0e7bbb549116..d3539f998f88 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -113,7 +113,7 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
                __flush_tlb();
 }
-static inline void native_flush_tlb_others(const cpumask_t *cpumask,
+static inline void native_flush_tlb_others(const struct cpumask *cpumask,
                                           struct mm_struct *mm,
                                           unsigned long va)
 {
@@ -142,31 +142,28 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
        flush_tlb_mm(vma->vm_mm);
 }
-void native_flush_tlb_others(const cpumask_t *cpumask, struct mm_struct *mm,
+void native_flush_tlb_others(const struct cpumask *cpumask,
-                             unsigned long va);
+                             struct mm_struct *mm, unsigned long va);
 #define TLBSTATE_OK     1
 #define TLBSTATE_LAZY   2
-#ifdef CONFIG_X86_32
 struct tlb_state {
        struct mm_struct *active_mm;
        int state;
-        char __cacheline_padding[L1_CACHE_BYTES-8];
 };
 DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
-void reset_lazy_tlbstate(void);
-#else
 static inline void reset_lazy_tlbstate(void)
 {
+        percpu_write(cpu_tlbstate.state, 0);
+        percpu_write(cpu_tlbstate.active_mm, &init_mm);
 }
-#endif
 #endif  /* SMP */
 #ifndef CONFIG_PARAVIRT
-#define flush_tlb_others(mask, mm, va)  native_flush_tlb_others(&mask, mm, va)
+#define flush_tlb_others(mask, mm, va)  native_flush_tlb_others(mask, mm, va)
 #endif
 static inline void flush_tlb_kernel_range(unsigned long start,
@@ -175,4 +172,6 @@ static inline void flush_tlb_kernel_range(unsigned long start,
        flush_tlb_all();
 }
+extern void zap_low_mappings(void);
 #endif /* _ASM_X86_TLBFLUSH_H */
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 4e2f2e0aab27..77cfb2cfb386 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -74,6 +74,8 @@ static inline const struct cpumask *cpumask_of_node(int node)
        return &node_to_cpumask_map[node];
 }
+static inline void setup_node_to_cpumask_map(void) { }
 #else /* CONFIG_X86_64 */
 /* Mappings between node number and cpus on that node. */
@@ -83,7 +85,8 @@ extern cpumask_t *node_to_cpumask_map;
 DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
 /* Returns the number of the current Node. */
-#define numa_node_id()          read_pda(nodenumber)
+DECLARE_PER_CPU(int, node_number);
+#define numa_node_id()          percpu_read(node_number)
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
 extern int cpu_to_node(int cpu);
@@ -102,10 +105,7 @@ static inline int cpu_to_node(int cpu)
 /* Same function but used if called before per_cpu areas are setup */
 static inline int early_cpu_to_node(int cpu)
 {
-        if (early_per_cpu_ptr(x86_cpu_to_node_map))
+        return early_per_cpu(x86_cpu_to_node_map, cpu);
-                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
-        return per_cpu(x86_cpu_to_node_map, cpu);
 }
 /* Returns a pointer to the cpumask of CPUs on Node 'node'. */
@@ -122,6 +122,8 @@ static inline cpumask_t node_to_cpumask(int node)
 #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
+extern void setup_node_to_cpumask_map(void);
 /*
 * Replace default node_to_cpumask_ptr with optimized version
 * Deprecated: use "const struct cpumask *mask = cpumask_of_node(node)"
@@ -192,9 +194,20 @@ extern int __node_distance(int, int);
 #else /* !CONFIG_NUMA */
-#define numa_node_id()          0
+static inline int numa_node_id(void)
-#define cpu_to_node(cpu)        0
+{
-#define early_cpu_to_node(cpu)  0
+        return 0;
+}
+static inline int cpu_to_node(int cpu)
+{
+        return 0;
+}
+static inline int early_cpu_to_node(int cpu)
+{
+        return 0;
+}
 static inline const cpumask_t *cpumask_of_node(int node)
 {
@@ -209,6 +222,8 @@ static inline int node_to_first_cpu(int node)
        return first_cpu(cpu_online_map);
 }
+static inline void setup_node_to_cpumask_map(void) { }
 /*
 * Replace default node_to_cpumask_ptr with optimized version
 * Deprecated: use "const struct cpumask *mask = cpumask_of_node(node)"
diff --git a/arch/x86/include/asm/trampoline.h b/arch/x86/include/asm/trampoline.h
index 780ba0ab94f9..90f06c25221d 100644
--- a/arch/x86/include/asm/trampoline.h
+++ b/arch/x86/include/asm/trampoline.h
@@ -13,6 +13,7 @@ extern unsigned char *trampoline_base;
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
+extern unsigned long initial_gs;
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 #define TRAMPOLINE_BASE 0x6000
diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h
new file mode 100644
index 000000000000..8ac1d7e312f3
--- /dev/null
+++ b/arch/x86/include/asm/uv/uv.h
@@ -0,0 +1,33 @@
+#ifndef _ASM_X86_UV_UV_H
+#define _ASM_X86_UV_UV_H
+enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
+#ifdef CONFIG_X86_UV
+extern enum uv_system_type get_uv_system_type(void);
+extern int is_uv_system(void);
+extern void uv_cpu_init(void);
+extern void uv_system_init(void);
+extern int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip);
+extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
+                                                 struct mm_struct *mm,
+                                                 unsigned long va,
+                                                 unsigned int cpu);
+#else   /* X86_UV */
+static inline enum uv_system_type get_uv_system_type(void) { return UV_NONE; }
+static inline int is_uv_system(void)    { return 0; }
+static inline void uv_cpu_init(void)    { }
+static inline void uv_system_init(void) { }
+static inline int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip)
+{ return 1; }
+static inline const struct cpumask *
+uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
+                    unsigned long va, unsigned int cpu)
+{ return cpumask; }
+#endif  /* X86_UV */
+#endif  /* _ASM_X86_UV_UV_H */
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index 50423c7b56b2..9b0e61bf7a88 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -325,7 +325,6 @@ static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
 #define cpubit_isset(cpu, bau_local_cpumask) \
        test_bit((cpu), (bau_local_cpumask).bits)
-extern int uv_flush_tlb_others(cpumask_t *, struct mm_struct *, unsigned long);
 extern void uv_bau_message_intr1(void);
 extern void uv_bau_timeout_intr1(void);
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index d364df03c1d6..37fa30bada17 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -23,11 +23,12 @@ nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_vsyscall_64.o    := $(PROFILING) -g0 $(nostackp)
 CFLAGS_hpet.o           := $(nostackp)
 CFLAGS_tsc.o            := $(nostackp)
+CFLAGS_paravirt.o       := $(nostackp)
 obj-y                   := process_$(BITS).o signal.o entry_$(BITS).o
 obj-y                   += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
 obj-y                   += time_$(BITS).o ioport.o ldt.o dumpstack.o
-obj-y                   += setup.o i8259.o irqinit_$(BITS).o setup_percpu.o
+obj-y                   += setup.o i8259.o irqinit_$(BITS).o
 obj-$(CONFIG_X86_VISWS) += visws_quirks.o
 obj-$(CONFIG_X86_32)    += probe_roms_32.o
 obj-$(CONFIG_X86_32)    += sys_i386_32.o i386_ksyms_32.o
@@ -57,9 +58,9 @@ obj-$(CONFIG_PCI)		+= early-quirks.o
 apm-y                           := apm_32.o
 obj-$(CONFIG_APM)               += apm.o
 obj-$(CONFIG_X86_SMP)           += smp.o
-obj-$(CONFIG_X86_SMP)           += smpboot.o tsc_sync.o ipi.o tlb_$(BITS).o
+obj-$(CONFIG_X86_SMP)           += smpboot.o tsc_sync.o ipi.o
-obj-$(CONFIG_X86_32_SMP)        += smpcommon.o
+obj-$(CONFIG_SMP)               += setup_percpu.o
-obj-$(CONFIG_X86_64_SMP)        += tsc_sync.o smpcommon.o
+obj-$(CONFIG_X86_64_SMP)        += tsc_sync.o
 obj-$(CONFIG_X86_TRAMPOLINE)    += trampoline_$(BITS).o
 obj-$(CONFIG_X86_MPPARSE)       += mpparse.o
 obj-$(CONFIG_X86_LOCAL_APIC)    += apic.o nmi.o
@@ -114,10 +115,11 @@ obj-$(CONFIG_SWIOTLB)			+= pci-swiotlb_64.o # NB rename without _64
 ###
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
-        obj-y                           += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o
+        obj-y                           += genapic_64.o genapic_flat_64.o
-        obj-y                           += bios_uv.o uv_irq.o uv_sysfs.o
        obj-y                           += genx2apic_cluster.o
        obj-y                           += genx2apic_phys.o
+        obj-$(CONFIG_X86_UV)            += genx2apic_uv_x.o tlb_uv.o
+        obj-$(CONFIG_X86_UV)            += bios_uv.o uv_irq.o uv_sysfs.o
        obj-$(CONFIG_X86_PM_TIMER)      += pmtimer_64.o
        obj-$(CONFIG_AUDIT)             += audit_64.o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index d37593c2f438..4cb5964f1499 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -912,8 +912,8 @@ static u8 __init uniq_ioapic_id(u8 id)
        DECLARE_BITMAP(used, 256);
        bitmap_zero(used, 256);
        for (i = 0; i < nr_ioapics; i++) {
-                struct mp_config_ioapic *ia = &mp_ioapics[i];
+                struct mpc_ioapic *ia = &mp_ioapics[i];
-                __set_bit(ia->mp_apicid, used);
+                __set_bit(ia->apicid, used);
        }
        if (!test_bit(id, used))
                return id;
@@ -945,47 +945,47 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
        idx = nr_ioapics;
-        mp_ioapics[idx].mp_type = MP_IOAPIC;
+        mp_ioapics[idx].type = MP_IOAPIC;
-        mp_ioapics[idx].mp_flags = MPC_APIC_USABLE;
+        mp_ioapics[idx].flags = MPC_APIC_USABLE;
-        mp_ioapics[idx].mp_apicaddr = address;
+        mp_ioapics[idx].apicaddr = address;
        set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
-        mp_ioapics[idx].mp_apicid = uniq_ioapic_id(id);
+        mp_ioapics[idx].apicid = uniq_ioapic_id(id);
 #ifdef CONFIG_X86_32
-        mp_ioapics[idx].mp_apicver = io_apic_get_version(idx);
+        mp_ioapics[idx].apicver = io_apic_get_version(idx);
 #else
-        mp_ioapics[idx].mp_apicver = 0;
+        mp_ioapics[idx].apicver = 0;
 #endif
        /*
         * Build basic GSI lookup table to facilitate gsi->io_apic lookups
         * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
         */
-        mp_ioapic_routing[idx].apic_id = mp_ioapics[idx].mp_apicid;
+        mp_ioapic_routing[idx].apic_id = mp_ioapics[idx].apicid;
        mp_ioapic_routing[idx].gsi_base = gsi_base;
        mp_ioapic_routing[idx].gsi_end = gsi_base +
            io_apic_get_redir_entries(idx);
-        printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%lx, "
+        printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
-               "GSI %d-%d\n", idx, mp_ioapics[idx].mp_apicid,
+               "GSI %d-%d\n", idx, mp_ioapics[idx].apicid,
-               mp_ioapics[idx].mp_apicver, mp_ioapics[idx].mp_apicaddr,
+               mp_ioapics[idx].apicver, mp_ioapics[idx].apicaddr,
               mp_ioapic_routing[idx].gsi_base, mp_ioapic_routing[idx].gsi_end);
        nr_ioapics++;
 }
-static void assign_to_mp_irq(struct mp_config_intsrc *m,
+static void assign_to_mp_irq(struct mpc_intsrc *m,
-                                    struct mp_config_intsrc *mp_irq)
+                                    struct mpc_intsrc *mp_irq)
 {
-        memcpy(mp_irq, m, sizeof(struct mp_config_intsrc));
+        memcpy(mp_irq, m, sizeof(struct mpc_intsrc));
 }
-static int mp_irq_cmp(struct mp_config_intsrc *mp_irq,
+static int mp_irq_cmp(struct mpc_intsrc *mp_irq,
-                                struct mp_config_intsrc *m)
+                                struct mpc_intsrc *m)
 {
-        return memcmp(mp_irq, m, sizeof(struct mp_config_intsrc));
+        return memcmp(mp_irq, m, sizeof(struct mpc_intsrc));
 }
-static void save_mp_irq(struct mp_config_intsrc *m)
+static void save_mp_irq(struct mpc_intsrc *m)
 {
        int i;
@@ -1003,7 +1003,7 @@ void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
 {
        int ioapic;
        int pin;
-        struct mp_config_intsrc mp_irq;
+        struct mpc_intsrc mp_irq;
        /*
         * Convert 'gsi' to 'ioapic.pin'.
@@ -1021,13 +1021,13 @@ void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
        if ((bus_irq == 0) && (trigger == 3))
                trigger = 1;
-        mp_irq.mp_type = MP_INTSRC;
+        mp_irq.type = MP_INTSRC;
-        mp_irq.mp_irqtype = mp_INT;
+        mp_irq.irqtype = mp_INT;
-        mp_irq.mp_irqflag = (trigger << 2) | polarity;
+        mp_irq.irqflag = (trigger << 2) | polarity;
-        mp_irq.mp_srcbus = MP_ISA_BUS;
+        mp_irq.srcbus = MP_ISA_BUS;
-        mp_irq.mp_srcbusirq = bus_irq;  /* IRQ */
+        mp_irq.srcbusirq = bus_irq;     /* IRQ */
-        mp_irq.mp_dstapic = mp_ioapics[ioapic].mp_apicid; /* APIC ID */
+        mp_irq.dstapic = mp_ioapics[ioapic].apicid; /* APIC ID */
-        mp_irq.mp_dstirq = pin; /* INTIN# */
+        mp_irq.dstirq = pin;    /* INTIN# */
        save_mp_irq(&mp_irq);
 }
@@ -1037,7 +1037,7 @@ void __init mp_config_acpi_legacy_irqs(void)
        int i;
        int ioapic;
        unsigned int dstapic;
-        struct mp_config_intsrc mp_irq;
+        struct mpc_intsrc mp_irq;
 #if defined (CONFIG_MCA) || defined (CONFIG_EISA)
        /*
@@ -1062,7 +1062,7 @@ void __init mp_config_acpi_legacy_irqs(void)
        ioapic = mp_find_ioapic(0);
        if (ioapic < 0)
                return;
-        dstapic = mp_ioapics[ioapic].mp_apicid;
+        dstapic = mp_ioapics[ioapic].apicid;
        /*
         * Use the default configuration for the IRQs 0-15.  Unless
@@ -1072,16 +1072,14 @@ void __init mp_config_acpi_legacy_irqs(void)
                int idx;
                for (idx = 0; idx < mp_irq_entries; idx++) {
-                        struct mp_config_intsrc *irq = mp_irqs + idx;
+                        struct mpc_intsrc *irq = mp_irqs + idx;
                        /* Do we already have a mapping for this ISA IRQ? */
-                        if (irq->mp_srcbus == MP_ISA_BUS
+                        if (irq->srcbus == MP_ISA_BUS && irq->srcbusirq == i)
-                            && irq->mp_srcbusirq == i)
                                break;
                        /* Do we already have a mapping for this IOAPIC pin */
-                        if (irq->mp_dstapic == dstapic &&
+                        if (irq->dstapic == dstapic && irq->dstirq == i)
-                            irq->mp_dstirq == i)
                                break;
                }
@@ -1090,13 +1088,13 @@ void __init mp_config_acpi_legacy_irqs(void)
                        continue;       /* IRQ already used */
                }
-                mp_irq.mp_type = MP_INTSRC;
+                mp_irq.type = MP_INTSRC;
-                mp_irq.mp_irqflag = 0;  /* Conforming */
+                mp_irq.irqflag = 0;     /* Conforming */
-                mp_irq.mp_srcbus = MP_ISA_BUS;
+                mp_irq.srcbus = MP_ISA_BUS;
-                mp_irq.mp_dstapic = dstapic;
+                mp_irq.dstapic = dstapic;
-                mp_irq.mp_irqtype = mp_INT;
+                mp_irq.irqtype = mp_INT;
-                mp_irq.mp_srcbusirq = i; /* Identity mapped */
+                mp_irq.srcbusirq = i; /* Identity mapped */
-                mp_irq.mp_dstirq = i;
+                mp_irq.dstirq = i;
                save_mp_irq(&mp_irq);
        }
@@ -1207,22 +1205,22 @@ int mp_config_acpi_gsi(unsigned char number, unsigned int devfn, u8 pin,
                        u32 gsi, int triggering, int polarity)
 {
 #ifdef CONFIG_X86_MPPARSE
-        struct mp_config_intsrc mp_irq;
+        struct mpc_intsrc mp_irq;
        int ioapic;
        if (!acpi_ioapic)
                return 0;
        /* print the entry should happen on mptable identically */
-        mp_irq.mp_type = MP_INTSRC;
+        mp_irq.type = MP_INTSRC;
-        mp_irq.mp_irqtype = mp_INT;
+        mp_irq.irqtype = mp_INT;
-        mp_irq.mp_irqflag = (triggering == ACPI_EDGE_SENSITIVE ? 4 : 0x0c) |
+        mp_irq.irqflag = (triggering == ACPI_EDGE_SENSITIVE ? 4 : 0x0c) |
                                (polarity == ACPI_ACTIVE_HIGH ? 1 : 3);
-        mp_irq.mp_srcbus = number;
+        mp_irq.srcbus = number;
-        mp_irq.mp_srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3);
+        mp_irq.srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3);
        ioapic = mp_find_ioapic(gsi);
-        mp_irq.mp_dstapic = mp_ioapic_routing[ioapic].apic_id;
+        mp_irq.dstapic = mp_ioapic_routing[ioapic].apic_id;
-        mp_irq.mp_dstirq = gsi - mp_ioapic_routing[ioapic].gsi_base;
+        mp_irq.dstirq = gsi - mp_ioapic_routing[ioapic].gsi_base;
        save_mp_irq(&mp_irq);
 #endif
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 707c1f6f95fa..4abff454c55b 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -101,6 +101,7 @@ int acpi_save_state_mem(void)
        stack_start.sp = temp_stack + sizeof(temp_stack);
        early_gdt_descr.address =
                        (unsigned long)get_cpu_gdt_table(smp_processor_id());
+        initial_gs = per_cpu_offset(smp_processor_id());
 #endif
        initial_code = (unsigned long)wakeup_long64;
        saved_magic = 0x123456789abcdef0;
diff --git a/arch/x86/kernel/apic.c b/arch/x86/kernel/apic.c
index 0f830e4f5675..c6f15647eba9 100644
--- a/arch/x86/kernel/apic.c
+++ b/arch/x86/kernel/apic.c
@@ -60,6 +60,24 @@
 # error SPURIOUS_APIC_VECTOR definition error
 #endif
+unsigned int num_processors;
+unsigned disabled_cpus __cpuinitdata;
+/* Processor that is doing the boot up */
+unsigned int boot_cpu_physical_apicid = -1U;
+EXPORT_SYMBOL(boot_cpu_physical_apicid);
+unsigned int max_physical_apicid;
+/* Bitmask of physically existing CPUs */
+physid_mask_t phys_cpu_present_map;
+/*
+ * Map cpu index to physical APIC ID
+ */
+DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
+EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
+EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
 #ifdef CONFIG_X86_32
 /*
 * Knob to control our willingness to enable the local APIC.
@@ -895,6 +913,10 @@ void disable_local_APIC(void)
 {
        unsigned int value;
+        /* APIC hasn't been mapped yet */
+        if (!apic_phys)
+                return;
        clear_local_APIC();
        /*
@@ -1126,6 +1148,13 @@ void __cpuinit setup_local_APIC(void)
        unsigned int value;
        int i, j;
+        if (disable_apic) {
+#ifdef CONFIG_X86_IO_APIC
+                disable_ioapic_setup();
+#endif
+                return;
+        }
 #ifdef CONFIG_X86_32
        /* Pound the ESR really hard over the head with a big hammer - mbligh */
        if (lapic_is_integrated() && esr_disable) {
@@ -1566,11 +1595,11 @@ int apic_version[MAX_APICS];
 int __init APIC_init_uniprocessor(void)
 {
-#ifdef CONFIG_X86_64
        if (disable_apic) {
                pr_info("Apic disabled\n");
                return -1;
        }
+#ifdef CONFIG_X86_64
        if (!cpu_has_apic) {
                disable_apic = 1;
                pr_info("Apic disabled by BIOS\n");
@@ -1833,6 +1862,11 @@ void __cpuinit generic_processor_info(int apicid, int version)
        num_processors++;
        cpu = cpumask_next_zero(-1, cpu_present_mask);
+        if (version != apic_version[boot_cpu_physical_apicid])
+                WARN_ONCE(1,
+                        "ACPI: apic version mismatch, bootcpu: %x cpu %d: %x\n",
+                        apic_version[boot_cpu_physical_apicid], cpu, version);
        physid_set(apicid, phys_cpu_present_map);
        if (apicid == boot_cpu_physical_apicid) {
                /*
@@ -1868,17 +1902,8 @@ void __cpuinit generic_processor_info(int apicid, int version)
 #endif
 #if defined(CONFIG_X86_SMP) || defined(CONFIG_X86_64)
-        /* are we being called early in kernel startup? */
+        early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
-        if (early_per_cpu_ptr(x86_cpu_to_apicid)) {
+        early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
-                u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
-                u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
-                cpu_to_apicid[cpu] = apicid;
-                bios_cpu_apicid[cpu] = apicid;
-        } else {
-                per_cpu(x86_cpu_to_apicid, cpu) = apicid;
-                per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
-        }
 #endif
        set_cpu_possible(cpu, true);
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 1d41d3f1edbc..8793ab33e2c1 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -11,7 +11,6 @@
 #include <linux/hardirq.h>
 #include <linux/suspend.h>
 #include <linux/kbuild.h>
-#include <asm/pda.h>
 #include <asm/processor.h>
 #include <asm/segment.h>
 #include <asm/thread_info.h>
@@ -48,16 +47,6 @@ int main(void)
 #endif
        BLANK();
 #undef ENTRY
-#define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))
-        ENTRY(kernelstack); 
-        ENTRY(oldrsp); 
-        ENTRY(pcurrent); 
-        ENTRY(irqcount);
-        ENTRY(cpunumber);
-        ENTRY(irqstackptr);
-        ENTRY(data_offset);
-        BLANK();
-#undef ENTRY
 #ifdef CONFIG_PARAVIRT
        BLANK();
        OFFSET(PARAVIRT_enabled, pv_info, paravirt_enabled);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 83492b1f93b1..652fdc9a757a 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -21,14 +21,16 @@
 #include <asm/asm.h>
 #include <asm/numa.h>
 #include <asm/smp.h>
+#include <asm/cpu.h>
+#include <asm/cpumask.h>
 #ifdef CONFIG_X86_LOCAL_APIC
 #include <asm/mpspec.h>
 #include <asm/apic.h>
 #include <mach_apic.h>
 #include <asm/genapic.h>
+#include <asm/uv/uv.h>
 #endif
-#include <asm/pda.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/desc.h>
@@ -50,6 +52,15 @@ cpumask_var_t cpu_initialized_mask;
 /* representing cpus for which sibling maps can be computed */
 cpumask_var_t cpu_sibling_setup_mask;
+/* correctly size the local cpu masks */
+void __init setup_cpu_local_masks(void)
+{
+        alloc_bootmem_cpumask_var(&cpu_initialized_mask);
+        alloc_bootmem_cpumask_var(&cpu_callin_mask);
+        alloc_bootmem_cpumask_var(&cpu_callout_mask);
+        alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
+}
 #else /* CONFIG_X86_32 */
 cpumask_t cpu_callin_map;
@@ -62,23 +73,23 @@ cpumask_t cpu_sibling_setup_map;
 static struct cpu_dev *this_cpu __cpuinitdata;
+DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 #ifdef CONFIG_X86_64
-/* We need valid kernel segments for data and code in long mode too
+        /*
- * IRET will check the segment types  kkeil 2000/10/28
+         * We need valid kernel segments for data and code in long mode too
- * Also sysret mandates a special GDT layout
+         * IRET will check the segment types  kkeil 2000/10/28
- */
+         * Also sysret mandates a special GDT layout
-/* The TLS descriptors are currently at a different place compared to i386.
+         *
-   Hopefully nobody expects them at a fixed place (Wine?) */
+         * The TLS descriptors are currently at a different place compared to i386.
-DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = {
+         * Hopefully nobody expects them at a fixed place (Wine?)
+         */
        [GDT_ENTRY_KERNEL32_CS] = { { { 0x0000ffff, 0x00cf9b00 } } },
        [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00af9b00 } } },
        [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9300 } } },
        [GDT_ENTRY_DEFAULT_USER32_CS] = { { { 0x0000ffff, 0x00cffb00 } } },
        [GDT_ENTRY_DEFAULT_USER_DS] = { { { 0x0000ffff, 0x00cff300 } } },
        [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00affb00 } } },
-} };
 #else
-DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
        [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00cf9a00 } } },
        [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9200 } } },
        [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00cffa00 } } },
@@ -110,9 +121,9 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
        [GDT_ENTRY_APMBIOS_BASE+2] = { { { 0x0000ffff, 0x00409200 } } },
        [GDT_ENTRY_ESPFIX_SS] = { { { 0x00000000, 0x00c09200 } } },
-        [GDT_ENTRY_PERCPU] = { { { 0x00000000, 0x00000000 } } },
+        [GDT_ENTRY_PERCPU] = { { { 0x0000ffff, 0x00cf9200 } } },
-} };
 #endif
+} };
 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
 #ifdef CONFIG_X86_32
@@ -247,12 +258,17 @@ __u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;
 void switch_to_new_gdt(void)
 {
        struct desc_ptr gdt_descr;
+        int cpu = smp_processor_id();
-        gdt_descr.address = (long)get_cpu_gdt_table(smp_processor_id());
+        gdt_descr.address = (long)get_cpu_gdt_table(cpu);
        gdt_descr.size = GDT_SIZE - 1;
        load_gdt(&gdt_descr);
+        /* Reload the per-cpu base */
 #ifdef CONFIG_X86_32
-        asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
+        loadsegment(fs, __KERNEL_PERCPU);
+#else
+        loadsegment(gs, 0);
+        wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
 #endif
 }
@@ -877,54 +893,26 @@ static __init int setup_disablecpuid(char *arg)
 __setup("clearcpuid=", setup_disablecpuid);
 #ifdef CONFIG_X86_64
-struct x8664_pda **_cpu_pda __read_mostly;
-EXPORT_SYMBOL(_cpu_pda);
 struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
-static char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss;
+DEFINE_PER_CPU_FIRST(union irq_stack_union,
+                     irq_stack_union) __aligned(PAGE_SIZE);
+#ifdef CONFIG_SMP
+DEFINE_PER_CPU(char *, irq_stack_ptr);  /* will be set during per cpu init */
+#else
+DEFINE_PER_CPU(char *, irq_stack_ptr) =
+        per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
+#endif
-void __cpuinit pda_init(int cpu)
+DEFINE_PER_CPU(unsigned long, kernel_stack) =
-{
+        (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
-        struct x8664_pda *pda = cpu_pda(cpu);
+EXPORT_PER_CPU_SYMBOL(kernel_stack);
-        /* Setup up data that may be needed in __get_free_pages early */
+DEFINE_PER_CPU(unsigned int, irq_count) = -1;
-        loadsegment(fs, 0);
-        loadsegment(gs, 0);
-        /* Memory clobbers used to order PDA accessed */
-        mb();
-        wrmsrl(MSR_GS_BASE, pda);
-        mb();
-        pda->cpunumber = cpu;
-        pda->irqcount = -1;
-        pda->kernelstack = (unsigned long)stack_thread_info() -
-                                 PDA_STACKOFFSET + THREAD_SIZE;
-        pda->active_mm = &init_mm;
-        pda->mmu_state = 0;
-        if (cpu == 0) {
-                /* others are initialized in smpboot.c */
-                pda->pcurrent = &init_task;
-                pda->irqstackptr = boot_cpu_stack;
-                pda->irqstackptr += IRQSTACKSIZE - 64;
-        } else {
-                if (!pda->irqstackptr) {
-                        pda->irqstackptr = (char *)
-                                __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
-                        if (!pda->irqstackptr)
-                                panic("cannot allocate irqstack for cpu %d",
-                                      cpu);
-                        pda->irqstackptr += IRQSTACKSIZE - 64;
-                }
-                if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE)
+static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
-                        pda->nodenumber = cpu_to_node(cpu);
+        [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ])
-        }
+        __aligned(PAGE_SIZE);
-}
-static char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ +
-                                  DEBUG_STKSZ] __page_aligned_bss;
 extern asmlinkage void ignore_sysret(void);
@@ -982,15 +970,14 @@ void __cpuinit cpu_init(void)
        struct tss_struct *t = &per_cpu(init_tss, cpu);
        struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu);
        unsigned long v;
-        char *estacks = NULL;
        struct task_struct *me;
        int i;
-        /* CPU 0 is initialised in head64.c */
+#ifdef CONFIG_NUMA
-        if (cpu != 0)
+        if (cpu != 0 && percpu_read(node_number) == 0 &&
-                pda_init(cpu);
+            cpu_to_node(cpu) != NUMA_NO_NODE)
-        else
+                percpu_write(node_number, cpu_to_node(cpu));
-                estacks = boot_exception_stacks;
+#endif
        me = current;
@@ -1007,6 +994,8 @@ void __cpuinit cpu_init(void)
         */
        switch_to_new_gdt();
+        loadsegment(fs, 0);
        load_idt((const struct desc_ptr *)&idt_descr);
        memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
@@ -1024,18 +1013,13 @@ void __cpuinit cpu_init(void)
         * set up and load the per-CPU TSS
         */
        if (!orig_ist->ist[0]) {
-                static const unsigned int order[N_EXCEPTION_STACKS] = {
+                static const unsigned int sizes[N_EXCEPTION_STACKS] = {
-                  [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
+                  [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ,
-                  [DEBUG_STACK - 1] = DEBUG_STACK_ORDER
+                  [DEBUG_STACK - 1] = DEBUG_STKSZ
                };
+                char *estacks = per_cpu(exception_stacks, cpu);
                for (v = 0; v < N_EXCEPTION_STACKS; v++) {
-                        if (cpu) {
+                        estacks += sizes[v];
-                                estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
-                                if (!estacks)
-                                        panic("Cannot allocate exception "
-                                              "stack %ld %d\n", v, cpu);
-                        }
-                        estacks += PAGE_SIZE << order[v];
                        orig_ist->ist[v] = t->x86_tss.ist[v] =
                                        (unsigned long)estacks;
                }
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
index 6f11e029e8c5..4b1c319d30c3 100644
--- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
+++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
@@ -145,13 +145,14 @@ typedef union {
 struct drv_cmd {
        unsigned int type;
-        cpumask_var_t mask;
+        const struct cpumask *mask;
        drv_addr_union addr;
        u32 val;
 };
-static void do_drv_read(struct drv_cmd *cmd)
+static long do_drv_read(void *_cmd)
 {
+        struct drv_cmd *cmd = _cmd;
        u32 h;
        switch (cmd->type) {
@@ -166,10 +167,12 @@ static void do_drv_read(struct drv_cmd *cmd)
        default:
                break;
        }
+        return 0;
 }
-static void do_drv_write(struct drv_cmd *cmd)
+static long do_drv_write(void *_cmd)
 {
+        struct drv_cmd *cmd = _cmd;
        u32 lo, hi;
        switch (cmd->type) {
@@ -186,30 +189,23 @@ static void do_drv_write(struct drv_cmd *cmd)
        default:
                break;
        }
+        return 0;
 }
 static void drv_read(struct drv_cmd *cmd)
 {
-        cpumask_t saved_mask = current->cpus_allowed;
        cmd->val = 0;
-        set_cpus_allowed_ptr(current, cmd->mask);
+        work_on_cpu(cpumask_any(cmd->mask), do_drv_read, cmd);
-        do_drv_read(cmd);
-        set_cpus_allowed_ptr(current, &saved_mask);
 }
 static void drv_write(struct drv_cmd *cmd)
 {
-        cpumask_t saved_mask = current->cpus_allowed;
        unsigned int i;
        for_each_cpu(i, cmd->mask) {
-                set_cpus_allowed_ptr(current, cpumask_of(i));
+                work_on_cpu(i, do_drv_write, cmd);
-                do_drv_write(cmd);
        }
-        set_cpus_allowed_ptr(current, &saved_mask);
-        return;
 }
 static u32 get_cur_val(const struct cpumask *mask)
@@ -235,8 +231,7 @@ static u32 get_cur_val(const struct cpumask *mask)
                return 0;
        }
-        cpumask_copy(cmd.mask, mask);
+        cmd.mask = mask;
        drv_read(&cmd);
        dprintk("get_cur_val = %u\n", cmd.val);
@@ -368,7 +363,7 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
        return freq;
 }
-static unsigned int check_freqs(const cpumask_t *mask, unsigned int freq,
+static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq,
                                struct acpi_cpufreq_data *data)
 {
        unsigned int cur_freq;
@@ -403,9 +398,6 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
                return -ENODEV;
        }
-        if (unlikely(!alloc_cpumask_var(&cmd.mask, GFP_KERNEL)))
-                return -ENOMEM;
        perf = data->acpi_data;
        result = cpufreq_frequency_table_target(policy,
                                                data->freq_table,
@@ -450,9 +442,9 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
        /* cpufreq holds the hotplug lock, so we are safe from here on */
        if (policy->shared_type != CPUFREQ_SHARED_TYPE_ANY)
-                cpumask_and(cmd.mask, cpu_online_mask, policy->cpus);
+                cmd.mask = policy->cpus;
        else
-                cpumask_copy(cmd.mask, cpumask_of(policy->cpu));
+                cmd.mask = cpumask_of(policy->cpu);
        freqs.old = perf->states[perf->state].core_frequency * 1000;
        freqs.new = data->freq_table[next_state].frequency;
@@ -479,7 +471,6 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
        perf->state = next_perf_state;
 out:
-        free_cpumask_var(cmd.mask);
        return result;
 }
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 8ea6929e974c..549f2ada55f5 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -29,6 +29,19 @@
 static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
 {
+        /* Unmask CPUID levels if masked: */
+        if (c->x86 == 6 && c->x86_model >= 15) {
+                u64 misc_enable;
+                rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
+                if (misc_enable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) {
+                        misc_enable &= ~MSR_IA32_MISC_ENABLE_LIMIT_CPUID;
+                        wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
+                        c->cpuid_level = cpuid_eax(0);
+                }
+        }
        if ((c->x86 == 0xf && c->x86_model >= 0x03) ||
                (c->x86 == 0x6 && c->x86_model >= 0x0e))
                set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 48533d77be78..58527a9fc404 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -132,7 +132,16 @@ struct _cpuid4_info {
        union _cpuid4_leaf_ecx ecx;
        unsigned long size;
        unsigned long can_disable;
-        cpumask_t shared_cpu_map;       /* future?: only cpus/node is needed */
+        DECLARE_BITMAP(shared_cpu_map, NR_CPUS);
+};
+/* subset of above _cpuid4_info w/o shared_cpu_map */
+struct _cpuid4_info_regs {
+        union _cpuid4_leaf_eax eax;
+        union _cpuid4_leaf_ebx ebx;
+        union _cpuid4_leaf_ecx ecx;
+        unsigned long size;
+        unsigned long can_disable;
 };
 #ifdef CONFIG_PCI
@@ -263,7 +272,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
 }
 static void __cpuinit
-amd_check_l3_disable(int index, struct _cpuid4_info *this_leaf)
+amd_check_l3_disable(int index, struct _cpuid4_info_regs *this_leaf)
 {
        if (index < 3)
                return;
@@ -271,7 +280,8 @@ amd_check_l3_disable(int index, struct _cpuid4_info *this_leaf)
 }
 static int
-__cpuinit cpuid4_cache_lookup(int index, struct _cpuid4_info *this_leaf)
+__cpuinit cpuid4_cache_lookup_regs(int index,
+                                   struct _cpuid4_info_regs *this_leaf)
 {
        union _cpuid4_leaf_eax  eax;
        union _cpuid4_leaf_ebx  ebx;
@@ -299,6 +309,15 @@ __cpuinit cpuid4_cache_lookup(int index, struct _cpuid4_info *this_leaf)
        return 0;
 }
+static int
+__cpuinit cpuid4_cache_lookup(int index, struct _cpuid4_info *this_leaf)
+{
+        struct _cpuid4_info_regs *leaf_regs =
+                (struct _cpuid4_info_regs *)this_leaf;
+        return cpuid4_cache_lookup_regs(index, leaf_regs);
+}
 static int __cpuinit find_num_cache_leaves(void)
 {
        unsigned int            eax, ebx, ecx, edx;
@@ -338,11 +357,10 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
                 * parameters cpuid leaf to find the cache details
                 */
                for (i = 0; i < num_cache_leaves; i++) {
-                        struct _cpuid4_info this_leaf;
+                        struct _cpuid4_info_regs this_leaf;
                        int retval;
-                        retval = cpuid4_cache_lookup(i, &this_leaf);
+                        retval = cpuid4_cache_lookup_regs(i, &this_leaf);
                        if (retval >= 0) {
                                switch(this_leaf.eax.split.level) {
                                    case 1:
@@ -491,17 +509,20 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
        num_threads_sharing = 1 + this_leaf->eax.split.num_threads_sharing;
        if (num_threads_sharing == 1)
-                cpu_set(cpu, this_leaf->shared_cpu_map);
+                cpumask_set_cpu(cpu, to_cpumask(this_leaf->shared_cpu_map));
        else {
                index_msb = get_count_order(num_threads_sharing);
                for_each_online_cpu(i) {
                        if (cpu_data(i).apicid >> index_msb ==
                            c->apicid >> index_msb) {
-                                cpu_set(i, this_leaf->shared_cpu_map);
+                                cpumask_set_cpu(i,
+                                        to_cpumask(this_leaf->shared_cpu_map));
                                if (i != cpu && per_cpu(cpuid4_info, i))  {
-                                        sibling_leaf = CPUID4_INFO_IDX(i, index);
+                                        sibling_leaf =
-                                        cpu_set(cpu, sibling_leaf->shared_cpu_map);
+                                                CPUID4_INFO_IDX(i, index);
+                                        cpumask_set_cpu(cpu, to_cpumask(
+                                                sibling_leaf->shared_cpu_map));
                                }
                        }
                }
@@ -513,9 +534,10 @@ static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index)
        int sibling;
        this_leaf = CPUID4_INFO_IDX(cpu, index);
-        for_each_cpu_mask_nr(sibling, this_leaf->shared_cpu_map) {
+        for_each_cpu(sibling, to_cpumask(this_leaf->shared_cpu_map)) {
                sibling_leaf = CPUID4_INFO_IDX(sibling, index);
-                cpu_clear(cpu, sibling_leaf->shared_cpu_map);
+                cpumask_clear_cpu(cpu,
+                                  to_cpumask(sibling_leaf->shared_cpu_map));
        }
 }
 #else
@@ -620,8 +642,9 @@ static ssize_t show_shared_cpu_map_func(struct _cpuid4_info *this_leaf,
        int n = 0;
        if (len > 1) {
-                cpumask_t *mask = &this_leaf->shared_cpu_map;
+                const struct cpumask *mask;
+                mask = to_cpumask(this_leaf->shared_cpu_map);
                n = type?
                        cpulist_scnprintf(buf, len-2, mask) :
                        cpumask_scnprintf(buf, len-2, mask);
@@ -684,7 +707,8 @@ static struct pci_dev *get_k8_northbridge(int node)
 static ssize_t show_cache_disable(struct _cpuid4_info *this_leaf, char *buf)
 {
-        int node = cpu_to_node(first_cpu(this_leaf->shared_cpu_map));
+        const struct cpumask *mask = to_cpumask(this_leaf->shared_cpu_map);
+        int node = cpu_to_node(cpumask_first(mask));
        struct pci_dev *dev = NULL;
        ssize_t ret = 0;
        int i;
@@ -718,7 +742,8 @@ static ssize_t
 store_cache_disable(struct _cpuid4_info *this_leaf, const char *buf,
                    size_t count)
 {
-        int node = cpu_to_node(first_cpu(this_leaf->shared_cpu_map));
+        const struct cpumask *mask = to_cpumask(this_leaf->shared_cpu_map);
+        int node = cpu_to_node(cpumask_first(mask));
        struct pci_dev *dev = NULL;
        unsigned int ret, index, val;
@@ -863,7 +888,7 @@ err_out:
        return -ENOMEM;
 }
-static cpumask_t cache_dev_map = CPU_MASK_NONE;
+static DECLARE_BITMAP(cache_dev_map, NR_CPUS);
 /* Add/Remove cache interface for CPU device */
 static int __cpuinit cache_add_dev(struct sys_device * sys_dev)
@@ -903,7 +928,7 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev)
                }
                kobject_uevent(&(this_object->kobj), KOBJ_ADD);
        }
-        cpu_set(cpu, cache_dev_map);
+        cpumask_set_cpu(cpu, to_cpumask(cache_dev_map));
        kobject_uevent(per_cpu(cache_kobject, cpu), KOBJ_ADD);
        return 0;
@@ -916,9 +941,9 @@ static void __cpuinit cache_remove_dev(struct sys_device * sys_dev)
        if (per_cpu(cpuid4_info, cpu) == NULL)
                return;
-        if (!cpu_isset(cpu, cache_dev_map))
+        if (!cpumask_test_cpu(cpu, to_cpumask(cache_dev_map)))
                return;
-        cpu_clear(cpu, cache_dev_map);
+        cpumask_clear_cpu(cpu, to_cpumask(cache_dev_map));
        for (i = 0; i < num_cache_leaves; i++)
                kobject_put(&(INDEX_KOBJECT_PTR(cpu,i)->kobj));
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
index 8ae8c4ff094d..4772e91e8246 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
@@ -67,7 +67,7 @@ static struct threshold_block threshold_defaults = {
 struct threshold_bank {
        struct kobject *kobj;
        struct threshold_block *blocks;
-        cpumask_t cpus;
+        cpumask_var_t cpus;
 };
 static DEFINE_PER_CPU(struct threshold_bank *, threshold_banks[NR_BANKS]);
@@ -481,7 +481,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 #ifdef CONFIG_SMP
        if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) {   /* symlink */
-                i = first_cpu(per_cpu(cpu_core_map, cpu));
+                i = cpumask_first(&per_cpu(cpu_core_map, cpu));
                /* first core not up yet */
                if (cpu_data(i).cpu_core_id)
@@ -501,7 +501,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
                if (err)
                        goto out;
-                b->cpus = per_cpu(cpu_core_map, cpu);
+                cpumask_copy(b->cpus, &per_cpu(cpu_core_map, cpu));
                per_cpu(threshold_banks, cpu)[bank] = b;
                goto out;
        }
@@ -512,15 +512,20 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
                err = -ENOMEM;
                goto out;
        }
+        if (!alloc_cpumask_var(&b->cpus, GFP_KERNEL)) {
+                kfree(b);
+                err = -ENOMEM;
+                goto out;
+        }
        b->kobj = kobject_create_and_add(name, &per_cpu(device_mce, cpu).kobj);
        if (!b->kobj)
                goto out_free;
 #ifndef CONFIG_SMP
-        b->cpus = CPU_MASK_ALL;
+        cpumask_setall(b->cpus);
 #else
-        b->cpus = per_cpu(cpu_core_map, cpu);
+        cpumask_copy(b->cpus, &per_cpu(cpu_core_map, cpu));
 #endif
        per_cpu(threshold_banks, cpu)[bank] = b;
@@ -529,7 +534,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
        if (err)
                goto out_free;
-        for_each_cpu_mask_nr(i, b->cpus) {
+        for_each_cpu(i, b->cpus) {
                if (i == cpu)
                        continue;
@@ -545,6 +550,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 out_free:
        per_cpu(threshold_banks, cpu)[bank] = NULL;
+        free_cpumask_var(b->cpus);
        kfree(b);
 out:
        return err;
@@ -619,7 +625,7 @@ static void threshold_remove_bank(unsigned int cpu, int bank)
 #endif
        /* remove all sibling symlinks before unregistering */
-        for_each_cpu_mask_nr(i, b->cpus) {
+        for_each_cpu(i, b->cpus) {
                if (i == cpu)
                        continue;
@@ -632,6 +638,7 @@ static void threshold_remove_bank(unsigned int cpu, int bank)
 free_out:
        kobject_del(b->kobj);
        kobject_put(b->kobj);
+        free_cpumask_var(b->cpus);
        kfree(b);
        per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
index 4b48f251fd39..5e8c79e748a6 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
@@ -7,6 +7,7 @@
 #include <linux/interrupt.h>
 #include <linux/percpu.h>
 #include <asm/processor.h>
+#include <asm/apic.h>
 #include <asm/msr.h>
 #include <asm/mce.h>
 #include <asm/hw_irq.h>
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index b59ddcc88cd8..0c0a455fe95c 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -33,11 +33,13 @@ u64 mtrr_tom2;
 struct mtrr_state_type mtrr_state = {};
 EXPORT_SYMBOL_GPL(mtrr_state);
-#undef MODULE_PARAM_PREFIX
+static int __initdata mtrr_show;
-#define MODULE_PARAM_PREFIX "mtrr."
+static int __init mtrr_debug(char *opt)
+{
-static int mtrr_show;
+        mtrr_show = 1;
-module_param_named(show, mtrr_show, bool, 0);
+        return 0;
+}
+early_param("mtrr.show", mtrr_debug);
 /*
 * Returns the effective MTRR type for the region
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index c689d19e35ab..11b93cabdf78 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -24,7 +24,7 @@
 #include <asm/apic.h>
 #include <asm/hpet.h>
 #include <linux/kdebug.h>
-#include <asm/smp.h>
+#include <asm/cpu.h>
 #include <asm/reboot.h>
 #include <asm/virtext.h>
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index c302d0707048..d35db5993fd6 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -106,7 +106,8 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                const struct stacktrace_ops *ops, void *data)
 {
        const unsigned cpu = get_cpu();
-        unsigned long *irqstack_end = (unsigned long *)cpu_pda(cpu)->irqstackptr;
+        unsigned long *irq_stack_end =
+                (unsigned long *)per_cpu(irq_stack_ptr, cpu);
        unsigned used = 0;
        struct thread_info *tinfo;
        int graph = 0;
@@ -160,23 +161,23 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                        stack = (unsigned long *) estack_end[-2];
                        continue;
                }
-                if (irqstack_end) {
+                if (irq_stack_end) {
-                        unsigned long *irqstack;
+                        unsigned long *irq_stack;
-                        irqstack = irqstack_end -
+                        irq_stack = irq_stack_end -
-                                (IRQSTACKSIZE - 64) / sizeof(*irqstack);
+                                (IRQ_STACK_SIZE - 64) / sizeof(*irq_stack);
-                        if (stack >= irqstack && stack < irqstack_end) {
+                        if (stack >= irq_stack && stack < irq_stack_end) {
                                if (ops->stack(data, "IRQ") < 0)
                                        break;
                                bp = print_context_stack(tinfo, stack, bp,
-                                        ops, data, irqstack_end, &graph);
+                                        ops, data, irq_stack_end, &graph);
                                /*
                                 * We link to the next stack (which would be
                                 * the process stack normally) the last
                                 * pointer (index -1 to end) in the IRQ stack:
                                 */
-                                stack = (unsigned long *) (irqstack_end[-1]);
+                                stack = (unsigned long *) (irq_stack_end[-1]);
-                                irqstack_end = NULL;
+                                irq_stack_end = NULL;
                                ops->stack(data, "EOI");
                                continue;
                        }
@@ -199,10 +200,10 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
        unsigned long *stack;
        int i;
        const int cpu = smp_processor_id();
-        unsigned long *irqstack_end =
+        unsigned long *irq_stack_end =
-                (unsigned long *) (cpu_pda(cpu)->irqstackptr);
+                (unsigned long *)(per_cpu(irq_stack_ptr, cpu));
-        unsigned long *irqstack =
+        unsigned long *irq_stack =
-                (unsigned long *) (cpu_pda(cpu)->irqstackptr - IRQSTACKSIZE);
+                (unsigned long *)(per_cpu(irq_stack_ptr, cpu) - IRQ_STACK_SIZE);
        /*
         * debugging aid: "show_stack(NULL, NULL);" prints the
@@ -218,9 +219,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
        stack = sp;
        for (i = 0; i < kstack_depth_to_print; i++) {
-                if (stack >= irqstack && stack <= irqstack_end) {
+                if (stack >= irq_stack && stack <= irq_stack_end) {
-                        if (stack == irqstack_end) {
+                        if (stack == irq_stack_end) {
-                                stack = (unsigned long *) (irqstack_end[-1]);
+                                stack = (unsigned long *) (irq_stack_end[-1]);
                                printk(" <EOI> ");
                        }
                } else {
@@ -241,7 +242,7 @@ void show_registers(struct pt_regs *regs)
        int i;
        unsigned long sp;
        const int cpu = smp_processor_id();
-        struct task_struct *cur = cpu_pda(cpu)->pcurrent;
+        struct task_struct *cur = current;
        sp = regs->sp;
        printk("CPU %d ", cpu);
diff --git a/arch/x86/kernel/efi.c b/arch/x86/kernel/efi.c
index 1119d247fe11..b205272ad394 100644
--- a/arch/x86/kernel/efi.c
+++ b/arch/x86/kernel/efi.c
@@ -366,10 +366,12 @@ void __init efi_init(void)
                                        SMBIOS_TABLE_GUID)) {
                        efi.smbios = config_tables[i].table;
                        printk(" SMBIOS=0x%lx ", config_tables[i].table);
+#ifdef CONFIG_X86_UV
                } else if (!efi_guidcmp(config_tables[i].guid,
                                        UV_SYSTEM_TABLE_GUID)) {
                        efi.uv_systab = config_tables[i].table;
                        printk(" UVsystab=0x%lx ", config_tables[i].table);
+#endif
                } else if (!efi_guidcmp(config_tables[i].guid,
                                        HCDP_TABLE_GUID)) {
                        efi.hcdp = config_tables[i].table;
diff --git a/arch/x86/kernel/efi_64.c b/arch/x86/kernel/efi_64.c
index 652c5287215f..a4ee29127fdf 100644
--- a/arch/x86/kernel/efi_64.c
+++ b/arch/x86/kernel/efi_64.c
@@ -36,6 +36,7 @@
 #include <asm/proto.h>
 #include <asm/efi.h>
 #include <asm/cacheflush.h>
+#include <asm/fixmap.h>
 static pgd_t save_pgd __initdata;
 static unsigned long efi_flags __initdata;
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 46469029e9d3..a0b91aac72a1 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -672,7 +672,7 @@ common_interrupt:
 ENDPROC(common_interrupt)
        CFI_ENDPROC
-#define BUILD_INTERRUPT(name, nr)       \
+#define BUILD_INTERRUPT3(name, nr, fn)  \
 ENTRY(name)                             \
        RING0_INT_FRAME;                \
        pushl $~(nr);                   \
@@ -680,11 +680,13 @@ ENTRY(name)				\
        SAVE_ALL;                       \
        TRACE_IRQS_OFF                  \
        movl %esp,%eax;                 \
-        call smp_##name;                \
+        call fn;                        \
        jmp ret_from_intr;              \
        CFI_ENDPROC;                    \
 ENDPROC(name)
+#define BUILD_INTERRUPT(name, nr)       BUILD_INTERRUPT3(name, nr, smp_##name)
 /* The include is where all of the SMP etc. interrupts come from */
 #include "entry_arch.h"
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e28c7a987793..a52703864a16 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -52,6 +52,7 @@
 #include <asm/irqflags.h>
 #include <asm/paravirt.h>
 #include <asm/ftrace.h>
+#include <asm/percpu.h>
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -209,7 +210,7 @@ ENTRY(native_usergs_sysret64)
        /* %rsp:at FRAMEEND */
        .macro FIXUP_TOP_OF_STACK tmp offset=0
-        movq %gs:pda_oldrsp,\tmp
+        movq PER_CPU_VAR(old_rsp),\tmp
        movq \tmp,RSP+\offset(%rsp)
        movq $__USER_DS,SS+\offset(%rsp)
        movq $__USER_CS,CS+\offset(%rsp)
@@ -220,7 +221,7 @@ ENTRY(native_usergs_sysret64)
        .macro RESTORE_TOP_OF_STACK tmp offset=0
        movq RSP+\offset(%rsp),\tmp
-        movq \tmp,%gs:pda_oldrsp
+        movq \tmp,PER_CPU_VAR(old_rsp)
        movq EFLAGS+\offset(%rsp),\tmp
        movq \tmp,R11+\offset(%rsp)
        .endm
@@ -336,15 +337,15 @@ ENTRY(save_args)
        je 1f
        SWAPGS
        /*
-         * irqcount is used to check if a CPU is already on an interrupt stack
+         * irq_count is used to check if a CPU is already on an interrupt stack
         * or not. While this is essentially redundant with preempt_count it is
         * a little cheaper to use a separate counter in the PDA (short of
         * moving irq_enter into assembly, which would be too much work)
         */
-1:      incl %gs:pda_irqcount
+1:      incl PER_CPU_VAR(irq_count)
        jne 2f
        popq_cfi %rax                   /* move return address... */
-        mov %gs:pda_irqstackptr,%rsp
+        mov PER_CPU_VAR(irq_stack_ptr),%rsp
        EMPTY_FRAME 0
        pushq_cfi %rax                  /* ... to the new stack */
        /*
@@ -467,7 +468,7 @@ END(ret_from_fork)
 ENTRY(system_call)
        CFI_STARTPROC   simple
        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
+        CFI_DEF_CFA     rsp,KERNEL_STACK_OFFSET
        CFI_REGISTER    rip,rcx
        /*CFI_REGISTER  rflags,r11*/
        SWAPGS_UNSAFE_STACK
@@ -478,8 +479,8 @@ ENTRY(system_call)
         */
 ENTRY(system_call_after_swapgs)
-        movq    %rsp,%gs:pda_oldrsp
+        movq    %rsp,PER_CPU_VAR(old_rsp)
-        movq    %gs:pda_kernelstack,%rsp
+        movq    PER_CPU_VAR(kernel_stack),%rsp
        /*
         * No need to follow this irqs off/on section - it's straight
         * and short:
@@ -522,7 +523,7 @@ sysret_check:
        CFI_REGISTER    rip,rcx
        RESTORE_ARGS 0,-ARG_SKIP,1
        /*CFI_REGISTER  rflags,r11*/
-        movq    %gs:pda_oldrsp, %rsp
+        movq    PER_CPU_VAR(old_rsp), %rsp
        USERGS_SYSRET64
        CFI_RESTORE_STATE
@@ -832,11 +833,11 @@ common_interrupt:
        XCPT_FRAME
        addq $-0x80,(%rsp)              /* Adjust vector to [-256,-1] range */
        interrupt do_IRQ
-        /* 0(%rsp): oldrsp-ARGOFFSET */
+        /* 0(%rsp): old_rsp-ARGOFFSET */
 ret_from_intr:
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
        leaveq
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
@@ -981,8 +982,10 @@ apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \
        irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
 #endif
+#ifdef CONFIG_X86_UV
 apicinterrupt UV_BAU_MESSAGE \
        uv_bau_message_intr1 uv_bau_message_interrupt
+#endif
 apicinterrupt LOCAL_TIMER_VECTOR \
        apic_timer_interrupt smp_apic_timer_interrupt
@@ -1072,10 +1075,10 @@ ENTRY(\sym)
        TRACE_IRQS_OFF
        movq %rsp,%rdi          /* pt_regs pointer */
        xorl %esi,%esi          /* no error code */
-        movq %gs:pda_data_offset, %rbp
+        PER_CPU(init_tss, %rbp)
-        subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        subq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%rbp)
        call \do_sym
-        addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        addq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%rbp)
        jmp paranoid_exit       /* %ebx: no swapgs flag */
        CFI_ENDPROC
 END(\sym)
@@ -1259,14 +1262,14 @@ ENTRY(call_softirq)
        CFI_REL_OFFSET rbp,0
        mov  %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
-        incl %gs:pda_irqcount
+        incl PER_CPU_VAR(irq_count)
-        cmove %gs:pda_irqstackptr,%rsp
+        cmove PER_CPU_VAR(irq_stack_ptr),%rsp
        push  %rbp                      # backlink for old unwinder
        call __do_softirq
        leaveq
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
        ret
        CFI_ENDPROC
 END(call_softirq)
@@ -1296,15 +1299,15 @@ ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
        movq %rdi, %rsp            # we don't return, adjust the stack frame
        CFI_ENDPROC
        DEFAULT_FRAME
-11:     incl %gs:pda_irqcount
+11:     incl PER_CPU_VAR(irq_count)
        movq %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
-        cmovzq %gs:pda_irqstackptr,%rsp
+        cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
        pushq %rbp                      # backlink for old unwinder
        call xen_evtchn_do_upcall
        popq %rsp
        CFI_DEF_CFA_REGISTER rsp
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
        jmp  error_exit
        CFI_ENDPROC
 END(do_hypervisor_callback)
diff --git a/arch/x86/kernel/genapic_64.c b/arch/x86/kernel/genapic_64.c
index 2bced78b0b8e..e656c2721154 100644
--- a/arch/x86/kernel/genapic_64.c
+++ b/arch/x86/kernel/genapic_64.c
@@ -32,7 +32,9 @@ extern struct genapic apic_x2apic_cluster;
 struct genapic __read_mostly *genapic = &apic_flat;
 static struct genapic *apic_probe[] __initdata = {
+#ifdef CONFIG_X86_UV
        &apic_x2apic_uv_x,
+#endif
        &apic_x2apic_phys,
        &apic_x2apic_cluster,
        &apic_physflat,
diff --git a/arch/x86/kernel/genx2apic_uv_x.c b/arch/x86/kernel/genx2apic_uv_x.c
index b193e082f6ce..bfe36249145c 100644
--- a/arch/x86/kernel/genx2apic_uv_x.c
+++ b/arch/x86/kernel/genx2apic_uv_x.c
@@ -25,6 +25,7 @@
 #include <asm/ipi.h>
 #include <asm/genapic.h>
 #include <asm/pgtable.h>
+#include <asm/uv/uv.h>
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
 #include <asm/uv/bios.h>
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index b9a4d8c4b935..f5b272247690 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -26,27 +26,6 @@
 #include <asm/bios_ebda.h>
 #include <asm/trampoline.h>
-/* boot cpu pda */
-static struct x8664_pda _boot_cpu_pda;
-#ifdef CONFIG_SMP
-/*
- * We install an empty cpu_pda pointer table to indicate to early users
- * (numa_set_node) that the cpu_pda pointer table for cpus other than
- * the boot cpu is not yet setup.
- */
-static struct x8664_pda *__cpu_pda[NR_CPUS] __initdata;
-#else
-static struct x8664_pda *__cpu_pda[NR_CPUS] __read_mostly;
-#endif
-void __init x86_64_init_pda(void)
-{
-        _cpu_pda = __cpu_pda;
-        cpu_pda(0) = &_boot_cpu_pda;
-        pda_init(0);
-}
 static void __init zap_identity_mappings(void)
 {
        pgd_t *pgd = pgd_offset_k(0UL);
@@ -112,8 +91,6 @@ void __init x86_64_start_kernel(char * real_mode_data)
        if (console_loglevel == 10)
                early_printk("Kernel alive\n");
-        x86_64_init_pda();
        x86_64_start_reservations(real_mode_data);
 }
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index e835b4eea70b..24c0e5cd71e3 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -429,12 +429,14 @@ is386:	movl $2,%ecx		# set MP
        ljmp $(__KERNEL_CS),$1f
 1:      movl $(__KERNEL_DS),%eax        # reload all the segment registers
        movl %eax,%ss                   # after changing gdt.
-        movl %eax,%fs                   # gets reset once there's real percpu
        movl $(__USER_DS),%eax          # DS/ES contains default USER segment
        movl %eax,%ds
        movl %eax,%es
+        movl $(__KERNEL_PERCPU), %eax
+        movl %eax,%fs                   # set this cpu's percpu
        xorl %eax,%eax                  # Clear GS and LDT
        movl %eax,%gs
        lldt %ax
@@ -446,8 +448,6 @@ is386:	movl $2,%ecx		# set MP
        movb $1, ready
        cmpb $0,%cl             # the first CPU calls start_kernel
        je   1f
-        movl $(__KERNEL_PERCPU), %eax
-        movl %eax,%fs           # set this cpu's percpu
        movl (stack_start), %esp
 1:
 #endif /* CONFIG_SMP */
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 0e275d495563..a0a2b5ca9b7d 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -19,6 +19,7 @@
 #include <asm/msr.h>
 #include <asm/cache.h>
 #include <asm/processor-flags.h>
+#include <asm/percpu.h>
 #ifdef CONFIG_PARAVIRT
 #include <asm/asm-offsets.h>
@@ -204,6 +205,19 @@ ENTRY(secondary_startup_64)
        pushq $0
        popfq
+#ifdef CONFIG_SMP
+        /*
+         * Fix up static pointers that need __per_cpu_load added.  The assembler
+         * is unable to do this directly.  This is only needed for the boot cpu.
+         * These values are set up with the correct base addresses by C code for
+         * secondary cpus.
+         */
+        movq    initial_gs(%rip), %rax
+        cmpl    $0, per_cpu__cpu_number(%rax)
+        jne     1f
+        addq    %rax, early_gdt_descr_base(%rip)
+1:
+#endif
        /*
         * We must switch to a new descriptor in kernel space for the GDT
         * because soon the kernel won't have access anymore to the userspace
@@ -226,12 +240,15 @@ ENTRY(secondary_startup_64)
        movl %eax,%fs
        movl %eax,%gs
-        /* 
+        /* Set up %gs.
-         * Setup up a dummy PDA. this is just for some early bootup code
+         *
-         * that does in_interrupt() 
+         * The base of %gs always points to the bottom of the irqstack
-         */ 
+         * union.  If the stack protector canary is enabled, it is
+         * located at %gs:40.  Note that, on SMP, the boot cpu uses
+         * init data section till per cpu areas are set up.
+         */
        movl    $MSR_GS_BASE,%ecx
-        movq    $empty_zero_page,%rax
+        movq    initial_gs(%rip),%rax
        movq    %rax,%rdx
        shrq    $32,%rdx
        wrmsr   
@@ -257,6 +274,12 @@ ENTRY(secondary_startup_64)
        .align  8
        ENTRY(initial_code)
        .quad   x86_64_start_kernel
+        ENTRY(initial_gs)
+#ifdef CONFIG_SMP
+        .quad   __per_cpu_load
+#else
+        .quad   PER_CPU_VAR(irq_stack_union)
+#endif
        __FINITDATA
        ENTRY(stack_start)
@@ -401,7 +424,8 @@ NEXT_PAGE(level2_spare_pgt)
        .globl early_gdt_descr
 early_gdt_descr:
        .word   GDT_ENTRIES*8-1
-        .quad   per_cpu__gdt_page
+early_gdt_descr_base:
+        .quad   per_cpu__gdt_page
 ENTRY(phys_base)
        /* This must match the first entry in level2_kernel_pgt */
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index cd759ad90690..64d5ad0b8add 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -628,11 +628,12 @@ static int hpet_cpuhp_notify(struct notifier_block *n,
        switch (action & 0xf) {
        case CPU_ONLINE:
-                INIT_DELAYED_WORK(&work.work, hpet_work);
+                INIT_DELAYED_WORK_ON_STACK(&work.work, hpet_work);
                init_completion(&work.complete);
                /* FIXME: add schedule_work_on() */
                schedule_delayed_work_on(cpu, &work.work, 0);
                wait_for_completion(&work.complete);
+                destroy_timer_on_stack(&work.work.timer);
                break;
        case CPU_DEAD:
                if (hdev) {
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c
index 1c4a1302536c..e4d36bd56b62 100644
--- a/arch/x86/kernel/io_apic.c
+++ b/arch/x86/kernel/io_apic.c
@@ -46,6 +46,7 @@
 #include <asm/idle.h>
 #include <asm/io.h>
 #include <asm/smp.h>
+#include <asm/cpu.h>
 #include <asm/desc.h>
 #include <asm/proto.h>
 #include <asm/acpi.h>
@@ -82,11 +83,11 @@ static DEFINE_SPINLOCK(vector_lock);
 int nr_ioapic_registers[MAX_IO_APICS];
 /* I/O APIC entries */
-struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
+struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
 int nr_ioapics;
 /* MP IRQ source entries */
-struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
+struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
 /* # of MP IRQ source entries */
 int mp_irq_entries;
@@ -356,7 +357,7 @@ set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
        if (!cfg->move_in_progress) {
                /* it means that domain is not changed */
-                if (!cpumask_intersects(&desc->affinity, mask))
+                if (!cpumask_intersects(desc->affinity, mask))
                        cfg->move_desc_pending = 1;
        }
 }
@@ -386,7 +387,7 @@ struct io_apic {
 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
 {
        return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
-                + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
+                + (mp_ioapics[idx].apicaddr & ~PAGE_MASK);
 }
 static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
@@ -579,9 +580,9 @@ set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
        if (assign_irq_vector(irq, cfg, mask))
                return BAD_APICID;
-        cpumask_and(&desc->affinity, cfg->domain, mask);
+        cpumask_and(desc->affinity, cfg->domain, mask);
        set_extra_move_desc(desc, mask);
-        return cpu_mask_to_apicid_and(&desc->affinity, cpu_online_mask);
+        return cpu_mask_to_apicid_and(desc->affinity, cpu_online_mask);
 }
 static void
@@ -944,10 +945,10 @@ static int find_irq_entry(int apic, int pin, int type)
        int i;
        for (i = 0; i < mp_irq_entries; i++)
-                if (mp_irqs[i].mp_irqtype == type &&
+                if (mp_irqs[i].irqtype == type &&
-                    (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
+                    (mp_irqs[i].dstapic == mp_ioapics[apic].apicid ||
-                     mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
+                     mp_irqs[i].dstapic == MP_APIC_ALL) &&
-                    mp_irqs[i].mp_dstirq == pin)
+                    mp_irqs[i].dstirq == pin)
                        return i;
        return -1;
@@ -961,13 +962,13 @@ static int __init find_isa_irq_pin(int irq, int type)
        int i;
        for (i = 0; i < mp_irq_entries; i++) {
-                int lbus = mp_irqs[i].mp_srcbus;
+                int lbus = mp_irqs[i].srcbus;
                if (test_bit(lbus, mp_bus_not_pci) &&
-                    (mp_irqs[i].mp_irqtype == type) &&
+                    (mp_irqs[i].irqtype == type) &&
-                    (mp_irqs[i].mp_srcbusirq == irq))
+                    (mp_irqs[i].srcbusirq == irq))
-                        return mp_irqs[i].mp_dstirq;
+                        return mp_irqs[i].dstirq;
        }
        return -1;
 }
@@ -977,17 +978,17 @@ static int __init find_isa_irq_apic(int irq, int type)
        int i;
        for (i = 0; i < mp_irq_entries; i++) {
-                int lbus = mp_irqs[i].mp_srcbus;
+                int lbus = mp_irqs[i].srcbus;
                if (test_bit(lbus, mp_bus_not_pci) &&
-                    (mp_irqs[i].mp_irqtype == type) &&
+                    (mp_irqs[i].irqtype == type) &&
-                    (mp_irqs[i].mp_srcbusirq == irq))
+                    (mp_irqs[i].srcbusirq == irq))
                        break;
        }
        if (i < mp_irq_entries) {
                int apic;
                for(apic = 0; apic < nr_ioapics; apic++) {
-                        if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
+                        if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic)
                                return apic;
                }
        }
@@ -1012,23 +1013,23 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
                return -1;
        }
        for (i = 0; i < mp_irq_entries; i++) {
-                int lbus = mp_irqs[i].mp_srcbus;
+                int lbus = mp_irqs[i].srcbus;
                for (apic = 0; apic < nr_ioapics; apic++)
-                        if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
+                        if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
-                            mp_irqs[i].mp_dstapic == MP_APIC_ALL)
+                            mp_irqs[i].dstapic == MP_APIC_ALL)
                                break;
                if (!test_bit(lbus, mp_bus_not_pci) &&
-                    !mp_irqs[i].mp_irqtype &&
+                    !mp_irqs[i].irqtype &&
                    (bus == lbus) &&
-                    (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
+                    (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
-                        int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
+                        int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
                        if (!(apic || IO_APIC_IRQ(irq)))
                                continue;
-                        if (pin == (mp_irqs[i].mp_srcbusirq & 3))
+                        if (pin == (mp_irqs[i].srcbusirq & 3))
                                return irq;
                        /*
                         * Use the first all-but-pin matching entry as a
@@ -1071,7 +1072,7 @@ static int EISA_ELCR(unsigned int irq)
 * EISA conforming in the MP table, that means its trigger type must
 * be read in from the ELCR */
-#define default_EISA_trigger(idx)       (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
+#define default_EISA_trigger(idx)       (EISA_ELCR(mp_irqs[idx].srcbusirq))
 #define default_EISA_polarity(idx)      default_ISA_polarity(idx)
 /* PCI interrupts are always polarity one level triggered,
@@ -1088,13 +1089,13 @@ static int EISA_ELCR(unsigned int irq)
 static int MPBIOS_polarity(int idx)
 {
-        int bus = mp_irqs[idx].mp_srcbus;
+        int bus = mp_irqs[idx].srcbus;
        int polarity;
        /*
         * Determine IRQ line polarity (high active or low active):
         */
-        switch (mp_irqs[idx].mp_irqflag & 3)
+        switch (mp_irqs[idx].irqflag & 3)
        {
                case 0: /* conforms, ie. bus-type dependent polarity */
                        if (test_bit(bus, mp_bus_not_pci))
@@ -1130,13 +1131,13 @@ static int MPBIOS_polarity(int idx)
 static int MPBIOS_trigger(int idx)
 {
-        int bus = mp_irqs[idx].mp_srcbus;
+        int bus = mp_irqs[idx].srcbus;
        int trigger;
        /*
         * Determine IRQ trigger mode (edge or level sensitive):
         */
-        switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
+        switch ((mp_irqs[idx].irqflag>>2) & 3)
        {
                case 0: /* conforms, ie. bus-type dependent */
                        if (test_bit(bus, mp_bus_not_pci))
@@ -1214,16 +1215,16 @@ int (*ioapic_renumber_irq)(int ioapic, int irq);
 static int pin_2_irq(int idx, int apic, int pin)
 {
        int irq, i;
-        int bus = mp_irqs[idx].mp_srcbus;
+        int bus = mp_irqs[idx].srcbus;
        /*
         * Debugging check, we are in big trouble if this message pops up!
         */
-        if (mp_irqs[idx].mp_dstirq != pin)
+        if (mp_irqs[idx].dstirq != pin)
                printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
        if (test_bit(bus, mp_bus_not_pci)) {
-                irq = mp_irqs[idx].mp_srcbusirq;
+                irq = mp_irqs[idx].srcbusirq;
        } else {
                /*
                 * PCI IRQs are mapped in order
@@ -1566,14 +1567,14 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_de
        apic_printk(APIC_VERBOSE,KERN_DEBUG
                    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
                    "IRQ %d Mode:%i Active:%i)\n",
-                    apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
+                    apic, mp_ioapics[apic].apicid, pin, cfg->vector,
                    irq, trigger, polarity);
-        if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
+        if (setup_ioapic_entry(mp_ioapics[apic].apicid, irq, &entry,
                               dest, trigger, polarity, cfg->vector)) {
                printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
-                       mp_ioapics[apic].mp_apicid, pin);
+                       mp_ioapics[apic].apicid, pin);
                __clear_irq_vector(irq, cfg);
                return;
        }
@@ -1604,12 +1605,10 @@ static void __init setup_IO_APIC_irqs(void)
                                        notcon = 1;
                                        apic_printk(APIC_VERBOSE,
                                                KERN_DEBUG " %d-%d",
-                                                mp_ioapics[apic].mp_apicid,
+                                                mp_ioapics[apic].apicid, pin);
-                                                pin);
                                } else
                                        apic_printk(APIC_VERBOSE, " %d-%d",
-                                                mp_ioapics[apic].mp_apicid,
+                                                mp_ioapics[apic].apicid, pin);
-                                                pin);
                                continue;
                        }
                        if (notcon) {
@@ -1699,7 +1698,7 @@ __apicdebuginit(void) print_IO_APIC(void)
        printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
        for (i = 0; i < nr_ioapics; i++)
                printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
-                       mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
+                       mp_ioapics[i].apicid, nr_ioapic_registers[i]);
        /*
         * We are a bit conservative about what we expect.  We have to
@@ -1719,7 +1718,7 @@ __apicdebuginit(void) print_IO_APIC(void)
        spin_unlock_irqrestore(&ioapic_lock, flags);
        printk("\n");
-        printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
+        printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid);
        printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
        printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
        printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
@@ -2121,14 +2120,14 @@ static void __init setup_ioapic_ids_from_mpc(void)
                reg_00.raw = io_apic_read(apic, 0);
                spin_unlock_irqrestore(&ioapic_lock, flags);
-                old_id = mp_ioapics[apic].mp_apicid;
+                old_id = mp_ioapics[apic].apicid;
-                if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
+                if (mp_ioapics[apic].apicid >= get_physical_broadcast()) {
                        printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
-                                apic, mp_ioapics[apic].mp_apicid);
+                                apic, mp_ioapics[apic].apicid);
                        printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
                                reg_00.bits.ID);
-                        mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
+                        mp_ioapics[apic].apicid = reg_00.bits.ID;
                }
                /*
@@ -2137,9 +2136,9 @@ static void __init setup_ioapic_ids_from_mpc(void)
                 * 'stuck on smp_invalidate_needed IPI wait' messages.
                 */
                if (check_apicid_used(phys_id_present_map,
-                                        mp_ioapics[apic].mp_apicid)) {
+                                        mp_ioapics[apic].apicid)) {
                        printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
-                                apic, mp_ioapics[apic].mp_apicid);
+                                apic, mp_ioapics[apic].apicid);
                        for (i = 0; i < get_physical_broadcast(); i++)
                                if (!physid_isset(i, phys_id_present_map))
                                        break;
@@ -2148,13 +2147,13 @@ static void __init setup_ioapic_ids_from_mpc(void)
                        printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
                                i);
                        physid_set(i, phys_id_present_map);
-                        mp_ioapics[apic].mp_apicid = i;
+                        mp_ioapics[apic].apicid = i;
                } else {
                        physid_mask_t tmp;
-                        tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
+                        tmp = apicid_to_cpu_present(mp_ioapics[apic].apicid);
                        apic_printk(APIC_VERBOSE, "Setting %d in the "
                                        "phys_id_present_map\n",
-                                        mp_ioapics[apic].mp_apicid);
+                                        mp_ioapics[apic].apicid);
                        physids_or(phys_id_present_map, phys_id_present_map, tmp);
                }
@@ -2163,11 +2162,11 @@ static void __init setup_ioapic_ids_from_mpc(void)
                 * We need to adjust the IRQ routing table
                 * if the ID changed.
                 */
-                if (old_id != mp_ioapics[apic].mp_apicid)
+                if (old_id != mp_ioapics[apic].apicid)
                        for (i = 0; i < mp_irq_entries; i++)
-                                if (mp_irqs[i].mp_dstapic == old_id)
+                                if (mp_irqs[i].dstapic == old_id)
-                                        mp_irqs[i].mp_dstapic
+                                        mp_irqs[i].dstapic
-                                                = mp_ioapics[apic].mp_apicid;
+                                                = mp_ioapics[apic].apicid;
                /*
                 * Read the right value from the MPC table and
@@ -2175,9 +2174,9 @@ static void __init setup_ioapic_ids_from_mpc(void)
                 */
                apic_printk(APIC_VERBOSE, KERN_INFO
                        "...changing IO-APIC physical APIC ID to %d ...",
-                        mp_ioapics[apic].mp_apicid);
+                        mp_ioapics[apic].apicid);
-                reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
+                reg_00.bits.ID = mp_ioapics[apic].apicid;
                spin_lock_irqsave(&ioapic_lock, flags);
                io_apic_write(apic, 0, reg_00.raw);
                spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -2188,7 +2187,7 @@ static void __init setup_ioapic_ids_from_mpc(void)
                spin_lock_irqsave(&ioapic_lock, flags);
                reg_00.raw = io_apic_read(apic, 0);
                spin_unlock_irqrestore(&ioapic_lock, flags);
-                if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
+                if (reg_00.bits.ID != mp_ioapics[apic].apicid)
                        printk("could not set ID!\n");
                else
                        apic_printk(APIC_VERBOSE, " ok.\n");
@@ -2383,7 +2382,7 @@ migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
        if (cfg->move_in_progress)
                send_cleanup_vector(cfg);
-        cpumask_copy(&desc->affinity, mask);
+        cpumask_copy(desc->affinity, mask);
 }
 static int migrate_irq_remapped_level_desc(struct irq_desc *desc)
@@ -2405,11 +2404,11 @@ static int migrate_irq_remapped_level_desc(struct irq_desc *desc)
        }
        /* everthing is clear. we have right of way */
-        migrate_ioapic_irq_desc(desc, &desc->pending_mask);
+        migrate_ioapic_irq_desc(desc, desc->pending_mask);
        ret = 0;
        desc->status &= ~IRQ_MOVE_PENDING;
-        cpumask_clear(&desc->pending_mask);
+        cpumask_clear(desc->pending_mask);
 unmask:
        unmask_IO_APIC_irq_desc(desc);
@@ -2434,7 +2433,7 @@ static void ir_irq_migration(struct work_struct *work)
                                continue;
                        }
-                        desc->chip->set_affinity(irq, &desc->pending_mask);
+                        desc->chip->set_affinity(irq, desc->pending_mask);
                        spin_unlock_irqrestore(&desc->lock, flags);
                }
        }
@@ -2448,7 +2447,7 @@ static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
 {
        if (desc->status & IRQ_LEVEL) {
                desc->status |= IRQ_MOVE_PENDING;
-                cpumask_copy(&desc->pending_mask, mask);
+                cpumask_copy(desc->pending_mask, mask);
                migrate_irq_remapped_level_desc(desc);
                return;
        }
@@ -2516,7 +2515,7 @@ static void irq_complete_move(struct irq_desc **descp)
                /* domain has not changed, but affinity did */
                me = smp_processor_id();
-                if (cpu_isset(me, desc->affinity)) {
+                if (cpumask_test_cpu(me, desc->affinity)) {
                        *descp = desc = move_irq_desc(desc, me);
                        /* get the new one */
                        cfg = desc->chip_data;
@@ -3117,8 +3116,8 @@ static int ioapic_resume(struct sys_device *dev)
        spin_lock_irqsave(&ioapic_lock, flags);
        reg_00.raw = io_apic_read(dev->id, 0);
-        if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
+        if (reg_00.bits.ID != mp_ioapics[dev->id].apicid) {
-                reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
+                reg_00.bits.ID = mp_ioapics[dev->id].apicid;
                io_apic_write(dev->id, 0, reg_00.raw);
        }
        spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -3183,7 +3182,7 @@ unsigned int create_irq_nr(unsigned int irq_want)
        irq = 0;
        spin_lock_irqsave(&vector_lock, flags);
-        for (new = irq_want; new < NR_IRQS; new++) {
+        for (new = irq_want; new < nr_irqs; new++) {
                if (platform_legacy_irq(new))
                        continue;
@@ -3258,6 +3257,9 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
        int err;
        unsigned dest;
+        if (disable_apic)
+                return -ENXIO;
        cfg = irq_cfg(irq);
        err = assign_irq_vector(irq, cfg, TARGET_CPUS);
        if (err)
@@ -3726,6 +3728,9 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
        struct irq_cfg *cfg;
        int err;
+        if (disable_apic)
+                return -ENXIO;
        cfg = irq_cfg(irq);
        err = assign_irq_vector(irq, cfg, TARGET_CPUS);
        if (!err) {
@@ -3760,7 +3765,7 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
 }
 #endif /* CONFIG_HT_IRQ */
-#ifdef CONFIG_X86_64
+#ifdef CONFIG_X86_UV
 /*
 * Re-target the irq to the specified CPU and enable the specified MMR located
 * on the specified blade to allow the sending of MSIs to the specified CPU.
@@ -3850,6 +3855,22 @@ void __init probe_nr_irqs_gsi(void)
                nr_irqs_gsi = nr;
 }
+#ifdef CONFIG_SPARSE_IRQ
+int __init arch_probe_nr_irqs(void)
+{
+        int nr;
+        nr = ((8 * nr_cpu_ids) > (32 * nr_ioapics) ?
+                (NR_VECTORS + (8 * nr_cpu_ids)) :
+                (NR_VECTORS + (32 * nr_ioapics)));
+        if (nr < nr_irqs && nr > nr_irqs_gsi)
+                nr_irqs = nr;
+        return 0;
+}
+#endif
 /* --------------------------------------------------------------------------
                          ACPI-based IOAPIC Configuration
   -------------------------------------------------------------------------- */
@@ -3984,8 +4005,8 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
                return -1;
        for (i = 0; i < mp_irq_entries; i++)
-                if (mp_irqs[i].mp_irqtype == mp_INT &&
+                if (mp_irqs[i].irqtype == mp_INT &&
-                    mp_irqs[i].mp_srcbusirq == bus_irq)
+                    mp_irqs[i].srcbusirq == bus_irq)
                        break;
        if (i >= mp_irq_entries)
                return -1;
@@ -4039,7 +4060,7 @@ void __init setup_ioapic_dest(void)
                         */
                        if (desc->status &
                            (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
-                                mask = &desc->affinity;
+                                mask = desc->affinity;
                        else
                                mask = TARGET_CPUS;
@@ -4100,7 +4121,7 @@ void __init ioapic_init_mappings(void)
        ioapic_res = ioapic_setup_resources();
        for (i = 0; i < nr_ioapics; i++) {
                if (smp_found_config) {
-                        ioapic_phys = mp_ioapics[i].mp_apicaddr;
+                        ioapic_phys = mp_ioapics[i].apicaddr;
 #ifdef CONFIG_X86_32
                        if (!ioapic_phys) {
                                printk(KERN_ERR
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 3973e2df7f87..8b30d0c2512c 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -36,11 +36,7 @@ void ack_bad_irq(unsigned int irq)
 #endif
 }
-#ifdef CONFIG_X86_32
+#define irq_stats(x)            (&per_cpu(irq_stat, x))
-# define irq_stats(x)           (&per_cpu(irq_stat, x))
-#else
-# define irq_stats(x)           cpu_pda(x)
-#endif
 /*
 * /proc/interrupts printing:
 */
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 74b9ff7341e9..e0f29be8ab0b 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -248,7 +248,7 @@ void fixup_irqs(void)
                if (irq == 2)
                        continue;
-                affinity = &desc->affinity;
+                affinity = desc->affinity;
                if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
                        printk("Breaking affinity for irq %i\n", irq);
                        affinity = cpu_all_mask;
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 63c88e6ec025..018963aa6ee3 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -18,6 +18,13 @@
 #include <linux/smp.h>
 #include <asm/io_apic.h>
 #include <asm/idle.h>
+#include <asm/apic.h>
+DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
+EXPORT_PER_CPU_SYMBOL(irq_stat);
+DEFINE_PER_CPU(struct pt_regs *, irq_regs);
+EXPORT_PER_CPU_SYMBOL(irq_regs);
 /*
 * Probabilistic stack overflow check:
@@ -100,7 +107,7 @@ void fixup_irqs(void)
                /* interrupt's are disabled at this point */
                spin_lock(&desc->lock);
-                affinity = &desc->affinity;
+                affinity = desc->affinity;
                if (!irq_has_action(irq) ||
                    cpumask_equal(affinity, cpu_online_mask)) {
                        spin_unlock(&desc->lock);
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
index 1507ad4e674d..bf629cadec1a 100644
--- a/arch/x86/kernel/irqinit_32.c
+++ b/arch/x86/kernel/irqinit_32.c
@@ -149,8 +149,15 @@ void __init native_init_IRQ(void)
         */
        alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
-        /* IPI for invalidation */
+        /* IPIs for invalidation */
-        alloc_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
        /* IPI for generic function call */
        alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index b7f4c929e615..5e9f4fc51385 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -87,9 +87,9 @@
 #include <linux/cpu.h>
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
+#include <linux/uaccess.h>
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/processor.h>
 #include <asm/microcode.h>
@@ -196,7 +196,7 @@ static inline int update_match_cpu(struct cpu_signature *csig, int sig, int pf)
        return (!sigmatch(sig, csig->sig, pf, csig->pf)) ? 0 : 1;
 }
-static inline int 
+static inline int
 update_match_revision(struct microcode_header_intel *mc_header, int rev)
 {
        return (mc_header->rev <= rev) ? 0 : 1;
@@ -442,8 +442,8 @@ static int request_microcode_fw(int cpu, struct device *device)
                return ret;
        }
-        ret = generic_load_microcode(cpu, (void*)firmware->data, firmware->size,
+        ret = generic_load_microcode(cpu, (void *)firmware->data,
-                        &get_ucode_fw);
+                                     firmware->size, &get_ucode_fw);
        release_firmware(firmware);
@@ -460,7 +460,7 @@ static int request_microcode_user(int cpu, const void __user *buf, size_t size)
        /* We should bind the task to the CPU */
        BUG_ON(cpu != raw_smp_processor_id());
-        return generic_load_microcode(cpu, (void*)buf, size, &get_ucode_user);
+        return generic_load_microcode(cpu, (void *)buf, size, &get_ucode_user);
 }
 static void microcode_fini_cpu(int cpu)
diff --git a/arch/x86/kernel/module_32.c b/arch/x86/kernel/module_32.c
index 3db0a5442eb1..0edd819050e7 100644
--- a/arch/x86/kernel/module_32.c
+++ b/arch/x86/kernel/module_32.c
@@ -42,7 +42,7 @@ void module_free(struct module *mod, void *module_region)
 {
        vfree(module_region);
        /* FIXME: If module_region == mod->init_region, trim exception
-           table entries. */
+           table entries. */
 }
 /* We don't need anything special. */
@@ -113,13 +113,13 @@ int module_finalize(const Elf_Ehdr *hdr,
                *para = NULL;
        char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
-        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 
+        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
                if (!strcmp(".text", secstrings + s->sh_name))
                        text = s;
                if (!strcmp(".altinstructions", secstrings + s->sh_name))
                        alt = s;
                if (!strcmp(".smp_locks", secstrings + s->sh_name))
-                        locks= s;
+                        locks = s;
                if (!strcmp(".parainstructions", secstrings + s->sh_name))
                        para = s;
        }
diff --git a/arch/x86/kernel/module_64.c b/arch/x86/kernel/module_64.c
index 6ba87830d4b1..c23880b90b5c 100644
--- a/arch/x86/kernel/module_64.c
+++ b/arch/x86/kernel/module_64.c
@@ -30,14 +30,14 @@
 #include <asm/page.h>
 #include <asm/pgtable.h>
-#define DEBUGP(fmt...) 
+#define DEBUGP(fmt...)
 #ifndef CONFIG_UML
 void module_free(struct module *mod, void *module_region)
 {
        vfree(module_region);
        /* FIXME: If module_region == mod->init_region, trim exception
-           table entries. */
+           table entries. */
 }
 void *module_alloc(unsigned long size)
@@ -77,7 +77,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
        Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;
        Elf64_Sym *sym;
        void *loc;
-        u64 val; 
+        u64 val;
        DEBUGP("Applying relocate section %u to %u\n", relsec,
               sechdrs[relsec].sh_info);
@@ -91,11 +91,11 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
                sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
                        + ELF64_R_SYM(rel[i].r_info);
-                DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
+                DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
-                       (int)ELF64_R_TYPE(rel[i].r_info), 
+                        (int)ELF64_R_TYPE(rel[i].r_info),
-                       sym->st_value, rel[i].r_addend, (u64)loc);
+                        sym->st_value, rel[i].r_addend, (u64)loc);
-                val = sym->st_value + rel[i].r_addend; 
+                val = sym->st_value + rel[i].r_addend;
                switch (ELF64_R_TYPE(rel[i].r_info)) {
                case R_X86_64_NONE:
@@ -113,16 +113,16 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
                        if ((s64)val != *(s32 *)loc)
                                goto overflow;
                        break;
-                case R_X86_64_PC32: 
+                case R_X86_64_PC32:
                        val -= (u64)loc;
                        *(u32 *)loc = val;
 #if 0
                        if ((s64)val != *(s32 *)loc)
-                                goto overflow; 
+                                goto overflow;
 #endif
                        break;
                default:
-                        printk(KERN_ERR "module %s: Unknown rela relocation: %Lu\n",
+                        printk(KERN_ERR "module %s: Unknown rela relocation: %llu\n",
                               me->name, ELF64_R_TYPE(rel[i].r_info));
                        return -ENOEXEC;
                }
@@ -130,7 +130,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
        return 0;
 overflow:
-        printk(KERN_ERR "overflow in relocation type %d val %Lx\n", 
+        printk(KERN_ERR "overflow in relocation type %d val %Lx\n",
               (int)ELF64_R_TYPE(rel[i].r_info), val);
        printk(KERN_ERR "`%s' likely not compiled with -mcmodel=kernel\n",
               me->name);
@@ -143,13 +143,13 @@ int apply_relocate(Elf_Shdr *sechdrs,
                   unsigned int relsec,
                   struct module *me)
 {
-        printk("non add relocation not supported\n");
+        printk(KERN_ERR "non add relocation not supported\n");
        return -ENOSYS;
-} 
+}
 int module_finalize(const Elf_Ehdr *hdr,
-                    const Elf_Shdr *sechdrs,
+                    const Elf_Shdr *sechdrs,
-                    struct module *me)
+                    struct module *me)
 {
        const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
                *para = NULL;
@@ -161,7 +161,7 @@ int module_finalize(const Elf_Ehdr *hdr,
                if (!strcmp(".altinstructions", secstrings + s->sh_name))
                        alt = s;
                if (!strcmp(".smp_locks", secstrings + s->sh_name))
-                        locks= s;
+                        locks = s;
                if (!strcmp(".parainstructions", secstrings + s->sh_name))
                        para = s;
        }
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index a649a4ccad43..fa6bb263892e 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -144,11 +144,11 @@ static void __init MP_ioapic_info(struct mpc_ioapic *m)
        if (bad_ioapic(m->apicaddr))
                return;
-        mp_ioapics[nr_ioapics].mp_apicaddr = m->apicaddr;
+        mp_ioapics[nr_ioapics].apicaddr = m->apicaddr;
-        mp_ioapics[nr_ioapics].mp_apicid = m->apicid;
+        mp_ioapics[nr_ioapics].apicid = m->apicid;
-        mp_ioapics[nr_ioapics].mp_type = m->type;
+        mp_ioapics[nr_ioapics].type = m->type;
-        mp_ioapics[nr_ioapics].mp_apicver = m->apicver;
+        mp_ioapics[nr_ioapics].apicver = m->apicver;
-        mp_ioapics[nr_ioapics].mp_flags = m->flags;
+        mp_ioapics[nr_ioapics].flags = m->flags;
        nr_ioapics++;
 }
@@ -160,55 +160,55 @@ static void print_MP_intsrc_info(struct mpc_intsrc *m)
                m->srcbusirq, m->dstapic, m->dstirq);
 }
-static void __init print_mp_irq_info(struct mp_config_intsrc *mp_irq)
+static void __init print_mp_irq_info(struct mpc_intsrc *mp_irq)
 {
        apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
                " IRQ %02x, APIC ID %x, APIC INT %02x\n",
-                mp_irq->mp_irqtype, mp_irq->mp_irqflag & 3,
+                mp_irq->irqtype, mp_irq->irqflag & 3,
-                (mp_irq->mp_irqflag >> 2) & 3, mp_irq->mp_srcbus,
+                (mp_irq->irqflag >> 2) & 3, mp_irq->srcbus,
-                mp_irq->mp_srcbusirq, mp_irq->mp_dstapic, mp_irq->mp_dstirq);
+                mp_irq->srcbusirq, mp_irq->dstapic, mp_irq->dstirq);
 }
 static void __init assign_to_mp_irq(struct mpc_intsrc *m,
-                                    struct mp_config_intsrc *mp_irq)
+                                    struct mpc_intsrc *mp_irq)
 {
-        mp_irq->mp_dstapic = m->dstapic;
+        mp_irq->dstapic = m->dstapic;
-        mp_irq->mp_type = m->type;
+        mp_irq->type = m->type;
-        mp_irq->mp_irqtype = m->irqtype;
+        mp_irq->irqtype = m->irqtype;
-        mp_irq->mp_irqflag = m->irqflag;
+        mp_irq->irqflag = m->irqflag;
-        mp_irq->mp_srcbus = m->srcbus;
+        mp_irq->srcbus = m->srcbus;
-        mp_irq->mp_srcbusirq = m->srcbusirq;
+        mp_irq->srcbusirq = m->srcbusirq;
-        mp_irq->mp_dstirq = m->dstirq;
+        mp_irq->dstirq = m->dstirq;
 }
-static void __init assign_to_mpc_intsrc(struct mp_config_intsrc *mp_irq,
+static void __init assign_to_mpc_intsrc(struct mpc_intsrc *mp_irq,
                                        struct mpc_intsrc *m)
 {
-        m->dstapic = mp_irq->mp_dstapic;
+        m->dstapic = mp_irq->dstapic;
-        m->type = mp_irq->mp_type;
+        m->type = mp_irq->type;
-        m->irqtype = mp_irq->mp_irqtype;
+        m->irqtype = mp_irq->irqtype;
-        m->irqflag = mp_irq->mp_irqflag;
+        m->irqflag = mp_irq->irqflag;
-        m->srcbus = mp_irq->mp_srcbus;
+        m->srcbus = mp_irq->srcbus;
-        m->srcbusirq = mp_irq->mp_srcbusirq;
+        m->srcbusirq = mp_irq->srcbusirq;
-        m->dstirq = mp_irq->mp_dstirq;
+        m->dstirq = mp_irq->dstirq;
 }
-static int __init mp_irq_mpc_intsrc_cmp(struct mp_config_intsrc *mp_irq,
+static int __init mp_irq_mpc_intsrc_cmp(struct mpc_intsrc *mp_irq,
                                        struct mpc_intsrc *m)
 {
-        if (mp_irq->mp_dstapic != m->dstapic)
+        if (mp_irq->dstapic != m->dstapic)
                return 1;
-        if (mp_irq->mp_type != m->type)
+        if (mp_irq->type != m->type)
                return 2;
-        if (mp_irq->mp_irqtype != m->irqtype)
+        if (mp_irq->irqtype != m->irqtype)
                return 3;
-        if (mp_irq->mp_irqflag != m->irqflag)
+        if (mp_irq->irqflag != m->irqflag)
                return 4;
-        if (mp_irq->mp_srcbus != m->srcbus)
+        if (mp_irq->srcbus != m->srcbus)
                return 5;
-        if (mp_irq->mp_srcbusirq != m->srcbusirq)
+        if (mp_irq->srcbusirq != m->srcbusirq)
                return 6;
-        if (mp_irq->mp_dstirq != m->dstirq)
+        if (mp_irq->dstirq != m->dstirq)
                return 7;
        return 0;
@@ -417,7 +417,7 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
        intsrc.type = MP_INTSRC;
        intsrc.irqflag = 0;     /* conforming */
        intsrc.srcbus = 0;
-        intsrc.dstapic = mp_ioapics[0].mp_apicid;
+        intsrc.dstapic = mp_ioapics[0].apicid;
        intsrc.irqtype = mp_INT;
@@ -570,14 +570,14 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
        }
 }
-static struct intel_mp_floating *mpf_found;
+static struct mpf_intel *mpf_found;
 /*
 * Scan the memory blocks for an SMP configuration block.
 */
 static void __init __get_smp_config(unsigned int early)
 {
-        struct intel_mp_floating *mpf = mpf_found;
+        struct mpf_intel *mpf = mpf_found;
        if (!mpf)
                return;
@@ -598,9 +598,9 @@ static void __init __get_smp_config(unsigned int early)
        }
        printk(KERN_INFO "Intel MultiProcessor Specification v1.%d\n",
-               mpf->mpf_specification);
+               mpf->specification);
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
-        if (mpf->mpf_feature2 & (1 << 7)) {
+        if (mpf->feature2 & (1 << 7)) {
                printk(KERN_INFO "    IMCR and PIC compatibility mode.\n");
                pic_mode = 1;
        } else {
@@ -611,7 +611,7 @@ static void __init __get_smp_config(unsigned int early)
        /*
         * Now see if we need to read further.
         */
-        if (mpf->mpf_feature1 != 0) {
+        if (mpf->feature1 != 0) {
                if (early) {
                        /*
                         * local APIC has default address
@@ -621,16 +621,16 @@ static void __init __get_smp_config(unsigned int early)
                }
                printk(KERN_INFO "Default MP configuration #%d\n",
-                       mpf->mpf_feature1);
+                       mpf->feature1);
-                construct_default_ISA_mptable(mpf->mpf_feature1);
+                construct_default_ISA_mptable(mpf->feature1);
-        } else if (mpf->mpf_physptr) {
+        } else if (mpf->physptr) {
                /*
                 * Read the physical hardware table.  Anything here will
                 * override the defaults.
                 */
-                if (!smp_read_mpc(phys_to_virt(mpf->mpf_physptr), early)) {
+                if (!smp_read_mpc(phys_to_virt(mpf->physptr), early)) {
 #ifdef CONFIG_X86_LOCAL_APIC
                        smp_found_config = 0;
 #endif
@@ -688,19 +688,19 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
                                  unsigned reserve)
 {
        unsigned int *bp = phys_to_virt(base);
-        struct intel_mp_floating *mpf;
+        struct mpf_intel *mpf;
        apic_printk(APIC_VERBOSE, "Scan SMP from %p for %ld bytes.\n",
                        bp, length);
        BUILD_BUG_ON(sizeof(*mpf) != 16);
        while (length > 0) {
-                mpf = (struct intel_mp_floating *)bp;
+                mpf = (struct mpf_intel *)bp;
                if ((*bp == SMP_MAGIC_IDENT) &&
-                    (mpf->mpf_length == 1) &&
+                    (mpf->length == 1) &&
                    !mpf_checksum((unsigned char *)bp, 16) &&
-                    ((mpf->mpf_specification == 1)
+                    ((mpf->specification == 1)
-                     || (mpf->mpf_specification == 4))) {
+                     || (mpf->specification == 4))) {
 #ifdef CONFIG_X86_LOCAL_APIC
                        smp_found_config = 1;
 #endif
@@ -713,7 +713,7 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
                                return 1;
                        reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
                                        BOOTMEM_DEFAULT);
-                        if (mpf->mpf_physptr) {
+                        if (mpf->physptr) {
                                unsigned long size = PAGE_SIZE;
 #ifdef CONFIG_X86_32
                                /*
@@ -722,14 +722,14 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
                                 * the bottom is mapped now.
                                 * PC-9800's MPC table places on the very last
                                 * of physical memory; so that simply reserving
-                                 * PAGE_SIZE from mpg->mpf_physptr yields BUG()
+                                 * PAGE_SIZE from mpf->physptr yields BUG()
                                 * in reserve_bootmem.
                                 */
                                unsigned long end = max_low_pfn * PAGE_SIZE;
-                                if (mpf->mpf_physptr + size > end)
+                                if (mpf->physptr + size > end)
-                                        size = end - mpf->mpf_physptr;
+                                        size = end - mpf->physptr;
 #endif
-                                reserve_bootmem_generic(mpf->mpf_physptr, size,
+                                reserve_bootmem_generic(mpf->physptr, size,
                                                BOOTMEM_DEFAULT);
                        }
@@ -809,15 +809,15 @@ static int  __init get_MP_intsrc_index(struct mpc_intsrc *m)
        /* not legacy */
        for (i = 0; i < mp_irq_entries; i++) {
-                if (mp_irqs[i].mp_irqtype != mp_INT)
+                if (mp_irqs[i].irqtype != mp_INT)
                        continue;
-                if (mp_irqs[i].mp_irqflag != 0x0f)
+                if (mp_irqs[i].irqflag != 0x0f)
                        continue;
-                if (mp_irqs[i].mp_srcbus != m->srcbus)
+                if (mp_irqs[i].srcbus != m->srcbus)
                        continue;
-                if (mp_irqs[i].mp_srcbusirq != m->srcbusirq)
+                if (mp_irqs[i].srcbusirq != m->srcbusirq)
                        continue;
                if (irq_used[i]) {
                        /* already claimed */
@@ -922,10 +922,10 @@ static int  __init replace_intsrc_all(struct mpc_table *mpc,
                if (irq_used[i])
                        continue;
-                if (mp_irqs[i].mp_irqtype != mp_INT)
+                if (mp_irqs[i].irqtype != mp_INT)
                        continue;
-                if (mp_irqs[i].mp_irqflag != 0x0f)
+                if (mp_irqs[i].irqflag != 0x0f)
                        continue;
                if (nr_m_spare > 0) {
@@ -1001,7 +1001,7 @@ static int __init update_mp_table(void)
 {
        char str[16];
        char oem[10];
-        struct intel_mp_floating *mpf;
+        struct mpf_intel *mpf;
        struct mpc_table *mpc, *mpc_new;
        if (!enable_update_mptable)
@@ -1014,19 +1014,19 @@ static int __init update_mp_table(void)
        /*
         * Now see if we need to go further.
         */
-        if (mpf->mpf_feature1 != 0)
+        if (mpf->feature1 != 0)
                return 0;
-        if (!mpf->mpf_physptr)
+        if (!mpf->physptr)
                return 0;
-        mpc = phys_to_virt(mpf->mpf_physptr);
+        mpc = phys_to_virt(mpf->physptr);
        if (!smp_check_mpc(mpc, oem, str))
                return 0;
        printk(KERN_INFO "mpf: %lx\n", virt_to_phys(mpf));
-        printk(KERN_INFO "mpf_physptr: %x\n", mpf->mpf_physptr);
+        printk(KERN_INFO "physptr: %x\n", mpf->physptr);
        if (mpc_new_phys && mpc->length > mpc_new_length) {
                mpc_new_phys = 0;
@@ -1047,23 +1047,23 @@ static int __init update_mp_table(void)
                }
                printk(KERN_INFO "use in-positon replacing\n");
        } else {
-                mpf->mpf_physptr = mpc_new_phys;
+                mpf->physptr = mpc_new_phys;
                mpc_new = phys_to_virt(mpc_new_phys);
                memcpy(mpc_new, mpc, mpc->length);
                mpc = mpc_new;
                /* check if we can modify that */
-                if (mpc_new_phys - mpf->mpf_physptr) {
+                if (mpc_new_phys - mpf->physptr) {
-                        struct intel_mp_floating *mpf_new;
+                        struct mpf_intel *mpf_new;
                        /* steal 16 bytes from [0, 1k) */
                        printk(KERN_INFO "mpf new: %x\n", 0x400 - 16);
                        mpf_new = phys_to_virt(0x400 - 16);
                        memcpy(mpf_new, mpf, 16);
                        mpf = mpf_new;
-                        mpf->mpf_physptr = mpc_new_phys;
+                        mpf->physptr = mpc_new_phys;
                }
-                mpf->mpf_checksum = 0;
+                mpf->checksum = 0;
-                mpf->mpf_checksum -= mpf_checksum((unsigned char *)mpf, 16);
+                mpf->checksum -= mpf_checksum((unsigned char *)mpf, 16);
-                printk(KERN_INFO "mpf_physptr new: %x\n", mpf->mpf_physptr);
+                printk(KERN_INFO "physptr new: %x\n", mpf->physptr);
        }
        /*
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 726266695b2c..3cf3413ec626 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -35,10 +35,10 @@
 #include <linux/device.h>
 #include <linux/cpu.h>
 #include <linux/notifier.h>
+#include <linux/uaccess.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/system.h>
 static struct class *msr_class;
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 7228979f1e7f..23b6d9e6e4f5 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -61,11 +61,7 @@ static int endflag __initdata;
 static inline unsigned int get_nmi_count(int cpu)
 {
-#ifdef CONFIG_X86_64
+        return per_cpu(irq_stat, cpu).__nmi_count;
-        return cpu_pda(cpu)->__nmi_count;
-#else
-        return nmi_count(cpu);
-#endif
 }
 static inline int mce_in_progress(void)
@@ -82,12 +78,8 @@ static inline int mce_in_progress(void)
 */
 static inline unsigned int get_timer_irqs(int cpu)
 {
-#ifdef CONFIG_X86_64
-        return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
-#else
        return per_cpu(irq_stat, cpu).apic_timer_irqs +
                per_cpu(irq_stat, cpu).irq0_irqs;
-#endif
 }
 #ifdef CONFIG_SMP
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index 00c2bcd41463..d5768b1af080 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -5,7 +5,7 @@
 * This allows to use PCI devices that only support 32bit addresses on systems
 * with more than 4GB.
 *
- * See Documentation/DMA-mapping.txt for the interface specification.
+ * See Documentation/PCI/PCI-DMA-mapping.txt for the interface specification.
 *
 * Copyright 2002 Andi Kleen, SuSE Labs.
 * Subject to the GNU General Public License v2 only.
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index a546f55c77b4..1a1ae8edc40c 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -66,9 +66,6 @@ asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
-DEFINE_PER_CPU(int, cpu_number);
-EXPORT_PER_CPU_SYMBOL(cpu_number);
 /*
 * Return saved PC of a blocked thread.
 */
@@ -111,7 +108,6 @@ void cpu_idle(void)
                                play_dead();
                        local_irq_disable();
-                        __get_cpu_var(irq_stat).idle_timestamp = jiffies;
                        /* Don't trace irqs off for idle */
                        stop_critical_timings();
                        pm_idle();
@@ -591,7 +587,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        if (prev->gs | next->gs)
                loadsegment(gs, next->gs);
-        x86_write_percpu(current_task, next_p);
+        percpu_write(current_task, next_p);
        return prev_p;
 }
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 416fb9282f4f..c422eebb0c58 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -16,6 +16,7 @@
 #include <stdarg.h>
+#include <linux/stackprotector.h>
 #include <linux/cpu.h>
 #include <linux/errno.h>
 #include <linux/sched.h>
@@ -46,7 +47,6 @@
 #include <asm/processor.h>
 #include <asm/i387.h>
 #include <asm/mmu_context.h>
-#include <asm/pda.h>
 #include <asm/prctl.h>
 #include <asm/desc.h>
 #include <asm/proto.h>
@@ -57,6 +57,12 @@
 asmlinkage extern void ret_from_fork(void);
+DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
+EXPORT_PER_CPU_SYMBOL(current_task);
+DEFINE_PER_CPU(unsigned long, old_rsp);
+static DEFINE_PER_CPU(unsigned char, is_idle);
 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
@@ -75,13 +81,13 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 void enter_idle(void)
 {
-        write_pda(isidle, 1);
+        percpu_write(is_idle, 1);
        atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
 }
 static void __exit_idle(void)
 {
-        if (test_and_clear_bit_pda(0, isidle) == 0)
+        if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
                return;
        atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
 }
@@ -111,6 +117,17 @@ static inline void play_dead(void)
 void cpu_idle(void)
 {
        current_thread_info()->status |= TS_POLLING;
+        /*
+         * If we're the non-boot CPU, nothing set the PDA stack
+         * canary up for us - and if we are the boot CPU we have
+         * a 0 stack canary. This is a good place for updating
+         * it, as we wont ever return from this function (so the
+         * invalid canaries already on the stack wont ever
+         * trigger):
+         */
+        boot_init_stack_canary();
        /* endless idle loop with no priority at all */
        while (1) {
                tick_nohz_stop_sched_tick(1);
@@ -392,7 +409,7 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
        load_gs_index(0);
        regs->ip                = new_ip;
        regs->sp                = new_sp;
-        write_pda(oldrsp, new_sp);
+        percpu_write(old_rsp, new_sp);
        regs->cs                = __USER_CS;
        regs->ss                = __USER_DS;
        regs->flags             = 0x200;
@@ -613,21 +630,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        /*
         * Switch the PDA and FPU contexts.
         */
-        prev->usersp = read_pda(oldrsp);
+        prev->usersp = percpu_read(old_rsp);
-        write_pda(oldrsp, next->usersp);
+        percpu_write(old_rsp, next->usersp);
-        write_pda(pcurrent, next_p);
+        percpu_write(current_task, next_p);
-        write_pda(kernelstack,
+        percpu_write(kernel_stack,
                  (unsigned long)task_stack_page(next_p) +
-                  THREAD_SIZE - PDA_STACKOFFSET);
+                  THREAD_SIZE - KERNEL_STACK_OFFSET);
-#ifdef CONFIG_CC_STACKPROTECTOR
-        write_pda(stack_canary, next_p->stack_canary);
-        /*
-         * Build time only check to make sure the stack_canary is at
-         * offset 40 in the pda; this is a gcc ABI requirement
-         */
-        BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
-#endif
        /*
         * Now maybe reload the debug registers and handle I/O bitmaps
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index 2b46eb41643b..f8536fee5c12 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -14,6 +14,7 @@
 #include <asm/reboot.h>
 #include <asm/pci_x86.h>
 #include <asm/virtext.h>
+#include <asm/cpu.h>
 #ifdef CONFIG_X86_32
 # include <linux/dmi.h>
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index ae0d8042cf69..f41c4486c270 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -89,7 +89,7 @@
 #include <asm/system.h>
 #include <asm/vsyscall.h>
-#include <asm/smp.h>
+#include <asm/cpu.h>
 #include <asm/desc.h>
 #include <asm/dma.h>
 #include <asm/iommu.h>
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 55c46074eba0..0d1e7ac439f4 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -13,145 +13,46 @@
 #include <asm/mpspec.h>
 #include <asm/apicdef.h>
 #include <asm/highmem.h>
+#include <asm/proto.h>
+#include <asm/cpumask.h>
+#include <asm/cpu.h>
-#ifdef CONFIG_X86_LOCAL_APIC
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
-unsigned int num_processors;
+# define DBG(x...) printk(KERN_DEBUG x)
-unsigned disabled_cpus __cpuinitdata;
-/* Processor that is doing the boot up */
-unsigned int boot_cpu_physical_apicid = -1U;
-EXPORT_SYMBOL(boot_cpu_physical_apicid);
-unsigned int max_physical_apicid;
-/* Bitmask of physically existing CPUs */
-physid_mask_t phys_cpu_present_map;
-#endif
-/* map cpu index to physical APIC ID */
-DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
-DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
-EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
-EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
-#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
-#define X86_64_NUMA     1
-/* map cpu index to node index */
-DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
-EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
-/* which logical CPUs are on which nodes */
-cpumask_t *node_to_cpumask_map;
-EXPORT_SYMBOL(node_to_cpumask_map);
-/* setup node_to_cpumask_map */
-static void __init setup_node_to_cpumask_map(void);
 #else
-static inline void setup_node_to_cpumask_map(void) { }
+# define DBG(x...)
 #endif
-#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_X86_SMP)
+DEFINE_PER_CPU(int, cpu_number);
-/*
+EXPORT_PER_CPU_SYMBOL(cpu_number);
- * Copy data used in early init routines from the initial arrays to the
- * per cpu data areas.  These arrays then become expendable and the
- * *_early_ptr's are zeroed indicating that the static arrays are gone.
- */
-static void __init setup_per_cpu_maps(void)
-{
-        int cpu;
-        for_each_possible_cpu(cpu) {
+#ifdef CONFIG_X86_64
-                per_cpu(x86_cpu_to_apicid, cpu) =
+#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
-                                early_per_cpu_map(x86_cpu_to_apicid, cpu);
+#else
-                per_cpu(x86_bios_cpu_apicid, cpu) =
+#define BOOT_PERCPU_OFFSET 0
-                                early_per_cpu_map(x86_bios_cpu_apicid, cpu);
-#ifdef X86_64_NUMA
-                per_cpu(x86_cpu_to_node_map, cpu) =
-                                early_per_cpu_map(x86_cpu_to_node_map, cpu);
 #endif
-        }
-        /* indicate the early static arrays will soon be gone */
+DEFINE_PER_CPU(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
-        early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
+EXPORT_PER_CPU_SYMBOL(this_cpu_off);
-        early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
-#ifdef X86_64_NUMA
-        early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
-#endif
-}
-#ifdef CONFIG_X86_32
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
-/*
+        [0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
- * Great future not-so-futuristic plan: make i386 and x86_64 do it
+};
- * the same way
- */
-unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 EXPORT_SYMBOL(__per_cpu_offset);
-static inline void setup_cpu_pda_map(void) { }
-#elif !defined(CONFIG_SMP)
-static inline void setup_cpu_pda_map(void) { }
-#else /* CONFIG_SMP && CONFIG_X86_64 */
-/*
- * Allocate cpu_pda pointer table and array via alloc_bootmem.
- */
-static void __init setup_cpu_pda_map(void)
-{
-        char *pda;
-        struct x8664_pda **new_cpu_pda;
-        unsigned long size;
-        int cpu;
-        size = roundup(sizeof(struct x8664_pda), cache_line_size());
-        /* allocate cpu_pda array and pointer table */
-        {
-                unsigned long tsize = nr_cpu_ids * sizeof(void *);
-                unsigned long asize = size * (nr_cpu_ids - 1);
-                tsize = roundup(tsize, cache_line_size());
+static inline void setup_percpu_segment(int cpu)
-                new_cpu_pda = alloc_bootmem(tsize + asize);
-                pda = (char *)new_cpu_pda + tsize;
-        }
-        /* initialize pointer table to static pda's */
-        for_each_possible_cpu(cpu) {
-                if (cpu == 0) {
-                        /* leave boot cpu pda in place */
-                        new_cpu_pda[0] = cpu_pda(0);
-                        continue;
-                }
-                new_cpu_pda[cpu] = (struct x8664_pda *)pda;
-                new_cpu_pda[cpu]->in_bootmem = 1;
-                pda += size;
-        }
-        /* point to new pointer table */
-        _cpu_pda = new_cpu_pda;
-}
-#endif /* CONFIG_SMP && CONFIG_X86_64 */
-#ifdef CONFIG_X86_64
-/* correctly size the local cpu masks */
-static void setup_cpu_local_masks(void)
 {
-        alloc_bootmem_cpumask_var(&cpu_initialized_mask);
+#ifdef CONFIG_X86_32
-        alloc_bootmem_cpumask_var(&cpu_callin_mask);
+        struct desc_struct gdt;
-        alloc_bootmem_cpumask_var(&cpu_callout_mask);
-        alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
-}
-#else /* CONFIG_X86_32 */
-static inline void setup_cpu_local_masks(void)
+        pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
-{
+                        0x2 | DESCTYPE_S, 0x8);
+        gdt.s = 1;
+        write_gdt_entry(get_cpu_gdt_table(cpu),
+                        GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
+#endif
 }
-#endif /* CONFIG_X86_32 */
 /*
 * Great future plan:
 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
@@ -159,18 +60,12 @@ static inline void setup_cpu_local_masks(void)
 */
 void __init setup_per_cpu_areas(void)
 {
-        ssize_t size, old_size;
+        ssize_t size;
        char *ptr;
        int cpu;
-        unsigned long align = 1;
-        /* Setup cpu_pda map */
-        setup_cpu_pda_map();
        /* Copy section for each CPU (we discard the original) */
-        old_size = PERCPU_ENOUGH_ROOM;
+        size = roundup(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
-        align = max_t(unsigned long, PAGE_SIZE, align);
-        size = roundup(old_size, align);
        pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
                NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
@@ -179,30 +74,67 @@ void __init setup_per_cpu_areas(void)
        for_each_possible_cpu(cpu) {
 #ifndef CONFIG_NEED_MULTIPLE_NODES
-                ptr = __alloc_bootmem(size, align,
+                ptr = alloc_bootmem_pages(size);
-                                 __pa(MAX_DMA_ADDRESS));
 #else
                int node = early_cpu_to_node(cpu);
                if (!node_online(node) || !NODE_DATA(node)) {
-                        ptr = __alloc_bootmem(size, align,
+                        ptr = alloc_bootmem_pages(size);
-                                         __pa(MAX_DMA_ADDRESS));
                        pr_info("cpu %d has no node %d or node-local memory\n",
                                cpu, node);
                        pr_debug("per cpu data for cpu%d at %016lx\n",
                                 cpu, __pa(ptr));
                } else {
-                        ptr = __alloc_bootmem_node(NODE_DATA(node), size, align,
+                        ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
-                                                        __pa(MAX_DMA_ADDRESS));
                        pr_debug("per cpu data for cpu%d on node%d at %016lx\n",
                                cpu, node, __pa(ptr));
                }
 #endif
+                memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start);
                per_cpu_offset(cpu) = ptr - __per_cpu_start;
-                memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+                per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
+                per_cpu(cpu_number, cpu) = cpu;
+                setup_percpu_segment(cpu);
+                /*
+                 * Copy data used in early init routines from the
+                 * initial arrays to the per cpu data areas.  These
+                 * arrays then become expendable and the *_early_ptr's
+                 * are zeroed indicating that the static arrays are
+                 * gone.
+                 */
+#ifdef CONFIG_X86_LOCAL_APIC
+                per_cpu(x86_cpu_to_apicid, cpu) =
+                        early_per_cpu_map(x86_cpu_to_apicid, cpu);
+                per_cpu(x86_bios_cpu_apicid, cpu) =
+                        early_per_cpu_map(x86_bios_cpu_apicid, cpu);
+#endif
+#ifdef CONFIG_X86_64
+                per_cpu(irq_stack_ptr, cpu) =
+                        per_cpu(irq_stack_union.irq_stack, cpu) +
+                        IRQ_STACK_SIZE - 64;
+#ifdef CONFIG_NUMA
+                per_cpu(x86_cpu_to_node_map, cpu) =
+                        early_per_cpu_map(x86_cpu_to_node_map, cpu);
+#endif
+#endif
+                /*
+                 * Up to this point, the boot CPU has been using .data.init
+                 * area.  Reload any changed state for the boot CPU.
+                 */
+                if (cpu == boot_cpu_id)
+                        switch_to_new_gdt();
+                DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
        }
-        /* Setup percpu data maps */
+        /* indicate the early static arrays will soon be gone */
-        setup_per_cpu_maps();
+#ifdef CONFIG_X86_LOCAL_APIC
+        early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
+        early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
+#endif
+#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
+        early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
+#endif
        /* Setup node to cpumask map */
        setup_node_to_cpumask_map();
@@ -210,199 +142,3 @@ void __init setup_per_cpu_areas(void)
        /* Setup cpu initialized, callin, callout masks */
        setup_cpu_local_masks();
 }
-#endif
-#ifdef X86_64_NUMA
-/*
- * Allocate node_to_cpumask_map based on number of available nodes
- * Requires node_possible_map to be valid.
- *
- * Note: node_to_cpumask() is not valid until after this is done.
- */
-static void __init setup_node_to_cpumask_map(void)
-{
-        unsigned int node, num = 0;
-        cpumask_t *map;
-        /* setup nr_node_ids if not done yet */
-        if (nr_node_ids == MAX_NUMNODES) {
-                for_each_node_mask(node, node_possible_map)
-                        num = node;
-                nr_node_ids = num + 1;
-        }
-        /* allocate the map */
-        map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
-        pr_debug("Node to cpumask map at %p for %d nodes\n",
-                 map, nr_node_ids);
-        /* node_to_cpumask() will now work */
-        node_to_cpumask_map = map;
-}
-void __cpuinit numa_set_node(int cpu, int node)
-{
-        int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
-        if (cpu_pda(cpu) && node != NUMA_NO_NODE)
-                cpu_pda(cpu)->nodenumber = node;
-        if (cpu_to_node_map)
-                cpu_to_node_map[cpu] = node;
-        else if (per_cpu_offset(cpu))
-                per_cpu(x86_cpu_to_node_map, cpu) = node;
-        else
-                pr_debug("Setting node for non-present cpu %d\n", cpu);
-}
-void __cpuinit numa_clear_node(int cpu)
-{
-        numa_set_node(cpu, NUMA_NO_NODE);
-}
-#ifndef CONFIG_DEBUG_PER_CPU_MAPS
-void __cpuinit numa_add_cpu(int cpu)
-{
-        cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
-}
-void __cpuinit numa_remove_cpu(int cpu)
-{
-        cpu_clear(cpu, node_to_cpumask_map[cpu_to_node(cpu)]);
-}
-#else /* CONFIG_DEBUG_PER_CPU_MAPS */
-/*
- * --------- debug versions of the numa functions ---------
- */
-static void __cpuinit numa_set_cpumask(int cpu, int enable)
-{
-        int node = cpu_to_node(cpu);
-        cpumask_t *mask;
-        char buf[64];
-        if (node_to_cpumask_map == NULL) {
-                printk(KERN_ERR "node_to_cpumask_map NULL\n");
-                dump_stack();
-                return;
-        }
-        mask = &node_to_cpumask_map[node];
-        if (enable)
-                cpu_set(cpu, *mask);
-        else
-                cpu_clear(cpu, *mask);
-        cpulist_scnprintf(buf, sizeof(buf), mask);
-        printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
-                enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf);
-}
-void __cpuinit numa_add_cpu(int cpu)
-{
-        numa_set_cpumask(cpu, 1);
-}
-void __cpuinit numa_remove_cpu(int cpu)
-{
-        numa_set_cpumask(cpu, 0);
-}
-int cpu_to_node(int cpu)
-{
-        if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
-                printk(KERN_WARNING
-                        "cpu_to_node(%d): usage too early!\n", cpu);
-                dump_stack();
-                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
-        }
-        return per_cpu(x86_cpu_to_node_map, cpu);
-}
-EXPORT_SYMBOL(cpu_to_node);
-/*
- * Same function as cpu_to_node() but used if called before the
- * per_cpu areas are setup.
- */
-int early_cpu_to_node(int cpu)
-{
-        if (early_per_cpu_ptr(x86_cpu_to_node_map))
-                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
-        if (!per_cpu_offset(cpu)) {
-                printk(KERN_WARNING
-                        "early_cpu_to_node(%d): no per_cpu area!\n", cpu);
-                dump_stack();
-                return NUMA_NO_NODE;
-        }
-        return per_cpu(x86_cpu_to_node_map, cpu);
-}
-/* empty cpumask */
-static const cpumask_t cpu_mask_none;
-/*
- * Returns a pointer to the bitmask of CPUs on Node 'node'.
- */
-const cpumask_t *cpumask_of_node(int node)
-{
-        if (node_to_cpumask_map == NULL) {
-                printk(KERN_WARNING
-                        "cpumask_of_node(%d): no node_to_cpumask_map!\n",
-                        node);
-                dump_stack();
-                return (const cpumask_t *)&cpu_online_map;
-        }
-        if (node >= nr_node_ids) {
-                printk(KERN_WARNING
-                        "cpumask_of_node(%d): node > nr_node_ids(%d)\n",
-                        node, nr_node_ids);
-                dump_stack();
-                return &cpu_mask_none;
-        }
-        return &node_to_cpumask_map[node];
-}
-EXPORT_SYMBOL(cpumask_of_node);
-/*
- * Returns a bitmask of CPUs on Node 'node'.
- *
- * Side note: this function creates the returned cpumask on the stack
- * so with a high NR_CPUS count, excessive stack space is used.  The
- * node_to_cpumask_ptr function should be used whenever possible.
- */
-cpumask_t node_to_cpumask(int node)
-{
-        if (node_to_cpumask_map == NULL) {
-                printk(KERN_WARNING
-                        "node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
-                dump_stack();
-                return cpu_online_map;
-        }
-        if (node >= nr_node_ids) {
-                printk(KERN_WARNING
-                        "node_to_cpumask(%d): node > nr_node_ids(%d)\n",
-                        node, nr_node_ids);
-                dump_stack();
-                return cpu_mask_none;
-        }
-        return node_to_cpumask_map[node];
-}
-EXPORT_SYMBOL(node_to_cpumask);
-/*
- * --------- end of debug versions of the numa functions ---------
- */
-#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
-#endif /* X86_64_NUMA */
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index 89bb7668041d..df0587f24c54 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -632,9 +632,16 @@ badframe:
 }
 #ifdef CONFIG_X86_32
-asmlinkage int sys_rt_sigreturn(struct pt_regs regs)
+/*
+ * Note: do not pass in pt_regs directly as with tail-call optimization
+ * GCC will incorrectly stomp on the caller's frame and corrupt user-space
+ * register state:
+ */
+asmlinkage int sys_rt_sigreturn(unsigned long __unused)
 {
-        return do_rt_sigreturn(&regs);
+        struct pt_regs *regs = (struct pt_regs *)&__unused;
+        return do_rt_sigreturn(regs);
 }
 #else /* !CONFIG_X86_32 */
 asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index bb1a3b1fc87f..f9dbcff43546 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -53,7 +53,6 @@
 #include <asm/nmi.h>
 #include <asm/irq.h>
 #include <asm/idle.h>
-#include <asm/smp.h>
 #include <asm/trampoline.h>
 #include <asm/cpu.h>
 #include <asm/numa.h>
@@ -63,6 +62,7 @@
 #include <asm/vmi.h>
 #include <asm/genapic.h>
 #include <asm/setup.h>
+#include <asm/uv/uv.h>
 #include <linux/mc146818rtc.h>
 #include <mach_apic.h>
@@ -745,52 +745,6 @@ static void __cpuinit do_fork_idle(struct work_struct *work)
        complete(&c_idle->done);
 }
-#ifdef CONFIG_X86_64
-/* __ref because it's safe to call free_bootmem when after_bootmem == 0. */
-static void __ref free_bootmem_pda(struct x8664_pda *oldpda)
-{
-        if (!after_bootmem)
-                free_bootmem((unsigned long)oldpda, sizeof(*oldpda));
-}
-/*
- * Allocate node local memory for the AP pda.
- *
- * Must be called after the _cpu_pda pointer table is initialized.
- */
-int __cpuinit get_local_pda(int cpu)
-{
-        struct x8664_pda *oldpda, *newpda;
-        unsigned long size = sizeof(struct x8664_pda);
-        int node = cpu_to_node(cpu);
-        if (cpu_pda(cpu) && !cpu_pda(cpu)->in_bootmem)
-                return 0;
-        oldpda = cpu_pda(cpu);
-        newpda = kmalloc_node(size, GFP_ATOMIC, node);
-        if (!newpda) {
-                printk(KERN_ERR "Could not allocate node local PDA "
-                        "for CPU %d on node %d\n", cpu, node);
-                if (oldpda)
-                        return 0;       /* have a usable pda */
-                else
-                        return -1;
-        }
-        if (oldpda) {
-                memcpy(newpda, oldpda, size);
-                free_bootmem_pda(oldpda);
-        }
-        newpda->in_bootmem = 0;
-        cpu_pda(cpu) = newpda;
-        return 0;
-}
-#endif /* CONFIG_X86_64 */
 static int __cpuinit do_boot_cpu(int apicid, int cpu)
 /*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
@@ -808,16 +762,6 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
        };
        INIT_WORK(&c_idle.work, do_fork_idle);
-#ifdef CONFIG_X86_64
-        /* Allocate node local memory for AP pdas */
-        if (cpu > 0) {
-                boot_error = get_local_pda(cpu);
-                if (boot_error)
-                        goto restore_state;
-                        /* if can't get pda memory, can't start cpu */
-        }
-#endif
        alternatives_smp_switch(1);
        c_idle.idle = get_idle_for_cpu(cpu);
@@ -847,14 +791,16 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
        set_idle_for_cpu(cpu, c_idle.idle);
 do_rest:
-#ifdef CONFIG_X86_32
        per_cpu(current_task, cpu) = c_idle.idle;
-        init_gdt(cpu);
+#ifdef CONFIG_X86_32
        /* Stack for startup_32 can be just as for start_secondary onwards */
        irq_ctx_init(cpu);
 #else
-        cpu_pda(cpu)->pcurrent = c_idle.idle;
        clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
+        initial_gs = per_cpu_offset(cpu);
+        per_cpu(kernel_stack, cpu) =
+                (unsigned long)task_stack_page(c_idle.idle) -
+                KERNEL_STACK_OFFSET + THREAD_SIZE;
 #endif
        early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
        initial_code = (unsigned long)start_secondary;
@@ -931,9 +877,7 @@ do_rest:
                                inquire_remote_apic(apicid);
                }
        }
-#ifdef CONFIG_X86_64
-restore_state:
-#endif
        if (boot_error) {
                /* Try to put things back the way they were before ... */
                numa_remove_cpu(cpu); /* was set by numa_add_cpu */
@@ -1125,6 +1069,7 @@ static int __init smp_sanity_check(unsigned max_cpus)
                printk(KERN_ERR "... forcing use of dummy APIC emulation."
                                "(tell your hw vendor)\n");
                smpboot_clear_io_apic();
+                disable_ioapic_setup();
                return -1;
        }
@@ -1240,9 +1185,6 @@ out:
 void __init native_smp_prepare_boot_cpu(void)
 {
        int me = smp_processor_id();
-#ifdef CONFIG_X86_32
-        init_gdt(me);
-#endif
        switch_to_new_gdt();
        /* already set me in cpu_online_mask in boot_cpu_init() */
        cpumask_set_cpu(me, cpu_callout_mask);
diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c
deleted file mode 100644
index 397e309839dd..000000000000
--- a/arch/x86/kernel/smpcommon.c
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * SMP stuff which is common to all sub-architectures.
- */
-#include <linux/module.h>
-#include <asm/smp.h>
-#ifdef CONFIG_X86_32
-DEFINE_PER_CPU(unsigned long, this_cpu_off);
-EXPORT_PER_CPU_SYMBOL(this_cpu_off);
-/*
- * Initialize the CPU's GDT.  This is either the boot CPU doing itself
- * (still using the master per-cpu area), or a CPU doing it for a
- * secondary which will soon come up.
- */
-__cpuinit void init_gdt(int cpu)
-{
-        struct desc_struct gdt;
-        pack_descriptor(&gdt, __per_cpu_offset[cpu], 0xFFFFF,
-                        0x2 | DESCTYPE_S, 0x8);
-        gdt.s = 1;
-        write_gdt_entry(get_cpu_gdt_table(cpu),
-                        GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
-        per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
-        per_cpu(cpu_number, cpu) = cpu;
-}
-#endif
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
deleted file mode 100644
index ce5054642247..000000000000
--- a/arch/x86/kernel/tlb_32.c
+++ /dev/null
@@ -1,256 +0,0 @@
-#include <linux/spinlock.h>
-#include <linux/cpu.h>
-#include <linux/interrupt.h>
-#include <asm/tlbflush.h>
-DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
-                        ____cacheline_aligned = { &init_mm, 0, };
-/* must come after the send_IPI functions above for inlining */
-#include <mach_ipi.h>
-/*
- *      Smarter SMP flushing macros.
- *              c/o Linus Torvalds.
- *
- *      These mean you can really definitely utterly forget about
- *      writing to user space from interrupts. (Its not allowed anyway).
- *
- *      Optimizations Manfred Spraul <manfred@colorfullife.com>
- */
-static cpumask_t flush_cpumask;
-static struct mm_struct *flush_mm;
-static unsigned long flush_va;
-static DEFINE_SPINLOCK(tlbstate_lock);
-/*
- * We cannot call mmdrop() because we are in interrupt context,
- * instead update mm->cpu_vm_mask.
- *
- * We need to reload %cr3 since the page tables may be going
- * away from under us..
- */
-void leave_mm(int cpu)
-{
-        BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK);
-        cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask);
-        load_cr3(swapper_pg_dir);
-}
-EXPORT_SYMBOL_GPL(leave_mm);
-/*
- *
- * The flush IPI assumes that a thread switch happens in this order:
- * [cpu0: the cpu that switches]
- * 1) switch_mm() either 1a) or 1b)
- * 1a) thread switch to a different mm
- * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
- *      Stop ipi delivery for the old mm. This is not synchronized with
- *      the other cpus, but smp_invalidate_interrupt ignore flush ipis
- *      for the wrong mm, and in the worst case we perform a superfluous
- *      tlb flush.
- * 1a2) set cpu_tlbstate to TLBSTATE_OK
- *      Now the smp_invalidate_interrupt won't call leave_mm if cpu0
- *      was in lazy tlb mode.
- * 1a3) update cpu_tlbstate[].active_mm
- *      Now cpu0 accepts tlb flushes for the new mm.
- * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
- *      Now the other cpus will send tlb flush ipis.
- * 1a4) change cr3.
- * 1b) thread switch without mm change
- *      cpu_tlbstate[].active_mm is correct, cpu0 already handles
- *      flush ipis.
- * 1b1) set cpu_tlbstate to TLBSTATE_OK
- * 1b2) test_and_set the cpu bit in cpu_vm_mask.
- *      Atomically set the bit [other cpus will start sending flush ipis],
- *      and test the bit.
- * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
- * 2) switch %%esp, ie current
- *
- * The interrupt must handle 2 special cases:
- * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
- * - the cpu performs speculative tlb reads, i.e. even if the cpu only
- *   runs in kernel space, the cpu could load tlb entries for user space
- *   pages.
- *
- * The good news is that cpu_tlbstate is local to each cpu, no
- * write/read ordering problems.
- */
-/*
- * TLB flush IPI:
- *
- * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
- * 2) Leave the mm if we are in the lazy tlb mode.
- */
-void smp_invalidate_interrupt(struct pt_regs *regs)
-{
-        unsigned long cpu;
-        cpu = get_cpu();
-        if (!cpu_isset(cpu, flush_cpumask))
-                goto out;
-                /*
-                 * This was a BUG() but until someone can quote me the
-                 * line from the intel manual that guarantees an IPI to
-                 * multiple CPUs is retried _only_ on the erroring CPUs
-                 * its staying as a return
-                 *
-                 * BUG();
-                 */
-        if (flush_mm == x86_read_percpu(cpu_tlbstate.active_mm)) {
-                if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK) {
-                        if (flush_va == TLB_FLUSH_ALL)
-                                local_flush_tlb();
-                        else
-                                __flush_tlb_one(flush_va);
-                } else
-                        leave_mm(cpu);
-        }
-        ack_APIC_irq();
-        smp_mb__before_clear_bit();
-        cpu_clear(cpu, flush_cpumask);
-        smp_mb__after_clear_bit();
-out:
-        put_cpu_no_resched();
-        inc_irq_stat(irq_tlb_count);
-}
-void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
-                             unsigned long va)
-{
-        cpumask_t cpumask = *cpumaskp;
-        /*
-         * A couple of (to be removed) sanity checks:
-         *
-         * - current CPU must not be in mask
-         * - mask must exist :)
-         */
-        BUG_ON(cpus_empty(cpumask));
-        BUG_ON(cpu_isset(smp_processor_id(), cpumask));
-        BUG_ON(!mm);
-#ifdef CONFIG_HOTPLUG_CPU
-        /* If a CPU which we ran on has gone down, OK. */
-        cpus_and(cpumask, cpumask, cpu_online_map);
-        if (unlikely(cpus_empty(cpumask)))
-                return;
-#endif
-        /*
-         * i'm not happy about this global shared spinlock in the
-         * MM hot path, but we'll see how contended it is.
-         * AK: x86-64 has a faster method that could be ported.
-         */
-        spin_lock(&tlbstate_lock);
-        flush_mm = mm;
-        flush_va = va;
-        cpus_or(flush_cpumask, cpumask, flush_cpumask);
-        /*
-         * Make the above memory operations globally visible before
-         * sending the IPI.
-         */
-        smp_mb();
-        /*
-         * We have to send the IPI only to
-         * CPUs affected.
-         */
-        send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR);
-        while (!cpus_empty(flush_cpumask))
-                /* nothing. lockup detection does not belong here */
-                cpu_relax();
-        flush_mm = NULL;
-        flush_va = 0;
-        spin_unlock(&tlbstate_lock);
-}
-void flush_tlb_current_task(void)
-{
-        struct mm_struct *mm = current->mm;
-        cpumask_t cpu_mask;
-        preempt_disable();
-        cpu_mask = mm->cpu_vm_mask;
-        cpu_clear(smp_processor_id(), cpu_mask);
-        local_flush_tlb();
-        if (!cpus_empty(cpu_mask))
-                flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
-        preempt_enable();
-}
-void flush_tlb_mm(struct mm_struct *mm)
-{
-        cpumask_t cpu_mask;
-        preempt_disable();
-        cpu_mask = mm->cpu_vm_mask;
-        cpu_clear(smp_processor_id(), cpu_mask);
-        if (current->active_mm == mm) {
-                if (current->mm)
-                        local_flush_tlb();
-                else
-                        leave_mm(smp_processor_id());
-        }
-        if (!cpus_empty(cpu_mask))
-                flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
-        preempt_enable();
-}
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
-{
-        struct mm_struct *mm = vma->vm_mm;
-        cpumask_t cpu_mask;
-        preempt_disable();
-        cpu_mask = mm->cpu_vm_mask;
-        cpu_clear(smp_processor_id(), cpu_mask);
-        if (current->active_mm == mm) {
-                if (current->mm)
-                        __flush_tlb_one(va);
-                 else
-                        leave_mm(smp_processor_id());
-        }
-        if (!cpus_empty(cpu_mask))
-                flush_tlb_others(cpu_mask, mm, va);
-        preempt_enable();
-}
-EXPORT_SYMBOL(flush_tlb_page);
-static void do_flush_tlb_all(void *info)
-{
-        unsigned long cpu = smp_processor_id();
-        __flush_tlb_all();
-        if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_LAZY)
-                leave_mm(cpu);
-}
-void flush_tlb_all(void)
-{
-        on_each_cpu(do_flush_tlb_all, NULL, 1);
-}
-void reset_lazy_tlbstate(void)
-{
-        int cpu = raw_smp_processor_id();
-        per_cpu(cpu_tlbstate, cpu).state = 0;
-        per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
-}
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index f885023167e0..89fce1b6d01f 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <asm/mmu_context.h>
+#include <asm/uv/uv.h>
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
 #include <asm/uv/uv_bau.h>
@@ -200,6 +201,7 @@ static int uv_wait_completion(struct bau_desc *bau_desc,
                                destination_timeouts = 0;
                        }
                }
+                cpu_relax();
        }
        return FLUSH_COMPLETE;
 }
@@ -209,14 +211,15 @@ static int uv_wait_completion(struct bau_desc *bau_desc,
 *
 * Send a broadcast and wait for a broadcast message to complete.
 *
- * The cpumaskp mask contains the cpus the broadcast was sent to.
+ * The flush_mask contains the cpus the broadcast was sent to.
 *
- * Returns 1 if all remote flushing was done. The mask is zeroed.
+ * Returns NULL if all remote flushing was done. The mask is zeroed.
- * Returns 0 if some remote flushing remains to be done. The mask is left
+ * Returns @flush_mask if some remote flushing remains to be done. The
- * unchanged.
+ * mask will have some bits still set.
 */
-int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
+const struct cpumask *uv_flush_send_and_wait(int cpu, int this_blade,
-                           cpumask_t *cpumaskp)
+                                             struct bau_desc *bau_desc,
+                                             struct cpumask *flush_mask)
 {
        int completion_status = 0;
        int right_shift;
@@ -263,59 +266,69 @@ int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
         * Success, so clear the remote cpu's from the mask so we don't
         * use the IPI method of shootdown on them.
         */
-        for_each_cpu_mask(bit, *cpumaskp) {
+        for_each_cpu(bit, flush_mask) {
                blade = uv_cpu_to_blade_id(bit);
                if (blade == this_blade)
                        continue;
-                cpu_clear(bit, *cpumaskp);
+                cpumask_clear_cpu(bit, flush_mask);
        }
-        if (!cpus_empty(*cpumaskp))
+        if (!cpumask_empty(flush_mask))
-                return 0;
+                return flush_mask;
-        return 1;
+        return NULL;
 }
 /**
 * uv_flush_tlb_others - globally purge translation cache of a virtual
 * address or all TLB's
- * @cpumaskp: mask of all cpu's in which the address is to be removed
+ * @cpumask: mask of all cpu's in which the address is to be removed
 * @mm: mm_struct containing virtual address range
 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
+ * @cpu: the current cpu
 *
 * This is the entry point for initiating any UV global TLB shootdown.
 *
 * Purges the translation caches of all specified processors of the given
 * virtual address, or purges all TLB's on specified processors.
 *
- * The caller has derived the cpumaskp from the mm_struct and has subtracted
+ * The caller has derived the cpumask from the mm_struct.  This function
- * the local cpu from the mask.  This function is called only if there
+ * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
- * are bits set in the mask. (e.g. flush_tlb_page())
 *
- * The cpumaskp is converted into a nodemask of the nodes containing
+ * The cpumask is converted into a nodemask of the nodes containing
 * the cpus.
 *
- * Returns 1 if all remote flushing was done.
+ * Note that this function should be called with preemption disabled.
- * Returns 0 if some remote flushing remains to be done.
+ *
+ * Returns NULL if all remote flushing was done.
+ * Returns pointer to cpumask if some remote flushing remains to be
+ * done.  The returned pointer is valid till preemption is re-enabled.
 */
-int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
+const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
-                        unsigned long va)
+                                          struct mm_struct *mm,
+                                          unsigned long va, unsigned int cpu)
 {
+        static DEFINE_PER_CPU(cpumask_t, flush_tlb_mask);
+        struct cpumask *flush_mask = &__get_cpu_var(flush_tlb_mask);
        int i;
        int bit;
        int blade;
-        int cpu;
+        int uv_cpu;
        int this_blade;
        int locals = 0;
        struct bau_desc *bau_desc;
-        cpu = uv_blade_processor_id();
+        WARN_ON(!in_atomic());
+        cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));
+        uv_cpu = uv_blade_processor_id();
        this_blade = uv_numa_blade_id();
        bau_desc = __get_cpu_var(bau_control).descriptor_base;
-        bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu;
+        bau_desc += UV_ITEMS_PER_DESCRIPTOR * uv_cpu;
        bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
        i = 0;
-        for_each_cpu_mask(bit, *cpumaskp) {
+        for_each_cpu(bit, flush_mask) {
                blade = uv_cpu_to_blade_id(bit);
                BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
                if (blade == this_blade) {
@@ -330,17 +343,17 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
                 * no off_node flushing; return status for local node
                 */
                if (locals)
-                        return 0;
+                        return flush_mask;
                else
-                        return 1;
+                        return NULL;
        }
        __get_cpu_var(ptcstats).requestor++;
        __get_cpu_var(ptcstats).ntargeted += i;
        bau_desc->payload.address = va;
-        bau_desc->payload.sending_cpu = smp_processor_id();
+        bau_desc->payload.sending_cpu = cpu;
-        return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp);
+        return uv_flush_send_and_wait(uv_cpu, this_blade, bau_desc, flush_mask);
 }
 /*
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 98c2d055284b..ed5aee5f3fcc 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -59,7 +59,6 @@
 #ifdef CONFIG_X86_64
 #include <asm/pgalloc.h>
 #include <asm/proto.h>
-#include <asm/pda.h>
 #else
 #include <asm/processor-flags.h>
 #include <asm/arch_hooks.h>
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
index 23206ba16874..1d3302cc2ddf 100644
--- a/arch/x86/kernel/vmi_32.c
+++ b/arch/x86/kernel/vmi_32.c
@@ -858,7 +858,7 @@ void __init vmi_init(void)
 #endif
 }
-void vmi_activate(void)
+void __init vmi_activate(void)
 {
        unsigned long flags;
diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S
index 82c67559dde7..3eba7f7bac05 100644
--- a/arch/x86/kernel/vmlinux_32.lds.S
+++ b/arch/x86/kernel/vmlinux_32.lds.S
@@ -178,14 +178,7 @@ SECTIONS
        __initramfs_end = .;
  }
 #endif
-  . = ALIGN(PAGE_SIZE);
+  PERCPU(PAGE_SIZE)
-  .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
-        __per_cpu_start = .;
-        *(.data.percpu.page_aligned)
-        *(.data.percpu)
-        *(.data.percpu.shared_aligned)
-        __per_cpu_end = .;
-  }
  . = ALIGN(PAGE_SIZE);
  /* freed after init ends here */
diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S
index 1a614c0e6bef..c9740996430a 100644
--- a/arch/x86/kernel/vmlinux_64.lds.S
+++ b/arch/x86/kernel/vmlinux_64.lds.S
@@ -5,6 +5,7 @@
 #define LOAD_OFFSET __START_KERNEL_map
 #include <asm-generic/vmlinux.lds.h>
+#include <asm/asm-offsets.h>
 #include <asm/page.h>
 #undef i386     /* in case the preprocessor is a 32bit one */
@@ -13,12 +14,14 @@ OUTPUT_FORMAT("elf64-x86-64", "elf64-x86-64", "elf64-x86-64")
 OUTPUT_ARCH(i386:x86-64)
 ENTRY(phys_startup_64)
 jiffies_64 = jiffies;
-_proxy_pda = 1;
 PHDRS {
        text PT_LOAD FLAGS(5);  /* R_E */
        data PT_LOAD FLAGS(7);  /* RWE */
        user PT_LOAD FLAGS(7);  /* RWE */
        data.init PT_LOAD FLAGS(7);     /* RWE */
+#ifdef CONFIG_SMP
+        percpu PT_LOAD FLAGS(7);        /* RWE */
+#endif
        note PT_NOTE FLAGS(0);  /* ___ */
 }
 SECTIONS
@@ -208,14 +211,28 @@ SECTIONS
  __initramfs_end = .;
 #endif
+#ifdef CONFIG_SMP
+  /*
+   * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
+   * output PHDR, so the next output section - __data_nosave - should
+   * switch it back to data.init.  Also, pda should be at the head of
+   * percpu area.  Preallocate it and define the percpu offset symbol
+   * so that it can be accessed as a percpu variable.
+   */
+  . = ALIGN(PAGE_SIZE);
+  PERCPU_VADDR(0, :percpu)
+#else
  PERCPU(PAGE_SIZE)
+#endif
  . = ALIGN(PAGE_SIZE);
  __init_end = .;
  . = ALIGN(PAGE_SIZE);
  __nosave_begin = .;
-  .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) { *(.data.nosave) }
+  .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
+      *(.data.nosave)
+  } :data.init  /* switch back to data.init, see PERCPU_VADDR() above */
  . = ALIGN(PAGE_SIZE);
  __nosave_end = .;
@@ -244,3 +261,8 @@ SECTIONS
 */
 ASSERT((_end - _text <= KERNEL_IMAGE_SIZE),
        "kernel image bigger than KERNEL_IMAGE_SIZE")
+#ifdef CONFIG_SMP
+ASSERT((per_cpu__irq_stack_union == 0),
+        "irq_stack_union is not at start of per-cpu area");
+#endif
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 695e426aa354..3909e3ba5ce3 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -58,5 +58,3 @@ EXPORT_SYMBOL(__memcpy);
 EXPORT_SYMBOL(empty_zero_page);
 EXPORT_SYMBOL(init_level4_pgt);
 EXPORT_SYMBOL(load_gs_index);
-EXPORT_SYMBOL(_proxy_pda);
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index a7ed208f81e3..92f1c6f3e19d 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -931,7 +931,7 @@ static void lguest_restart(char *reason)
 * that we can fit comfortably.
 *
 * First we need assembly templates of each of the patchable Guest operations,
- * and these are in lguest_asm.S. */
+ * and these are in i386_head.S. */
 /*G:060 We construct a table from the assembler templates: */
 static const struct lguest_insns
@@ -1093,7 +1093,7 @@ __init void lguest_init(void)
        acpi_ht = 0;
 #endif
-        /* We set the perferred console to "hvc".  This is the "hypervisor
+        /* We set the preferred console to "hvc".  This is the "hypervisor
         * virtual console" driver written by the PowerPC people, which we also
         * adapted for lguest's use. */
        add_preferred_console("hvc", 0, NULL);
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 4a20b2f9a381..7c8ca91bb9ec 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -56,7 +56,7 @@ do {									   \
                "       jmp 2b\n"                                          \
                ".previous\n"                                              \
                _ASM_EXTABLE(0b,3b)                                        \
-                : "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1),      \
+                : "=&d"(res), "=&c"(count), "=&a" (__d0), "=&S" (__d1),    \
                  "=&D" (__d2)                                             \
                : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
                : "memory");                                               \
@@ -218,7 +218,7 @@ long strnlen_user(const char __user *s, long n)
                "       .align 4\n"
                "       .long 0b,2b\n"
                ".previous"
-                :"=r" (n), "=D" (s), "=a" (res), "=c" (tmp)
+                :"=&r" (n), "=&D" (s), "=&a" (res), "=&c" (tmp)
                :"0" (n), "1" (s), "2" (0), "3" (mask)
                :"cc");
        return res & mask;
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 64d6c84e6353..ec13cb5f17ed 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -32,7 +32,7 @@ do {									   \
                "       jmp 2b\n"                                          \
                ".previous\n"                                              \
                _ASM_EXTABLE(0b,3b)                                        \
-                : "=r"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1),      \
+                : "=&r"(res), "=&c"(count), "=&a" (__d0), "=&S" (__d1),    \
                  "=&D" (__d2)                                             \
                : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
                : "memory");                                               \
@@ -86,7 +86,7 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
                ".previous\n"
                _ASM_EXTABLE(0b,3b)
                _ASM_EXTABLE(1b,2b)
-                : [size8] "=c"(size), [dst] "=&D" (__d0)
+                : [size8] "=&c"(size), [dst] "=&D" (__d0)
                : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr),
                  [zero] "r" (0UL), [eight] "r" (8UL));
        return size;
diff --git a/arch/x86/mach-voyager/setup.c b/arch/x86/mach-voyager/setup.c
index a580b9562e76..0ade62555ff3 100644
--- a/arch/x86/mach-voyager/setup.c
+++ b/arch/x86/mach-voyager/setup.c
@@ -9,6 +9,7 @@
 #include <asm/e820.h>
 #include <asm/io.h>
 #include <asm/setup.h>
+#include <asm/cpu.h>
 void __init pre_intr_init_hook(void)
 {
diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c
index 9840b7ec749a..331cd6d56483 100644
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -402,7 +402,7 @@ void __init find_smp_config(void)
             VOYAGER_SUS_IN_CONTROL_PORT);
        current_thread_info()->cpu = boot_cpu_id;
-        x86_write_percpu(cpu_number, boot_cpu_id);
+        percpu_write(cpu_number, boot_cpu_id);
 }
 /*
@@ -530,7 +530,6 @@ static void __init do_boot_cpu(__u8 cpu)
        /* init_tasks (in sched.c) is indexed logically */
        stack_start.sp = (void *)idle->thread.sp;
-        init_gdt(cpu);
        per_cpu(current_task, cpu) = idle;
        early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
        irq_ctx_init(cpu);
@@ -1747,7 +1746,6 @@ static void __init voyager_smp_prepare_cpus(unsigned int max_cpus)
 static void __cpuinit voyager_smp_prepare_boot_cpu(void)
 {
-        init_gdt(smp_processor_id());
        switch_to_new_gdt();
        cpu_set(smp_processor_id(), cpu_online_map);
@@ -1780,7 +1778,6 @@ static void __init voyager_smp_cpus_done(unsigned int max_cpus)
 void __init smp_setup_processor_id(void)
 {
        current_thread_info()->cpu = hard_smp_processor_id();
-        x86_write_percpu(cpu_number, hard_smp_processor_id());
 }
 static void voyager_send_call_func(cpumask_t callmask)
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index d8cc96a2738f..9f05157220f5 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -1,6 +1,8 @@
 obj-y   :=  init_$(BITS).o fault.o ioremap.o extable.o pageattr.o mmap.o \
            pat.o pgtable.o gup.o
+obj-$(CONFIG_X86_SMP)           += tlb.o
 obj-$(CONFIG_X86_32)            += pgtable_32.o iomap_32.o
 obj-$(CONFIG_HUGETLB_PAGE)      += hugetlbpage.o
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 90dfae511a41..65709a6aa6ee 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -26,6 +26,7 @@
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/kdebug.h>
+#include <linux/magic.h>
 #include <asm/system.h>
 #include <asm/desc.h>
@@ -91,8 +92,8 @@ static inline int notify_page_fault(struct pt_regs *regs)
 *
 * Opcode checker based on code by Richard Brunner
 */
-static int is_prefetch(struct pt_regs *regs, unsigned long addr,
+static int is_prefetch(struct pt_regs *regs, unsigned long error_code,
-                       unsigned long error_code)
+                        unsigned long addr)
 {
        unsigned char *instr;
        int scan_more = 1;
@@ -409,15 +410,15 @@ static void show_fault_oops(struct pt_regs *regs, unsigned long error_code,
 }
 #ifdef CONFIG_X86_64
-static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
+static noinline void pgtable_bad(struct pt_regs *regs,
-                                 unsigned long error_code)
+                         unsigned long error_code, unsigned long address)
 {
        unsigned long flags = oops_begin();
        int sig = SIGKILL;
-        struct task_struct *tsk;
+        struct task_struct *tsk = current;
        printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
-               current->comm, address);
+               tsk->comm, address);
        dump_pagetable(address);
        tsk = current;
        tsk->thread.cr2 = address;
@@ -429,6 +430,196 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 }
 #endif
+static noinline void no_context(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        struct task_struct *tsk = current;
+        unsigned long *stackend;
+#ifdef CONFIG_X86_64
+        unsigned long flags;
+        int sig;
+#endif
+        /* Are we prepared to handle this kernel fault?  */
+        if (fixup_exception(regs))
+                return;
+        /*
+         * X86_32
+         * Valid to do another page fault here, because if this fault
+         * had been triggered by is_prefetch fixup_exception would have
+         * handled it.
+         *
+         * X86_64
+         * Hall of shame of CPU/BIOS bugs.
+         */
+        if (is_prefetch(regs, error_code, address))
+                return;
+        if (is_errata93(regs, address))
+                return;
+        /*
+         * Oops. The kernel tried to access some bad page. We'll have to
+         * terminate things with extreme prejudice.
+         */
+#ifdef CONFIG_X86_32
+        bust_spinlocks(1);
+#else
+        flags = oops_begin();
+#endif
+        show_fault_oops(regs, error_code, address);
+        stackend = end_of_stack(tsk);
+        if (*stackend != STACK_END_MAGIC)
+                printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
+        tsk->thread.cr2 = address;
+        tsk->thread.trap_no = 14;
+        tsk->thread.error_code = error_code;
+#ifdef CONFIG_X86_32
+        die("Oops", regs, error_code);
+        bust_spinlocks(0);
+        do_exit(SIGKILL);
+#else
+        sig = SIGKILL;
+        if (__die("Oops", regs, error_code))
+                sig = 0;
+        /* Executive summary in case the body of the oops scrolled away */
+        printk(KERN_EMERG "CR2: %016lx\n", address);
+        oops_end(flags, regs, sig);
+#endif
+}
+static void __bad_area_nosemaphore(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address,
+                        int si_code)
+{
+        struct task_struct *tsk = current;
+        /* User mode accesses just cause a SIGSEGV */
+        if (error_code & PF_USER) {
+                /*
+                 * It's possible to have interrupts off here.
+                 */
+                local_irq_enable();
+                /*
+                 * Valid to do another page fault here because this one came
+                 * from user space.
+                 */
+                if (is_prefetch(regs, error_code, address))
+                        return;
+                if (is_errata100(regs, address))
+                        return;
+                if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
+                    printk_ratelimit()) {
+                        printk(
+                        "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
+                        task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
+                        tsk->comm, task_pid_nr(tsk), address,
+                        (void *) regs->ip, (void *) regs->sp, error_code);
+                        print_vma_addr(" in ", regs->ip);
+                        printk("\n");
+                }
+                tsk->thread.cr2 = address;
+                /* Kernel addresses are always protection faults */
+                tsk->thread.error_code = error_code | (address >= TASK_SIZE);
+                tsk->thread.trap_no = 14;
+                force_sig_info_fault(SIGSEGV, si_code, address, tsk);
+                return;
+        }
+        if (is_f00f_bug(regs, address))
+                return;
+        no_context(regs, error_code, address);
+}
+static noinline void bad_area_nosemaphore(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
+}
+static void __bad_area(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address,
+                        int si_code)
+{
+        struct mm_struct *mm = current->mm;
+        /*
+         * Something tried to access memory that isn't in our memory map..
+         * Fix it, but check if it's kernel or user first..
+         */
+        up_read(&mm->mmap_sem);
+        __bad_area_nosemaphore(regs, error_code, address, si_code);
+}
+static noinline void bad_area(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area(regs, error_code, address, SEGV_MAPERR);
+}
+static noinline void bad_area_access_error(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area(regs, error_code, address, SEGV_ACCERR);
+}
+/* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */
+static void out_of_memory(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        /*
+         * We ran out of memory, call the OOM killer, and return the userspace
+         * (which will retry the fault, or kill us if we got oom-killed).
+         */
+        up_read(&current->mm->mmap_sem);
+        pagefault_out_of_memory();
+}
+static void do_sigbus(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        struct task_struct *tsk = current;
+        struct mm_struct *mm = tsk->mm;
+        up_read(&mm->mmap_sem);
+        /* Kernel mode? Handle exceptions or die */
+        if (!(error_code & PF_USER))
+                no_context(regs, error_code, address);
+#ifdef CONFIG_X86_32
+        /* User space => ok to do another page fault */
+        if (is_prefetch(regs, error_code, address))
+                return;
+#endif
+        tsk->thread.cr2 = address;
+        tsk->thread.error_code = error_code;
+        tsk->thread.trap_no = 14;
+        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
+}
+static noinline void mm_fault_error(struct pt_regs *regs,
+                unsigned long error_code, unsigned long address, unsigned int fault)
+{
+        if (fault & VM_FAULT_OOM)
+                out_of_memory(regs, error_code, address);
+        else if (fault & VM_FAULT_SIGBUS)
+                do_sigbus(regs, error_code, address);
+        else
+                BUG();
+}
 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 {
        if ((error_code & PF_WRITE) && !pte_write(*pte))
@@ -448,8 +639,8 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 * There are no security implications to leaving a stale TLB when
 * increasing the permissions on a page.
 */
-static int spurious_fault(unsigned long address,
+static noinline int spurious_fault(unsigned long error_code,
-                          unsigned long error_code)
+                                unsigned long address)
 {
        pgd_t *pgd;
        pud_t *pud;
@@ -494,7 +685,7 @@ static int spurious_fault(unsigned long address,
 *
 * This assumes no large pages in there.
 */
-static int vmalloc_fault(unsigned long address)
+static noinline int vmalloc_fault(unsigned long address)
 {
 #ifdef CONFIG_X86_32
        unsigned long pgd_paddr;
@@ -573,6 +764,25 @@ static int vmalloc_fault(unsigned long address)
 int show_unhandled_signals = 1;
+static inline int access_error(unsigned long error_code, int write,
+                                struct vm_area_struct *vma)
+{
+        if (write) {
+                /* write, present and write, not present */
+                if (unlikely(!(vma->vm_flags & VM_WRITE)))
+                        return 1;
+        } else if (unlikely(error_code & PF_PROT)) {
+                /* read, present */
+                return 1;
+        } else {
+                /* read, not present */
+                if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
+                        return 1;
+        }
+        return 0;
+}
 /*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
@@ -583,16 +793,12 @@ asmlinkage
 #endif
 void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 {
+        unsigned long address;
        struct task_struct *tsk;
        struct mm_struct *mm;
        struct vm_area_struct *vma;
-        unsigned long address;
+        int write;
-        int write, si_code;
        int fault;
-#ifdef CONFIG_X86_64
-        unsigned long flags;
-        int sig;
-#endif
        tsk = current;
        mm = tsk->mm;
@@ -601,9 +807,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
        /* get the address */
        address = read_cr2();
-        si_code = SEGV_MAPERR;
+        if (unlikely(notify_page_fault(regs)))
-        if (notify_page_fault(regs))
                return;
        if (unlikely(kmmio_fault(regs, address)))
                return;
@@ -631,17 +835,17 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                        return;
                /* Can handle a stale RO->RW TLB */
-                if (spurious_fault(address, error_code))
+                if (spurious_fault(error_code, address))
                        return;
                /*
                 * Don't take the mm semaphore here. If we fixup a prefetch
                 * fault we could otherwise deadlock.
                 */
-                goto bad_area_nosemaphore;
+                bad_area_nosemaphore(regs, error_code, address);
+                return;
        }
        /*
         * It's safe to allow irq's after cr2 has been saved and the
         * vmalloc fault has been handled.
@@ -657,15 +861,17 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 #ifdef CONFIG_X86_64
        if (unlikely(error_code & PF_RSVD))
-                pgtable_bad(address, regs, error_code);
+                pgtable_bad(regs, error_code, address);
 #endif
        /*
         * If we're in an interrupt, have no user context or are running in an
         * atomic region then we must not take the fault.
         */
-        if (unlikely(in_atomic() || !mm))
+        if (unlikely(in_atomic() || !mm)) {
-                goto bad_area_nosemaphore;
+                bad_area_nosemaphore(regs, error_code, address);
+                return;
+        }
        /*
         * When running in the kernel we expect faults to occur only to
@@ -683,20 +889,26 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
         * source.  If this is invalid we can skip the address space check,
         * thus avoiding the deadlock.
         */
-        if (!down_read_trylock(&mm->mmap_sem)) {
+        if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
                if ((error_code & PF_USER) == 0 &&
-                    !search_exception_tables(regs->ip))
+                    !search_exception_tables(regs->ip)) {
-                        goto bad_area_nosemaphore;
+                        bad_area_nosemaphore(regs, error_code, address);
+                        return;
+                }
                down_read(&mm->mmap_sem);
        }
        vma = find_vma(mm, address);
-        if (!vma)
+        if (unlikely(!vma)) {
-                goto bad_area;
+                bad_area(regs, error_code, address);
-        if (vma->vm_start <= address)
+                return;
+        }
+        if (likely(vma->vm_start <= address))
                goto good_area;
-        if (!(vma->vm_flags & VM_GROWSDOWN))
+        if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
-                goto bad_area;
+                bad_area(regs, error_code, address);
+                return;
+        }
        if (error_code & PF_USER) {
                /*
                 * Accessing the stack below %sp is always a bug.
@@ -704,31 +916,25 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                 * and pusha to work.  ("enter $65535,$31" pushes
                 * 32 pointers and then decrements %sp by 65535.)
                 */
-                if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
+                if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
-                        goto bad_area;
+                        bad_area(regs, error_code, address);
+                        return;
+                }
        }
-        if (expand_stack(vma, address))
+        if (unlikely(expand_stack(vma, address))) {
-                goto bad_area;
+                bad_area(regs, error_code, address);
-/*
+                return;
- * Ok, we have a good vm_area for this memory access, so
+        }
- * we can handle it..
- */
+        /*
+         * Ok, we have a good vm_area for this memory access, so
+         * we can handle it..
+         */
 good_area:
-        si_code = SEGV_ACCERR;
+        write = error_code & PF_WRITE;
-        write = 0;
+        if (unlikely(access_error(error_code, write, vma))) {
-        switch (error_code & (PF_PROT|PF_WRITE)) {
+                bad_area_access_error(regs, error_code, address);
-        default:        /* 3: write, present */
+                return;
-                /* fall through */
-        case PF_WRITE:          /* write, not present */
-                if (!(vma->vm_flags & VM_WRITE))
-                        goto bad_area;
-                write++;
-                break;
-        case PF_PROT:           /* read, present */
-                goto bad_area;
-        case 0:                 /* read, not present */
-                if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
-                        goto bad_area;
        }
        /*
@@ -738,11 +944,8 @@ good_area:
         */
        fault = handle_mm_fault(mm, vma, address, write);
        if (unlikely(fault & VM_FAULT_ERROR)) {
-                if (fault & VM_FAULT_OOM)
+                mm_fault_error(regs, error_code, address, fault);
-                        goto out_of_memory;
+                return;
-                else if (fault & VM_FAULT_SIGBUS)
-                        goto do_sigbus;
-                BUG();
        }
        if (fault & VM_FAULT_MAJOR)
                tsk->maj_flt++;
@@ -760,128 +963,6 @@ good_area:
        }
 #endif
        up_read(&mm->mmap_sem);
-        return;
-/*
- * Something tried to access memory that isn't in our memory map..
- * Fix it, but check if it's kernel or user first..
- */
-bad_area:
-        up_read(&mm->mmap_sem);
-bad_area_nosemaphore:
-        /* User mode accesses just cause a SIGSEGV */
-        if (error_code & PF_USER) {
-                /*
-                 * It's possible to have interrupts off here.
-                 */
-                local_irq_enable();
-                /*
-                 * Valid to do another page fault here because this one came
-                 * from user space.
-                 */
-                if (is_prefetch(regs, address, error_code))
-                        return;
-                if (is_errata100(regs, address))
-                        return;
-                if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
-                    printk_ratelimit()) {
-                        printk(
-                        "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
-                        task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
-                        tsk->comm, task_pid_nr(tsk), address,
-                        (void *) regs->ip, (void *) regs->sp, error_code);
-                        print_vma_addr(" in ", regs->ip);
-                        printk("\n");
-                }
-                tsk->thread.cr2 = address;
-                /* Kernel addresses are always protection faults */
-                tsk->thread.error_code = error_code | (address >= TASK_SIZE);
-                tsk->thread.trap_no = 14;
-                force_sig_info_fault(SIGSEGV, si_code, address, tsk);
-                return;
-        }
-        if (is_f00f_bug(regs, address))
-                return;
-no_context:
-        /* Are we prepared to handle this kernel fault?  */
-        if (fixup_exception(regs))
-                return;
-        /*
-         * X86_32
-         * Valid to do another page fault here, because if this fault
-         * had been triggered by is_prefetch fixup_exception would have
-         * handled it.
-         *
-         * X86_64
-         * Hall of shame of CPU/BIOS bugs.
-         */
-        if (is_prefetch(regs, address, error_code))
-                return;
-        if (is_errata93(regs, address))
-                return;
-/*
- * Oops. The kernel tried to access some bad page. We'll have to
- * terminate things with extreme prejudice.
- */
-#ifdef CONFIG_X86_32
-        bust_spinlocks(1);
-#else
-        flags = oops_begin();
-#endif
-        show_fault_oops(regs, error_code, address);
-        tsk->thread.cr2 = address;
-        tsk->thread.trap_no = 14;
-        tsk->thread.error_code = error_code;
-#ifdef CONFIG_X86_32
-        die("Oops", regs, error_code);
-        bust_spinlocks(0);
-        do_exit(SIGKILL);
-#else
-        sig = SIGKILL;
-        if (__die("Oops", regs, error_code))
-                sig = 0;
-        /* Executive summary in case the body of the oops scrolled away */
-        printk(KERN_EMERG "CR2: %016lx\n", address);
-        oops_end(flags, regs, sig);
-#endif
-out_of_memory:
-        /*
-         * We ran out of memory, call the OOM killer, and return the userspace
-         * (which will retry the fault, or kill us if we got oom-killed).
-         */
-        up_read(&mm->mmap_sem);
-        pagefault_out_of_memory();
-        return;
-do_sigbus:
-        up_read(&mm->mmap_sem);
-        /* Kernel mode? Handle exceptions or die */
-        if (!(error_code & PF_USER))
-                goto no_context;
-#ifdef CONFIG_X86_32
-        /* User space => ok to do another page fault */
-        if (is_prefetch(regs, address, error_code))
-                return;
-#endif
-        tsk->thread.cr2 = address;
-        tsk->thread.error_code = error_code;
-        tsk->thread.trap_no = 14;
-        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
 }
 DEFINE_SPINLOCK(pgd_lock);
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 88f1b10de3be..00263bf07a88 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -49,7 +49,6 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 #include <asm/cacheflush.h>
-#include <asm/smp.h>
 unsigned int __VMALLOC_RESERVE = 128 << 20;
@@ -138,6 +137,47 @@ static pte_t * __init one_page_table_init(pmd_t *pmd)
        return pte_offset_kernel(pmd, 0);
 }
+static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
+                                           unsigned long vaddr, pte_t *lastpte)
+{
+#ifdef CONFIG_HIGHMEM
+        /*
+         * Something (early fixmap) may already have put a pte
+         * page here, which causes the page table allocation
+         * to become nonlinear. Attempt to fix it, and if it
+         * is still nonlinear then we have to bug.
+         */
+        int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
+        int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;
+        if (pmd_idx_kmap_begin != pmd_idx_kmap_end
+            && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
+            && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end
+            && ((__pa(pte) >> PAGE_SHIFT) < table_start
+                || (__pa(pte) >> PAGE_SHIFT) >= table_end)) {
+                pte_t *newpte;
+                int i;
+                BUG_ON(after_init_bootmem);
+                newpte = alloc_low_page();
+                for (i = 0; i < PTRS_PER_PTE; i++)
+                        set_pte(newpte + i, pte[i]);
+                paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
+                set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
+                BUG_ON(newpte != pte_offset_kernel(pmd, 0));
+                __flush_tlb_all();
+                paravirt_release_pte(__pa(pte) >> PAGE_SHIFT);
+                pte = newpte;
+        }
+        BUG_ON(vaddr < fix_to_virt(FIX_KMAP_BEGIN - 1)
+               && vaddr > fix_to_virt(FIX_KMAP_END)
+               && lastpte && lastpte + PTRS_PER_PTE != pte);
+#endif
+        return pte;
+}
 /*
 * This function initializes a certain range of kernel virtual memory
 * with new bootmem page tables, everywhere page tables are missing in
@@ -154,6 +194,7 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
        unsigned long vaddr;
        pgd_t *pgd;
        pmd_t *pmd;
+        pte_t *pte = NULL;
        vaddr = start;
        pgd_idx = pgd_index(vaddr);
@@ -165,7 +206,8 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
                pmd = pmd + pmd_index(vaddr);
                for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
                                                        pmd++, pmd_idx++) {
-                        one_page_table_init(pmd);
+                        pte = page_table_kmap_check(one_page_table_init(pmd),
+                                                    pmd, vaddr, pte);
                        vaddr += PMD_SIZE;
                }
@@ -508,7 +550,6 @@ static void __init early_ioremap_page_table_range_init(pgd_t *pgd_base)
         * Fixed mappings, only the page table structure has to be
         * created - mappings will be set by set_fixmap():
         */
-        early_ioremap_clear();
        vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
        end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
        page_table_range_init(vaddr, end, pgd_base);
@@ -801,7 +842,7 @@ static void __init find_early_table_space(unsigned long end, int use_pse)
        tables += PAGE_ALIGN(ptes * sizeof(pte_t));
        /* for fixmap */
-        tables += PAGE_SIZE * 2;
+        tables += PAGE_ALIGN(__end_of_fixed_addresses * sizeof(pte_t));
        /*
         * RED-PEN putting page tables only on node 0 could
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 23f68e77ad1f..e6d36b490250 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -596,7 +596,7 @@ static void __init init_gbpages(void)
                direct_gbpages = 0;
 }
-static unsigned long __init kernel_physical_mapping_init(unsigned long start,
+static unsigned long __meminit kernel_physical_mapping_init(unsigned long start,
                                                unsigned long end,
                                                unsigned long page_size_mask)
 {
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
index d0151d8ce452..ca53224fc56c 100644
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -17,6 +17,7 @@
 */
 #include <asm/iomap.h>
+#include <asm/pat.h>
 #include <linux/module.h>
 /* Map 'pfn' using fixed map 'type' and protections 'prot'
@@ -29,6 +30,15 @@ iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot)
        pagefault_disable();
+        /*
+         * For non-PAT systems, promote PAGE_KERNEL_WC to PAGE_KERNEL_UC_MINUS.
+         * PAGE_KERNEL_WC maps to PWT, which translates to uncached if the
+         * MTRR is UC or WC.  UC_MINUS gets the real intention, of the
+         * user, which is "WC if the MTRR is WC, UC if you can't do that."
+         */
+        if (!pat_enabled && pgprot_val(prot) == pgprot_val(PAGE_KERNEL_WC))
+                prot = PAGE_KERNEL_UC_MINUS;
        idx = type + KM_TYPE_NR*smp_processor_id();
        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
        set_pte(kmap_pte-idx, pfn_pte(pfn, prot));
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index bd85d42819e1..af750ab973b6 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -557,34 +557,9 @@ void __init early_ioremap_init(void)
        }
 }
-void __init early_ioremap_clear(void)
-{
-        pmd_t *pmd;
-        if (early_ioremap_debug)
-                printk(KERN_INFO "early_ioremap_clear()\n");
-        pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
-        pmd_clear(pmd);
-        paravirt_release_pte(__pa(bm_pte) >> PAGE_SHIFT);
-        __flush_tlb_all();
-}
 void __init early_ioremap_reset(void)
 {
-        enum fixed_addresses idx;
-        unsigned long addr, phys;
-        pte_t *pte;
        after_paging_init = 1;
-        for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
-                addr = fix_to_virt(idx);
-                pte = early_ioremap_pte(addr);
-                if (pte_present(*pte)) {
-                        phys = pte_val(*pte) & PAGE_MASK;
-                        set_fixmap(idx, phys);
-                }
-        }
 }
 static void __init __early_set_fixmap(enum fixed_addresses idx,
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 71a14f89f89e..08d140fbc31b 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -20,6 +20,12 @@
 #include <asm/acpi.h>
 #include <asm/k8.h>
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+# define DBG(x...) printk(KERN_DEBUG x)
+#else
+# define DBG(x...)
+#endif
 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
 EXPORT_SYMBOL(node_data);
@@ -33,6 +39,21 @@ int numa_off __initdata;
 static unsigned long __initdata nodemap_addr;
 static unsigned long __initdata nodemap_size;
+DEFINE_PER_CPU(int, node_number) = 0;
+EXPORT_PER_CPU_SYMBOL(node_number);
+/*
+ * Map cpu index to node index
+ */
+DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
+EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
+/*
+ * Which logical CPUs are on which nodes
+ */
+cpumask_t *node_to_cpumask_map;
+EXPORT_SYMBOL(node_to_cpumask_map);
 /*
 * Given a shift value, try to populate memnodemap[]
 * Returns :
@@ -640,3 +661,199 @@ void __init init_cpu_to_node(void)
 #endif
+/*
+ * Allocate node_to_cpumask_map based on number of available nodes
+ * Requires node_possible_map to be valid.
+ *
+ * Note: node_to_cpumask() is not valid until after this is done.
+ * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
+ */
+void __init setup_node_to_cpumask_map(void)
+{
+        unsigned int node, num = 0;
+        cpumask_t *map;
+        /* setup nr_node_ids if not done yet */
+        if (nr_node_ids == MAX_NUMNODES) {
+                for_each_node_mask(node, node_possible_map)
+                        num = node;
+                nr_node_ids = num + 1;
+        }
+        /* allocate the map */
+        map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
+        DBG("node_to_cpumask_map at %p for %d nodes\n", map, nr_node_ids);
+        pr_debug("Node to cpumask map at %p for %d nodes\n",
+                 map, nr_node_ids);
+        /* node_to_cpumask() will now work */
+        node_to_cpumask_map = map;
+}
+void __cpuinit numa_set_node(int cpu, int node)
+{
+        int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
+        /* early setting, no percpu area yet */
+        if (cpu_to_node_map) {
+                cpu_to_node_map[cpu] = node;
+                return;
+        }
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+        if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
+                printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
+                dump_stack();
+                return;
+        }
+#endif
+        per_cpu(x86_cpu_to_node_map, cpu) = node;
+        if (node != NUMA_NO_NODE)
+                per_cpu(node_number, cpu) = node;
+}
+void __cpuinit numa_clear_node(int cpu)
+{
+        numa_set_node(cpu, NUMA_NO_NODE);
+}
+#ifndef CONFIG_DEBUG_PER_CPU_MAPS
+void __cpuinit numa_add_cpu(int cpu)
+{
+        cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
+}
+void __cpuinit numa_remove_cpu(int cpu)
+{
+        cpu_clear(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
+}
+#else /* CONFIG_DEBUG_PER_CPU_MAPS */
+/*
+ * --------- debug versions of the numa functions ---------
+ */
+static void __cpuinit numa_set_cpumask(int cpu, int enable)
+{
+        int node = early_cpu_to_node(cpu);
+        cpumask_t *mask;
+        char buf[64];
+        if (node_to_cpumask_map == NULL) {
+                printk(KERN_ERR "node_to_cpumask_map NULL\n");
+                dump_stack();
+                return;
+        }
+        mask = &node_to_cpumask_map[node];
+        if (enable)
+                cpu_set(cpu, *mask);
+        else
+                cpu_clear(cpu, *mask);
+        cpulist_scnprintf(buf, sizeof(buf), mask);
+        printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
+                enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf);
+}
+void __cpuinit numa_add_cpu(int cpu)
+{
+        numa_set_cpumask(cpu, 1);
+}
+void __cpuinit numa_remove_cpu(int cpu)
+{
+        numa_set_cpumask(cpu, 0);
+}
+int cpu_to_node(int cpu)
+{
+        if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
+                printk(KERN_WARNING
+                        "cpu_to_node(%d): usage too early!\n", cpu);
+                dump_stack();
+                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
+        }
+        return per_cpu(x86_cpu_to_node_map, cpu);
+}
+EXPORT_SYMBOL(cpu_to_node);
+/*
+ * Same function as cpu_to_node() but used if called before the
+ * per_cpu areas are setup.
+ */
+int early_cpu_to_node(int cpu)
+{
+        if (early_per_cpu_ptr(x86_cpu_to_node_map))
+                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
+        if (!per_cpu_offset(cpu)) {
+                printk(KERN_WARNING
+                        "early_cpu_to_node(%d): no per_cpu area!\n", cpu);
+                dump_stack();
+                return NUMA_NO_NODE;
+        }
+        return per_cpu(x86_cpu_to_node_map, cpu);
+}
+/* empty cpumask */
+static const cpumask_t cpu_mask_none;
+/*
+ * Returns a pointer to the bitmask of CPUs on Node 'node'.
+ */
+const cpumask_t *cpumask_of_node(int node)
+{
+        if (node_to_cpumask_map == NULL) {
+                printk(KERN_WARNING
+                        "cpumask_of_node(%d): no node_to_cpumask_map!\n",
+                        node);
+                dump_stack();
+                return (const cpumask_t *)&cpu_online_map;
+        }
+        if (node >= nr_node_ids) {
+                printk(KERN_WARNING
+                        "cpumask_of_node(%d): node > nr_node_ids(%d)\n",
+                        node, nr_node_ids);
+                dump_stack();
+                return &cpu_mask_none;
+        }
+        return &node_to_cpumask_map[node];
+}
+EXPORT_SYMBOL(cpumask_of_node);
+/*
+ * Returns a bitmask of CPUs on Node 'node'.
+ *
+ * Side note: this function creates the returned cpumask on the stack
+ * so with a high NR_CPUS count, excessive stack space is used.  The
+ * node_to_cpumask_ptr function should be used whenever possible.
+ */
+cpumask_t node_to_cpumask(int node)
+{
+        if (node_to_cpumask_map == NULL) {
+                printk(KERN_WARNING
+                        "node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
+                dump_stack();
+                return cpu_online_map;
+        }
+        if (node >= nr_node_ids) {
+                printk(KERN_WARNING
+                        "node_to_cpumask(%d): node > nr_node_ids(%d)\n",
+                        node, nr_node_ids);
+                dump_stack();
+                return cpu_mask_none;
+        }
+        return node_to_cpumask_map[node];
+}
+EXPORT_SYMBOL(node_to_cpumask);
+/*
+ * --------- end of debug versions of the numa functions ---------
+ */
+#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index e89d24815f26..84ba74820ad6 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -534,6 +534,36 @@ out_unlock:
        return 0;
 }
+static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
+                               int primary)
+{
+        /*
+         * Ignore all non primary paths.
+         */
+        if (!primary)
+                return 0;
+        /*
+         * Ignore the NULL PTE for kernel identity mapping, as it is expected
+         * to have holes.
+         * Also set numpages to '1' indicating that we processed cpa req for
+         * one virtual address page and its pfn. TBD: numpages can be set based
+         * on the initial value and the level returned by lookup_address().
+         */
+        if (within(vaddr, PAGE_OFFSET,
+                   PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
+                cpa->numpages = 1;
+                cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
+                return 0;
+        } else {
+                WARN(1, KERN_WARNING "CPA: called for zero pte. "
+                        "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
+                        *cpa->vaddr);
+                return -EFAULT;
+        }
+}
 static int __change_page_attr(struct cpa_data *cpa, int primary)
 {
        unsigned long address;
@@ -549,17 +579,11 @@ static int __change_page_attr(struct cpa_data *cpa, int primary)
 repeat:
        kpte = lookup_address(address, &level);
        if (!kpte)
-                return 0;
+                return __cpa_process_fault(cpa, address, primary);
        old_pte = *kpte;
-        if (!pte_val(old_pte)) {
+        if (!pte_val(old_pte))
-                if (!primary)
+                return __cpa_process_fault(cpa, address, primary);
-                        return 0;
-                WARN(1, KERN_WARNING "CPA: called for zero pte. "
-                       "vaddr = %lx cpa->vaddr = %lx\n", address,
-                       *cpa->vaddr);
-                return -EINVAL;
-        }
        if (level == PG_LEVEL_4K) {
                pte_t new_pte;
@@ -657,12 +681,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
                vaddr = *cpa->vaddr;
        if (!(within(vaddr, PAGE_OFFSET,
-                    PAGE_OFFSET + (max_low_pfn_mapped << PAGE_SHIFT))
+                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
-#ifdef CONFIG_X86_64
-                || within(vaddr, PAGE_OFFSET + (1UL<<32),
-                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))
-#endif
-        )) {
                alias_cpa = *cpa;
                temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT);
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 8b08fb955274..7b61036427df 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -333,11 +333,23 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                                              req_type & _PAGE_CACHE_MASK);
        }
-        is_range_ram = pagerange_is_ram(start, end);
+        if (new_type)
-        if (is_range_ram == 1)
+                *new_type = actual_type;
-                return reserve_ram_pages_type(start, end, req_type, new_type);
-        else if (is_range_ram < 0)
+        /*
-                return -EINVAL;
+         * For legacy reasons, some parts of the physical address range in the
+         * legacy 1MB region is treated as non-RAM (even when listed as RAM in
+         * the e820 tables).  So we will track the memory attributes of this
+         * legacy 1MB region using the linear memtype_list always.
+         */
+        if (end >= ISA_END_ADDRESS) {
+                is_range_ram = pagerange_is_ram(start, end);
+                if (is_range_ram == 1)
+                        return reserve_ram_pages_type(start, end, req_type,
+                                                      new_type);
+                else if (is_range_ram < 0)
+                        return -EINVAL;
+        }
        new  = kmalloc(sizeof(struct memtype), GFP_KERNEL);
        if (!new)
@@ -347,9 +359,6 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
        new->end        = end;
        new->type       = actual_type;
-        if (new_type)
-                *new_type = actual_type;
        spin_lock(&memtype_lock);
        if (cached_entry && start >= cached_start)
@@ -437,11 +446,19 @@ int free_memtype(u64 start, u64 end)
        if (is_ISA_range(start, end - 1))
                return 0;
-        is_range_ram = pagerange_is_ram(start, end);
+        /*
-        if (is_range_ram == 1)
+         * For legacy reasons, some parts of the physical address range in the
-                return free_ram_pages_type(start, end);
+         * legacy 1MB region is treated as non-RAM (even when listed as RAM in
-        else if (is_range_ram < 0)
+         * the e820 tables).  So we will track the memory attributes of this
-                return -EINVAL;
+         * legacy 1MB region using the linear memtype_list always.
+         */
+        if (end >= ISA_END_ADDRESS) {
+                is_range_ram = pagerange_is_ram(start, end);
+                if (is_range_ram == 1)
+                        return free_ram_pages_type(start, end);
+                else if (is_range_ram < 0)
+                        return -EINVAL;
+        }
        spin_lock(&memtype_lock);
        list_for_each_entry(entry, &memtype_list, nd) {
diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c
index 09737c8af074..15df1baee100 100644
--- a/arch/x86/mm/srat_64.c
+++ b/arch/x86/mm/srat_64.c
@@ -21,6 +21,7 @@
 #include <asm/numa.h>
 #include <asm/e820.h>
 #include <asm/genapic.h>
+#include <asm/uv/uv.h>
 int acpi_numa __initdata;
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/mm/tlb.c
index f8be6f1d2e48..72a6d4ebe34d 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/mm/tlb.c
@@ -1,22 +1,18 @@
 #include <linux/init.h>
 #include <linux/mm.h>
-#include <linux/delay.h>
 #include <linux/spinlock.h>
 #include <linux/smp.h>
-#include <linux/kernel_stat.h>
-#include <linux/mc146818rtc.h>
 #include <linux/interrupt.h>
+#include <linux/module.h>
-#include <asm/mtrr.h>
-#include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
-#include <asm/proto.h>
+#include <asm/apic.h>
-#include <asm/apicdef.h>
+#include <asm/uv/uv.h>
-#include <asm/idle.h>
-#include <asm/uv/uv_hub.h>
+DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
-#include <asm/uv/uv_bau.h>
+                        = { &init_mm, 0, };
 #include <mach_ipi.h>
 /*
@@ -33,7 +29,7 @@
 *      To avoid global state use 8 different call vectors.
 *      Each CPU uses a specific vector to trigger flushes on other
 *      CPUs. Depending on the received vector the target CPUs look into
- *      the right per cpu variable for the flush data.
+ *      the right array slot for the flush data.
 *
 *      With more than 8 CPUs they are hashed to the 8 available
 *      vectors. The limited global vector space forces us to this right now.
@@ -43,18 +39,18 @@
 union smp_flush_state {
        struct {
-                cpumask_t flush_cpumask;
                struct mm_struct *flush_mm;
                unsigned long flush_va;
                spinlock_t tlbstate_lock;
+                DECLARE_BITMAP(flush_cpumask, NR_CPUS);
        };
-        char pad[SMP_CACHE_BYTES];
+        char pad[CONFIG_X86_INTERNODE_CACHE_BYTES];
-} ____cacheline_aligned;
+} ____cacheline_internodealigned_in_smp;
 /* State is put into the per CPU data section, but padded
   to a full cache line because other CPUs can access it and we don't
   want false sharing in the per cpu data segment. */
-static DEFINE_PER_CPU(union smp_flush_state, flush_state);
+static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
 /*
 * We cannot call mmdrop() because we are in interrupt context,
@@ -62,9 +58,9 @@ static DEFINE_PER_CPU(union smp_flush_state, flush_state);
 */
 void leave_mm(int cpu)
 {
-        if (read_pda(mmu_state) == TLBSTATE_OK)
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                BUG();
-        cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
+        cpu_clear(cpu, percpu_read(cpu_tlbstate.active_mm)->cpu_vm_mask);
        load_cr3(swapper_pg_dir);
 }
 EXPORT_SYMBOL_GPL(leave_mm);
@@ -117,10 +113,20 @@ EXPORT_SYMBOL_GPL(leave_mm);
 * Interrupts are disabled.
 */
-asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
+/*
+ * FIXME: use of asmlinkage is not consistent.  On x86_64 it's noop
+ * but still used for documentation purpose but the usage is slightly
+ * inconsistent.  On x86_32, asmlinkage is regparm(0) but interrupt
+ * entry calls in with the first parameter in %eax.  Maybe define
+ * intrlinkage?
+ */
+#ifdef CONFIG_X86_64
+asmlinkage
+#endif
+void smp_invalidate_interrupt(struct pt_regs *regs)
 {
-        int cpu;
+        unsigned int cpu;
-        int sender;
+        unsigned int sender;
        union smp_flush_state *f;
        cpu = smp_processor_id();
@@ -129,9 +135,9 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
         * Use that to determine where the sender put the data.
         */
        sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
-        f = &per_cpu(flush_state, sender);
+        f = &flush_state[sender];
-        if (!cpu_isset(cpu, f->flush_cpumask))
+        if (!cpumask_test_cpu(cpu, to_cpumask(f->flush_cpumask)))
                goto out;
                /*
                 * This was a BUG() but until someone can quote me the
@@ -142,8 +148,8 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
                 * BUG();
                 */
-        if (f->flush_mm == read_pda(active_mm)) {
+        if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
-                if (read_pda(mmu_state) == TLBSTATE_OK) {
+                if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
                        if (f->flush_va == TLB_FLUSH_ALL)
                                local_flush_tlb();
                        else
@@ -153,23 +159,21 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
        }
 out:
        ack_APIC_irq();
-        cpu_clear(cpu, f->flush_cpumask);
+        smp_mb__before_clear_bit();
+        cpumask_clear_cpu(cpu, to_cpumask(f->flush_cpumask));
+        smp_mb__after_clear_bit();
        inc_irq_stat(irq_tlb_count);
 }
-void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
+static void flush_tlb_others_ipi(const struct cpumask *cpumask,
-                             unsigned long va)
+                                 struct mm_struct *mm, unsigned long va)
 {
-        int sender;
+        unsigned int sender;
        union smp_flush_state *f;
-        cpumask_t cpumask = *cpumaskp;
-        if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va))
-                return;
        /* Caller has disabled preemption */
        sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
-        f = &per_cpu(flush_state, sender);
+        f = &flush_state[sender];
        /*
         * Could avoid this lock when
@@ -180,7 +184,8 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
        f->flush_mm = mm;
        f->flush_va = va;
-        cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask);
+        cpumask_andnot(to_cpumask(f->flush_cpumask),
+                       cpumask, cpumask_of(smp_processor_id()));
        /*
         * Make the above memory operations globally visible before
@@ -191,9 +196,10 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
         * We have to send the IPI only to
         * CPUs affected.
         */
-        send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR_START + sender);
+        send_IPI_mask(to_cpumask(f->flush_cpumask),
+                      INVALIDATE_TLB_VECTOR_START + sender);
-        while (!cpus_empty(f->flush_cpumask))
+        while (!cpumask_empty(to_cpumask(f->flush_cpumask)))
                cpu_relax();
        f->flush_mm = NULL;
@@ -201,12 +207,28 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
        spin_unlock(&f->tlbstate_lock);
 }
+void native_flush_tlb_others(const struct cpumask *cpumask,
+                             struct mm_struct *mm, unsigned long va)
+{
+        if (is_uv_system()) {
+                unsigned int cpu;
+                cpu = get_cpu();
+                cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu);
+                if (cpumask)
+                        flush_tlb_others_ipi(cpumask, mm, va);
+                put_cpu();
+                return;
+        }
+        flush_tlb_others_ipi(cpumask, mm, va);
+}
 static int __cpuinit init_smp_flush(void)
 {
        int i;
-        for_each_possible_cpu(i)
+        for (i = 0; i < ARRAY_SIZE(flush_state); i++)
-                spin_lock_init(&per_cpu(flush_state, i).tlbstate_lock);
+                spin_lock_init(&flush_state[i].tlbstate_lock);
        return 0;
 }
@@ -215,25 +237,18 @@ core_initcall(init_smp_flush);
 void flush_tlb_current_task(void)
 {
        struct mm_struct *mm = current->mm;
-        cpumask_t cpu_mask;
        preempt_disable();
-        cpu_mask = mm->cpu_vm_mask;
-        cpu_clear(smp_processor_id(), cpu_mask);
        local_flush_tlb();
-        if (!cpus_empty(cpu_mask))
+        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
+                flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
        preempt_enable();
 }
 void flush_tlb_mm(struct mm_struct *mm)
 {
-        cpumask_t cpu_mask;
        preempt_disable();
-        cpu_mask = mm->cpu_vm_mask;
-        cpu_clear(smp_processor_id(), cpu_mask);
        if (current->active_mm == mm) {
                if (current->mm)
@@ -241,8 +256,8 @@ void flush_tlb_mm(struct mm_struct *mm)
                else
                        leave_mm(smp_processor_id());
        }
-        if (!cpus_empty(cpu_mask))
+        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
+                flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
        preempt_enable();
 }
@@ -250,11 +265,8 @@ void flush_tlb_mm(struct mm_struct *mm)
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
 {
        struct mm_struct *mm = vma->vm_mm;
-        cpumask_t cpu_mask;
        preempt_disable();
-        cpu_mask = mm->cpu_vm_mask;
-        cpu_clear(smp_processor_id(), cpu_mask);
        if (current->active_mm == mm) {
                if (current->mm)
@@ -263,8 +275,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
                        leave_mm(smp_processor_id());
        }
-        if (!cpus_empty(cpu_mask))
+        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(cpu_mask, mm, va);
+                flush_tlb_others(&mm->cpu_vm_mask, mm, va);
        preempt_enable();
 }
@@ -274,7 +286,7 @@ static void do_flush_tlb_all(void *info)
        unsigned long cpu = smp_processor_id();
        __flush_tlb_all();
-        if (read_pda(mmu_state) == TLBSTATE_LAZY)
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
                leave_mm(cpu);
 }
diff --git a/arch/x86/scripts/strip-symbols b/arch/x86/scripts/strip-symbols
deleted file mode 100644
index a2f1ccb827c7..000000000000
--- a/arch/x86/scripts/strip-symbols
+++ /dev/null
@@ -1 +0,0 @@
-__cpu_vendor_dev_X86_VENDOR_*
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 6f1bb71aa13a..6b3f7eef57e3 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -634,35 +634,27 @@ static void xen_flush_tlb_single(unsigned long addr)
        preempt_enable();
 }
-static void xen_flush_tlb_others(const cpumask_t *cpus, struct mm_struct *mm,
+static void xen_flush_tlb_others(const struct cpumask *cpus,
-                                 unsigned long va)
+                                 struct mm_struct *mm, unsigned long va)
 {
        struct {
                struct mmuext_op op;
-                cpumask_t mask;
+                DECLARE_BITMAP(mask, NR_CPUS);
        } *args;
-        cpumask_t cpumask = *cpus;
        struct multicall_space mcs;
-        /*
+        BUG_ON(cpumask_empty(cpus));
-         * A couple of (to be removed) sanity checks:
-         *
-         * - current CPU must not be in mask
-         * - mask must exist :)
-         */
-        BUG_ON(cpus_empty(cpumask));
-        BUG_ON(cpu_isset(smp_processor_id(), cpumask));
        BUG_ON(!mm);
-        /* If a CPU which we ran on has gone down, OK. */
-        cpus_and(cpumask, cpumask, cpu_online_map);
-        if (cpus_empty(cpumask))
-                return;
        mcs = xen_mc_entry(sizeof(*args));
        args = mcs.args;
-        args->mask = cpumask;
+        args->op.arg2.vcpumask = to_cpumask(args->mask);
-        args->op.arg2.vcpumask = &args->mask;
+        /* Remove us, and any offline CPUS. */
+        cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
+        cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
+        if (unlikely(cpumask_empty(to_cpumask(args->mask))))
+                goto issue;
        if (va == TLB_FLUSH_ALL) {
                args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
@@ -673,6 +665,7 @@ static void xen_flush_tlb_others(const cpumask_t *cpus, struct mm_struct *mm,
        MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
+issue:
        xen_mc_issue(PARAVIRT_LAZY_MMU);
 }
@@ -702,17 +695,17 @@ static void xen_write_cr0(unsigned long cr0)
 static void xen_write_cr2(unsigned long cr2)
 {
-        x86_read_percpu(xen_vcpu)->arch.cr2 = cr2;
+        percpu_read(xen_vcpu)->arch.cr2 = cr2;
 }
 static unsigned long xen_read_cr2(void)
 {
-        return x86_read_percpu(xen_vcpu)->arch.cr2;
+        return percpu_read(xen_vcpu)->arch.cr2;
 }
 static unsigned long xen_read_cr2_direct(void)
 {
-        return x86_read_percpu(xen_vcpu_info.arch.cr2);
+        return percpu_read(xen_vcpu_info.arch.cr2);
 }
 static void xen_write_cr4(unsigned long cr4)
@@ -725,12 +718,12 @@ static void xen_write_cr4(unsigned long cr4)
 static unsigned long xen_read_cr3(void)
 {
-        return x86_read_percpu(xen_cr3);
+        return percpu_read(xen_cr3);
 }
 static void set_current_cr3(void *v)
 {
-        x86_write_percpu(xen_current_cr3, (unsigned long)v);
+        percpu_write(xen_current_cr3, (unsigned long)v);
 }
 static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -755,7 +748,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
        MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
        if (kernel) {
-                x86_write_percpu(xen_cr3, cr3);
+                percpu_write(xen_cr3, cr3);
                /* Update xen_current_cr3 once the batch has actually
                   been submitted. */
@@ -771,7 +764,7 @@ static void xen_write_cr3(unsigned long cr3)
        /* Update while interrupts are disabled, so its atomic with
           respect to ipis */
-        x86_write_percpu(xen_cr3, cr3);
+        percpu_write(xen_cr3, cr3);
        __xen_write_cr3(true, cr3);
@@ -1651,7 +1644,6 @@ asmlinkage void __init xen_start_kernel(void)
 #ifdef CONFIG_X86_64
        /* Disable until direct per-cpu data access. */
        have_vcpu_info_placement = 0;
-        x86_64_init_pda();
 #endif
        xen_smp_init();
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index bb042608c602..2e8271431e1a 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -39,7 +39,7 @@ static unsigned long xen_save_fl(void)
        struct vcpu_info *vcpu;
        unsigned long flags;
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
        /* flag has opposite sense of mask */
        flags = !vcpu->evtchn_upcall_mask;
@@ -62,7 +62,7 @@ static void xen_restore_fl(unsigned long flags)
           make sure we're don't switch CPUs between getting the vcpu
           pointer and updating the mask. */
        preempt_disable();
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
        vcpu->evtchn_upcall_mask = flags;
        preempt_enable_no_resched();
@@ -83,7 +83,7 @@ static void xen_irq_disable(void)
           make sure we're don't switch CPUs between getting the vcpu
           pointer and updating the mask. */
        preempt_disable();
-        x86_read_percpu(xen_vcpu)->evtchn_upcall_mask = 1;
+        percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
        preempt_enable_no_resched();
 }
@@ -96,7 +96,7 @@ static void xen_irq_enable(void)
           the caller is confused and is trying to re-enable interrupts
           on an indeterminate processor. */
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
        vcpu->evtchn_upcall_mask = 0;
        /* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 503c240e26c7..98cb9869eb24 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1063,18 +1063,14 @@ static void drop_other_mm_ref(void *info)
        struct mm_struct *mm = info;
        struct mm_struct *active_mm;
-#ifdef CONFIG_X86_64
+        active_mm = percpu_read(cpu_tlbstate.active_mm);
-        active_mm = read_pda(active_mm);
-#else
-        active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
-#endif
        if (active_mm == mm)
                leave_mm(smp_processor_id());
        /* If this cpu still has a stale cr3 reference, then make sure
           it has been flushed. */
-        if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
+        if (percpu_read(xen_current_cr3) == __pa(mm->pgd)) {
                load_cr3(swapper_pg_dir);
                arch_flush_lazy_cpu_mode();
        }
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index 858938241616..e786fa7f2615 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -39,7 +39,7 @@ static inline void xen_mc_issue(unsigned mode)
                xen_mc_flush();
        /* restore flags saved in xen_mc_batch */
-        local_irq_restore(x86_read_percpu(xen_mc_irq_flags));
+        local_irq_restore(percpu_read(xen_mc_irq_flags));
 }
 /* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index c44e2069c7c7..7735e3dd359c 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -50,11 +50,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
 */
 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
 {
-#ifdef CONFIG_X86_32
+        inc_irq_stat(irq_resched_count);
-        __get_cpu_var(irq_stat).irq_resched_count++;
-#else
-        add_pda(irq_resched_count, 1);
-#endif
        return IRQ_HANDLED;
 }
@@ -78,7 +74,7 @@ static __cpuinit void cpu_bringup(void)
        xen_setup_cpu_clockevents();
        cpu_set(cpu, cpu_online_map);
-        x86_write_percpu(cpu_state, CPU_ONLINE);
+        percpu_write(cpu_state, CPU_ONLINE);
        wmb();
        /* We can take interrupts now: we're officially "up". */
@@ -283,22 +279,10 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
        struct task_struct *idle = idle_task(cpu);
        int rc;
-#ifdef CONFIG_X86_64
-        /* Allocate node local memory for AP pdas */
-        WARN_ON(cpu == 0);
-        if (cpu > 0) {
-                rc = get_local_pda(cpu);
-                if (rc)
-                        return rc;
-        }
-#endif
-#ifdef CONFIG_X86_32
-        init_gdt(cpu);
        per_cpu(current_task, cpu) = idle;
+#ifdef CONFIG_X86_32
        irq_ctx_init(cpu);
 #else
-        cpu_pda(cpu)->pcurrent = idle;
        clear_tsk_thread_flag(idle, TIF_FORK);
 #endif
        xen_setup_timer(cpu);
@@ -445,11 +429,7 @@ static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
 {
        irq_enter();
        generic_smp_call_function_interrupt();
-#ifdef CONFIG_X86_32
+        inc_irq_stat(irq_call_count);
-        __get_cpu_var(irq_stat).irq_call_count++;
-#else
-        add_pda(irq_call_count, 1);
-#endif
        irq_exit();
        return IRQ_HANDLED;
@@ -459,11 +439,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
 {
        irq_enter();
        generic_smp_call_function_single_interrupt();
-#ifdef CONFIG_X86_32
+        inc_irq_stat(irq_call_count);
-        __get_cpu_var(irq_stat).irq_call_count++;
-#else
-        add_pda(irq_call_count, 1);
-#endif
        irq_exit();
        return IRQ_HANDLED;
diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c
index 212ffe012b76..95be7b434724 100644
--- a/arch/x86/xen/suspend.c
+++ b/arch/x86/xen/suspend.c
@@ -6,6 +6,7 @@
 #include <asm/xen/hypercall.h>
 #include <asm/xen/page.h>
+#include <asm/fixmap.h>
 #include "xen-ops.h"
 #include "mmu.h"
diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index 05794c566e87..d6fc51f4ce85 100644
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -17,6 +17,7 @@
 #include <asm/processor-flags.h>
 #include <asm/errno.h>
 #include <asm/segment.h>
+#include <asm/percpu.h>
 #include <xen/interface/xen.h>
@@ -28,12 +29,10 @@
 #if 1
 /*
-        x86-64 does not yet support direct access to percpu variables
+        FIXME: x86_64 now can support direct access to percpu variables
-        via a segment override, so we just need to make sure this code
+        via a segment override.  Update xen accordingly.
-        never gets used
 */
 #define BUG                     ud2a
-#define PER_CPU_VAR(var, off)   0xdeadbeef
 #endif
 /*
@@ -45,14 +44,14 @@ ENTRY(xen_irq_enable_direct)
        BUG
        /* Unmask events */
-        movb $0, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
        /* Preempt here doesn't matter because that will deal with
           any pending interrupts.  The pending check may end up being
           run on the wrong CPU, but that doesn't hurt. */
        /* Test for pending */
-        testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending)
+        testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
        jz 1f
 2:      call check_events
@@ -69,7 +68,7 @@ ENDPATCH(xen_irq_enable_direct)
 ENTRY(xen_irq_disable_direct)
        BUG
-        movb $1, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 ENDPATCH(xen_irq_disable_direct)
        ret
        ENDPROC(xen_irq_disable_direct)
@@ -87,7 +86,7 @@ ENDPATCH(xen_irq_disable_direct)
 ENTRY(xen_save_fl_direct)
        BUG
-        testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
        setz %ah
        addb %ah,%ah
 ENDPATCH(xen_save_fl_direct)
@@ -107,13 +106,13 @@ ENTRY(xen_restore_fl_direct)
        BUG
        testb $X86_EFLAGS_IF>>8, %ah
-        setz PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
        /* Preempt here doesn't matter because that will deal with
           any pending interrupts.  The pending check may end up being
           run on the wrong CPU, but that doesn't hurt. */
        /* check for unmasked and pending */
-        cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending)
+        cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
        jz 1f
 2:      call check_events
 1:
@@ -195,11 +194,11 @@ RELOC(xen_sysexit, 1b+1)
 ENTRY(xen_sysret64)
        /* We're already on the usermode stack at this point, but still
           with the kernel gs, so we can easily switch back */
-        movq %rsp, %gs:pda_oldrsp
+        movq %rsp, PER_CPU_VAR(old_rsp)
-        movq %gs:pda_kernelstack,%rsp
+        movq PER_CPU_VAR(kernel_stack),%rsp
        pushq $__USER_DS
-        pushq %gs:pda_oldrsp
+        pushq PER_CPU_VAR(old_rsp)
        pushq %r11
        pushq $__USER_CS
        pushq %rcx
@@ -212,11 +211,11 @@ RELOC(xen_sysret64, 1b+1)
 ENTRY(xen_sysret32)
        /* We're already on the usermode stack at this point, but still
           with the kernel gs, so we can easily switch back */
-        movq %rsp, %gs:pda_oldrsp
+        movq %rsp, PER_CPU_VAR(old_rsp)
-        movq %gs:pda_kernelstack, %rsp
+        movq PER_CPU_VAR(kernel_stack), %rsp
        pushq $__USER32_DS
-        pushq %gs:pda_oldrsp
+        pushq PER_CPU_VAR(old_rsp)
        pushq %r11
        pushq $__USER32_CS
        pushq %rcx
author	H. Peter Anvin <hpa@linux.intel.com>	2009-01-30 17:50:57 -0500
committer	H. Peter Anvin <hpa@linux.intel.com>	2009-01-30 17:50:57 -0500
commit	9b7ed8faa034fc2d350e2eff5c68680eb5c43a07 (patch)
tree	5c94c34ad30e312604c1ce4f08ab6631b64a94f5 /arch/x86
parent	6522869c34664dd5f05a0a327e93915b1281c90d (diff)
parent	c43e0e46adf79c321ed3fbf0351e1005fb8a2413 (diff)