106 files changed, 1286 insertions, 1577 deletions
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index 00216071eaf7..85598f7da407 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -65,6 +65,7 @@ SECTIONS
 #endif
                . = ALIGN(4096);
                __per_cpu_start = .;
+                        *(.data.percpu.page_aligned)
                        *(.data.percpu)
                        *(.data.percpu.shared_aligned)
                __per_cpu_end = .;
diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S
index 10a7d47e8510..f45e4e508eca 100644
--- a/arch/ia64/kernel/vmlinux.lds.S
+++ b/arch/ia64/kernel/vmlinux.lds.S
@@ -219,6 +219,7 @@ SECTIONS
  .data.percpu PERCPU_ADDR : AT(__phys_per_cpu_start - LOAD_OFFSET)
        {
                __per_cpu_start = .;
+                *(.data.percpu.page_aligned)
                *(.data.percpu)
                *(.data.percpu.shared_aligned)
                __per_cpu_end = .;
diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index 47bf15cd2c9e..04e8ecea9b40 100644
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -182,6 +182,7 @@ SECTIONS
        . = ALIGN(PAGE_SIZE);
        .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
                __per_cpu_start = .;
+                *(.data.percpu.page_aligned)
                *(.data.percpu)
                *(.data.percpu.shared_aligned)
                __per_cpu_end = .;
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 73f7fe8fd4d1..ef27aed6ff74 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1340,13 +1340,17 @@ config SECCOMP
          If unsure, say Y. Only embedded should say N here.
+config CC_STACKPROTECTOR_ALL
+        bool
 config CC_STACKPROTECTOR
        bool "Enable -fstack-protector buffer overflow detection (EXPERIMENTAL)"
-        depends on X86_64 && EXPERIMENTAL && BROKEN
+        depends on X86_64
+        select CC_STACKPROTECTOR_ALL
        help
-         This option turns on the -fstack-protector GCC feature. This
+          This option turns on the -fstack-protector GCC feature. This
-          feature puts, at the beginning of critical functions, a canary
+          feature puts, at the beginning of functions, a canary value on
-          value on the stack just before the return address, and validates
+          the stack just before the return address, and validates
          the value just before actually returning.  Stack based buffer
          overflows (that need to overwrite this return address) now also
          overwrite the canary, which gets detected and the attack is then
@@ -1354,15 +1358,8 @@ config CC_STACKPROTECTOR
          This feature requires gcc version 4.2 or above, or a distribution
          gcc with the feature backported. Older versions are automatically
-          detected and for those versions, this configuration option is ignored.
+          detected and for those versions, this configuration option is
+          ignored. (and a warning is printed during bootup)
-config CC_STACKPROTECTOR_ALL
-        bool "Use stack-protector for all functions"
-        depends on CC_STACKPROTECTOR
-        help
-          Normally, GCC only inserts the canary value protection for
-          functions that use large-ish on-stack buffers. By enabling
-          this option, GCC will be asked to do this for ALL functions.
 source kernel/Kconfig.hz
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 10d6cc3fd052..28f111461ca8 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -117,6 +117,7 @@ config DEBUG_RODATA
 config DEBUG_RODATA_TEST
        bool "Testcase for the DEBUG_RODATA feature"
        depends on DEBUG_RODATA
+        default y
        help
          This option enables a testcase for the DEBUG_RODATA
          feature as well as for the change_page_attr() infrastructure.
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index d1a47adb5aec..cacee981d166 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -73,7 +73,7 @@ else
        stackp := $(CONFIG_SHELL) $(srctree)/scripts/gcc-x86_64-has-stack-protector.sh
        stackp-$(CONFIG_CC_STACKPROTECTOR) := $(shell $(stackp) \
-                "$(CC)" -fstack-protector )
+                "$(CC)" "-fstack-protector -DGCC_HAS_SP" )
        stackp-$(CONFIG_CC_STACKPROTECTOR_ALL) += $(shell $(stackp) \
                "$(CC)" -fstack-protector-all )
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 256b00b61892..9c79b2477008 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -112,8 +112,8 @@ ENTRY(ia32_sysenter_target)
        CFI_DEF_CFA     rsp,0
        CFI_REGISTER    rsp,rbp
        SWAPGS_UNSAFE_STACK
-        movq    %gs:pda_kernelstack, %rsp
+        movq    PER_CPU_VAR(kernel_stack), %rsp
-        addq    $(PDA_STACKOFFSET),%rsp 
+        addq    $(KERNEL_STACK_OFFSET),%rsp
        /*
         * No need to follow this irqs on/off section: the syscall
         * disabled irqs, here we enable it straight after entry:
@@ -273,13 +273,13 @@ ENDPROC(ia32_sysenter_target)
 ENTRY(ia32_cstar_target)
        CFI_STARTPROC32 simple
        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
+        CFI_DEF_CFA     rsp,KERNEL_STACK_OFFSET
        CFI_REGISTER    rip,rcx
        /*CFI_REGISTER  rflags,r11*/
        SWAPGS_UNSAFE_STACK
        movl    %esp,%r8d
        CFI_REGISTER    rsp,r8
-        movq    %gs:pda_kernelstack,%rsp
+        movq    PER_CPU_VAR(kernel_stack),%rsp
        /*
         * No need to follow this irqs on/off section: the syscall
         * disabled irqs and here we enable it straight after entry:
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index e02a359d2aa5..02b47a603fc8 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -3,6 +3,9 @@
 /*
 * Copyright 1992, Linus Torvalds.
+ *
+ * Note: inlines with more than a single statement should be marked
+ * __always_inline to avoid problems with older gcc's inlining heuristics.
 */
 #ifndef _LINUX_BITOPS_H
@@ -53,7 +56,8 @@
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
-static inline void set_bit(unsigned int nr, volatile unsigned long *addr)
+static __always_inline void
+set_bit(unsigned int nr, volatile unsigned long *addr)
 {
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "orb %1,%0"
@@ -90,7 +94,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr)
 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
 * in order to ensure changes are visible on other processors.
 */
-static inline void clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+clear_bit(int nr, volatile unsigned long *addr)
 {
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "andb %1,%0"
@@ -204,7 +209,8 @@ static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
 *
 * This is the same as test_and_set_bit on x86.
 */
-static inline int test_and_set_bit_lock(int nr, volatile unsigned long *addr)
+static __always_inline int
+test_and_set_bit_lock(int nr, volatile unsigned long *addr)
 {
        return test_and_set_bit(nr, addr);
 }
@@ -300,7 +306,7 @@ static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
        return oldbit;
 }
-static inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
+static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
 {
        return ((1UL << (nr % BITS_PER_LONG)) &
                (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h
index bae482df6039..f03b23e32864 100644
--- a/arch/x86/include/asm/cpu.h
+++ b/arch/x86/include/asm/cpu.h
@@ -7,6 +7,20 @@
 #include <linux/nodemask.h>
 #include <linux/percpu.h>
+#ifdef CONFIG_SMP
+extern void prefill_possible_map(void);
+#else /* CONFIG_SMP */
+static inline void prefill_possible_map(void) {}
+#define cpu_physical_id(cpu)                    boot_cpu_physical_apicid
+#define safe_smp_processor_id()                 0
+#define stack_smp_processor_id()                0
+#endif /* CONFIG_SMP */
 struct x86_cpu {
        struct cpu cpu;
 };
@@ -17,4 +31,11 @@ extern void arch_unregister_cpu(int);
 #endif
 DECLARE_PER_CPU(int, cpu_state);
+#ifdef CONFIG_X86_HAS_BOOT_CPU_ID
+extern unsigned char boot_cpu_id;
+#else
+#define boot_cpu_id                             0
+#endif
 #endif /* _ASM_X86_CPU_H */
diff --git a/arch/x86/include/asm/cpumask.h b/arch/x86/include/asm/cpumask.h
new file mode 100644
index 000000000000..26c6dad90479
--- /dev/null
+++ b/arch/x86/include/asm/cpumask.h
@@ -0,0 +1,28 @@
+#ifndef _ASM_X86_CPUMASK_H
+#define _ASM_X86_CPUMASK_H
+#ifndef __ASSEMBLY__
+#include <linux/cpumask.h>
+#ifdef CONFIG_X86_64
+extern cpumask_var_t cpu_callin_mask;
+extern cpumask_var_t cpu_callout_mask;
+extern cpumask_var_t cpu_initialized_mask;
+extern cpumask_var_t cpu_sibling_setup_mask;
+#else /* CONFIG_X86_32 */
+extern cpumask_t cpu_callin_map;
+extern cpumask_t cpu_callout_map;
+extern cpumask_t cpu_initialized;
+extern cpumask_t cpu_sibling_setup_map;
+#define cpu_callin_mask         ((struct cpumask *)&cpu_callin_map)
+#define cpu_callout_mask        ((struct cpumask *)&cpu_callout_map)
+#define cpu_initialized_mask    ((struct cpumask *)&cpu_initialized)
+#define cpu_sibling_setup_mask  ((struct cpumask *)&cpu_sibling_setup_map)
+#endif /* CONFIG_X86_32 */
+#endif /* __ASSEMBLY__ */
+#endif /* _ASM_X86_CPUMASK_H */
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 0930b4f8d672..c68c361697e1 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -1,39 +1,21 @@
 #ifndef _ASM_X86_CURRENT_H
 #define _ASM_X86_CURRENT_H
-#ifdef CONFIG_X86_32
 #include <linux/compiler.h>
 #include <asm/percpu.h>
+#ifndef __ASSEMBLY__
 struct task_struct;
 DECLARE_PER_CPU(struct task_struct *, current_task);
-static __always_inline struct task_struct *get_current(void)
-{
-        return x86_read_percpu(current_task);
-}
-#else /* X86_32 */
-#ifndef __ASSEMBLY__
-#include <asm/pda.h>
-struct task_struct;
 static __always_inline struct task_struct *get_current(void)
 {
-        return read_pda(pcurrent);
+        return percpu_read(current_task);
 }
-#else /* __ASSEMBLY__ */
+#define current get_current()
-#include <asm/asm-offsets.h>
-#define GET_CURRENT(reg) movq %gs:(pda_pcurrent),reg
 #endif /* __ASSEMBLY__ */
-#endif /* X86_32 */
-#define current get_current()
 #endif /* _ASM_X86_CURRENT_H */
diff --git a/arch/x86/include/asm/genapic_32.h b/arch/x86/include/asm/genapic_32.h
index 2c05b737ee22..4334502d3664 100644
--- a/arch/x86/include/asm/genapic_32.h
+++ b/arch/x86/include/asm/genapic_32.h
@@ -138,11 +138,4 @@ struct genapic {
 extern struct genapic *genapic;
 extern void es7000_update_genapic_to_cluster(void);
-enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
-#define get_uv_system_type()            UV_NONE
-#define is_uv_system()                  0
-#define uv_wakeup_secondary(a, b)       1
-#define uv_system_init()                do {} while (0)
 #endif /* _ASM_X86_GENAPIC_32_H */
diff --git a/arch/x86/include/asm/genapic_64.h b/arch/x86/include/asm/genapic_64.h
index adf32fb56aa6..7bb092c59055 100644
--- a/arch/x86/include/asm/genapic_64.h
+++ b/arch/x86/include/asm/genapic_64.h
@@ -51,15 +51,9 @@ extern struct genapic apic_x2apic_phys;
 extern int acpi_madt_oem_check(char *, char *);
 extern void apic_send_IPI_self(int vector);
-enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
-extern enum uv_system_type get_uv_system_type(void);
-extern int is_uv_system(void);
 extern struct genapic apic_x2apic_uv_x;
 DECLARE_PER_CPU(int, x2apic_extra_bits);
-extern void uv_cpu_init(void);
-extern void uv_system_init(void);
-extern int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip);
 extern void setup_apic_routing(void);
diff --git a/arch/x86/include/asm/hardirq_32.h b/arch/x86/include/asm/hardirq_32.h
index cf7954d1405f..d4b5d731073f 100644
--- a/arch/x86/include/asm/hardirq_32.h
+++ b/arch/x86/include/asm/hardirq_32.h
@@ -19,6 +19,9 @@ typedef struct {
 DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
+/* We can have at most NR_VECTORS irqs routed to a cpu at a time */
+#define MAX_HARDIRQS_PER_CPU NR_VECTORS
 #define __ARCH_IRQ_STAT
 #define __IRQ_STAT(cpu, member) (per_cpu(irq_stat, cpu).member)
diff --git a/arch/x86/include/asm/hardirq_64.h b/arch/x86/include/asm/hardirq_64.h
index b5a6b5d56704..a65bab20f6ce 100644
--- a/arch/x86/include/asm/hardirq_64.h
+++ b/arch/x86/include/asm/hardirq_64.h
@@ -3,22 +3,36 @@
 #include <linux/threads.h>
 #include <linux/irq.h>
-#include <asm/pda.h>
 #include <asm/apic.h>
+typedef struct {
+        unsigned int __softirq_pending;
+        unsigned int __nmi_count;       /* arch dependent */
+        unsigned int apic_timer_irqs;   /* arch dependent */
+        unsigned int irq0_irqs;
+        unsigned int irq_resched_count;
+        unsigned int irq_call_count;
+        unsigned int irq_tlb_count;
+        unsigned int irq_thermal_count;
+        unsigned int irq_spurious_count;
+        unsigned int irq_threshold_count;
+} ____cacheline_aligned irq_cpustat_t;
+DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
 /* We can have at most NR_VECTORS irqs routed to a cpu at a time */
 #define MAX_HARDIRQS_PER_CPU NR_VECTORS
 #define __ARCH_IRQ_STAT 1
-#define inc_irq_stat(member)    add_pda(member, 1)
+#define inc_irq_stat(member)    percpu_add(irq_stat.member, 1)
-#define local_softirq_pending() read_pda(__softirq_pending)
+#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
 #define __ARCH_SET_SOFTIRQ_PENDING 1
-#define set_softirq_pending(x) write_pda(__softirq_pending, (x))
+#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x)  or_pda(__softirq_pending, (x))
+#define or_softirq_pending(x)  percpu_or(irq_stat.__softirq_pending, (x))
 extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 7a1f44ac1f17..08ec793aa043 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -114,38 +114,16 @@ struct IR_IO_APIC_route_entry {
 extern int nr_ioapics;
 extern int nr_ioapic_registers[MAX_IO_APICS];
-/*
- * MP-BIOS irq configuration table structures:
- */
 #define MP_MAX_IOAPIC_PIN 127
-struct mp_config_ioapic {
-        unsigned long mp_apicaddr;
-        unsigned int mp_apicid;
-        unsigned char mp_type;
-        unsigned char mp_apicver;
-        unsigned char mp_flags;
-};
-struct mp_config_intsrc {
-        unsigned int mp_dstapic;
-        unsigned char mp_type;
-        unsigned char mp_irqtype;
-        unsigned short mp_irqflag;
-        unsigned char mp_srcbus;
-        unsigned char mp_srcbusirq;
-        unsigned char mp_dstirq;
-};
 /* I/O APIC entries */
-extern struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
+extern struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
 /* # of MP IRQ source entries */
 extern int mp_irq_entries;
 /* MP IRQ source entries */
-extern struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
+extern struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
 /* non-0 if default (table-less) MP configuration */
 extern int mpc_default_type;
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 89c898ab298b..77843225b7ea 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -1,5 +1,31 @@
-#ifdef CONFIG_X86_32
+/*
-# include "irq_regs_32.h"
+ * Per-cpu current frame pointer - the location of the last exception frame on
-#else
+ * the stack, stored in the per-cpu area.
-# include "irq_regs_64.h"
+ *
-#endif
+ * Jeremy Fitzhardinge <jeremy@goop.org>
+ */
+#ifndef _ASM_X86_IRQ_REGS_H
+#define _ASM_X86_IRQ_REGS_H
+#include <asm/percpu.h>
+#define ARCH_HAS_OWN_IRQ_REGS
+DECLARE_PER_CPU(struct pt_regs *, irq_regs);
+static inline struct pt_regs *get_irq_regs(void)
+{
+        return percpu_read(irq_regs);
+}
+static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
+{
+        struct pt_regs *old_regs;
+        old_regs = get_irq_regs();
+        percpu_write(irq_regs, new_regs);
+        return old_regs;
+}
+#endif /* _ASM_X86_IRQ_REGS_32_H */
diff --git a/arch/x86/include/asm/irq_regs_32.h b/arch/x86/include/asm/irq_regs_32.h
deleted file mode 100644
index 86afd7473457..000000000000
--- a/arch/x86/include/asm/irq_regs_32.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Per-cpu current frame pointer - the location of the last exception frame on
- * the stack, stored in the per-cpu area.
- *
- * Jeremy Fitzhardinge <jeremy@goop.org>
- */
-#ifndef _ASM_X86_IRQ_REGS_32_H
-#define _ASM_X86_IRQ_REGS_32_H
-#include <asm/percpu.h>
-#define ARCH_HAS_OWN_IRQ_REGS
-DECLARE_PER_CPU(struct pt_regs *, irq_regs);
-static inline struct pt_regs *get_irq_regs(void)
-{
-        return x86_read_percpu(irq_regs);
-}
-static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
-{
-        struct pt_regs *old_regs;
-        old_regs = get_irq_regs();
-        x86_write_percpu(irq_regs, new_regs);
-        return old_regs;
-}
-#endif /* _ASM_X86_IRQ_REGS_32_H */
diff --git a/arch/x86/include/asm/irq_regs_64.h b/arch/x86/include/asm/irq_regs_64.h
deleted file mode 100644
index 3dd9c0b70270..000000000000
--- a/arch/x86/include/asm/irq_regs_64.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <asm-generic/irq_regs.h>
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index a16a2ab2b429..9a83a10a5d51 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -49,31 +49,33 @@
 *  some of the following vectors are 'rare', they are merged
 *  into a single vector (CALL_FUNCTION_VECTOR) to save vector space.
 *  TLB, reschedule and local APIC vectors are performance-critical.
- *
- *  Vectors 0xf0-0xfa are free (reserved for future Linux use).
 */
 #ifdef CONFIG_X86_32
 # define SPURIOUS_APIC_VECTOR           0xff
 # define ERROR_APIC_VECTOR              0xfe
-# define INVALIDATE_TLB_VECTOR          0xfd
+# define RESCHEDULE_VECTOR              0xfd
-# define RESCHEDULE_VECTOR              0xfc
+# define CALL_FUNCTION_VECTOR           0xfc
-# define CALL_FUNCTION_VECTOR           0xfb
+# define CALL_FUNCTION_SINGLE_VECTOR    0xfb
-# define CALL_FUNCTION_SINGLE_VECTOR    0xfa
+# define THERMAL_APIC_VECTOR            0xfa
-# define THERMAL_APIC_VECTOR            0xf0
+/* 0xf8 - 0xf9 : free */
+# define INVALIDATE_TLB_VECTOR_END      0xf7
+# define INVALIDATE_TLB_VECTOR_START    0xf0    /* f0-f7 used for TLB flush */
+# define NUM_INVALIDATE_TLB_VECTORS     8
 #else
-#define SPURIOUS_APIC_VECTOR            0xff
+# define SPURIOUS_APIC_VECTOR           0xff
-#define ERROR_APIC_VECTOR               0xfe
+# define ERROR_APIC_VECTOR              0xfe
-#define RESCHEDULE_VECTOR               0xfd
+# define RESCHEDULE_VECTOR              0xfd
-#define CALL_FUNCTION_VECTOR            0xfc
+# define CALL_FUNCTION_VECTOR           0xfc
-#define CALL_FUNCTION_SINGLE_VECTOR     0xfb
+# define CALL_FUNCTION_SINGLE_VECTOR    0xfb
-#define THERMAL_APIC_VECTOR             0xfa
+# define THERMAL_APIC_VECTOR            0xfa
-#define THRESHOLD_APIC_VECTOR           0xf9
+# define THRESHOLD_APIC_VECTOR          0xf9
-#define UV_BAU_MESSAGE                  0xf8
+# define UV_BAU_MESSAGE                 0xf8
-#define INVALIDATE_TLB_VECTOR_END       0xf7
+# define INVALIDATE_TLB_VECTOR_END      0xf7
-#define INVALIDATE_TLB_VECTOR_START     0xf0    /* f0-f7 used for TLB flush */
+# define INVALIDATE_TLB_VECTOR_START    0xf0    /* f0-f7 used for TLB flush */
 #define NUM_INVALIDATE_TLB_VECTORS      8
diff --git a/arch/x86/include/asm/mach-default/entry_arch.h b/arch/x86/include/asm/mach-default/entry_arch.h
index 6b1add8e31dd..6fa399ad1de2 100644
--- a/arch/x86/include/asm/mach-default/entry_arch.h
+++ b/arch/x86/include/asm/mach-default/entry_arch.h
@@ -11,10 +11,26 @@
 */
 #ifdef CONFIG_X86_SMP
 BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR)
-BUILD_INTERRUPT(invalidate_interrupt,INVALIDATE_TLB_VECTOR)
 BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR)
 BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR)
 BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR)
+BUILD_INTERRUPT3(invalidate_interrupt0,INVALIDATE_TLB_VECTOR_START+0,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt1,INVALIDATE_TLB_VECTOR_START+1,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt2,INVALIDATE_TLB_VECTOR_START+2,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt3,INVALIDATE_TLB_VECTOR_START+3,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt4,INVALIDATE_TLB_VECTOR_START+4,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt5,INVALIDATE_TLB_VECTOR_START+5,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt6,INVALIDATE_TLB_VECTOR_START+6,
+                 smp_invalidate_interrupt)
+BUILD_INTERRUPT3(invalidate_interrupt7,INVALIDATE_TLB_VECTOR_START+7,
+                 smp_invalidate_interrupt)
 #endif
 /*
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 8aeeb3fd73db..52948df9cd1d 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -21,11 +21,54 @@ static inline void paravirt_activate_mm(struct mm_struct *prev,
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
 void destroy_context(struct mm_struct *mm);
-#ifdef CONFIG_X86_32
-# include "mmu_context_32.h"
+static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-#else
+{
-# include "mmu_context_64.h"
+#ifdef CONFIG_SMP
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+                percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+#endif
+}
+static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
+                             struct task_struct *tsk)
+{
+        unsigned cpu = smp_processor_id();
+        if (likely(prev != next)) {
+                /* stop flush ipis for the previous mm */
+                cpu_clear(cpu, prev->cpu_vm_mask);
+#ifdef CONFIG_SMP
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                percpu_write(cpu_tlbstate.active_mm, next);
 #endif
+                cpu_set(cpu, next->cpu_vm_mask);
+                /* Re-load page tables */
+                load_cr3(next->pgd);
+                /*
+                 * load the LDT, if the LDT is different:
+                 */
+                if (unlikely(prev->context.ldt != next->context.ldt))
+                        load_LDT_nolock(&next->context);
+        }
+#ifdef CONFIG_SMP
+        else {
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
+                        /* We were in lazy tlb mode and leave_mm disabled
+                         * tlb flush IPI delivery. We must reload CR3
+                         * to make sure to use no freed page tables.
+                         */
+                        load_cr3(next->pgd);
+                        load_LDT_nolock(&next->context);
+                }
+        }
+#endif
+}
 #define activate_mm(prev, next)                 \
 do {                                            \
@@ -33,5 +76,17 @@ do {						\
        switch_mm((prev), (next), NULL);        \
 } while (0);
+#ifdef CONFIG_X86_32
+#define deactivate_mm(tsk, mm)                  \
+do {                                            \
+        loadsegment(gs, 0);                     \
+} while (0)
+#else
+#define deactivate_mm(tsk, mm)                  \
+do {                                            \
+        load_gs_index(0);                       \
+        loadsegment(fs, 0);                     \
+} while (0)
+#endif
 #endif /* _ASM_X86_MMU_CONTEXT_H */
diff --git a/arch/x86/include/asm/mmu_context_32.h b/arch/x86/include/asm/mmu_context_32.h
deleted file mode 100644
index 7e98ce1d2c0e..000000000000
--- a/arch/x86/include/asm/mmu_context_32.h
+++ /dev/null
@@ -1,55 +0,0 @@
-#ifndef _ASM_X86_MMU_CONTEXT_32_H
-#define _ASM_X86_MMU_CONTEXT_32_H
-static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-{
-#ifdef CONFIG_SMP
-        if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK)
-                x86_write_percpu(cpu_tlbstate.state, TLBSTATE_LAZY);
-#endif
-}
-static inline void switch_mm(struct mm_struct *prev,
-                             struct mm_struct *next,
-                             struct task_struct *tsk)
-{
-        int cpu = smp_processor_id();
-        if (likely(prev != next)) {
-                /* stop flush ipis for the previous mm */
-                cpu_clear(cpu, prev->cpu_vm_mask);
-#ifdef CONFIG_SMP
-                x86_write_percpu(cpu_tlbstate.state, TLBSTATE_OK);
-                x86_write_percpu(cpu_tlbstate.active_mm, next);
-#endif
-                cpu_set(cpu, next->cpu_vm_mask);
-                /* Re-load page tables */
-                load_cr3(next->pgd);
-                /*
-                 * load the LDT, if the LDT is different:
-                 */
-                if (unlikely(prev->context.ldt != next->context.ldt))
-                        load_LDT_nolock(&next->context);
-        }
-#ifdef CONFIG_SMP
-        else {
-                x86_write_percpu(cpu_tlbstate.state, TLBSTATE_OK);
-                BUG_ON(x86_read_percpu(cpu_tlbstate.active_mm) != next);
-                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
-                        /* We were in lazy tlb mode and leave_mm disabled
-                         * tlb flush IPI delivery. We must reload %cr3.
-                         */
-                        load_cr3(next->pgd);
-                        load_LDT_nolock(&next->context);
-                }
-        }
-#endif
-}
-#define deactivate_mm(tsk, mm)                  \
-        asm("movl %0,%%gs": :"r" (0));
-#endif /* _ASM_X86_MMU_CONTEXT_32_H */
diff --git a/arch/x86/include/asm/mmu_context_64.h b/arch/x86/include/asm/mmu_context_64.h
deleted file mode 100644
index 677d36e9540a..000000000000
--- a/arch/x86/include/asm/mmu_context_64.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#ifndef _ASM_X86_MMU_CONTEXT_64_H
-#define _ASM_X86_MMU_CONTEXT_64_H
-#include <asm/pda.h>
-static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-{
-#ifdef CONFIG_SMP
-        if (read_pda(mmu_state) == TLBSTATE_OK)
-                write_pda(mmu_state, TLBSTATE_LAZY);
-#endif
-}
-static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
-                             struct task_struct *tsk)
-{
-        unsigned cpu = smp_processor_id();
-        if (likely(prev != next)) {
-                /* stop flush ipis for the previous mm */
-                cpu_clear(cpu, prev->cpu_vm_mask);
-#ifdef CONFIG_SMP
-                write_pda(mmu_state, TLBSTATE_OK);
-                write_pda(active_mm, next);
-#endif
-                cpu_set(cpu, next->cpu_vm_mask);
-                load_cr3(next->pgd);
-                if (unlikely(next->context.ldt != prev->context.ldt))
-                        load_LDT_nolock(&next->context);
-        }
-#ifdef CONFIG_SMP
-        else {
-                write_pda(mmu_state, TLBSTATE_OK);
-                if (read_pda(active_mm) != next)
-                        BUG();
-                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
-                        /* We were in lazy tlb mode and leave_mm disabled
-                         * tlb flush IPI delivery. We must reload CR3
-                         * to make sure to use no freed page tables.
-                         */
-                        load_cr3(next->pgd);
-                        load_LDT_nolock(&next->context);
-                }
-        }
-#endif
-}
-#define deactivate_mm(tsk, mm)                  \
-do {                                            \
-        load_gs_index(0);                       \
-        asm volatile("movl %0,%%fs"::"r"(0));   \
-} while (0)
-#endif /* _ASM_X86_MMU_CONTEXT_64_H */
diff --git a/arch/x86/include/asm/mpspec_def.h b/arch/x86/include/asm/mpspec_def.h
index 59568bc4767f..4a7f96d7c188 100644
--- a/arch/x86/include/asm/mpspec_def.h
+++ b/arch/x86/include/asm/mpspec_def.h
@@ -24,17 +24,18 @@
 # endif
 #endif
-struct intel_mp_floating {
+/* Intel MP Floating Pointer Structure */
-        char mpf_signature[4];          /* "_MP_"                       */
+struct mpf_intel {
-        unsigned int mpf_physptr;       /* Configuration table address  */
+        char signature[4];              /* "_MP_"                       */
-        unsigned char mpf_length;       /* Our length (paragraphs)      */
+        unsigned int physptr;           /* Configuration table address  */
-        unsigned char mpf_specification;/* Specification version        */
+        unsigned char length;           /* Our length (paragraphs)      */
-        unsigned char mpf_checksum;     /* Checksum (makes sum 0)       */
+        unsigned char specification;    /* Specification version        */
-        unsigned char mpf_feature1;     /* Standard or configuration ?  */
+        unsigned char checksum;         /* Checksum (makes sum 0)       */
-        unsigned char mpf_feature2;     /* Bit7 set for IMCR|PIC        */
+        unsigned char feature1;         /* Standard or configuration ?  */
-        unsigned char mpf_feature3;     /* Unused (0)                   */
+        unsigned char feature2;         /* Bit7 set for IMCR|PIC        */
-        unsigned char mpf_feature4;     /* Unused (0)                   */
+        unsigned char feature3;         /* Unused (0)                   */
-        unsigned char mpf_feature5;     /* Unused (0)                   */
+        unsigned char feature4;         /* Unused (0)                   */
+        unsigned char feature5;         /* Unused (0)                   */
 };
 #define MPC_SIGNATURE "PCMP"
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 5ebca29f44f0..e27fdbe5f9e4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -13,8 +13,8 @@
 #define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1)
 #define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER)
-#define IRQSTACK_ORDER 2
+#define IRQ_STACK_ORDER 2
-#define IRQSTACKSIZE (PAGE_SIZE << IRQSTACK_ORDER)
+#define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER)
 #define STACKFAULT_STACK 1
 #define DOUBLEFAULT_STACK 2
diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
deleted file mode 100644
index 2fbfff88df37..000000000000
--- a/arch/x86/include/asm/pda.h
+++ /dev/null
@@ -1,137 +0,0 @@
-#ifndef _ASM_X86_PDA_H
-#define _ASM_X86_PDA_H
-#ifndef __ASSEMBLY__
-#include <linux/stddef.h>
-#include <linux/types.h>
-#include <linux/cache.h>
-#include <asm/page.h>
-/* Per processor datastructure. %gs points to it while the kernel runs */
-struct x8664_pda {
-        struct task_struct *pcurrent;   /* 0  Current process */
-        unsigned long data_offset;      /* 8 Per cpu data offset from linker
-                                           address */
-        unsigned long kernelstack;      /* 16 top of kernel stack for current */
-        unsigned long oldrsp;           /* 24 user rsp for system call */
-        int irqcount;                   /* 32 Irq nesting counter. Starts -1 */
-        unsigned int cpunumber;         /* 36 Logical CPU number */
-#ifdef CONFIG_CC_STACKPROTECTOR
-        unsigned long stack_canary;     /* 40 stack canary value */
-                                        /* gcc-ABI: this canary MUST be at
-                                           offset 40!!! */
-#endif
-        char *irqstackptr;
-        short nodenumber;               /* number of current node (32k max) */
-        short in_bootmem;               /* pda lives in bootmem */
-        unsigned int __softirq_pending;
-        unsigned int __nmi_count;       /* number of NMI on this CPUs */
-        short mmu_state;
-        short isidle;
-        struct mm_struct *active_mm;
-        unsigned apic_timer_irqs;
-        unsigned irq0_irqs;
-        unsigned irq_resched_count;
-        unsigned irq_call_count;
-        unsigned irq_tlb_count;
-        unsigned irq_thermal_count;
-        unsigned irq_threshold_count;
-        unsigned irq_spurious_count;
-} ____cacheline_aligned_in_smp;
-extern struct x8664_pda **_cpu_pda;
-extern void pda_init(int);
-#define cpu_pda(i) (_cpu_pda[i])
-/*
- * There is no fast way to get the base address of the PDA, all the accesses
- * have to mention %fs/%gs.  So it needs to be done this Torvaldian way.
- */
-extern void __bad_pda_field(void) __attribute__((noreturn));
-/*
- * proxy_pda doesn't actually exist, but tell gcc it is accessed for
- * all PDA accesses so it gets read/write dependencies right.
- */
-extern struct x8664_pda _proxy_pda;
-#define pda_offset(field) offsetof(struct x8664_pda, field)
-#define pda_to_op(op, field, val)                                       \
-do {                                                                    \
-        typedef typeof(_proxy_pda.field) T__;                           \
-        if (0) { T__ tmp__; tmp__ = (val); }    /* type checking */     \
-        switch (sizeof(_proxy_pda.field)) {                             \
-        case 2:                                                         \
-                asm(op "w %1,%%gs:%c2" :                                \
-                    "+m" (_proxy_pda.field) :                           \
-                    "ri" ((T__)val),                                    \
-                    "i"(pda_offset(field)));                            \
-                break;                                                  \
-        case 4:                                                         \
-                asm(op "l %1,%%gs:%c2" :                                \
-                    "+m" (_proxy_pda.field) :                           \
-                    "ri" ((T__)val),                                    \
-                    "i" (pda_offset(field)));                           \
-                break;                                                  \
-        case 8:                                                         \
-                asm(op "q %1,%%gs:%c2":                                 \
-                    "+m" (_proxy_pda.field) :                           \
-                    "ri" ((T__)val),                                    \
-                    "i"(pda_offset(field)));                            \
-                break;                                                  \
-        default:                                                        \
-                __bad_pda_field();                                      \
-        }                                                               \
-} while (0)
-#define pda_from_op(op, field)                  \
-({                                              \
-        typeof(_proxy_pda.field) ret__;         \
-        switch (sizeof(_proxy_pda.field)) {     \
-        case 2:                                 \
-                asm(op "w %%gs:%c1,%0" :        \
-                    "=r" (ret__) :              \
-                    "i" (pda_offset(field)),    \
-                    "m" (_proxy_pda.field));    \
-                break;                          \
-        case 4:                                 \
-                asm(op "l %%gs:%c1,%0":         \
-                    "=r" (ret__):               \
-                    "i" (pda_offset(field)),    \
-                    "m" (_proxy_pda.field));    \
-                break;                          \
-        case 8:                                 \
-                asm(op "q %%gs:%c1,%0":         \
-                    "=r" (ret__) :              \
-                    "i" (pda_offset(field)),    \
-                    "m" (_proxy_pda.field));    \
-                break;                          \
-        default:                                \
-                __bad_pda_field();              \
-        }                                       \
-        ret__;                                  \
-})
-#define read_pda(field)         pda_from_op("mov", field)
-#define write_pda(field, val)   pda_to_op("mov", field, val)
-#define add_pda(field, val)     pda_to_op("add", field, val)
-#define sub_pda(field, val)     pda_to_op("sub", field, val)
-#define or_pda(field, val)      pda_to_op("or", field, val)
-/* This is not atomic against other CPUs -- CPU preemption needs to be off */
-#define test_and_clear_bit_pda(bit, field)                              \
-({                                                                      \
-        int old__;                                                      \
-        asm volatile("btr %2,%%gs:%c3\n\tsbbl %0,%0"                    \
-                     : "=r" (old__), "+m" (_proxy_pda.field)            \
-                     : "dIr" (bit), "i" (pda_offset(field)) : "memory");\
-        old__;                                                          \
-})
-#endif
-#define PDA_STACKOFFSET (5*8)
-#endif /* _ASM_X86_PDA_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index ece72053ba63..0b64af4f13ac 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -2,53 +2,12 @@
 #define _ASM_X86_PERCPU_H
 #ifdef CONFIG_X86_64
-#include <linux/compiler.h>
+#define __percpu_seg            gs
+#define __percpu_mov_op         movq
-/* Same as asm-generic/percpu.h, except that we store the per cpu offset
+#else
-   in the PDA. Longer term the PDA and every per cpu variable
+#define __percpu_seg            fs
-   should be just put into a single section and referenced directly
+#define __percpu_mov_op         movl
-   from %gs */
-#ifdef CONFIG_SMP
-#include <asm/pda.h>
-#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
-#define __my_cpu_offset read_pda(data_offset)
-#define per_cpu_offset(x) (__per_cpu_offset(x))
 #endif
-#include <asm-generic/percpu.h>
-DECLARE_PER_CPU(struct x8664_pda, pda);
-/*
- * These are supposed to be implemented as a single instruction which
- * operates on the per-cpu data base segment.  x86-64 doesn't have
- * that yet, so this is a fairly inefficient workaround for the
- * meantime.  The single instruction is atomic with respect to
- * preemption and interrupts, so we need to explicitly disable
- * interrupts here to achieve the same effect.  However, because it
- * can be used from within interrupt-disable/enable, we can't actually
- * disable interrupts; disabling preemption is enough.
- */
-#define x86_read_percpu(var)                                            \
-        ({                                                              \
-                typeof(per_cpu_var(var)) __tmp;                         \
-                preempt_disable();                                      \
-                __tmp = __get_cpu_var(var);                             \
-                preempt_enable();                                       \
-                __tmp;                                                  \
-        })
-#define x86_write_percpu(var, val)                                      \
-        do {                                                            \
-                preempt_disable();                                      \
-                __get_cpu_var(var) = (val);                             \
-                preempt_enable();                                       \
-        } while(0)
-#else /* CONFIG_X86_64 */
 #ifdef __ASSEMBLY__
@@ -65,47 +24,26 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
 #ifdef CONFIG_SMP
-#define PER_CPU(var, reg)                               \
+#define PER_CPU(var, reg)                                               \
-        movl %fs:per_cpu__##this_cpu_off, reg;          \
+        __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg;       \
        lea per_cpu__##var(reg), reg
-#define PER_CPU_VAR(var)        %fs:per_cpu__##var
+#define PER_CPU_VAR(var)        %__percpu_seg:per_cpu__##var
 #else /* ! SMP */
-#define PER_CPU(var, reg)                       \
+#define PER_CPU(var, reg)                                               \
-        movl $per_cpu__##var, reg
+        __percpu_mov_op $per_cpu__##var, reg
 #define PER_CPU_VAR(var)        per_cpu__##var
 #endif  /* SMP */
 #else /* ...!ASSEMBLY */
-/*
+#include <linux/stringify.h>
- * PER_CPU finds an address of a per-cpu variable.
- *
- * Args:
- *    var - variable name
- *    cpu - 32bit register containing the current CPU number
- *
- * The resulting address is stored in the "cpu" argument.
- *
- * Example:
- *    PER_CPU(cpu_gdt_descr, %ebx)
- */
-#ifdef CONFIG_SMP
-#define __my_cpu_offset x86_read_percpu(this_cpu_off)
-/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
+#ifdef CONFIG_SMP
-#define __percpu_seg "%%fs:"
+#define __percpu_arg(x)         "%%"__stringify(__percpu_seg)":%P" #x
+#define __my_cpu_offset         percpu_read(this_cpu_off)
-#else  /* !SMP */
+#else
+#define __percpu_arg(x)         "%" #x
-#define __percpu_seg ""
+#endif
-#endif  /* SMP */
-#include <asm-generic/percpu.h>
-/* We can use this directly for local CPU (faster). */
-DECLARE_PER_CPU(unsigned long, this_cpu_off);
 /* For arch-specific code, we can use direct single-insn ops (they
 * don't give an lvalue though). */
@@ -120,20 +58,25 @@ do {							\
        }                                               \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b %1,"__percpu_seg"%0"          \
+                asm(op "b %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 2:                                         \
-                asm(op "w %1,"__percpu_seg"%0"          \
+                asm(op "w %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 4:                                         \
-                asm(op "l %1,"__percpu_seg"%0"          \
+                asm(op "l %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
+        case 8:                                         \
+                asm(op "q %1,"__percpu_arg(0)           \
+                    : "+m" (var)                        \
+                    : "re" ((T__)val));                 \
+                break;                                  \
        default: __bad_percpu_size();                   \
        }                                               \
 } while (0)
@@ -143,17 +86,22 @@ do {							\
        typeof(var) ret__;                              \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b "__percpu_seg"%1,%0"          \
+                asm(op "b "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 2:                                         \
-                asm(op "w "__percpu_seg"%1,%0"          \
+                asm(op "w "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 4:                                         \
-                asm(op "l "__percpu_seg"%1,%0"          \
+                asm(op "l "__percpu_arg(1)",%0"         \
+                    : "=r" (ret__)                      \
+                    : "m" (var));                       \
+                break;                                  \
+        case 8:                                         \
+                asm(op "q "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
@@ -162,13 +110,30 @@ do {							\
        ret__;                                          \
 })
-#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
+#define percpu_read(var)        percpu_from_op("mov", per_cpu__##var)
-#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val)
+#define percpu_write(var, val)  percpu_to_op("mov", per_cpu__##var, val)
-#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)
+#define percpu_add(var, val)    percpu_to_op("add", per_cpu__##var, val)
-#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)
+#define percpu_sub(var, val)    percpu_to_op("sub", per_cpu__##var, val)
-#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
+#define percpu_and(var, val)    percpu_to_op("and", per_cpu__##var, val)
+#define percpu_or(var, val)     percpu_to_op("or", per_cpu__##var, val)
+#define percpu_xor(var, val)    percpu_to_op("xor", per_cpu__##var, val)
+/* This is not atomic against other CPUs -- CPU preemption needs to be off */
+#define x86_test_and_clear_bit_percpu(bit, var)                         \
+({                                                                      \
+        int old__;                                                      \
+        asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"           \
+                     : "=r" (old__), "+m" (per_cpu__##var)              \
+                     : "dIr" (bit));                                    \
+        old__;                                                          \
+})
+#include <asm-generic/percpu.h>
+/* We can use this directly for local CPU (faster). */
+DECLARE_PER_CPU(unsigned long, this_cpu_off);
 #endif /* !__ASSEMBLY__ */
-#endif /* !CONFIG_X86_64 */
 #ifdef CONFIG_SMP
@@ -195,9 +160,9 @@ do {							\
 #define early_per_cpu_ptr(_name) (_name##_early_ptr)
 #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
 #define early_per_cpu(_name, _cpu)                              \
-        (early_per_cpu_ptr(_name) ?                             \
+        *(early_per_cpu_ptr(_name) ?                            \
-                early_per_cpu_ptr(_name)[_cpu] :                \
+                &early_per_cpu_ptr(_name)[_cpu] :               \
-                per_cpu(_name, _cpu))
+                &per_cpu(_name, _cpu))
 #else   /* !CONFIG_SMP */
 #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)          \
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index ba09289accaa..1df9637dfda3 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -11,7 +11,6 @@
 #include <asm/processor.h>
 #include <linux/bitops.h>
 #include <linux/threads.h>
-#include <asm/pda.h>
 extern pud_t level3_kernel_pgt[512];
 extern pud_t level3_ident_pgt[512];
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 091cd8855f2e..48676b943b92 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -378,6 +378,30 @@ union thread_xstate {
 #ifdef CONFIG_X86_64
 DECLARE_PER_CPU(struct orig_ist, orig_ist);
+union irq_stack_union {
+        char irq_stack[IRQ_STACK_SIZE];
+        /*
+         * GCC hardcodes the stack canary as %gs:40.  Since the
+         * irq_stack is the object at %gs:0, we reserve the bottom
+         * 48 bytes of the irq stack for the canary.
+         */
+        struct {
+                char gs_base[40];
+                unsigned long stack_canary;
+        };
+};
+DECLARE_PER_CPU(union irq_stack_union, irq_stack_union);
+DECLARE_PER_CPU(char *, irq_stack_ptr);
+static inline void load_gs_base(int cpu)
+{
+        /* Memory clobbers used to order pda/percpu accesses */
+        mb();
+        wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
+        mb();
+}
 #endif
 extern void print_cpu_info(struct cpuinfo_x86 *);
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index ebe858cdc8a3..536949749bc2 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -100,7 +100,6 @@ extern unsigned long init_pg_tables_start;
 extern unsigned long init_pg_tables_end;
 #else
-void __init x86_64_init_pda(void);
 void __init x86_64_start_kernel(char *real_mode);
 void __init x86_64_start_reservations(char *real_mode_data);
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 19953df61c52..45ef8a1b9d7c 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -15,34 +15,8 @@
 #  include <asm/io_apic.h>
 # endif
 #endif
-#include <asm/pda.h>
 #include <asm/thread_info.h>
+#include <asm/cpumask.h>
-#ifdef CONFIG_X86_64
-extern cpumask_var_t cpu_callin_mask;
-extern cpumask_var_t cpu_callout_mask;
-extern cpumask_var_t cpu_initialized_mask;
-extern cpumask_var_t cpu_sibling_setup_mask;
-#else /* CONFIG_X86_32 */
-extern cpumask_t cpu_callin_map;
-extern cpumask_t cpu_callout_map;
-extern cpumask_t cpu_initialized;
-extern cpumask_t cpu_sibling_setup_map;
-#define cpu_callin_mask         ((struct cpumask *)&cpu_callin_map)
-#define cpu_callout_mask        ((struct cpumask *)&cpu_callout_map)
-#define cpu_initialized_mask    ((struct cpumask *)&cpu_initialized)
-#define cpu_sibling_setup_mask  ((struct cpumask *)&cpu_sibling_setup_map)
-#endif /* CONFIG_X86_32 */
-extern void (*mtrr_hook)(void);
-extern void zap_low_mappings(void);
-extern int __cpuinit get_local_pda(int cpu);
 extern int smp_num_siblings;
 extern unsigned int num_processors;
@@ -50,9 +24,7 @@ extern unsigned int num_processors;
 DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
 DECLARE_PER_CPU(cpumask_t, cpu_core_map);
 DECLARE_PER_CPU(u16, cpu_llc_id);
-#ifdef CONFIG_X86_32
 DECLARE_PER_CPU(int, cpu_number);
-#endif
 static inline struct cpumask *cpu_sibling_mask(int cpu)
 {
@@ -167,8 +139,6 @@ void play_dead_common(void);
 void native_send_call_func_ipi(const struct cpumask *mask);
 void native_send_call_func_single_ipi(int cpu);
-extern void prefill_possible_map(void);
 void smp_store_cpu_info(int id);
 #define cpu_physical_id(cpu)    per_cpu(x86_cpu_to_apicid, cpu)
@@ -177,10 +147,6 @@ static inline int num_booting_cpus(void)
 {
        return cpumask_weight(cpu_callout_mask);
 }
-#else
-static inline void prefill_possible_map(void)
-{
-}
 #endif /* CONFIG_SMP */
 extern unsigned disabled_cpus __cpuinitdata;
@@ -191,11 +157,11 @@ extern unsigned disabled_cpus __cpuinitdata;
 * from the initial startup. We map APIC_BASE very early in page_setup(),
 * so this is correct in the x86 case.
 */
-#define raw_smp_processor_id() (x86_read_percpu(cpu_number))
+#define raw_smp_processor_id() (percpu_read(cpu_number))
 extern int safe_smp_processor_id(void);
 #elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id()  read_pda(cpunumber)
+#define raw_smp_processor_id() (percpu_read(cpu_number))
 #define stack_smp_processor_id()                                        \
 ({                                                              \
@@ -205,10 +171,6 @@ extern int safe_smp_processor_id(void);
 })
 #define safe_smp_processor_id()         smp_processor_id()
-#else /* !CONFIG_X86_32_SMP && !CONFIG_X86_64_SMP */
-#define cpu_physical_id(cpu)            boot_cpu_physical_apicid
-#define safe_smp_processor_id()         0
-#define stack_smp_processor_id()        0
 #endif
 #ifdef CONFIG_X86_LOCAL_APIC
@@ -251,11 +213,5 @@ static inline int hard_smp_processor_id(void)
 #endif /* CONFIG_X86_LOCAL_APIC */
-#ifdef CONFIG_X86_HAS_BOOT_CPU_ID
-extern unsigned char boot_cpu_id;
-#else
-#define boot_cpu_id     0
-#endif
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_X86_SMP_H */
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
new file mode 100644
index 000000000000..36a700acaf2b
--- /dev/null
+++ b/arch/x86/include/asm/stackprotector.h
@@ -0,0 +1,38 @@
+#ifndef _ASM_STACKPROTECTOR_H
+#define _ASM_STACKPROTECTOR_H 1
+#include <asm/tsc.h>
+#include <asm/processor.h>
+/*
+ * Initialize the stackprotector canary value.
+ *
+ * NOTE: this must only be called from functions that never return,
+ * and it must always be inlined.
+ */
+static __always_inline void boot_init_stack_canary(void)
+{
+        u64 canary;
+        u64 tsc;
+        /*
+         * Build time only check to make sure the stack_canary is at
+         * offset 40 in the pda; this is a gcc ABI requirement
+         */
+        BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
+        /*
+         * We both use the random pool and the current TSC as a source
+         * of randomness. The TSC only matters for very early init,
+         * there it already has some randomness on most systems. Later
+         * on during the bootup the random pool has true entropy too.
+         */
+        get_random_bytes(&canary, sizeof(canary));
+        tsc = __native_read_tsc();
+        canary += tsc + (tsc << 32UL);
+        current->stack_canary = canary;
+        percpu_write(irq_stack_union.stack_canary, canary);
+}
+#endif
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 8e626ea33a1a..2fcc70bc85f3 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -86,27 +86,44 @@ do {									\
        , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
          "r12", "r13", "r14", "r15"
+#ifdef CONFIG_CC_STACKPROTECTOR
+#define __switch_canary                                                   \
+        "movq %P[task_canary](%%rsi),%%r8\n\t"                            \
+        "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
+#define __switch_canary_oparam                                            \
+        , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary))
+#define __switch_canary_iparam                                            \
+        , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
+#else   /* CC_STACKPROTECTOR */
+#define __switch_canary
+#define __switch_canary_oparam
+#define __switch_canary_iparam
+#endif  /* CC_STACKPROTECTOR */
 /* Save restore flags to clear handle leaking NT */
 #define switch_to(prev, next, last) \
-        asm volatile(SAVE_CONTEXT                                                   \
+        asm volatile(SAVE_CONTEXT                                         \
             "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */       \
             "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */    \
             "call __switch_to\n\t"                                       \
             ".globl thread_return\n"                                     \
             "thread_return:\n\t"                                         \
-             "movq %%gs:%P[pda_pcurrent],%%rsi\n\t"                       \
+             "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
+             __switch_canary                                              \
             "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
             LOCK_PREFIX "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"        \
             "movq %%rax,%%rdi\n\t"                                       \
             "jc   ret_from_fork\n\t"                                     \
             RESTORE_CONTEXT                                              \
             : "=a" (last)                                                \
+               __switch_canary_oparam                                     \
             : [next] "S" (next), [prev] "D" (prev),                      \
               [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
               [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
               [tif_fork] "i" (TIF_FORK),                                 \
               [thread_info] "i" (offsetof(struct task_struct, stack)),   \
-               [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent))  \
+               [current_task] "m" (per_cpu_var(current_task))             \
+               __switch_canary_iparam                                     \
             : "memory", "cc" __EXTRA_CLOBBER)
 #endif
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 98789647baa9..b46f8ca007b5 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -194,25 +194,21 @@ static inline struct thread_info *current_thread_info(void)
 #else /* X86_32 */
-#include <asm/pda.h>
+#include <asm/percpu.h>
+#define KERNEL_STACK_OFFSET (5*8)
 /*
 * macros/functions for gaining access to the thread information structure
 * preempt_count needs to be 1 initially, until the scheduler is functional.
 */
 #ifndef __ASSEMBLY__
-static inline struct thread_info *current_thread_info(void)
+DECLARE_PER_CPU(unsigned long, kernel_stack);
-{
-        struct thread_info *ti;
-        ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
-        return ti;
-}
-/* do not use in interrupt context */
+static inline struct thread_info *current_thread_info(void)
-static inline struct thread_info *stack_thread_info(void)
 {
        struct thread_info *ti;
-        asm("andq %%rsp,%0; " : "=r" (ti) : "0" (~(THREAD_SIZE - 1)));
+        ti = (void *)(percpu_read(kernel_stack) +
+                      KERNEL_STACK_OFFSET - THREAD_SIZE);
        return ti;
 }
@@ -220,8 +216,8 @@ static inline struct thread_info *stack_thread_info(void)
 /* how to get the thread information struct from ASM */
 #define GET_THREAD_INFO(reg) \
-        movq %gs:pda_kernelstack,reg ; \
+        movq PER_CPU_VAR(kernel_stack),reg ; \
-        subq $(THREAD_SIZE-PDA_STACKOFFSET),reg
+        subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
 #endif
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index f4e1b550ce61..d3539f998f88 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
 #define TLBSTATE_OK     1
 #define TLBSTATE_LAZY   2
-#ifdef CONFIG_X86_32
 struct tlb_state {
        struct mm_struct *active_mm;
        int state;
-        char __cacheline_padding[L1_CACHE_BYTES-8];
 };
 DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
-void reset_lazy_tlbstate(void);
-#else
 static inline void reset_lazy_tlbstate(void)
 {
+        percpu_write(cpu_tlbstate.state, 0);
+        percpu_write(cpu_tlbstate.active_mm, &init_mm);
 }
-#endif
 #endif  /* SMP */
@@ -175,4 +172,6 @@ static inline void flush_tlb_kernel_range(unsigned long start,
        flush_tlb_all();
 }
+extern void zap_low_mappings(void);
 #endif /* _ASM_X86_TLBFLUSH_H */
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index d0c68e291635..10022ed3a4b6 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -83,7 +83,8 @@ extern cpumask_t *node_to_cpumask_map;
 DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
 /* Returns the number of the current Node. */
-#define numa_node_id()          read_pda(nodenumber)
+DECLARE_PER_CPU(int, node_number);
+#define numa_node_id()          percpu_read(node_number)
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
 extern int cpu_to_node(int cpu);
@@ -102,10 +103,7 @@ static inline int cpu_to_node(int cpu)
 /* Same function but used if called before per_cpu areas are setup */
 static inline int early_cpu_to_node(int cpu)
 {
-        if (early_per_cpu_ptr(x86_cpu_to_node_map))
+        return early_per_cpu(x86_cpu_to_node_map, cpu);
-                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
-        return per_cpu(x86_cpu_to_node_map, cpu);
 }
 /* Returns a pointer to the cpumask of CPUs on Node 'node'. */
diff --git a/arch/x86/include/asm/trampoline.h b/arch/x86/include/asm/trampoline.h
index 780ba0ab94f9..90f06c25221d 100644
--- a/arch/x86/include/asm/trampoline.h
+++ b/arch/x86/include/asm/trampoline.h
@@ -13,6 +13,7 @@ extern unsigned char *trampoline_base;
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
+extern unsigned long initial_gs;
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 #define TRAMPOLINE_BASE 0x6000
diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h
new file mode 100644
index 000000000000..dce5fe350134
--- /dev/null
+++ b/arch/x86/include/asm/uv/uv.h
@@ -0,0 +1,33 @@
+#ifndef _ASM_X86_UV_UV_H
+#define _ASM_X86_UV_UV_H
+enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC};
+#ifdef CONFIG_X86_64
+extern enum uv_system_type get_uv_system_type(void);
+extern int is_uv_system(void);
+extern void uv_cpu_init(void);
+extern void uv_system_init(void);
+extern int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip);
+extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
+                                                 struct mm_struct *mm,
+                                                 unsigned long va,
+                                                 unsigned int cpu);
+#else   /* X86_64 */
+static inline enum uv_system_type get_uv_system_type(void) { return UV_NONE; }
+static inline int is_uv_system(void)    { return 0; }
+static inline void uv_cpu_init(void)    { }
+static inline void uv_system_init(void) { }
+static inline int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip)
+{ return 1; }
+static inline const struct cpumask *
+uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
+                    unsigned long va, unsigned int cpu)
+{ return cpumask; }
+#endif  /* X86_64 */
+#endif  /* _ASM_X86_UV_UV_H */
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index 74e6393bfddb..9b0e61bf7a88 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -325,8 +325,6 @@ static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
 #define cpubit_isset(cpu, bau_local_cpumask) \
        test_bit((cpu), (bau_local_cpumask).bits)
-extern int uv_flush_tlb_others(struct cpumask *,
-                               struct mm_struct *, unsigned long);
 extern void uv_bau_message_intr1(void);
 extern void uv_bau_timeout_intr1(void);
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index d364df03c1d6..0626a88fbb46 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -23,6 +23,7 @@ nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_vsyscall_64.o    := $(PROFILING) -g0 $(nostackp)
 CFLAGS_hpet.o           := $(nostackp)
 CFLAGS_tsc.o            := $(nostackp)
+CFLAGS_paravirt.o       := $(nostackp)
 obj-y                   := process_$(BITS).o signal.o entry_$(BITS).o
 obj-y                   += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
@@ -57,7 +58,7 @@ obj-$(CONFIG_PCI)		+= early-quirks.o
 apm-y                           := apm_32.o
 obj-$(CONFIG_APM)               += apm.o
 obj-$(CONFIG_X86_SMP)           += smp.o
-obj-$(CONFIG_X86_SMP)           += smpboot.o tsc_sync.o ipi.o tlb_$(BITS).o
+obj-$(CONFIG_X86_SMP)           += smpboot.o tsc_sync.o ipi.o tlb.o
 obj-$(CONFIG_X86_32_SMP)        += smpcommon.o
 obj-$(CONFIG_X86_64_SMP)        += tsc_sync.o smpcommon.o
 obj-$(CONFIG_X86_TRAMPOLINE)    += trampoline_$(BITS).o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index d37593c2f438..4cb5964f1499 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -912,8 +912,8 @@ static u8 __init uniq_ioapic_id(u8 id)
        DECLARE_BITMAP(used, 256);
        bitmap_zero(used, 256);
        for (i = 0; i < nr_ioapics; i++) {
-                struct mp_config_ioapic *ia = &mp_ioapics[i];
+                struct mpc_ioapic *ia = &mp_ioapics[i];
-                __set_bit(ia->mp_apicid, used);
+                __set_bit(ia->apicid, used);
        }
        if (!test_bit(id, used))
                return id;
@@ -945,47 +945,47 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
        idx = nr_ioapics;
-        mp_ioapics[idx].mp_type = MP_IOAPIC;
+        mp_ioapics[idx].type = MP_IOAPIC;
-        mp_ioapics[idx].mp_flags = MPC_APIC_USABLE;
+        mp_ioapics[idx].flags = MPC_APIC_USABLE;
-        mp_ioapics[idx].mp_apicaddr = address;
+        mp_ioapics[idx].apicaddr = address;
        set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
-        mp_ioapics[idx].mp_apicid = uniq_ioapic_id(id);
+        mp_ioapics[idx].apicid = uniq_ioapic_id(id);
 #ifdef CONFIG_X86_32
-        mp_ioapics[idx].mp_apicver = io_apic_get_version(idx);
+        mp_ioapics[idx].apicver = io_apic_get_version(idx);
 #else
-        mp_ioapics[idx].mp_apicver = 0;
+        mp_ioapics[idx].apicver = 0;
 #endif
        /*
         * Build basic GSI lookup table to facilitate gsi->io_apic lookups
         * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
         */
-        mp_ioapic_routing[idx].apic_id = mp_ioapics[idx].mp_apicid;
+        mp_ioapic_routing[idx].apic_id = mp_ioapics[idx].apicid;
        mp_ioapic_routing[idx].gsi_base = gsi_base;
        mp_ioapic_routing[idx].gsi_end = gsi_base +
            io_apic_get_redir_entries(idx);
-        printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%lx, "
+        printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
-               "GSI %d-%d\n", idx, mp_ioapics[idx].mp_apicid,
+               "GSI %d-%d\n", idx, mp_ioapics[idx].apicid,
-               mp_ioapics[idx].mp_apicver, mp_ioapics[idx].mp_apicaddr,
+               mp_ioapics[idx].apicver, mp_ioapics[idx].apicaddr,
               mp_ioapic_routing[idx].gsi_base, mp_ioapic_routing[idx].gsi_end);
        nr_ioapics++;
 }
-static void assign_to_mp_irq(struct mp_config_intsrc *m,
+static void assign_to_mp_irq(struct mpc_intsrc *m,
-                                    struct mp_config_intsrc *mp_irq)
+                                    struct mpc_intsrc *mp_irq)
 {
-        memcpy(mp_irq, m, sizeof(struct mp_config_intsrc));
+        memcpy(mp_irq, m, sizeof(struct mpc_intsrc));
 }
-static int mp_irq_cmp(struct mp_config_intsrc *mp_irq,
+static int mp_irq_cmp(struct mpc_intsrc *mp_irq,
-                                struct mp_config_intsrc *m)
+                                struct mpc_intsrc *m)
 {
-        return memcmp(mp_irq, m, sizeof(struct mp_config_intsrc));
+        return memcmp(mp_irq, m, sizeof(struct mpc_intsrc));
 }
-static void save_mp_irq(struct mp_config_intsrc *m)
+static void save_mp_irq(struct mpc_intsrc *m)
 {
        int i;
@@ -1003,7 +1003,7 @@ void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
 {
        int ioapic;
        int pin;
-        struct mp_config_intsrc mp_irq;
+        struct mpc_intsrc mp_irq;
        /*
         * Convert 'gsi' to 'ioapic.pin'.
@@ -1021,13 +1021,13 @@ void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
        if ((bus_irq == 0) && (trigger == 3))
                trigger = 1;
-        mp_irq.mp_type = MP_INTSRC;
+        mp_irq.type = MP_INTSRC;
-        mp_irq.mp_irqtype = mp_INT;
+        mp_irq.irqtype = mp_INT;
-        mp_irq.mp_irqflag = (trigger << 2) | polarity;
+        mp_irq.irqflag = (trigger << 2) | polarity;
-        mp_irq.mp_srcbus = MP_ISA_BUS;
+        mp_irq.srcbus = MP_ISA_BUS;
-        mp_irq.mp_srcbusirq = bus_irq;  /* IRQ */
+        mp_irq.srcbusirq = bus_irq;     /* IRQ */
-        mp_irq.mp_dstapic = mp_ioapics[ioapic].mp_apicid; /* APIC ID */
+        mp_irq.dstapic = mp_ioapics[ioapic].apicid; /* APIC ID */
-        mp_irq.mp_dstirq = pin; /* INTIN# */
+        mp_irq.dstirq = pin;    /* INTIN# */
        save_mp_irq(&mp_irq);
 }
@@ -1037,7 +1037,7 @@ void __init mp_config_acpi_legacy_irqs(void)
        int i;
        int ioapic;
        unsigned int dstapic;
-        struct mp_config_intsrc mp_irq;
+        struct mpc_intsrc mp_irq;
 #if defined (CONFIG_MCA) || defined (CONFIG_EISA)
        /*
@@ -1062,7 +1062,7 @@ void __init mp_config_acpi_legacy_irqs(void)
        ioapic = mp_find_ioapic(0);
        if (ioapic < 0)
                return;
-        dstapic = mp_ioapics[ioapic].mp_apicid;
+        dstapic = mp_ioapics[ioapic].apicid;
        /*
         * Use the default configuration for the IRQs 0-15.  Unless
@@ -1072,16 +1072,14 @@ void __init mp_config_acpi_legacy_irqs(void)
                int idx;
                for (idx = 0; idx < mp_irq_entries; idx++) {
-                        struct mp_config_intsrc *irq = mp_irqs + idx;
+                        struct mpc_intsrc *irq = mp_irqs + idx;
                        /* Do we already have a mapping for this ISA IRQ? */
-                        if (irq->mp_srcbus == MP_ISA_BUS
+                        if (irq->srcbus == MP_ISA_BUS && irq->srcbusirq == i)
-                            && irq->mp_srcbusirq == i)
                                break;
                        /* Do we already have a mapping for this IOAPIC pin */
-                        if (irq->mp_dstapic == dstapic &&
+                        if (irq->dstapic == dstapic && irq->dstirq == i)
-                            irq->mp_dstirq == i)
                                break;
                }
@@ -1090,13 +1088,13 @@ void __init mp_config_acpi_legacy_irqs(void)
                        continue;       /* IRQ already used */
                }
-                mp_irq.mp_type = MP_INTSRC;
+                mp_irq.type = MP_INTSRC;
-                mp_irq.mp_irqflag = 0;  /* Conforming */
+                mp_irq.irqflag = 0;     /* Conforming */
-                mp_irq.mp_srcbus = MP_ISA_BUS;
+                mp_irq.srcbus = MP_ISA_BUS;
-                mp_irq.mp_dstapic = dstapic;
+                mp_irq.dstapic = dstapic;
-                mp_irq.mp_irqtype = mp_INT;
+                mp_irq.irqtype = mp_INT;
-                mp_irq.mp_srcbusirq = i; /* Identity mapped */
+                mp_irq.srcbusirq = i; /* Identity mapped */
-                mp_irq.mp_dstirq = i;
+                mp_irq.dstirq = i;
                save_mp_irq(&mp_irq);
        }
@@ -1207,22 +1205,22 @@ int mp_config_acpi_gsi(unsigned char number, unsigned int devfn, u8 pin,
                        u32 gsi, int triggering, int polarity)
 {
 #ifdef CONFIG_X86_MPPARSE
-        struct mp_config_intsrc mp_irq;
+        struct mpc_intsrc mp_irq;
        int ioapic;
        if (!acpi_ioapic)
                return 0;
        /* print the entry should happen on mptable identically */
-        mp_irq.mp_type = MP_INTSRC;
+        mp_irq.type = MP_INTSRC;
-        mp_irq.mp_irqtype = mp_INT;
+        mp_irq.irqtype = mp_INT;
-        mp_irq.mp_irqflag = (triggering == ACPI_EDGE_SENSITIVE ? 4 : 0x0c) |
+        mp_irq.irqflag = (triggering == ACPI_EDGE_SENSITIVE ? 4 : 0x0c) |
                                (polarity == ACPI_ACTIVE_HIGH ? 1 : 3);
-        mp_irq.mp_srcbus = number;
+        mp_irq.srcbus = number;
-        mp_irq.mp_srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3);
+        mp_irq.srcbusirq = (((devfn >> 3) & 0x1f) << 2) | ((pin - 1) & 3);
        ioapic = mp_find_ioapic(gsi);
-        mp_irq.mp_dstapic = mp_ioapic_routing[ioapic].apic_id;
+        mp_irq.dstapic = mp_ioapic_routing[ioapic].apic_id;
-        mp_irq.mp_dstirq = gsi - mp_ioapic_routing[ioapic].gsi_base;
+        mp_irq.dstirq = gsi - mp_ioapic_routing[ioapic].gsi_base;
        save_mp_irq(&mp_irq);
 #endif
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 707c1f6f95fa..4abff454c55b 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -101,6 +101,7 @@ int acpi_save_state_mem(void)
        stack_start.sp = temp_stack + sizeof(temp_stack);
        early_gdt_descr.address =
                        (unsigned long)get_cpu_gdt_table(smp_processor_id());
+        initial_gs = per_cpu_offset(smp_processor_id());
 #endif
        initial_code = (unsigned long)wakeup_long64;
        saved_magic = 0x123456789abcdef0;
diff --git a/arch/x86/kernel/apic.c b/arch/x86/kernel/apic.c
index db0998641c58..1df341a528a1 100644
--- a/arch/x86/kernel/apic.c
+++ b/arch/x86/kernel/apic.c
@@ -895,6 +895,10 @@ void disable_local_APIC(void)
 {
        unsigned int value;
+        /* APIC hasn't been mapped yet */
+        if (!apic_phys)
+                return;
        clear_local_APIC();
        /*
@@ -1126,6 +1130,13 @@ void __cpuinit setup_local_APIC(void)
        unsigned int value;
        int i, j;
+        if (disable_apic) {
+#ifdef CONFIG_X86_IO_APIC
+                disable_ioapic_setup();
+#endif
+                return;
+        }
 #ifdef CONFIG_X86_32
        /* Pound the ESR really hard over the head with a big hammer - mbligh */
        if (lapic_is_integrated() && esr_disable) {
@@ -1566,11 +1577,11 @@ int apic_version[MAX_APICS];
 int __init APIC_init_uniprocessor(void)
 {
-#ifdef CONFIG_X86_64
        if (disable_apic) {
                pr_info("Apic disabled\n");
                return -1;
        }
+#ifdef CONFIG_X86_64
        if (!cpu_has_apic) {
                disable_apic = 1;
                pr_info("Apic disabled by BIOS\n");
@@ -1873,17 +1884,8 @@ void __cpuinit generic_processor_info(int apicid, int version)
 #endif
 #if defined(CONFIG_X86_SMP) || defined(CONFIG_X86_64)
-        /* are we being called early in kernel startup? */
+        early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
-        if (early_per_cpu_ptr(x86_cpu_to_apicid)) {
+        early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
-                u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
-                u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
-                cpu_to_apicid[cpu] = apicid;
-                bios_cpu_apicid[cpu] = apicid;
-        } else {
-                per_cpu(x86_cpu_to_apicid, cpu) = apicid;
-                per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
-        }
 #endif
        set_cpu_possible(cpu, true);
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 1d41d3f1edbc..8793ab33e2c1 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -11,7 +11,6 @@
 #include <linux/hardirq.h>
 #include <linux/suspend.h>
 #include <linux/kbuild.h>
-#include <asm/pda.h>
 #include <asm/processor.h>
 #include <asm/segment.h>
 #include <asm/thread_info.h>
@@ -48,16 +47,6 @@ int main(void)
 #endif
        BLANK();
 #undef ENTRY
-#define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))
-        ENTRY(kernelstack); 
-        ENTRY(oldrsp); 
-        ENTRY(pcurrent); 
-        ENTRY(irqcount);
-        ENTRY(cpunumber);
-        ENTRY(irqstackptr);
-        ENTRY(data_offset);
-        BLANK();
-#undef ENTRY
 #ifdef CONFIG_PARAVIRT
        BLANK();
        OFFSET(PARAVIRT_enabled, pv_info, paravirt_enabled);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 83492b1f93b1..99904f288d6a 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -21,14 +21,16 @@
 #include <asm/asm.h>
 #include <asm/numa.h>
 #include <asm/smp.h>
+#include <asm/cpu.h>
+#include <asm/cpumask.h>
 #ifdef CONFIG_X86_LOCAL_APIC
 #include <asm/mpspec.h>
 #include <asm/apic.h>
 #include <mach_apic.h>
 #include <asm/genapic.h>
+#include <asm/uv/uv.h>
 #endif
-#include <asm/pda.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/desc.h>
@@ -62,23 +64,23 @@ cpumask_t cpu_sibling_setup_map;
 static struct cpu_dev *this_cpu __cpuinitdata;
+DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 #ifdef CONFIG_X86_64
-/* We need valid kernel segments for data and code in long mode too
+        /*
- * IRET will check the segment types  kkeil 2000/10/28
+         * We need valid kernel segments for data and code in long mode too
- * Also sysret mandates a special GDT layout
+         * IRET will check the segment types  kkeil 2000/10/28
- */
+         * Also sysret mandates a special GDT layout
-/* The TLS descriptors are currently at a different place compared to i386.
+         *
-   Hopefully nobody expects them at a fixed place (Wine?) */
+         * The TLS descriptors are currently at a different place compared to i386.
-DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = {
+         * Hopefully nobody expects them at a fixed place (Wine?)
+         */
        [GDT_ENTRY_KERNEL32_CS] = { { { 0x0000ffff, 0x00cf9b00 } } },
        [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00af9b00 } } },
        [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9300 } } },
        [GDT_ENTRY_DEFAULT_USER32_CS] = { { { 0x0000ffff, 0x00cffb00 } } },
        [GDT_ENTRY_DEFAULT_USER_DS] = { { { 0x0000ffff, 0x00cff300 } } },
        [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00affb00 } } },
-} };
 #else
-DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
        [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00cf9a00 } } },
        [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9200 } } },
        [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00cffa00 } } },
@@ -110,9 +112,9 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
        [GDT_ENTRY_APMBIOS_BASE+2] = { { { 0x0000ffff, 0x00409200 } } },
        [GDT_ENTRY_ESPFIX_SS] = { { { 0x00000000, 0x00c09200 } } },
-        [GDT_ENTRY_PERCPU] = { { { 0x00000000, 0x00000000 } } },
+        [GDT_ENTRY_PERCPU] = { { { 0x0000ffff, 0x00cf9200 } } },
-} };
 #endif
+} };
 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
 #ifdef CONFIG_X86_32
@@ -877,54 +879,26 @@ static __init int setup_disablecpuid(char *arg)
 __setup("clearcpuid=", setup_disablecpuid);
 #ifdef CONFIG_X86_64
-struct x8664_pda **_cpu_pda __read_mostly;
-EXPORT_SYMBOL(_cpu_pda);
 struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
-static char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss;
+DEFINE_PER_CPU_FIRST(union irq_stack_union,
+                     irq_stack_union) __aligned(PAGE_SIZE);
-void __cpuinit pda_init(int cpu)
+#ifdef CONFIG_SMP
-{
+DEFINE_PER_CPU(char *, irq_stack_ptr);  /* will be set during per cpu init */
-        struct x8664_pda *pda = cpu_pda(cpu);
+#else
+DEFINE_PER_CPU(char *, irq_stack_ptr) =
+        per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
+#endif
-        /* Setup up data that may be needed in __get_free_pages early */
+DEFINE_PER_CPU(unsigned long, kernel_stack) =
-        loadsegment(fs, 0);
+        (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
-        loadsegment(gs, 0);
+EXPORT_PER_CPU_SYMBOL(kernel_stack);
-        /* Memory clobbers used to order PDA accessed */
-        mb();
-        wrmsrl(MSR_GS_BASE, pda);
-        mb();
-        pda->cpunumber = cpu;
-        pda->irqcount = -1;
-        pda->kernelstack = (unsigned long)stack_thread_info() -
-                                 PDA_STACKOFFSET + THREAD_SIZE;
-        pda->active_mm = &init_mm;
-        pda->mmu_state = 0;
-        if (cpu == 0) {
-                /* others are initialized in smpboot.c */
-                pda->pcurrent = &init_task;
-                pda->irqstackptr = boot_cpu_stack;
-                pda->irqstackptr += IRQSTACKSIZE - 64;
-        } else {
-                if (!pda->irqstackptr) {
-                        pda->irqstackptr = (char *)
-                                __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
-                        if (!pda->irqstackptr)
-                                panic("cannot allocate irqstack for cpu %d",
-                                      cpu);
-                        pda->irqstackptr += IRQSTACKSIZE - 64;
-                }
-                if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE)
+DEFINE_PER_CPU(unsigned int, irq_count) = -1;
-                        pda->nodenumber = cpu_to_node(cpu);
-        }
-}
-static char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ +
+static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
-                                  DEBUG_STKSZ] __page_aligned_bss;
+        [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ])
+        __aligned(PAGE_SIZE);
 extern asmlinkage void ignore_sysret(void);
@@ -982,15 +956,18 @@ void __cpuinit cpu_init(void)
        struct tss_struct *t = &per_cpu(init_tss, cpu);
        struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu);
        unsigned long v;
-        char *estacks = NULL;
        struct task_struct *me;
        int i;
-        /* CPU 0 is initialised in head64.c */
+        loadsegment(fs, 0);
-        if (cpu != 0)
+        loadsegment(gs, 0);
-                pda_init(cpu);
+        load_gs_base(cpu);
-        else
-                estacks = boot_exception_stacks;
+#ifdef CONFIG_NUMA
+        if (cpu != 0 && percpu_read(node_number) == 0 &&
+            cpu_to_node(cpu) != NUMA_NO_NODE)
+                percpu_write(node_number, cpu_to_node(cpu));
+#endif
        me = current;
@@ -1024,18 +1001,13 @@ void __cpuinit cpu_init(void)
         * set up and load the per-CPU TSS
         */
        if (!orig_ist->ist[0]) {
-                static const unsigned int order[N_EXCEPTION_STACKS] = {
+                static const unsigned int sizes[N_EXCEPTION_STACKS] = {
-                  [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
+                  [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ,
-                  [DEBUG_STACK - 1] = DEBUG_STACK_ORDER
+                  [DEBUG_STACK - 1] = DEBUG_STKSZ
                };
+                char *estacks = per_cpu(exception_stacks, cpu);
                for (v = 0; v < N_EXCEPTION_STACKS; v++) {
-                        if (cpu) {
+                        estacks += sizes[v];
-                                estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
-                                if (!estacks)
-                                        panic("Cannot allocate exception "
-                                              "stack %ld %d\n", v, cpu);
-                        }
-                        estacks += PAGE_SIZE << order[v];
                        orig_ist->ist[v] = t->x86_tss.ist[v] =
                                        (unsigned long)estacks;
                }
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index c689d19e35ab..11b93cabdf78 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -24,7 +24,7 @@
 #include <asm/apic.h>
 #include <asm/hpet.h>
 #include <linux/kdebug.h>
-#include <asm/smp.h>
+#include <asm/cpu.h>
 #include <asm/reboot.h>
 #include <asm/virtext.h>
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index c302d0707048..d35db5993fd6 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -106,7 +106,8 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                const struct stacktrace_ops *ops, void *data)
 {
        const unsigned cpu = get_cpu();
-        unsigned long *irqstack_end = (unsigned long *)cpu_pda(cpu)->irqstackptr;
+        unsigned long *irq_stack_end =
+                (unsigned long *)per_cpu(irq_stack_ptr, cpu);
        unsigned used = 0;
        struct thread_info *tinfo;
        int graph = 0;
@@ -160,23 +161,23 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                        stack = (unsigned long *) estack_end[-2];
                        continue;
                }
-                if (irqstack_end) {
+                if (irq_stack_end) {
-                        unsigned long *irqstack;
+                        unsigned long *irq_stack;
-                        irqstack = irqstack_end -
+                        irq_stack = irq_stack_end -
-                                (IRQSTACKSIZE - 64) / sizeof(*irqstack);
+                                (IRQ_STACK_SIZE - 64) / sizeof(*irq_stack);
-                        if (stack >= irqstack && stack < irqstack_end) {
+                        if (stack >= irq_stack && stack < irq_stack_end) {
                                if (ops->stack(data, "IRQ") < 0)
                                        break;
                                bp = print_context_stack(tinfo, stack, bp,
-                                        ops, data, irqstack_end, &graph);
+                                        ops, data, irq_stack_end, &graph);
                                /*
                                 * We link to the next stack (which would be
                                 * the process stack normally) the last
                                 * pointer (index -1 to end) in the IRQ stack:
                                 */
-                                stack = (unsigned long *) (irqstack_end[-1]);
+                                stack = (unsigned long *) (irq_stack_end[-1]);
-                                irqstack_end = NULL;
+                                irq_stack_end = NULL;
                                ops->stack(data, "EOI");
                                continue;
                        }
@@ -199,10 +200,10 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
        unsigned long *stack;
        int i;
        const int cpu = smp_processor_id();
-        unsigned long *irqstack_end =
+        unsigned long *irq_stack_end =
-                (unsigned long *) (cpu_pda(cpu)->irqstackptr);
+                (unsigned long *)(per_cpu(irq_stack_ptr, cpu));
-        unsigned long *irqstack =
+        unsigned long *irq_stack =
-                (unsigned long *) (cpu_pda(cpu)->irqstackptr - IRQSTACKSIZE);
+                (unsigned long *)(per_cpu(irq_stack_ptr, cpu) - IRQ_STACK_SIZE);
        /*
         * debugging aid: "show_stack(NULL, NULL);" prints the
@@ -218,9 +219,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
        stack = sp;
        for (i = 0; i < kstack_depth_to_print; i++) {
-                if (stack >= irqstack && stack <= irqstack_end) {
+                if (stack >= irq_stack && stack <= irq_stack_end) {
-                        if (stack == irqstack_end) {
+                        if (stack == irq_stack_end) {
-                                stack = (unsigned long *) (irqstack_end[-1]);
+                                stack = (unsigned long *) (irq_stack_end[-1]);
                                printk(" <EOI> ");
                        }
                } else {
@@ -241,7 +242,7 @@ void show_registers(struct pt_regs *regs)
        int i;
        unsigned long sp;
        const int cpu = smp_processor_id();
-        struct task_struct *cur = cpu_pda(cpu)->pcurrent;
+        struct task_struct *cur = current;
        sp = regs->sp;
        printk("CPU %d ", cpu);
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 46469029e9d3..a0b91aac72a1 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -672,7 +672,7 @@ common_interrupt:
 ENDPROC(common_interrupt)
        CFI_ENDPROC
-#define BUILD_INTERRUPT(name, nr)       \
+#define BUILD_INTERRUPT3(name, nr, fn)  \
 ENTRY(name)                             \
        RING0_INT_FRAME;                \
        pushl $~(nr);                   \
@@ -680,11 +680,13 @@ ENTRY(name)				\
        SAVE_ALL;                       \
        TRACE_IRQS_OFF                  \
        movl %esp,%eax;                 \
-        call smp_##name;                \
+        call fn;                        \
        jmp ret_from_intr;              \
        CFI_ENDPROC;                    \
 ENDPROC(name)
+#define BUILD_INTERRUPT(name, nr)       BUILD_INTERRUPT3(name, nr, smp_##name)
 /* The include is where all of the SMP etc. interrupts come from */
 #include "entry_arch.h"
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e28c7a987793..c52b60919163 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -52,6 +52,7 @@
 #include <asm/irqflags.h>
 #include <asm/paravirt.h>
 #include <asm/ftrace.h>
+#include <asm/percpu.h>
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -209,7 +210,7 @@ ENTRY(native_usergs_sysret64)
        /* %rsp:at FRAMEEND */
        .macro FIXUP_TOP_OF_STACK tmp offset=0
-        movq %gs:pda_oldrsp,\tmp
+        movq PER_CPU_VAR(old_rsp),\tmp
        movq \tmp,RSP+\offset(%rsp)
        movq $__USER_DS,SS+\offset(%rsp)
        movq $__USER_CS,CS+\offset(%rsp)
@@ -220,7 +221,7 @@ ENTRY(native_usergs_sysret64)
        .macro RESTORE_TOP_OF_STACK tmp offset=0
        movq RSP+\offset(%rsp),\tmp
-        movq \tmp,%gs:pda_oldrsp
+        movq \tmp,PER_CPU_VAR(old_rsp)
        movq EFLAGS+\offset(%rsp),\tmp
        movq \tmp,R11+\offset(%rsp)
        .endm
@@ -336,15 +337,15 @@ ENTRY(save_args)
        je 1f
        SWAPGS
        /*
-         * irqcount is used to check if a CPU is already on an interrupt stack
+         * irq_count is used to check if a CPU is already on an interrupt stack
         * or not. While this is essentially redundant with preempt_count it is
         * a little cheaper to use a separate counter in the PDA (short of
         * moving irq_enter into assembly, which would be too much work)
         */
-1:      incl %gs:pda_irqcount
+1:      incl PER_CPU_VAR(irq_count)
        jne 2f
        popq_cfi %rax                   /* move return address... */
-        mov %gs:pda_irqstackptr,%rsp
+        mov PER_CPU_VAR(irq_stack_ptr),%rsp
        EMPTY_FRAME 0
        pushq_cfi %rax                  /* ... to the new stack */
        /*
@@ -467,7 +468,7 @@ END(ret_from_fork)
 ENTRY(system_call)
        CFI_STARTPROC   simple
        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
+        CFI_DEF_CFA     rsp,KERNEL_STACK_OFFSET
        CFI_REGISTER    rip,rcx
        /*CFI_REGISTER  rflags,r11*/
        SWAPGS_UNSAFE_STACK
@@ -478,8 +479,8 @@ ENTRY(system_call)
         */
 ENTRY(system_call_after_swapgs)
-        movq    %rsp,%gs:pda_oldrsp
+        movq    %rsp,PER_CPU_VAR(old_rsp)
-        movq    %gs:pda_kernelstack,%rsp
+        movq    PER_CPU_VAR(kernel_stack),%rsp
        /*
         * No need to follow this irqs off/on section - it's straight
         * and short:
@@ -522,7 +523,7 @@ sysret_check:
        CFI_REGISTER    rip,rcx
        RESTORE_ARGS 0,-ARG_SKIP,1
        /*CFI_REGISTER  rflags,r11*/
-        movq    %gs:pda_oldrsp, %rsp
+        movq    PER_CPU_VAR(old_rsp), %rsp
        USERGS_SYSRET64
        CFI_RESTORE_STATE
@@ -832,11 +833,11 @@ common_interrupt:
        XCPT_FRAME
        addq $-0x80,(%rsp)              /* Adjust vector to [-256,-1] range */
        interrupt do_IRQ
-        /* 0(%rsp): oldrsp-ARGOFFSET */
+        /* 0(%rsp): old_rsp-ARGOFFSET */
 ret_from_intr:
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
        leaveq
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
@@ -1072,10 +1073,10 @@ ENTRY(\sym)
        TRACE_IRQS_OFF
        movq %rsp,%rdi          /* pt_regs pointer */
        xorl %esi,%esi          /* no error code */
-        movq %gs:pda_data_offset, %rbp
+        PER_CPU(init_tss, %rbp)
-        subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        subq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%rbp)
        call \do_sym
-        addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        addq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%rbp)
        jmp paranoid_exit       /* %ebx: no swapgs flag */
        CFI_ENDPROC
 END(\sym)
@@ -1259,14 +1260,14 @@ ENTRY(call_softirq)
        CFI_REL_OFFSET rbp,0
        mov  %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
-        incl %gs:pda_irqcount
+        incl PER_CPU_VAR(irq_count)
-        cmove %gs:pda_irqstackptr,%rsp
+        cmove PER_CPU_VAR(irq_stack_ptr),%rsp
        push  %rbp                      # backlink for old unwinder
        call __do_softirq
        leaveq
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
        ret
        CFI_ENDPROC
 END(call_softirq)
@@ -1296,15 +1297,15 @@ ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
        movq %rdi, %rsp            # we don't return, adjust the stack frame
        CFI_ENDPROC
        DEFAULT_FRAME
-11:     incl %gs:pda_irqcount
+11:     incl PER_CPU_VAR(irq_count)
        movq %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
-        cmovzq %gs:pda_irqstackptr,%rsp
+        cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
        pushq %rbp                      # backlink for old unwinder
        call xen_evtchn_do_upcall
        popq %rsp
        CFI_DEF_CFA_REGISTER rsp
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
        jmp  error_exit
        CFI_ENDPROC
 END(do_hypervisor_callback)
diff --git a/arch/x86/kernel/genx2apic_uv_x.c b/arch/x86/kernel/genx2apic_uv_x.c
index b193e082f6ce..bfe36249145c 100644
--- a/arch/x86/kernel/genx2apic_uv_x.c
+++ b/arch/x86/kernel/genx2apic_uv_x.c
@@ -25,6 +25,7 @@
 #include <asm/ipi.h>
 #include <asm/genapic.h>
 #include <asm/pgtable.h>
+#include <asm/uv/uv.h>
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
 #include <asm/uv/bios.h>
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index b9a4d8c4b935..f5b272247690 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -26,27 +26,6 @@
 #include <asm/bios_ebda.h>
 #include <asm/trampoline.h>
-/* boot cpu pda */
-static struct x8664_pda _boot_cpu_pda;
-#ifdef CONFIG_SMP
-/*
- * We install an empty cpu_pda pointer table to indicate to early users
- * (numa_set_node) that the cpu_pda pointer table for cpus other than
- * the boot cpu is not yet setup.
- */
-static struct x8664_pda *__cpu_pda[NR_CPUS] __initdata;
-#else
-static struct x8664_pda *__cpu_pda[NR_CPUS] __read_mostly;
-#endif
-void __init x86_64_init_pda(void)
-{
-        _cpu_pda = __cpu_pda;
-        cpu_pda(0) = &_boot_cpu_pda;
-        pda_init(0);
-}
 static void __init zap_identity_mappings(void)
 {
        pgd_t *pgd = pgd_offset_k(0UL);
@@ -112,8 +91,6 @@ void __init x86_64_start_kernel(char * real_mode_data)
        if (console_loglevel == 10)
                early_printk("Kernel alive\n");
-        x86_64_init_pda();
        x86_64_start_reservations(real_mode_data);
 }
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index e835b4eea70b..24c0e5cd71e3 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -429,12 +429,14 @@ is386:	movl $2,%ecx		# set MP
        ljmp $(__KERNEL_CS),$1f
 1:      movl $(__KERNEL_DS),%eax        # reload all the segment registers
        movl %eax,%ss                   # after changing gdt.
-        movl %eax,%fs                   # gets reset once there's real percpu
        movl $(__USER_DS),%eax          # DS/ES contains default USER segment
        movl %eax,%ds
        movl %eax,%es
+        movl $(__KERNEL_PERCPU), %eax
+        movl %eax,%fs                   # set this cpu's percpu
        xorl %eax,%eax                  # Clear GS and LDT
        movl %eax,%gs
        lldt %ax
@@ -446,8 +448,6 @@ is386:	movl $2,%ecx		# set MP
        movb $1, ready
        cmpb $0,%cl             # the first CPU calls start_kernel
        je   1f
-        movl $(__KERNEL_PERCPU), %eax
-        movl %eax,%fs           # set this cpu's percpu
        movl (stack_start), %esp
 1:
 #endif /* CONFIG_SMP */
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 0e275d495563..a0a2b5ca9b7d 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -19,6 +19,7 @@
 #include <asm/msr.h>
 #include <asm/cache.h>
 #include <asm/processor-flags.h>
+#include <asm/percpu.h>
 #ifdef CONFIG_PARAVIRT
 #include <asm/asm-offsets.h>
@@ -204,6 +205,19 @@ ENTRY(secondary_startup_64)
        pushq $0
        popfq
+#ifdef CONFIG_SMP
+        /*
+         * Fix up static pointers that need __per_cpu_load added.  The assembler
+         * is unable to do this directly.  This is only needed for the boot cpu.
+         * These values are set up with the correct base addresses by C code for
+         * secondary cpus.
+         */
+        movq    initial_gs(%rip), %rax
+        cmpl    $0, per_cpu__cpu_number(%rax)
+        jne     1f
+        addq    %rax, early_gdt_descr_base(%rip)
+1:
+#endif
        /*
         * We must switch to a new descriptor in kernel space for the GDT
         * because soon the kernel won't have access anymore to the userspace
@@ -226,12 +240,15 @@ ENTRY(secondary_startup_64)
        movl %eax,%fs
        movl %eax,%gs
-        /* 
+        /* Set up %gs.
-         * Setup up a dummy PDA. this is just for some early bootup code
+         *
-         * that does in_interrupt() 
+         * The base of %gs always points to the bottom of the irqstack
-         */ 
+         * union.  If the stack protector canary is enabled, it is
+         * located at %gs:40.  Note that, on SMP, the boot cpu uses
+         * init data section till per cpu areas are set up.
+         */
        movl    $MSR_GS_BASE,%ecx
-        movq    $empty_zero_page,%rax
+        movq    initial_gs(%rip),%rax
        movq    %rax,%rdx
        shrq    $32,%rdx
        wrmsr   
@@ -257,6 +274,12 @@ ENTRY(secondary_startup_64)
        .align  8
        ENTRY(initial_code)
        .quad   x86_64_start_kernel
+        ENTRY(initial_gs)
+#ifdef CONFIG_SMP
+        .quad   __per_cpu_load
+#else
+        .quad   PER_CPU_VAR(irq_stack_union)
+#endif
        __FINITDATA
        ENTRY(stack_start)
@@ -401,7 +424,8 @@ NEXT_PAGE(level2_spare_pgt)
        .globl early_gdt_descr
 early_gdt_descr:
        .word   GDT_ENTRIES*8-1
-        .quad   per_cpu__gdt_page
+early_gdt_descr_base:
+        .quad   per_cpu__gdt_page
 ENTRY(phys_base)
        /* This must match the first entry in level2_kernel_pgt */
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c
index 157986916cd1..f79660390724 100644
--- a/arch/x86/kernel/io_apic.c
+++ b/arch/x86/kernel/io_apic.c
@@ -46,6 +46,7 @@
 #include <asm/idle.h>
 #include <asm/io.h>
 #include <asm/smp.h>
+#include <asm/cpu.h>
 #include <asm/desc.h>
 #include <asm/proto.h>
 #include <asm/acpi.h>
@@ -82,11 +83,11 @@ static DEFINE_SPINLOCK(vector_lock);
 int nr_ioapic_registers[MAX_IO_APICS];
 /* I/O APIC entries */
-struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
+struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
 int nr_ioapics;
 /* MP IRQ source entries */
-struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
+struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
 /* # of MP IRQ source entries */
 int mp_irq_entries;
@@ -386,7 +387,7 @@ struct io_apic {
 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
 {
        return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
-                + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
+                + (mp_ioapics[idx].apicaddr & ~PAGE_MASK);
 }
 static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
@@ -944,10 +945,10 @@ static int find_irq_entry(int apic, int pin, int type)
        int i;
        for (i = 0; i < mp_irq_entries; i++)
-                if (mp_irqs[i].mp_irqtype == type &&
+                if (mp_irqs[i].irqtype == type &&
-                    (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
+                    (mp_irqs[i].dstapic == mp_ioapics[apic].apicid ||
-                     mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
+                     mp_irqs[i].dstapic == MP_APIC_ALL) &&
-                    mp_irqs[i].mp_dstirq == pin)
+                    mp_irqs[i].dstirq == pin)
                        return i;
        return -1;
@@ -961,13 +962,13 @@ static int __init find_isa_irq_pin(int irq, int type)
        int i;
        for (i = 0; i < mp_irq_entries; i++) {
-                int lbus = mp_irqs[i].mp_srcbus;
+                int lbus = mp_irqs[i].srcbus;
                if (test_bit(lbus, mp_bus_not_pci) &&
-                    (mp_irqs[i].mp_irqtype == type) &&
+                    (mp_irqs[i].irqtype == type) &&
-                    (mp_irqs[i].mp_srcbusirq == irq))
+                    (mp_irqs[i].srcbusirq == irq))
-                        return mp_irqs[i].mp_dstirq;
+                        return mp_irqs[i].dstirq;
        }
        return -1;
 }
@@ -977,17 +978,17 @@ static int __init find_isa_irq_apic(int irq, int type)
        int i;
        for (i = 0; i < mp_irq_entries; i++) {
-                int lbus = mp_irqs[i].mp_srcbus;
+                int lbus = mp_irqs[i].srcbus;
                if (test_bit(lbus, mp_bus_not_pci) &&
-                    (mp_irqs[i].mp_irqtype == type) &&
+                    (mp_irqs[i].irqtype == type) &&
-                    (mp_irqs[i].mp_srcbusirq == irq))
+                    (mp_irqs[i].srcbusirq == irq))
                        break;
        }
        if (i < mp_irq_entries) {
                int apic;
                for(apic = 0; apic < nr_ioapics; apic++) {
-                        if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
+                        if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic)
                                return apic;
                }
        }
@@ -1012,23 +1013,23 @@ int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
                return -1;
        }
        for (i = 0; i < mp_irq_entries; i++) {
-                int lbus = mp_irqs[i].mp_srcbus;
+                int lbus = mp_irqs[i].srcbus;
                for (apic = 0; apic < nr_ioapics; apic++)
-                        if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
+                        if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
-                            mp_irqs[i].mp_dstapic == MP_APIC_ALL)
+                            mp_irqs[i].dstapic == MP_APIC_ALL)
                                break;
                if (!test_bit(lbus, mp_bus_not_pci) &&
-                    !mp_irqs[i].mp_irqtype &&
+                    !mp_irqs[i].irqtype &&
                    (bus == lbus) &&
-                    (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
+                    (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
-                        int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
+                        int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
                        if (!(apic || IO_APIC_IRQ(irq)))
                                continue;
-                        if (pin == (mp_irqs[i].mp_srcbusirq & 3))
+                        if (pin == (mp_irqs[i].srcbusirq & 3))
                                return irq;
                        /*
                         * Use the first all-but-pin matching entry as a
@@ -1071,7 +1072,7 @@ static int EISA_ELCR(unsigned int irq)
 * EISA conforming in the MP table, that means its trigger type must
 * be read in from the ELCR */
-#define default_EISA_trigger(idx)       (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
+#define default_EISA_trigger(idx)       (EISA_ELCR(mp_irqs[idx].srcbusirq))
 #define default_EISA_polarity(idx)      default_ISA_polarity(idx)
 /* PCI interrupts are always polarity one level triggered,
@@ -1088,13 +1089,13 @@ static int EISA_ELCR(unsigned int irq)
 static int MPBIOS_polarity(int idx)
 {
-        int bus = mp_irqs[idx].mp_srcbus;
+        int bus = mp_irqs[idx].srcbus;
        int polarity;
        /*
         * Determine IRQ line polarity (high active or low active):
         */
-        switch (mp_irqs[idx].mp_irqflag & 3)
+        switch (mp_irqs[idx].irqflag & 3)
        {
                case 0: /* conforms, ie. bus-type dependent polarity */
                        if (test_bit(bus, mp_bus_not_pci))
@@ -1130,13 +1131,13 @@ static int MPBIOS_polarity(int idx)
 static int MPBIOS_trigger(int idx)
 {
-        int bus = mp_irqs[idx].mp_srcbus;
+        int bus = mp_irqs[idx].srcbus;
        int trigger;
        /*
         * Determine IRQ trigger mode (edge or level sensitive):
         */
-        switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
+        switch ((mp_irqs[idx].irqflag>>2) & 3)
        {
                case 0: /* conforms, ie. bus-type dependent */
                        if (test_bit(bus, mp_bus_not_pci))
@@ -1214,16 +1215,16 @@ int (*ioapic_renumber_irq)(int ioapic, int irq);
 static int pin_2_irq(int idx, int apic, int pin)
 {
        int irq, i;
-        int bus = mp_irqs[idx].mp_srcbus;
+        int bus = mp_irqs[idx].srcbus;
        /*
         * Debugging check, we are in big trouble if this message pops up!
         */
-        if (mp_irqs[idx].mp_dstirq != pin)
+        if (mp_irqs[idx].dstirq != pin)
                printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
        if (test_bit(bus, mp_bus_not_pci)) {
-                irq = mp_irqs[idx].mp_srcbusirq;
+                irq = mp_irqs[idx].srcbusirq;
        } else {
                /*
                 * PCI IRQs are mapped in order
@@ -1566,14 +1567,14 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_de
        apic_printk(APIC_VERBOSE,KERN_DEBUG
                    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
                    "IRQ %d Mode:%i Active:%i)\n",
-                    apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
+                    apic, mp_ioapics[apic].apicid, pin, cfg->vector,
                    irq, trigger, polarity);
-        if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
+        if (setup_ioapic_entry(mp_ioapics[apic].apicid, irq, &entry,
                               dest, trigger, polarity, cfg->vector)) {
                printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
-                       mp_ioapics[apic].mp_apicid, pin);
+                       mp_ioapics[apic].apicid, pin);
                __clear_irq_vector(irq, cfg);
                return;
        }
@@ -1604,12 +1605,10 @@ static void __init setup_IO_APIC_irqs(void)
                                        notcon = 1;
                                        apic_printk(APIC_VERBOSE,
                                                KERN_DEBUG " %d-%d",
-                                                mp_ioapics[apic].mp_apicid,
+                                                mp_ioapics[apic].apicid, pin);
-                                                pin);
                                } else
                                        apic_printk(APIC_VERBOSE, " %d-%d",
-                                                mp_ioapics[apic].mp_apicid,
+                                                mp_ioapics[apic].apicid, pin);
-                                                pin);
                                continue;
                        }
                        if (notcon) {
@@ -1699,7 +1698,7 @@ __apicdebuginit(void) print_IO_APIC(void)
        printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
        for (i = 0; i < nr_ioapics; i++)
                printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
-                       mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
+                       mp_ioapics[i].apicid, nr_ioapic_registers[i]);
        /*
         * We are a bit conservative about what we expect.  We have to
@@ -1719,7 +1718,7 @@ __apicdebuginit(void) print_IO_APIC(void)
        spin_unlock_irqrestore(&ioapic_lock, flags);
        printk("\n");
-        printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
+        printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid);
        printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
        printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
        printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
@@ -2121,14 +2120,14 @@ static void __init setup_ioapic_ids_from_mpc(void)
                reg_00.raw = io_apic_read(apic, 0);
                spin_unlock_irqrestore(&ioapic_lock, flags);
-                old_id = mp_ioapics[apic].mp_apicid;
+                old_id = mp_ioapics[apic].apicid;
-                if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
+                if (mp_ioapics[apic].apicid >= get_physical_broadcast()) {
                        printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
-                                apic, mp_ioapics[apic].mp_apicid);
+                                apic, mp_ioapics[apic].apicid);
                        printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
                                reg_00.bits.ID);
-                        mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
+                        mp_ioapics[apic].apicid = reg_00.bits.ID;
                }
                /*
@@ -2137,9 +2136,9 @@ static void __init setup_ioapic_ids_from_mpc(void)
                 * 'stuck on smp_invalidate_needed IPI wait' messages.
                 */
                if (check_apicid_used(phys_id_present_map,
-                                        mp_ioapics[apic].mp_apicid)) {
+                                        mp_ioapics[apic].apicid)) {
                        printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
-                                apic, mp_ioapics[apic].mp_apicid);
+                                apic, mp_ioapics[apic].apicid);
                        for (i = 0; i < get_physical_broadcast(); i++)
                                if (!physid_isset(i, phys_id_present_map))
                                        break;
@@ -2148,13 +2147,13 @@ static void __init setup_ioapic_ids_from_mpc(void)
                        printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
                                i);
                        physid_set(i, phys_id_present_map);
-                        mp_ioapics[apic].mp_apicid = i;
+                        mp_ioapics[apic].apicid = i;
                } else {
                        physid_mask_t tmp;
-                        tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
+                        tmp = apicid_to_cpu_present(mp_ioapics[apic].apicid);
                        apic_printk(APIC_VERBOSE, "Setting %d in the "
                                        "phys_id_present_map\n",
-                                        mp_ioapics[apic].mp_apicid);
+                                        mp_ioapics[apic].apicid);
                        physids_or(phys_id_present_map, phys_id_present_map, tmp);
                }
@@ -2163,11 +2162,11 @@ static void __init setup_ioapic_ids_from_mpc(void)
                 * We need to adjust the IRQ routing table
                 * if the ID changed.
                 */
-                if (old_id != mp_ioapics[apic].mp_apicid)
+                if (old_id != mp_ioapics[apic].apicid)
                        for (i = 0; i < mp_irq_entries; i++)
-                                if (mp_irqs[i].mp_dstapic == old_id)
+                                if (mp_irqs[i].dstapic == old_id)
-                                        mp_irqs[i].mp_dstapic
+                                        mp_irqs[i].dstapic
-                                                = mp_ioapics[apic].mp_apicid;
+                                                = mp_ioapics[apic].apicid;
                /*
                 * Read the right value from the MPC table and
@@ -2175,9 +2174,9 @@ static void __init setup_ioapic_ids_from_mpc(void)
                 */
                apic_printk(APIC_VERBOSE, KERN_INFO
                        "...changing IO-APIC physical APIC ID to %d ...",
-                        mp_ioapics[apic].mp_apicid);
+                        mp_ioapics[apic].apicid);
-                reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
+                reg_00.bits.ID = mp_ioapics[apic].apicid;
                spin_lock_irqsave(&ioapic_lock, flags);
                io_apic_write(apic, 0, reg_00.raw);
                spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -2188,7 +2187,7 @@ static void __init setup_ioapic_ids_from_mpc(void)
                spin_lock_irqsave(&ioapic_lock, flags);
                reg_00.raw = io_apic_read(apic, 0);
                spin_unlock_irqrestore(&ioapic_lock, flags);
-                if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
+                if (reg_00.bits.ID != mp_ioapics[apic].apicid)
                        printk("could not set ID!\n");
                else
                        apic_printk(APIC_VERBOSE, " ok.\n");
@@ -3117,8 +3116,8 @@ static int ioapic_resume(struct sys_device *dev)
        spin_lock_irqsave(&ioapic_lock, flags);
        reg_00.raw = io_apic_read(dev->id, 0);
-        if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
+        if (reg_00.bits.ID != mp_ioapics[dev->id].apicid) {
-                reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
+                reg_00.bits.ID = mp_ioapics[dev->id].apicid;
                io_apic_write(dev->id, 0, reg_00.raw);
        }
        spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -3258,6 +3257,9 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
        int err;
        unsigned dest;
+        if (disable_apic)
+                return -ENXIO;
        cfg = irq_cfg(irq);
        err = assign_irq_vector(irq, cfg, TARGET_CPUS);
        if (err)
@@ -3726,6 +3728,9 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
        struct irq_cfg *cfg;
        int err;
+        if (disable_apic)
+                return -ENXIO;
        cfg = irq_cfg(irq);
        err = assign_irq_vector(irq, cfg, TARGET_CPUS);
        if (!err) {
@@ -4000,8 +4005,8 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
                return -1;
        for (i = 0; i < mp_irq_entries; i++)
-                if (mp_irqs[i].mp_irqtype == mp_INT &&
+                if (mp_irqs[i].irqtype == mp_INT &&
-                    mp_irqs[i].mp_srcbusirq == bus_irq)
+                    mp_irqs[i].srcbusirq == bus_irq)
                        break;
        if (i >= mp_irq_entries)
                return -1;
@@ -4116,7 +4121,7 @@ void __init ioapic_init_mappings(void)
        ioapic_res = ioapic_setup_resources();
        for (i = 0; i < nr_ioapics; i++) {
                if (smp_found_config) {
-                        ioapic_phys = mp_ioapics[i].mp_apicaddr;
+                        ioapic_phys = mp_ioapics[i].apicaddr;
 #ifdef CONFIG_X86_32
                        if (!ioapic_phys) {
                                printk(KERN_ERR
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 3973e2df7f87..8b30d0c2512c 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -36,11 +36,7 @@ void ack_bad_irq(unsigned int irq)
 #endif
 }
-#ifdef CONFIG_X86_32
+#define irq_stats(x)            (&per_cpu(irq_stat, x))
-# define irq_stats(x)           (&per_cpu(irq_stat, x))
-#else
-# define irq_stats(x)           cpu_pda(x)
-#endif
 /*
 * /proc/interrupts printing:
 */
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 0b21cb1ea11f..0b254de84083 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -19,6 +19,12 @@
 #include <asm/io_apic.h>
 #include <asm/idle.h>
+DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
+EXPORT_PER_CPU_SYMBOL(irq_stat);
+DEFINE_PER_CPU(struct pt_regs *, irq_regs);
+EXPORT_PER_CPU_SYMBOL(irq_regs);
 /*
 * Probabilistic stack overflow check:
 *
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
index 1507ad4e674d..bf629cadec1a 100644
--- a/arch/x86/kernel/irqinit_32.c
+++ b/arch/x86/kernel/irqinit_32.c
@@ -149,8 +149,15 @@ void __init native_init_IRQ(void)
         */
        alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
-        /* IPI for invalidation */
+        /* IPIs for invalidation */
-        alloc_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
+        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
        /* IPI for generic function call */
        alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index b7f4c929e615..5e9f4fc51385 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -87,9 +87,9 @@
 #include <linux/cpu.h>
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
+#include <linux/uaccess.h>
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/processor.h>
 #include <asm/microcode.h>
@@ -196,7 +196,7 @@ static inline int update_match_cpu(struct cpu_signature *csig, int sig, int pf)
        return (!sigmatch(sig, csig->sig, pf, csig->pf)) ? 0 : 1;
 }
-static inline int 
+static inline int
 update_match_revision(struct microcode_header_intel *mc_header, int rev)
 {
        return (mc_header->rev <= rev) ? 0 : 1;
@@ -442,8 +442,8 @@ static int request_microcode_fw(int cpu, struct device *device)
                return ret;
        }
-        ret = generic_load_microcode(cpu, (void*)firmware->data, firmware->size,
+        ret = generic_load_microcode(cpu, (void *)firmware->data,
-                        &get_ucode_fw);
+                                     firmware->size, &get_ucode_fw);
        release_firmware(firmware);
@@ -460,7 +460,7 @@ static int request_microcode_user(int cpu, const void __user *buf, size_t size)
        /* We should bind the task to the CPU */
        BUG_ON(cpu != raw_smp_processor_id());
-        return generic_load_microcode(cpu, (void*)buf, size, &get_ucode_user);
+        return generic_load_microcode(cpu, (void *)buf, size, &get_ucode_user);
 }
 static void microcode_fini_cpu(int cpu)
diff --git a/arch/x86/kernel/module_32.c b/arch/x86/kernel/module_32.c
index 3db0a5442eb1..0edd819050e7 100644
--- a/arch/x86/kernel/module_32.c
+++ b/arch/x86/kernel/module_32.c
@@ -42,7 +42,7 @@ void module_free(struct module *mod, void *module_region)
 {
        vfree(module_region);
        /* FIXME: If module_region == mod->init_region, trim exception
-           table entries. */
+           table entries. */
 }
 /* We don't need anything special. */
@@ -113,13 +113,13 @@ int module_finalize(const Elf_Ehdr *hdr,
                *para = NULL;
        char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
-        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 
+        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
                if (!strcmp(".text", secstrings + s->sh_name))
                        text = s;
                if (!strcmp(".altinstructions", secstrings + s->sh_name))
                        alt = s;
                if (!strcmp(".smp_locks", secstrings + s->sh_name))
-                        locks= s;
+                        locks = s;
                if (!strcmp(".parainstructions", secstrings + s->sh_name))
                        para = s;
        }
diff --git a/arch/x86/kernel/module_64.c b/arch/x86/kernel/module_64.c
index 6ba87830d4b1..c23880b90b5c 100644
--- a/arch/x86/kernel/module_64.c
+++ b/arch/x86/kernel/module_64.c
@@ -30,14 +30,14 @@
 #include <asm/page.h>
 #include <asm/pgtable.h>
-#define DEBUGP(fmt...) 
+#define DEBUGP(fmt...)
 #ifndef CONFIG_UML
 void module_free(struct module *mod, void *module_region)
 {
        vfree(module_region);
        /* FIXME: If module_region == mod->init_region, trim exception
-           table entries. */
+           table entries. */
 }
 void *module_alloc(unsigned long size)
@@ -77,7 +77,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
        Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;
        Elf64_Sym *sym;
        void *loc;
-        u64 val; 
+        u64 val;
        DEBUGP("Applying relocate section %u to %u\n", relsec,
               sechdrs[relsec].sh_info);
@@ -91,11 +91,11 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
                sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
                        + ELF64_R_SYM(rel[i].r_info);
-                DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
+                DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
-                       (int)ELF64_R_TYPE(rel[i].r_info), 
+                        (int)ELF64_R_TYPE(rel[i].r_info),
-                       sym->st_value, rel[i].r_addend, (u64)loc);
+                        sym->st_value, rel[i].r_addend, (u64)loc);
-                val = sym->st_value + rel[i].r_addend; 
+                val = sym->st_value + rel[i].r_addend;
                switch (ELF64_R_TYPE(rel[i].r_info)) {
                case R_X86_64_NONE:
@@ -113,16 +113,16 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
                        if ((s64)val != *(s32 *)loc)
                                goto overflow;
                        break;
-                case R_X86_64_PC32: 
+                case R_X86_64_PC32:
                        val -= (u64)loc;
                        *(u32 *)loc = val;
 #if 0
                        if ((s64)val != *(s32 *)loc)
-                                goto overflow; 
+                                goto overflow;
 #endif
                        break;
                default:
-                        printk(KERN_ERR "module %s: Unknown rela relocation: %Lu\n",
+                        printk(KERN_ERR "module %s: Unknown rela relocation: %llu\n",
                               me->name, ELF64_R_TYPE(rel[i].r_info));
                        return -ENOEXEC;
                }
@@ -130,7 +130,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
        return 0;
 overflow:
-        printk(KERN_ERR "overflow in relocation type %d val %Lx\n", 
+        printk(KERN_ERR "overflow in relocation type %d val %Lx\n",
               (int)ELF64_R_TYPE(rel[i].r_info), val);
        printk(KERN_ERR "`%s' likely not compiled with -mcmodel=kernel\n",
               me->name);
@@ -143,13 +143,13 @@ int apply_relocate(Elf_Shdr *sechdrs,
                   unsigned int relsec,
                   struct module *me)
 {
-        printk("non add relocation not supported\n");
+        printk(KERN_ERR "non add relocation not supported\n");
        return -ENOSYS;
-} 
+}
 int module_finalize(const Elf_Ehdr *hdr,
-                    const Elf_Shdr *sechdrs,
+                    const Elf_Shdr *sechdrs,
-                    struct module *me)
+                    struct module *me)
 {
        const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
                *para = NULL;
@@ -161,7 +161,7 @@ int module_finalize(const Elf_Ehdr *hdr,
                if (!strcmp(".altinstructions", secstrings + s->sh_name))
                        alt = s;
                if (!strcmp(".smp_locks", secstrings + s->sh_name))
-                        locks= s;
+                        locks = s;
                if (!strcmp(".parainstructions", secstrings + s->sh_name))
                        para = s;
        }
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index a649a4ccad43..fa6bb263892e 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -144,11 +144,11 @@ static void __init MP_ioapic_info(struct mpc_ioapic *m)
        if (bad_ioapic(m->apicaddr))
                return;
-        mp_ioapics[nr_ioapics].mp_apicaddr = m->apicaddr;
+        mp_ioapics[nr_ioapics].apicaddr = m->apicaddr;
-        mp_ioapics[nr_ioapics].mp_apicid = m->apicid;
+        mp_ioapics[nr_ioapics].apicid = m->apicid;
-        mp_ioapics[nr_ioapics].mp_type = m->type;
+        mp_ioapics[nr_ioapics].type = m->type;
-        mp_ioapics[nr_ioapics].mp_apicver = m->apicver;
+        mp_ioapics[nr_ioapics].apicver = m->apicver;
-        mp_ioapics[nr_ioapics].mp_flags = m->flags;
+        mp_ioapics[nr_ioapics].flags = m->flags;
        nr_ioapics++;
 }
@@ -160,55 +160,55 @@ static void print_MP_intsrc_info(struct mpc_intsrc *m)
                m->srcbusirq, m->dstapic, m->dstirq);
 }
-static void __init print_mp_irq_info(struct mp_config_intsrc *mp_irq)
+static void __init print_mp_irq_info(struct mpc_intsrc *mp_irq)
 {
        apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
                " IRQ %02x, APIC ID %x, APIC INT %02x\n",
-                mp_irq->mp_irqtype, mp_irq->mp_irqflag & 3,
+                mp_irq->irqtype, mp_irq->irqflag & 3,
-                (mp_irq->mp_irqflag >> 2) & 3, mp_irq->mp_srcbus,
+                (mp_irq->irqflag >> 2) & 3, mp_irq->srcbus,
-                mp_irq->mp_srcbusirq, mp_irq->mp_dstapic, mp_irq->mp_dstirq);
+                mp_irq->srcbusirq, mp_irq->dstapic, mp_irq->dstirq);
 }
 static void __init assign_to_mp_irq(struct mpc_intsrc *m,
-                                    struct mp_config_intsrc *mp_irq)
+                                    struct mpc_intsrc *mp_irq)
 {
-        mp_irq->mp_dstapic = m->dstapic;
+        mp_irq->dstapic = m->dstapic;
-        mp_irq->mp_type = m->type;
+        mp_irq->type = m->type;
-        mp_irq->mp_irqtype = m->irqtype;
+        mp_irq->irqtype = m->irqtype;
-        mp_irq->mp_irqflag = m->irqflag;
+        mp_irq->irqflag = m->irqflag;
-        mp_irq->mp_srcbus = m->srcbus;
+        mp_irq->srcbus = m->srcbus;
-        mp_irq->mp_srcbusirq = m->srcbusirq;
+        mp_irq->srcbusirq = m->srcbusirq;
-        mp_irq->mp_dstirq = m->dstirq;
+        mp_irq->dstirq = m->dstirq;
 }
-static void __init assign_to_mpc_intsrc(struct mp_config_intsrc *mp_irq,
+static void __init assign_to_mpc_intsrc(struct mpc_intsrc *mp_irq,
                                        struct mpc_intsrc *m)
 {
-        m->dstapic = mp_irq->mp_dstapic;
+        m->dstapic = mp_irq->dstapic;
-        m->type = mp_irq->mp_type;
+        m->type = mp_irq->type;
-        m->irqtype = mp_irq->mp_irqtype;
+        m->irqtype = mp_irq->irqtype;
-        m->irqflag = mp_irq->mp_irqflag;
+        m->irqflag = mp_irq->irqflag;
-        m->srcbus = mp_irq->mp_srcbus;
+        m->srcbus = mp_irq->srcbus;
-        m->srcbusirq = mp_irq->mp_srcbusirq;
+        m->srcbusirq = mp_irq->srcbusirq;
-        m->dstirq = mp_irq->mp_dstirq;
+        m->dstirq = mp_irq->dstirq;
 }
-static int __init mp_irq_mpc_intsrc_cmp(struct mp_config_intsrc *mp_irq,
+static int __init mp_irq_mpc_intsrc_cmp(struct mpc_intsrc *mp_irq,
                                        struct mpc_intsrc *m)
 {
-        if (mp_irq->mp_dstapic != m->dstapic)
+        if (mp_irq->dstapic != m->dstapic)
                return 1;
-        if (mp_irq->mp_type != m->type)
+        if (mp_irq->type != m->type)
                return 2;
-        if (mp_irq->mp_irqtype != m->irqtype)
+        if (mp_irq->irqtype != m->irqtype)
                return 3;
-        if (mp_irq->mp_irqflag != m->irqflag)
+        if (mp_irq->irqflag != m->irqflag)
                return 4;
-        if (mp_irq->mp_srcbus != m->srcbus)
+        if (mp_irq->srcbus != m->srcbus)
                return 5;
-        if (mp_irq->mp_srcbusirq != m->srcbusirq)
+        if (mp_irq->srcbusirq != m->srcbusirq)
                return 6;
-        if (mp_irq->mp_dstirq != m->dstirq)
+        if (mp_irq->dstirq != m->dstirq)
                return 7;
        return 0;
@@ -417,7 +417,7 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
        intsrc.type = MP_INTSRC;
        intsrc.irqflag = 0;     /* conforming */
        intsrc.srcbus = 0;
-        intsrc.dstapic = mp_ioapics[0].mp_apicid;
+        intsrc.dstapic = mp_ioapics[0].apicid;
        intsrc.irqtype = mp_INT;
@@ -570,14 +570,14 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
        }
 }
-static struct intel_mp_floating *mpf_found;
+static struct mpf_intel *mpf_found;
 /*
 * Scan the memory blocks for an SMP configuration block.
 */
 static void __init __get_smp_config(unsigned int early)
 {
-        struct intel_mp_floating *mpf = mpf_found;
+        struct mpf_intel *mpf = mpf_found;
        if (!mpf)
                return;
@@ -598,9 +598,9 @@ static void __init __get_smp_config(unsigned int early)
        }
        printk(KERN_INFO "Intel MultiProcessor Specification v1.%d\n",
-               mpf->mpf_specification);
+               mpf->specification);
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
-        if (mpf->mpf_feature2 & (1 << 7)) {
+        if (mpf->feature2 & (1 << 7)) {
                printk(KERN_INFO "    IMCR and PIC compatibility mode.\n");
                pic_mode = 1;
        } else {
@@ -611,7 +611,7 @@ static void __init __get_smp_config(unsigned int early)
        /*
         * Now see if we need to read further.
         */
-        if (mpf->mpf_feature1 != 0) {
+        if (mpf->feature1 != 0) {
                if (early) {
                        /*
                         * local APIC has default address
@@ -621,16 +621,16 @@ static void __init __get_smp_config(unsigned int early)
                }
                printk(KERN_INFO "Default MP configuration #%d\n",
-                       mpf->mpf_feature1);
+                       mpf->feature1);
-                construct_default_ISA_mptable(mpf->mpf_feature1);
+                construct_default_ISA_mptable(mpf->feature1);
-        } else if (mpf->mpf_physptr) {
+        } else if (mpf->physptr) {
                /*
                 * Read the physical hardware table.  Anything here will
                 * override the defaults.
                 */
-                if (!smp_read_mpc(phys_to_virt(mpf->mpf_physptr), early)) {
+                if (!smp_read_mpc(phys_to_virt(mpf->physptr), early)) {
 #ifdef CONFIG_X86_LOCAL_APIC
                        smp_found_config = 0;
 #endif
@@ -688,19 +688,19 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
                                  unsigned reserve)
 {
        unsigned int *bp = phys_to_virt(base);
-        struct intel_mp_floating *mpf;
+        struct mpf_intel *mpf;
        apic_printk(APIC_VERBOSE, "Scan SMP from %p for %ld bytes.\n",
                        bp, length);
        BUILD_BUG_ON(sizeof(*mpf) != 16);
        while (length > 0) {
-                mpf = (struct intel_mp_floating *)bp;
+                mpf = (struct mpf_intel *)bp;
                if ((*bp == SMP_MAGIC_IDENT) &&
-                    (mpf->mpf_length == 1) &&
+                    (mpf->length == 1) &&
                    !mpf_checksum((unsigned char *)bp, 16) &&
-                    ((mpf->mpf_specification == 1)
+                    ((mpf->specification == 1)
-                     || (mpf->mpf_specification == 4))) {
+                     || (mpf->specification == 4))) {
 #ifdef CONFIG_X86_LOCAL_APIC
                        smp_found_config = 1;
 #endif
@@ -713,7 +713,7 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
                                return 1;
                        reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
                                        BOOTMEM_DEFAULT);
-                        if (mpf->mpf_physptr) {
+                        if (mpf->physptr) {
                                unsigned long size = PAGE_SIZE;
 #ifdef CONFIG_X86_32
                                /*
@@ -722,14 +722,14 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
                                 * the bottom is mapped now.
                                 * PC-9800's MPC table places on the very last
                                 * of physical memory; so that simply reserving
-                                 * PAGE_SIZE from mpg->mpf_physptr yields BUG()
+                                 * PAGE_SIZE from mpf->physptr yields BUG()
                                 * in reserve_bootmem.
                                 */
                                unsigned long end = max_low_pfn * PAGE_SIZE;
-                                if (mpf->mpf_physptr + size > end)
+                                if (mpf->physptr + size > end)
-                                        size = end - mpf->mpf_physptr;
+                                        size = end - mpf->physptr;
 #endif
-                                reserve_bootmem_generic(mpf->mpf_physptr, size,
+                                reserve_bootmem_generic(mpf->physptr, size,
                                                BOOTMEM_DEFAULT);
                        }
@@ -809,15 +809,15 @@ static int  __init get_MP_intsrc_index(struct mpc_intsrc *m)
        /* not legacy */
        for (i = 0; i < mp_irq_entries; i++) {
-                if (mp_irqs[i].mp_irqtype != mp_INT)
+                if (mp_irqs[i].irqtype != mp_INT)
                        continue;
-                if (mp_irqs[i].mp_irqflag != 0x0f)
+                if (mp_irqs[i].irqflag != 0x0f)
                        continue;
-                if (mp_irqs[i].mp_srcbus != m->srcbus)
+                if (mp_irqs[i].srcbus != m->srcbus)
                        continue;
-                if (mp_irqs[i].mp_srcbusirq != m->srcbusirq)
+                if (mp_irqs[i].srcbusirq != m->srcbusirq)
                        continue;
                if (irq_used[i]) {
                        /* already claimed */
@@ -922,10 +922,10 @@ static int  __init replace_intsrc_all(struct mpc_table *mpc,
                if (irq_used[i])
                        continue;
-                if (mp_irqs[i].mp_irqtype != mp_INT)
+                if (mp_irqs[i].irqtype != mp_INT)
                        continue;
-                if (mp_irqs[i].mp_irqflag != 0x0f)
+                if (mp_irqs[i].irqflag != 0x0f)
                        continue;
                if (nr_m_spare > 0) {
@@ -1001,7 +1001,7 @@ static int __init update_mp_table(void)
 {
        char str[16];
        char oem[10];
-        struct intel_mp_floating *mpf;
+        struct mpf_intel *mpf;
        struct mpc_table *mpc, *mpc_new;
        if (!enable_update_mptable)
@@ -1014,19 +1014,19 @@ static int __init update_mp_table(void)
        /*
         * Now see if we need to go further.
         */
-        if (mpf->mpf_feature1 != 0)
+        if (mpf->feature1 != 0)
                return 0;
-        if (!mpf->mpf_physptr)
+        if (!mpf->physptr)
                return 0;
-        mpc = phys_to_virt(mpf->mpf_physptr);
+        mpc = phys_to_virt(mpf->physptr);
        if (!smp_check_mpc(mpc, oem, str))
                return 0;
        printk(KERN_INFO "mpf: %lx\n", virt_to_phys(mpf));
-        printk(KERN_INFO "mpf_physptr: %x\n", mpf->mpf_physptr);
+        printk(KERN_INFO "physptr: %x\n", mpf->physptr);
        if (mpc_new_phys && mpc->length > mpc_new_length) {
                mpc_new_phys = 0;
@@ -1047,23 +1047,23 @@ static int __init update_mp_table(void)
                }
                printk(KERN_INFO "use in-positon replacing\n");
        } else {
-                mpf->mpf_physptr = mpc_new_phys;
+                mpf->physptr = mpc_new_phys;
                mpc_new = phys_to_virt(mpc_new_phys);
                memcpy(mpc_new, mpc, mpc->length);
                mpc = mpc_new;
                /* check if we can modify that */
-                if (mpc_new_phys - mpf->mpf_physptr) {
+                if (mpc_new_phys - mpf->physptr) {
-                        struct intel_mp_floating *mpf_new;
+                        struct mpf_intel *mpf_new;
                        /* steal 16 bytes from [0, 1k) */
                        printk(KERN_INFO "mpf new: %x\n", 0x400 - 16);
                        mpf_new = phys_to_virt(0x400 - 16);
                        memcpy(mpf_new, mpf, 16);
                        mpf = mpf_new;
-                        mpf->mpf_physptr = mpc_new_phys;
+                        mpf->physptr = mpc_new_phys;
                }
-                mpf->mpf_checksum = 0;
+                mpf->checksum = 0;
-                mpf->mpf_checksum -= mpf_checksum((unsigned char *)mpf, 16);
+                mpf->checksum -= mpf_checksum((unsigned char *)mpf, 16);
-                printk(KERN_INFO "mpf_physptr new: %x\n", mpf->mpf_physptr);
+                printk(KERN_INFO "physptr new: %x\n", mpf->physptr);
        }
        /*
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 726266695b2c..3cf3413ec626 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -35,10 +35,10 @@
 #include <linux/device.h>
 #include <linux/cpu.h>
 #include <linux/notifier.h>
+#include <linux/uaccess.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/system.h>
 static struct class *msr_class;
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 7228979f1e7f..23b6d9e6e4f5 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -61,11 +61,7 @@ static int endflag __initdata;
 static inline unsigned int get_nmi_count(int cpu)
 {
-#ifdef CONFIG_X86_64
+        return per_cpu(irq_stat, cpu).__nmi_count;
-        return cpu_pda(cpu)->__nmi_count;
-#else
-        return nmi_count(cpu);
-#endif
 }
 static inline int mce_in_progress(void)
@@ -82,12 +78,8 @@ static inline int mce_in_progress(void)
 */
 static inline unsigned int get_timer_irqs(int cpu)
 {
-#ifdef CONFIG_X86_64
-        return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
-#else
        return per_cpu(irq_stat, cpu).apic_timer_irqs +
                per_cpu(irq_stat, cpu).irq0_irqs;
-#endif
 }
 #ifdef CONFIG_SMP
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index a546f55c77b4..2c00a57ccb90 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -66,9 +66,6 @@ asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
-DEFINE_PER_CPU(int, cpu_number);
-EXPORT_PER_CPU_SYMBOL(cpu_number);
 /*
 * Return saved PC of a blocked thread.
 */
@@ -591,7 +588,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        if (prev->gs | next->gs)
                loadsegment(gs, next->gs);
-        x86_write_percpu(current_task, next_p);
+        percpu_write(current_task, next_p);
        return prev_p;
 }
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 416fb9282f4f..c422eebb0c58 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -16,6 +16,7 @@
 #include <stdarg.h>
+#include <linux/stackprotector.h>
 #include <linux/cpu.h>
 #include <linux/errno.h>
 #include <linux/sched.h>
@@ -46,7 +47,6 @@
 #include <asm/processor.h>
 #include <asm/i387.h>
 #include <asm/mmu_context.h>
-#include <asm/pda.h>
 #include <asm/prctl.h>
 #include <asm/desc.h>
 #include <asm/proto.h>
@@ -57,6 +57,12 @@
 asmlinkage extern void ret_from_fork(void);
+DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
+EXPORT_PER_CPU_SYMBOL(current_task);
+DEFINE_PER_CPU(unsigned long, old_rsp);
+static DEFINE_PER_CPU(unsigned char, is_idle);
 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
@@ -75,13 +81,13 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 void enter_idle(void)
 {
-        write_pda(isidle, 1);
+        percpu_write(is_idle, 1);
        atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
 }
 static void __exit_idle(void)
 {
-        if (test_and_clear_bit_pda(0, isidle) == 0)
+        if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
                return;
        atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
 }
@@ -111,6 +117,17 @@ static inline void play_dead(void)
 void cpu_idle(void)
 {
        current_thread_info()->status |= TS_POLLING;
+        /*
+         * If we're the non-boot CPU, nothing set the PDA stack
+         * canary up for us - and if we are the boot CPU we have
+         * a 0 stack canary. This is a good place for updating
+         * it, as we wont ever return from this function (so the
+         * invalid canaries already on the stack wont ever
+         * trigger):
+         */
+        boot_init_stack_canary();
        /* endless idle loop with no priority at all */
        while (1) {
                tick_nohz_stop_sched_tick(1);
@@ -392,7 +409,7 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
        load_gs_index(0);
        regs->ip                = new_ip;
        regs->sp                = new_sp;
-        write_pda(oldrsp, new_sp);
+        percpu_write(old_rsp, new_sp);
        regs->cs                = __USER_CS;
        regs->ss                = __USER_DS;
        regs->flags             = 0x200;
@@ -613,21 +630,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        /*
         * Switch the PDA and FPU contexts.
         */
-        prev->usersp = read_pda(oldrsp);
+        prev->usersp = percpu_read(old_rsp);
-        write_pda(oldrsp, next->usersp);
+        percpu_write(old_rsp, next->usersp);
-        write_pda(pcurrent, next_p);
+        percpu_write(current_task, next_p);
-        write_pda(kernelstack,
+        percpu_write(kernel_stack,
                  (unsigned long)task_stack_page(next_p) +
-                  THREAD_SIZE - PDA_STACKOFFSET);
+                  THREAD_SIZE - KERNEL_STACK_OFFSET);
-#ifdef CONFIG_CC_STACKPROTECTOR
-        write_pda(stack_canary, next_p->stack_canary);
-        /*
-         * Build time only check to make sure the stack_canary is at
-         * offset 40 in the pda; this is a gcc ABI requirement
-         */
-        BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
-#endif
        /*
         * Now maybe reload the debug registers and handle I/O bitmaps
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index 2b46eb41643b..f8536fee5c12 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -14,6 +14,7 @@
 #include <asm/reboot.h>
 #include <asm/pci_x86.h>
 #include <asm/virtext.h>
+#include <asm/cpu.h>
 #ifdef CONFIG_X86_32
 # include <linux/dmi.h>
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index ae0d8042cf69..f41c4486c270 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -89,7 +89,7 @@
 #include <asm/system.h>
 #include <asm/vsyscall.h>
-#include <asm/smp.h>
+#include <asm/cpu.h>
 #include <asm/desc.h>
 #include <asm/dma.h>
 #include <asm/iommu.h>
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 55c46074eba0..90b8e154bb53 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -13,6 +13,23 @@
 #include <asm/mpspec.h>
 #include <asm/apicdef.h>
 #include <asm/highmem.h>
+#include <asm/proto.h>
+#include <asm/cpumask.h>
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+# define DBG(x...) printk(KERN_DEBUG x)
+#else
+# define DBG(x...)
+#endif
+/*
+ * Could be inside CONFIG_HAVE_SETUP_PER_CPU_AREA with other stuff but
+ * voyager wants cpu_number too.
+ */
+#ifdef CONFIG_SMP
+DEFINE_PER_CPU(int, cpu_number);
+EXPORT_PER_CPU_SYMBOL(cpu_number);
+#endif
 #ifdef CONFIG_X86_LOCAL_APIC
 unsigned int num_processors;
@@ -26,31 +43,60 @@ unsigned int max_physical_apicid;
 physid_mask_t phys_cpu_present_map;
 #endif
-/* map cpu index to physical APIC ID */
+/*
+ * Map cpu index to physical APIC ID
+ */
 DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
 DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
 #if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
-#define X86_64_NUMA     1
+#define X86_64_NUMA     1       /* (used later) */
+DEFINE_PER_CPU(int, node_number) = 0;
+EXPORT_PER_CPU_SYMBOL(node_number);
-/* map cpu index to node index */
+/*
+ * Map cpu index to node index
+ */
 DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
-/* which logical CPUs are on which nodes */
+/*
+ * Which logical CPUs are on which nodes
+ */
 cpumask_t *node_to_cpumask_map;
 EXPORT_SYMBOL(node_to_cpumask_map);
-/* setup node_to_cpumask_map */
+/*
+ * Setup node_to_cpumask_map
+ */
 static void __init setup_node_to_cpumask_map(void);
 #else
 static inline void setup_node_to_cpumask_map(void) { }
 #endif
-#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_X86_SMP)
+#ifdef CONFIG_X86_64
+/* correctly size the local cpu masks */
+static void setup_cpu_local_masks(void)
+{
+        alloc_bootmem_cpumask_var(&cpu_initialized_mask);
+        alloc_bootmem_cpumask_var(&cpu_callin_mask);
+        alloc_bootmem_cpumask_var(&cpu_callout_mask);
+        alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
+}
+#else /* CONFIG_X86_32 */
+static inline void setup_cpu_local_masks(void)
+{
+}
+#endif /* CONFIG_X86_32 */
+#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 /*
 * Copy data used in early init routines from the initial arrays to the
 * per cpu data areas.  These arrays then become expendable and the
@@ -79,78 +125,14 @@ static void __init setup_per_cpu_maps(void)
 #endif
 }
-#ifdef CONFIG_X86_32
+#ifdef CONFIG_X86_64
-/*
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
- * Great future not-so-futuristic plan: make i386 and x86_64 do it
+        [0] = (unsigned long)__per_cpu_load,
- * the same way
+};
- */
+#else
 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+#endif
 EXPORT_SYMBOL(__per_cpu_offset);
-static inline void setup_cpu_pda_map(void) { }
-#elif !defined(CONFIG_SMP)
-static inline void setup_cpu_pda_map(void) { }
-#else /* CONFIG_SMP && CONFIG_X86_64 */
-/*
- * Allocate cpu_pda pointer table and array via alloc_bootmem.
- */
-static void __init setup_cpu_pda_map(void)
-{
-        char *pda;
-        struct x8664_pda **new_cpu_pda;
-        unsigned long size;
-        int cpu;
-        size = roundup(sizeof(struct x8664_pda), cache_line_size());
-        /* allocate cpu_pda array and pointer table */
-        {
-                unsigned long tsize = nr_cpu_ids * sizeof(void *);
-                unsigned long asize = size * (nr_cpu_ids - 1);
-                tsize = roundup(tsize, cache_line_size());
-                new_cpu_pda = alloc_bootmem(tsize + asize);
-                pda = (char *)new_cpu_pda + tsize;
-        }
-        /* initialize pointer table to static pda's */
-        for_each_possible_cpu(cpu) {
-                if (cpu == 0) {
-                        /* leave boot cpu pda in place */
-                        new_cpu_pda[0] = cpu_pda(0);
-                        continue;
-                }
-                new_cpu_pda[cpu] = (struct x8664_pda *)pda;
-                new_cpu_pda[cpu]->in_bootmem = 1;
-                pda += size;
-        }
-        /* point to new pointer table */
-        _cpu_pda = new_cpu_pda;
-}
-#endif /* CONFIG_SMP && CONFIG_X86_64 */
-#ifdef CONFIG_X86_64
-/* correctly size the local cpu masks */
-static void setup_cpu_local_masks(void)
-{
-        alloc_bootmem_cpumask_var(&cpu_initialized_mask);
-        alloc_bootmem_cpumask_var(&cpu_callin_mask);
-        alloc_bootmem_cpumask_var(&cpu_callout_mask);
-        alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
-}
-#else /* CONFIG_X86_32 */
-static inline void setup_cpu_local_masks(void)
-{
-}
-#endif /* CONFIG_X86_32 */
 /*
 * Great future plan:
@@ -164,9 +146,6 @@ void __init setup_per_cpu_areas(void)
        int cpu;
        unsigned long align = 1;
-        /* Setup cpu_pda map */
-        setup_cpu_pda_map();
        /* Copy section for each CPU (we discard the original) */
        old_size = PERCPU_ENOUGH_ROOM;
        align = max_t(unsigned long, PAGE_SIZE, align);
@@ -197,8 +176,23 @@ void __init setup_per_cpu_areas(void)
                                cpu, node, __pa(ptr));
                }
 #endif
+                memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start);
                per_cpu_offset(cpu) = ptr - __per_cpu_start;
-                memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+                per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
+                per_cpu(cpu_number, cpu) = cpu;
+#ifdef CONFIG_X86_64
+                per_cpu(irq_stack_ptr, cpu) =
+                        per_cpu(irq_stack_union.irq_stack, cpu) + IRQ_STACK_SIZE - 64;
+                /*
+                 * Up to this point, CPU0 has been using .data.init
+                 * area.  Reload %gs offset for CPU0.
+                 */
+                if (cpu == 0)
+                        load_gs_base(cpu);
+#endif
+                DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
        }
        /* Setup percpu data maps */
@@ -220,6 +214,7 @@ void __init setup_per_cpu_areas(void)
 * Requires node_possible_map to be valid.
 *
 * Note: node_to_cpumask() is not valid until after this is done.
+ * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
 */
 static void __init setup_node_to_cpumask_map(void)
 {
@@ -235,6 +230,7 @@ static void __init setup_node_to_cpumask_map(void)
        /* allocate the map */
        map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
+        DBG("node_to_cpumask_map at %p for %d nodes\n", map, nr_node_ids);
        pr_debug("Node to cpumask map at %p for %d nodes\n",
                 map, nr_node_ids);
@@ -247,17 +243,23 @@ void __cpuinit numa_set_node(int cpu, int node)
 {
        int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
-        if (cpu_pda(cpu) && node != NUMA_NO_NODE)
+        /* early setting, no percpu area yet */
-                cpu_pda(cpu)->nodenumber = node;
+        if (cpu_to_node_map) {
-        if (cpu_to_node_map)
                cpu_to_node_map[cpu] = node;
+                return;
+        }
-        else if (per_cpu_offset(cpu))
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
-                per_cpu(x86_cpu_to_node_map, cpu) = node;
+        if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
+                printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
+                dump_stack();
+                return;
+        }
+#endif
+        per_cpu(x86_cpu_to_node_map, cpu) = node;
-        else
+        if (node != NUMA_NO_NODE)
-                pr_debug("Setting node for non-present cpu %d\n", cpu);
+                per_cpu(node_number, cpu) = node;
 }
 void __cpuinit numa_clear_node(int cpu)
@@ -274,7 +276,7 @@ void __cpuinit numa_add_cpu(int cpu)
 void __cpuinit numa_remove_cpu(int cpu)
 {
-        cpu_clear(cpu, node_to_cpumask_map[cpu_to_node(cpu)]);
+        cpu_clear(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
 }
 #else /* CONFIG_DEBUG_PER_CPU_MAPS */
@@ -284,7 +286,7 @@ void __cpuinit numa_remove_cpu(int cpu)
 */
 static void __cpuinit numa_set_cpumask(int cpu, int enable)
 {
-        int node = cpu_to_node(cpu);
+        int node = early_cpu_to_node(cpu);
        cpumask_t *mask;
        char buf[64];
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index bb1a3b1fc87f..def770b57b5a 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -53,7 +53,6 @@
 #include <asm/nmi.h>
 #include <asm/irq.h>
 #include <asm/idle.h>
-#include <asm/smp.h>
 #include <asm/trampoline.h>
 #include <asm/cpu.h>
 #include <asm/numa.h>
@@ -63,6 +62,7 @@
 #include <asm/vmi.h>
 #include <asm/genapic.h>
 #include <asm/setup.h>
+#include <asm/uv/uv.h>
 #include <linux/mc146818rtc.h>
 #include <mach_apic.h>
@@ -745,52 +745,6 @@ static void __cpuinit do_fork_idle(struct work_struct *work)
        complete(&c_idle->done);
 }
-#ifdef CONFIG_X86_64
-/* __ref because it's safe to call free_bootmem when after_bootmem == 0. */
-static void __ref free_bootmem_pda(struct x8664_pda *oldpda)
-{
-        if (!after_bootmem)
-                free_bootmem((unsigned long)oldpda, sizeof(*oldpda));
-}
-/*
- * Allocate node local memory for the AP pda.
- *
- * Must be called after the _cpu_pda pointer table is initialized.
- */
-int __cpuinit get_local_pda(int cpu)
-{
-        struct x8664_pda *oldpda, *newpda;
-        unsigned long size = sizeof(struct x8664_pda);
-        int node = cpu_to_node(cpu);
-        if (cpu_pda(cpu) && !cpu_pda(cpu)->in_bootmem)
-                return 0;
-        oldpda = cpu_pda(cpu);
-        newpda = kmalloc_node(size, GFP_ATOMIC, node);
-        if (!newpda) {
-                printk(KERN_ERR "Could not allocate node local PDA "
-                        "for CPU %d on node %d\n", cpu, node);
-                if (oldpda)
-                        return 0;       /* have a usable pda */
-                else
-                        return -1;
-        }
-        if (oldpda) {
-                memcpy(newpda, oldpda, size);
-                free_bootmem_pda(oldpda);
-        }
-        newpda->in_bootmem = 0;
-        cpu_pda(cpu) = newpda;
-        return 0;
-}
-#endif /* CONFIG_X86_64 */
 static int __cpuinit do_boot_cpu(int apicid, int cpu)
 /*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
@@ -808,16 +762,6 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
        };
        INIT_WORK(&c_idle.work, do_fork_idle);
-#ifdef CONFIG_X86_64
-        /* Allocate node local memory for AP pdas */
-        if (cpu > 0) {
-                boot_error = get_local_pda(cpu);
-                if (boot_error)
-                        goto restore_state;
-                        /* if can't get pda memory, can't start cpu */
-        }
-#endif
        alternatives_smp_switch(1);
        c_idle.idle = get_idle_for_cpu(cpu);
@@ -847,14 +791,17 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
        set_idle_for_cpu(cpu, c_idle.idle);
 do_rest:
-#ifdef CONFIG_X86_32
        per_cpu(current_task, cpu) = c_idle.idle;
+#ifdef CONFIG_X86_32
        init_gdt(cpu);
        /* Stack for startup_32 can be just as for start_secondary onwards */
        irq_ctx_init(cpu);
 #else
-        cpu_pda(cpu)->pcurrent = c_idle.idle;
        clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
+        initial_gs = per_cpu_offset(cpu);
+        per_cpu(kernel_stack, cpu) =
+                (unsigned long)task_stack_page(c_idle.idle) -
+                KERNEL_STACK_OFFSET + THREAD_SIZE;
 #endif
        early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
        initial_code = (unsigned long)start_secondary;
@@ -931,9 +878,7 @@ do_rest:
                                inquire_remote_apic(apicid);
                }
        }
-#ifdef CONFIG_X86_64
-restore_state:
-#endif
        if (boot_error) {
                /* Try to put things back the way they were before ... */
                numa_remove_cpu(cpu); /* was set by numa_add_cpu */
@@ -1125,6 +1070,7 @@ static int __init smp_sanity_check(unsigned max_cpus)
                printk(KERN_ERR "... forcing use of dummy APIC emulation."
                                "(tell your hw vendor)\n");
                smpboot_clear_io_apic();
+                disable_ioapic_setup();
                return -1;
        }
diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c
index 397e309839dd..add36b4e37c9 100644
--- a/arch/x86/kernel/smpcommon.c
+++ b/arch/x86/kernel/smpcommon.c
@@ -3,11 +3,16 @@
 */
 #include <linux/module.h>
 #include <asm/smp.h>
+#include <asm/sections.h>
-#ifdef CONFIG_X86_32
+#ifdef CONFIG_X86_64
+DEFINE_PER_CPU(unsigned long, this_cpu_off) = (unsigned long)__per_cpu_load;
+#else
 DEFINE_PER_CPU(unsigned long, this_cpu_off);
+#endif
 EXPORT_PER_CPU_SYMBOL(this_cpu_off);
+#ifdef CONFIG_X86_32
 /*
 * Initialize the CPU's GDT.  This is either the boot CPU doing itself
 * (still using the master per-cpu area), or a CPU doing it for a
@@ -23,8 +28,5 @@ __cpuinit void init_gdt(int cpu)
        write_gdt_entry(get_cpu_gdt_table(cpu),
                        GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
-        per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
-        per_cpu(cpu_number, cpu) = cpu;
 }
 #endif
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb.c
index 7f4141d3b661..b3ca1b940654 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/kernel/tlb.c
@@ -1,22 +1,18 @@
 #include <linux/init.h>
 #include <linux/mm.h>
-#include <linux/delay.h>
 #include <linux/spinlock.h>
 #include <linux/smp.h>
-#include <linux/kernel_stat.h>
-#include <linux/mc146818rtc.h>
 #include <linux/interrupt.h>
+#include <linux/module.h>
-#include <asm/mtrr.h>
-#include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
-#include <asm/proto.h>
+#include <asm/apic.h>
-#include <asm/apicdef.h>
+#include <asm/uv/uv.h>
-#include <asm/idle.h>
-#include <asm/uv/uv_hub.h>
+DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
-#include <asm/uv/uv_bau.h>
+                        = { &init_mm, 0, };
 #include <mach_ipi.h>
 /*
@@ -62,9 +58,9 @@ static DEFINE_PER_CPU(union smp_flush_state, flush_state);
 */
 void leave_mm(int cpu)
 {
-        if (read_pda(mmu_state) == TLBSTATE_OK)
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                BUG();
-        cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
+        cpu_clear(cpu, percpu_read(cpu_tlbstate.active_mm)->cpu_vm_mask);
        load_cr3(swapper_pg_dir);
 }
 EXPORT_SYMBOL_GPL(leave_mm);
@@ -117,10 +113,20 @@ EXPORT_SYMBOL_GPL(leave_mm);
 * Interrupts are disabled.
 */
-asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
+/*
+ * FIXME: use of asmlinkage is not consistent.  On x86_64 it's noop
+ * but still used for documentation purpose but the usage is slightly
+ * inconsistent.  On x86_32, asmlinkage is regparm(0) but interrupt
+ * entry calls in with the first parameter in %eax.  Maybe define
+ * intrlinkage?
+ */
+#ifdef CONFIG_X86_64
+asmlinkage
+#endif
+void smp_invalidate_interrupt(struct pt_regs *regs)
 {
-        int cpu;
+        unsigned int cpu;
-        int sender;
+        unsigned int sender;
        union smp_flush_state *f;
        cpu = smp_processor_id();
@@ -142,8 +148,8 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
                 * BUG();
                 */
-        if (f->flush_mm == read_pda(active_mm)) {
+        if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
-                if (read_pda(mmu_state) == TLBSTATE_OK) {
+                if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
                        if (f->flush_va == TLB_FLUSH_ALL)
                                local_flush_tlb();
                        else
@@ -153,14 +159,16 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
        }
 out:
        ack_APIC_irq();
+        smp_mb__before_clear_bit();
        cpumask_clear_cpu(cpu, to_cpumask(f->flush_cpumask));
+        smp_mb__after_clear_bit();
        inc_irq_stat(irq_tlb_count);
 }
 static void flush_tlb_others_ipi(const struct cpumask *cpumask,
                                 struct mm_struct *mm, unsigned long va)
 {
-        int sender;
+        unsigned int sender;
        union smp_flush_state *f;
        /* Caller has disabled preemption */
@@ -203,16 +211,13 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
                             struct mm_struct *mm, unsigned long va)
 {
        if (is_uv_system()) {
-                /* FIXME: could be an percpu_alloc'd thing */
+                unsigned int cpu;
-                static DEFINE_PER_CPU(cpumask_t, flush_tlb_mask);
-                struct cpumask *after_uv_flush = &get_cpu_var(flush_tlb_mask);
-                cpumask_andnot(after_uv_flush, cpumask,
-                               cpumask_of(smp_processor_id()));
-                if (!uv_flush_tlb_others(after_uv_flush, mm, va))
-                        flush_tlb_others_ipi(after_uv_flush, mm, va);
-                put_cpu_var(flush_tlb_uv_cpumask);
+                cpu = get_cpu();
+                cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu);
+                if (cpumask)
+                        flush_tlb_others_ipi(cpumask, mm, va);
+                put_cpu();
                return;
        }
        flush_tlb_others_ipi(cpumask, mm, va);
@@ -281,7 +286,7 @@ static void do_flush_tlb_all(void *info)
        unsigned long cpu = smp_processor_id();
        __flush_tlb_all();
-        if (read_pda(mmu_state) == TLBSTATE_LAZY)
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
                leave_mm(cpu);
 }
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
deleted file mode 100644
index d37bbfcb813d..000000000000
--- a/arch/x86/kernel/tlb_32.c
+++ /dev/null
@@ -1,250 +0,0 @@
-#include <linux/spinlock.h>
-#include <linux/cpu.h>
-#include <linux/interrupt.h>
-#include <asm/tlbflush.h>
-DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
-                        ____cacheline_aligned = { &init_mm, 0, };
-/* must come after the send_IPI functions above for inlining */
-#include <mach_ipi.h>
-/*
- *      Smarter SMP flushing macros.
- *              c/o Linus Torvalds.
- *
- *      These mean you can really definitely utterly forget about
- *      writing to user space from interrupts. (Its not allowed anyway).
- *
- *      Optimizations Manfred Spraul <manfred@colorfullife.com>
- */
-static cpumask_var_t flush_cpumask;
-static struct mm_struct *flush_mm;
-static unsigned long flush_va;
-static DEFINE_SPINLOCK(tlbstate_lock);
-/*
- * We cannot call mmdrop() because we are in interrupt context,
- * instead update mm->cpu_vm_mask.
- *
- * We need to reload %cr3 since the page tables may be going
- * away from under us..
- */
-void leave_mm(int cpu)
-{
-        BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK);
-        cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask);
-        load_cr3(swapper_pg_dir);
-}
-EXPORT_SYMBOL_GPL(leave_mm);
-/*
- *
- * The flush IPI assumes that a thread switch happens in this order:
- * [cpu0: the cpu that switches]
- * 1) switch_mm() either 1a) or 1b)
- * 1a) thread switch to a different mm
- * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
- *      Stop ipi delivery for the old mm. This is not synchronized with
- *      the other cpus, but smp_invalidate_interrupt ignore flush ipis
- *      for the wrong mm, and in the worst case we perform a superfluous
- *      tlb flush.
- * 1a2) set cpu_tlbstate to TLBSTATE_OK
- *      Now the smp_invalidate_interrupt won't call leave_mm if cpu0
- *      was in lazy tlb mode.
- * 1a3) update cpu_tlbstate[].active_mm
- *      Now cpu0 accepts tlb flushes for the new mm.
- * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
- *      Now the other cpus will send tlb flush ipis.
- * 1a4) change cr3.
- * 1b) thread switch without mm change
- *      cpu_tlbstate[].active_mm is correct, cpu0 already handles
- *      flush ipis.
- * 1b1) set cpu_tlbstate to TLBSTATE_OK
- * 1b2) test_and_set the cpu bit in cpu_vm_mask.
- *      Atomically set the bit [other cpus will start sending flush ipis],
- *      and test the bit.
- * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
- * 2) switch %%esp, ie current
- *
- * The interrupt must handle 2 special cases:
- * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
- * - the cpu performs speculative tlb reads, i.e. even if the cpu only
- *   runs in kernel space, the cpu could load tlb entries for user space
- *   pages.
- *
- * The good news is that cpu_tlbstate is local to each cpu, no
- * write/read ordering problems.
- */
-/*
- * TLB flush IPI:
- *
- * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
- * 2) Leave the mm if we are in the lazy tlb mode.
- */
-void smp_invalidate_interrupt(struct pt_regs *regs)
-{
-        unsigned long cpu;
-        cpu = get_cpu();
-        if (!cpumask_test_cpu(cpu, flush_cpumask))
-                goto out;
-                /*
-                 * This was a BUG() but until someone can quote me the
-                 * line from the intel manual that guarantees an IPI to
-                 * multiple CPUs is retried _only_ on the erroring CPUs
-                 * its staying as a return
-                 *
-                 * BUG();
-                 */
-        if (flush_mm == x86_read_percpu(cpu_tlbstate.active_mm)) {
-                if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK) {
-                        if (flush_va == TLB_FLUSH_ALL)
-                                local_flush_tlb();
-                        else
-                                __flush_tlb_one(flush_va);
-                } else
-                        leave_mm(cpu);
-        }
-        ack_APIC_irq();
-        smp_mb__before_clear_bit();
-        cpumask_clear_cpu(cpu, flush_cpumask);
-        smp_mb__after_clear_bit();
-out:
-        put_cpu_no_resched();
-        inc_irq_stat(irq_tlb_count);
-}
-void native_flush_tlb_others(const struct cpumask *cpumask,
-                             struct mm_struct *mm, unsigned long va)
-{
-        /*
-         * mm must exist :)
-         */
-        BUG_ON(!mm);
-        /*
-         * i'm not happy about this global shared spinlock in the
-         * MM hot path, but we'll see how contended it is.
-         * AK: x86-64 has a faster method that could be ported.
-         */
-        spin_lock(&tlbstate_lock);
-        cpumask_andnot(flush_cpumask, cpumask, cpumask_of(smp_processor_id()));
-        cpumask_and(flush_cpumask, flush_cpumask, cpu_online_mask);
-        /*
-         * If a task whose mm mask we are looking at has descheduled and
-         * has cleared its presence from the mask, or if a CPU which we ran
-         * on has gone down then there might be no flush work left:
-         */
-        if (unlikely(cpumask_empty(flush_cpumask))) {
-                spin_unlock(&tlbstate_lock);
-                return;
-        }
-        flush_mm = mm;
-        flush_va = va;
-        /*
-         * Make the above memory operations globally visible before
-         * sending the IPI.
-         */
-        smp_mb();
-        /*
-         * We have to send the IPI only to
-         * CPUs affected.
-         */
-        send_IPI_mask(flush_cpumask, INVALIDATE_TLB_VECTOR);
-        while (!cpumask_empty(flush_cpumask))
-                /* nothing. lockup detection does not belong here */
-                cpu_relax();
-        flush_mm = NULL;
-        flush_va = 0;
-        spin_unlock(&tlbstate_lock);
-}
-void flush_tlb_current_task(void)
-{
-        struct mm_struct *mm = current->mm;
-        preempt_disable();
-        local_flush_tlb();
-        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
-        preempt_enable();
-}
-void flush_tlb_mm(struct mm_struct *mm)
-{
-        preempt_disable();
-        if (current->active_mm == mm) {
-                if (current->mm)
-                        local_flush_tlb();
-                else
-                        leave_mm(smp_processor_id());
-        }
-        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
-        preempt_enable();
-}
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
-{
-        struct mm_struct *mm = vma->vm_mm;
-        preempt_disable();
-        if (current->active_mm == mm) {
-                if (current->mm)
-                        __flush_tlb_one(va);
-                 else
-                        leave_mm(smp_processor_id());
-        }
-        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(&mm->cpu_vm_mask, mm, va);
-        preempt_enable();
-}
-EXPORT_SYMBOL(flush_tlb_page);
-static void do_flush_tlb_all(void *info)
-{
-        unsigned long cpu = smp_processor_id();
-        __flush_tlb_all();
-        if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_LAZY)
-                leave_mm(cpu);
-}
-void flush_tlb_all(void)
-{
-        on_each_cpu(do_flush_tlb_all, NULL, 1);
-}
-void reset_lazy_tlbstate(void)
-{
-        int cpu = raw_smp_processor_id();
-        per_cpu(cpu_tlbstate, cpu).state = 0;
-        per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
-}
-static int init_flush_cpumask(void)
-{
-        alloc_cpumask_var(&flush_cpumask, GFP_KERNEL);
-        return 0;
-}
-early_initcall(init_flush_cpumask);
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index 690dcf1a27d4..aae15dd72604 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <asm/mmu_context.h>
+#include <asm/uv/uv.h>
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
 #include <asm/uv/uv_bau.h>
@@ -209,14 +210,15 @@ static int uv_wait_completion(struct bau_desc *bau_desc,
 *
 * Send a broadcast and wait for a broadcast message to complete.
 *
- * The cpumaskp mask contains the cpus the broadcast was sent to.
+ * The flush_mask contains the cpus the broadcast was sent to.
 *
- * Returns 1 if all remote flushing was done. The mask is zeroed.
+ * Returns NULL if all remote flushing was done. The mask is zeroed.
- * Returns 0 if some remote flushing remains to be done. The mask will have
+ * Returns @flush_mask if some remote flushing remains to be done. The
- * some bits still set.
+ * mask will have some bits still set.
 */
-int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
+const struct cpumask *uv_flush_send_and_wait(int cpu, int this_blade,
-                           struct cpumask *cpumaskp)
+                                             struct bau_desc *bau_desc,
+                                             struct cpumask *flush_mask)
 {
        int completion_status = 0;
        int right_shift;
@@ -263,59 +265,69 @@ int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
         * Success, so clear the remote cpu's from the mask so we don't
         * use the IPI method of shootdown on them.
         */
-        for_each_cpu(bit, cpumaskp) {
+        for_each_cpu(bit, flush_mask) {
                blade = uv_cpu_to_blade_id(bit);
                if (blade == this_blade)
                        continue;
-                cpumask_clear_cpu(bit, cpumaskp);
+                cpumask_clear_cpu(bit, flush_mask);
        }
-        if (!cpumask_empty(cpumaskp))
+        if (!cpumask_empty(flush_mask))
-                return 0;
+                return flush_mask;
-        return 1;
+        return NULL;
 }
 /**
 * uv_flush_tlb_others - globally purge translation cache of a virtual
 * address or all TLB's
- * @cpumaskp: mask of all cpu's in which the address is to be removed
+ * @cpumask: mask of all cpu's in which the address is to be removed
 * @mm: mm_struct containing virtual address range
 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
+ * @cpu: the current cpu
 *
 * This is the entry point for initiating any UV global TLB shootdown.
 *
 * Purges the translation caches of all specified processors of the given
 * virtual address, or purges all TLB's on specified processors.
 *
- * The caller has derived the cpumaskp from the mm_struct and has subtracted
+ * The caller has derived the cpumask from the mm_struct.  This function
- * the local cpu from the mask.  This function is called only if there
+ * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
- * are bits set in the mask. (e.g. flush_tlb_page())
 *
- * The cpumaskp is converted into a nodemask of the nodes containing
+ * The cpumask is converted into a nodemask of the nodes containing
 * the cpus.
 *
- * Returns 1 if all remote flushing was done.
+ * Note that this function should be called with preemption disabled.
- * Returns 0 if some remote flushing remains to be done.
+ *
+ * Returns NULL if all remote flushing was done.
+ * Returns pointer to cpumask if some remote flushing remains to be
+ * done.  The returned pointer is valid till preemption is re-enabled.
 */
-int uv_flush_tlb_others(struct cpumask *cpumaskp, struct mm_struct *mm,
+const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
-                        unsigned long va)
+                                          struct mm_struct *mm,
+                                          unsigned long va, unsigned int cpu)
 {
+        static DEFINE_PER_CPU(cpumask_t, flush_tlb_mask);
+        struct cpumask *flush_mask = &__get_cpu_var(flush_tlb_mask);
        int i;
        int bit;
        int blade;
-        int cpu;
+        int uv_cpu;
        int this_blade;
        int locals = 0;
        struct bau_desc *bau_desc;
-        cpu = uv_blade_processor_id();
+        WARN_ON(!in_atomic());
+        cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));
+        uv_cpu = uv_blade_processor_id();
        this_blade = uv_numa_blade_id();
        bau_desc = __get_cpu_var(bau_control).descriptor_base;
-        bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu;
+        bau_desc += UV_ITEMS_PER_DESCRIPTOR * uv_cpu;
        bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
        i = 0;
-        for_each_cpu(bit, cpumaskp) {
+        for_each_cpu(bit, flush_mask) {
                blade = uv_cpu_to_blade_id(bit);
                BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
                if (blade == this_blade) {
@@ -330,17 +342,17 @@ int uv_flush_tlb_others(struct cpumask *cpumaskp, struct mm_struct *mm,
                 * no off_node flushing; return status for local node
                 */
                if (locals)
-                        return 0;
+                        return flush_mask;
                else
-                        return 1;
+                        return NULL;
        }
        __get_cpu_var(ptcstats).requestor++;
        __get_cpu_var(ptcstats).ntargeted += i;
        bau_desc->payload.address = va;
-        bau_desc->payload.sending_cpu = smp_processor_id();
+        bau_desc->payload.sending_cpu = cpu;
-        return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp);
+        return uv_flush_send_and_wait(uv_cpu, this_blade, bau_desc, flush_mask);
 }
 /*
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 98c2d055284b..ed5aee5f3fcc 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -59,7 +59,6 @@
 #ifdef CONFIG_X86_64
 #include <asm/pgalloc.h>
 #include <asm/proto.h>
-#include <asm/pda.h>
 #else
 #include <asm/processor-flags.h>
 #include <asm/arch_hooks.h>
diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S
index 82c67559dde7..3eba7f7bac05 100644
--- a/arch/x86/kernel/vmlinux_32.lds.S
+++ b/arch/x86/kernel/vmlinux_32.lds.S
@@ -178,14 +178,7 @@ SECTIONS
        __initramfs_end = .;
  }
 #endif
-  . = ALIGN(PAGE_SIZE);
+  PERCPU(PAGE_SIZE)
-  .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
-        __per_cpu_start = .;
-        *(.data.percpu.page_aligned)
-        *(.data.percpu)
-        *(.data.percpu.shared_aligned)
-        __per_cpu_end = .;
-  }
  . = ALIGN(PAGE_SIZE);
  /* freed after init ends here */
diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S
index 1a614c0e6bef..c9740996430a 100644
--- a/arch/x86/kernel/vmlinux_64.lds.S
+++ b/arch/x86/kernel/vmlinux_64.lds.S
@@ -5,6 +5,7 @@
 #define LOAD_OFFSET __START_KERNEL_map
 #include <asm-generic/vmlinux.lds.h>
+#include <asm/asm-offsets.h>
 #include <asm/page.h>
 #undef i386     /* in case the preprocessor is a 32bit one */
@@ -13,12 +14,14 @@ OUTPUT_FORMAT("elf64-x86-64", "elf64-x86-64", "elf64-x86-64")
 OUTPUT_ARCH(i386:x86-64)
 ENTRY(phys_startup_64)
 jiffies_64 = jiffies;
-_proxy_pda = 1;
 PHDRS {
        text PT_LOAD FLAGS(5);  /* R_E */
        data PT_LOAD FLAGS(7);  /* RWE */
        user PT_LOAD FLAGS(7);  /* RWE */
        data.init PT_LOAD FLAGS(7);     /* RWE */
+#ifdef CONFIG_SMP
+        percpu PT_LOAD FLAGS(7);        /* RWE */
+#endif
        note PT_NOTE FLAGS(0);  /* ___ */
 }
 SECTIONS
@@ -208,14 +211,28 @@ SECTIONS
  __initramfs_end = .;
 #endif
+#ifdef CONFIG_SMP
+  /*
+   * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
+   * output PHDR, so the next output section - __data_nosave - should
+   * switch it back to data.init.  Also, pda should be at the head of
+   * percpu area.  Preallocate it and define the percpu offset symbol
+   * so that it can be accessed as a percpu variable.
+   */
+  . = ALIGN(PAGE_SIZE);
+  PERCPU_VADDR(0, :percpu)
+#else
  PERCPU(PAGE_SIZE)
+#endif
  . = ALIGN(PAGE_SIZE);
  __init_end = .;
  . = ALIGN(PAGE_SIZE);
  __nosave_begin = .;
-  .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) { *(.data.nosave) }
+  .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
+      *(.data.nosave)
+  } :data.init  /* switch back to data.init, see PERCPU_VADDR() above */
  . = ALIGN(PAGE_SIZE);
  __nosave_end = .;
@@ -244,3 +261,8 @@ SECTIONS
 */
 ASSERT((_end - _text <= KERNEL_IMAGE_SIZE),
        "kernel image bigger than KERNEL_IMAGE_SIZE")
+#ifdef CONFIG_SMP
+ASSERT((per_cpu__irq_stack_union == 0),
+        "irq_stack_union is not at start of per-cpu area");
+#endif
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 695e426aa354..3909e3ba5ce3 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -58,5 +58,3 @@ EXPORT_SYMBOL(__memcpy);
 EXPORT_SYMBOL(empty_zero_page);
 EXPORT_SYMBOL(init_level4_pgt);
 EXPORT_SYMBOL(load_gs_index);
-EXPORT_SYMBOL(_proxy_pda);
diff --git a/arch/x86/mach-voyager/setup.c b/arch/x86/mach-voyager/setup.c
index a580b9562e76..0ade62555ff3 100644
--- a/arch/x86/mach-voyager/setup.c
+++ b/arch/x86/mach-voyager/setup.c
@@ -9,6 +9,7 @@
 #include <asm/e820.h>
 #include <asm/io.h>
 #include <asm/setup.h>
+#include <asm/cpu.h>
 void __init pre_intr_init_hook(void)
 {
diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c
index 9840b7ec749a..96f15b09a4c5 100644
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -402,7 +402,7 @@ void __init find_smp_config(void)
             VOYAGER_SUS_IN_CONTROL_PORT);
        current_thread_info()->cpu = boot_cpu_id;
-        x86_write_percpu(cpu_number, boot_cpu_id);
+        percpu_write(cpu_number, boot_cpu_id);
 }
 /*
@@ -531,6 +531,7 @@ static void __init do_boot_cpu(__u8 cpu)
        stack_start.sp = (void *)idle->thread.sp;
        init_gdt(cpu);
+        per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
        per_cpu(current_task, cpu) = idle;
        early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
        irq_ctx_init(cpu);
@@ -1748,6 +1749,7 @@ static void __init voyager_smp_prepare_cpus(unsigned int max_cpus)
 static void __cpuinit voyager_smp_prepare_boot_cpu(void)
 {
        init_gdt(smp_processor_id());
+        per_cpu(this_cpu_off, cpu) = __per_cpu_offset[cpu];
        switch_to_new_gdt();
        cpu_set(smp_processor_id(), cpu_online_map);
@@ -1780,7 +1782,7 @@ static void __init voyager_smp_cpus_done(unsigned int max_cpus)
 void __init smp_setup_processor_id(void)
 {
        current_thread_info()->cpu = hard_smp_processor_id();
-        x86_write_percpu(cpu_number, hard_smp_processor_id());
+        percpu_write(cpu_number, hard_smp_processor_id());
 }
 static void voyager_send_call_func(cpumask_t callmask)
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 90dfae511a41..37242c405f16 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -26,6 +26,7 @@
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/kdebug.h>
+#include <linux/magic.h>
 #include <asm/system.h>
 #include <asm/desc.h>
@@ -589,6 +590,8 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
        unsigned long address;
        int write, si_code;
        int fault;
+        unsigned long *stackend;
 #ifdef CONFIG_X86_64
        unsigned long flags;
        int sig;
@@ -841,6 +844,10 @@ no_context:
        show_fault_oops(regs, error_code, address);
+        stackend = end_of_stack(tsk);
+        if (*stackend != STACK_END_MAGIC)
+                printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
        tsk->thread.cr2 = address;
        tsk->thread.trap_no = 14;
        tsk->thread.error_code = error_code;
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 88f1b10de3be..4a6989e47a53 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -49,7 +49,6 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 #include <asm/cacheflush.h>
-#include <asm/smp.h>
 unsigned int __VMALLOC_RESERVE = 128 << 20;
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 160c42d3eb8f..3be399013de6 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -333,11 +333,20 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                                              req_type & _PAGE_CACHE_MASK);
        }
-        is_range_ram = pagerange_is_ram(start, end);
+        /*
-        if (is_range_ram == 1)
+         * For legacy reasons, some parts of the physical address range in the
-                return reserve_ram_pages_type(start, end, req_type, new_type);
+         * legacy 1MB region is treated as non-RAM (even when listed as RAM in
-        else if (is_range_ram < 0)
+         * the e820 tables).  So we will track the memory attributes of this
-                return -EINVAL;
+         * legacy 1MB region using the linear memtype_list always.
+         */
+        if (end >= ISA_END_ADDRESS) {
+                is_range_ram = pagerange_is_ram(start, end);
+                if (is_range_ram == 1)
+                        return reserve_ram_pages_type(start, end, req_type,
+                                                      new_type);
+                else if (is_range_ram < 0)
+                        return -EINVAL;
+        }
        new  = kmalloc(sizeof(struct memtype), GFP_KERNEL);
        if (!new)
@@ -437,11 +446,19 @@ int free_memtype(u64 start, u64 end)
        if (is_ISA_range(start, end - 1))
                return 0;
-        is_range_ram = pagerange_is_ram(start, end);
+        /*
-        if (is_range_ram == 1)
+         * For legacy reasons, some parts of the physical address range in the
-                return free_ram_pages_type(start, end);
+         * legacy 1MB region is treated as non-RAM (even when listed as RAM in
-        else if (is_range_ram < 0)
+         * the e820 tables).  So we will track the memory attributes of this
-                return -EINVAL;
+         * legacy 1MB region using the linear memtype_list always.
+         */
+        if (end >= ISA_END_ADDRESS) {
+                is_range_ram = pagerange_is_ram(start, end);
+                if (is_range_ram == 1)
+                        return free_ram_pages_type(start, end);
+                else if (is_range_ram < 0)
+                        return -EINVAL;
+        }
        spin_lock(&memtype_lock);
        list_for_each_entry(entry, &memtype_list, nd) {
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 965539ec425f..bef941f61451 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -695,17 +695,17 @@ static void xen_write_cr0(unsigned long cr0)
 static void xen_write_cr2(unsigned long cr2)
 {
-        x86_read_percpu(xen_vcpu)->arch.cr2 = cr2;
+        percpu_read(xen_vcpu)->arch.cr2 = cr2;
 }
 static unsigned long xen_read_cr2(void)
 {
-        return x86_read_percpu(xen_vcpu)->arch.cr2;
+        return percpu_read(xen_vcpu)->arch.cr2;
 }
 static unsigned long xen_read_cr2_direct(void)
 {
-        return x86_read_percpu(xen_vcpu_info.arch.cr2);
+        return percpu_read(xen_vcpu_info.arch.cr2);
 }
 static void xen_write_cr4(unsigned long cr4)
@@ -718,12 +718,12 @@ static void xen_write_cr4(unsigned long cr4)
 static unsigned long xen_read_cr3(void)
 {
-        return x86_read_percpu(xen_cr3);
+        return percpu_read(xen_cr3);
 }
 static void set_current_cr3(void *v)
 {
-        x86_write_percpu(xen_current_cr3, (unsigned long)v);
+        percpu_write(xen_current_cr3, (unsigned long)v);
 }
 static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -748,7 +748,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
        MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
        if (kernel) {
-                x86_write_percpu(xen_cr3, cr3);
+                percpu_write(xen_cr3, cr3);
                /* Update xen_current_cr3 once the batch has actually
                   been submitted. */
@@ -764,7 +764,7 @@ static void xen_write_cr3(unsigned long cr3)
        /* Update while interrupts are disabled, so its atomic with
           respect to ipis */
-        x86_write_percpu(xen_cr3, cr3);
+        percpu_write(xen_cr3, cr3);
        __xen_write_cr3(true, cr3);
@@ -1645,7 +1645,6 @@ asmlinkage void __init xen_start_kernel(void)
 #ifdef CONFIG_X86_64
        /* Disable until direct per-cpu data access. */
        have_vcpu_info_placement = 0;
-        x86_64_init_pda();
 #endif
        xen_smp_init();
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index bb042608c602..2e8271431e1a 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -39,7 +39,7 @@ static unsigned long xen_save_fl(void)
        struct vcpu_info *vcpu;
        unsigned long flags;
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
        /* flag has opposite sense of mask */
        flags = !vcpu->evtchn_upcall_mask;
@@ -62,7 +62,7 @@ static void xen_restore_fl(unsigned long flags)
           make sure we're don't switch CPUs between getting the vcpu
           pointer and updating the mask. */
        preempt_disable();
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
        vcpu->evtchn_upcall_mask = flags;
        preempt_enable_no_resched();
@@ -83,7 +83,7 @@ static void xen_irq_disable(void)
           make sure we're don't switch CPUs between getting the vcpu
           pointer and updating the mask. */
        preempt_disable();
-        x86_read_percpu(xen_vcpu)->evtchn_upcall_mask = 1;
+        percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
        preempt_enable_no_resched();
 }
@@ -96,7 +96,7 @@ static void xen_irq_enable(void)
           the caller is confused and is trying to re-enable interrupts
           on an indeterminate processor. */
-        vcpu = x86_read_percpu(xen_vcpu);
+        vcpu = percpu_read(xen_vcpu);
        vcpu->evtchn_upcall_mask = 0;
        /* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 503c240e26c7..98cb9869eb24 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1063,18 +1063,14 @@ static void drop_other_mm_ref(void *info)
        struct mm_struct *mm = info;
        struct mm_struct *active_mm;
-#ifdef CONFIG_X86_64
+        active_mm = percpu_read(cpu_tlbstate.active_mm);
-        active_mm = read_pda(active_mm);
-#else
-        active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
-#endif
        if (active_mm == mm)
                leave_mm(smp_processor_id());
        /* If this cpu still has a stale cr3 reference, then make sure
           it has been flushed. */
-        if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
+        if (percpu_read(xen_current_cr3) == __pa(mm->pgd)) {
                load_cr3(swapper_pg_dir);
                arch_flush_lazy_cpu_mode();
        }
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index 858938241616..e786fa7f2615 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -39,7 +39,7 @@ static inline void xen_mc_issue(unsigned mode)
                xen_mc_flush();
        /* restore flags saved in xen_mc_batch */
-        local_irq_restore(x86_read_percpu(xen_mc_irq_flags));
+        local_irq_restore(percpu_read(xen_mc_irq_flags));
 }
 /* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index c44e2069c7c7..72c2eb9b64cd 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -50,11 +50,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
 */
 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
 {
-#ifdef CONFIG_X86_32
+        inc_irq_stat(irq_resched_count);
-        __get_cpu_var(irq_stat).irq_resched_count++;
-#else
-        add_pda(irq_resched_count, 1);
-#endif
        return IRQ_HANDLED;
 }
@@ -78,7 +74,7 @@ static __cpuinit void cpu_bringup(void)
        xen_setup_cpu_clockevents();
        cpu_set(cpu, cpu_online_map);
-        x86_write_percpu(cpu_state, CPU_ONLINE);
+        percpu_write(cpu_state, CPU_ONLINE);
        wmb();
        /* We can take interrupts now: we're officially "up". */
@@ -283,22 +279,11 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
        struct task_struct *idle = idle_task(cpu);
        int rc;
-#ifdef CONFIG_X86_64
+        per_cpu(current_task, cpu) = idle;
-        /* Allocate node local memory for AP pdas */
-        WARN_ON(cpu == 0);
-        if (cpu > 0) {
-                rc = get_local_pda(cpu);
-                if (rc)
-                        return rc;
-        }
-#endif
 #ifdef CONFIG_X86_32
        init_gdt(cpu);
-        per_cpu(current_task, cpu) = idle;
        irq_ctx_init(cpu);
 #else
-        cpu_pda(cpu)->pcurrent = idle;
        clear_tsk_thread_flag(idle, TIF_FORK);
 #endif
        xen_setup_timer(cpu);
@@ -445,11 +430,7 @@ static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
 {
        irq_enter();
        generic_smp_call_function_interrupt();
-#ifdef CONFIG_X86_32
+        inc_irq_stat(irq_call_count);
-        __get_cpu_var(irq_stat).irq_call_count++;
-#else
-        add_pda(irq_call_count, 1);
-#endif
        irq_exit();
        return IRQ_HANDLED;
@@ -459,11 +440,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
 {
        irq_enter();
        generic_smp_call_function_single_interrupt();
-#ifdef CONFIG_X86_32
+        inc_irq_stat(irq_call_count);
-        __get_cpu_var(irq_stat).irq_call_count++;
-#else
-        add_pda(irq_call_count, 1);
-#endif
        irq_exit();
        return IRQ_HANDLED;
diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index 05794c566e87..d6fc51f4ce85 100644
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -17,6 +17,7 @@
 #include <asm/processor-flags.h>
 #include <asm/errno.h>
 #include <asm/segment.h>
+#include <asm/percpu.h>
 #include <xen/interface/xen.h>
@@ -28,12 +29,10 @@
 #if 1
 /*
-        x86-64 does not yet support direct access to percpu variables
+        FIXME: x86_64 now can support direct access to percpu variables
-        via a segment override, so we just need to make sure this code
+        via a segment override.  Update xen accordingly.
-        never gets used
 */
 #define BUG                     ud2a
-#define PER_CPU_VAR(var, off)   0xdeadbeef
 #endif
 /*
@@ -45,14 +44,14 @@ ENTRY(xen_irq_enable_direct)
        BUG
        /* Unmask events */
-        movb $0, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
        /* Preempt here doesn't matter because that will deal with
           any pending interrupts.  The pending check may end up being
           run on the wrong CPU, but that doesn't hurt. */
        /* Test for pending */
-        testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending)
+        testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
        jz 1f
 2:      call check_events
@@ -69,7 +68,7 @@ ENDPATCH(xen_irq_enable_direct)
 ENTRY(xen_irq_disable_direct)
        BUG
-        movb $1, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 ENDPATCH(xen_irq_disable_direct)
        ret
        ENDPROC(xen_irq_disable_direct)
@@ -87,7 +86,7 @@ ENDPATCH(xen_irq_disable_direct)
 ENTRY(xen_save_fl_direct)
        BUG
-        testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
        setz %ah
        addb %ah,%ah
 ENDPATCH(xen_save_fl_direct)
@@ -107,13 +106,13 @@ ENTRY(xen_restore_fl_direct)
        BUG
        testb $X86_EFLAGS_IF>>8, %ah
-        setz PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask)
+        setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
        /* Preempt here doesn't matter because that will deal with
           any pending interrupts.  The pending check may end up being
           run on the wrong CPU, but that doesn't hurt. */
        /* check for unmasked and pending */
-        cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending)
+        cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
        jz 1f
 2:      call check_events
 1:
@@ -195,11 +194,11 @@ RELOC(xen_sysexit, 1b+1)
 ENTRY(xen_sysret64)
        /* We're already on the usermode stack at this point, but still
           with the kernel gs, so we can easily switch back */
-        movq %rsp, %gs:pda_oldrsp
+        movq %rsp, PER_CPU_VAR(old_rsp)
-        movq %gs:pda_kernelstack,%rsp
+        movq PER_CPU_VAR(kernel_stack),%rsp
        pushq $__USER_DS
-        pushq %gs:pda_oldrsp
+        pushq PER_CPU_VAR(old_rsp)
        pushq %r11
        pushq $__USER_CS
        pushq %rcx
@@ -212,11 +211,11 @@ RELOC(xen_sysret64, 1b+1)
 ENTRY(xen_sysret32)
        /* We're already on the usermode stack at this point, but still
           with the kernel gs, so we can easily switch back */
-        movq %rsp, %gs:pda_oldrsp
+        movq %rsp, PER_CPU_VAR(old_rsp)
-        movq %gs:pda_kernelstack, %rsp
+        movq PER_CPU_VAR(kernel_stack), %rsp
        pushq $__USER32_DS
-        pushq %gs:pda_oldrsp
+        pushq PER_CPU_VAR(old_rsp)
        pushq %r11
        pushq $__USER32_CS
        pushq %rcx
diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c
index f78371b22529..5a57753ea9fc 100644
--- a/drivers/pci/intr_remapping.c
+++ b/drivers/pci/intr_remapping.c
@@ -6,6 +6,7 @@
 #include <linux/irq.h>
 #include <asm/io_apic.h>
 #include <asm/smp.h>
+#include <asm/cpu.h>
 #include <linux/intel-iommu.h>
 #include "intr_remapping.h"
diff --git a/include/asm-generic/bitops/__ffs.h b/include/asm-generic/bitops/__ffs.h
index 9a3274aecf83..937d7c435575 100644
--- a/include/asm-generic/bitops/__ffs.h
+++ b/include/asm-generic/bitops/__ffs.h
@@ -9,7 +9,7 @@
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
-static inline unsigned long __ffs(unsigned long word)
+static __always_inline unsigned long __ffs(unsigned long word)
 {
        int num = 0;
diff --git a/include/asm-generic/bitops/__fls.h b/include/asm-generic/bitops/__fls.h
index be24465403d6..a60a7ccb6782 100644
--- a/include/asm-generic/bitops/__fls.h
+++ b/include/asm-generic/bitops/__fls.h
@@ -9,7 +9,7 @@
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
-static inline unsigned long __fls(unsigned long word)
+static __always_inline unsigned long __fls(unsigned long word)
 {
        int num = BITS_PER_LONG - 1;
diff --git a/include/asm-generic/bitops/fls.h b/include/asm-generic/bitops/fls.h
index 850859bc5069..0576d1f42f43 100644
--- a/include/asm-generic/bitops/fls.h
+++ b/include/asm-generic/bitops/fls.h
@@ -9,7 +9,7 @@
 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 */
-static inline int fls(int x)
+static __always_inline int fls(int x)
 {
        int r = 32;
diff --git a/include/asm-generic/bitops/fls64.h b/include/asm-generic/bitops/fls64.h
index 86d403f8b256..b097cf8444e3 100644
--- a/include/asm-generic/bitops/fls64.h
+++ b/include/asm-generic/bitops/fls64.h
@@ -15,7 +15,7 @@
 * at position 64.
 */
 #if BITS_PER_LONG == 32
-static inline int fls64(__u64 x)
+static __always_inline int fls64(__u64 x)
 {
        __u32 h = x >> 32;
        if (h)
@@ -23,7 +23,7 @@ static inline int fls64(__u64 x)
        return fls(x);
 }
 #elif BITS_PER_LONG == 64
-static inline int fls64(__u64 x)
+static __always_inline int fls64(__u64 x)
 {
        if (x == 0)
                return 0;
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index b0e63c672ebd..00f45ff081a6 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -80,4 +80,56 @@ extern void setup_per_cpu_areas(void);
 #define DECLARE_PER_CPU(type, name) extern PER_CPU_ATTRIBUTES \
                                        __typeof__(type) per_cpu_var(name)
+/*
+ * Optional methods for optimized non-lvalue per-cpu variable access.
+ *
+ * @var can be a percpu variable or a field of it and its size should
+ * equal char, int or long.  percpu_read() evaluates to a lvalue and
+ * all others to void.
+ *
+ * These operations are guaranteed to be atomic w.r.t. preemption.
+ * The generic versions use plain get/put_cpu_var().  Archs are
+ * encouraged to implement single-instruction alternatives which don't
+ * require preemption protection.
+ */
+#ifndef percpu_read
+# define percpu_read(var)                                               \
+  ({                                                                    \
+        typeof(per_cpu_var(var)) __tmp_var__;                           \
+        __tmp_var__ = get_cpu_var(var);                                 \
+        put_cpu_var(var);                                               \
+        __tmp_var__;                                                    \
+  })
+#endif
+#define __percpu_generic_to_op(var, val, op)                            \
+do {                                                                    \
+        get_cpu_var(var) op val;                                        \
+        put_cpu_var(var);                                               \
+} while (0)
+#ifndef percpu_write
+# define percpu_write(var, val)         __percpu_generic_to_op(var, (val), =)
+#endif
+#ifndef percpu_add
+# define percpu_add(var, val)           __percpu_generic_to_op(var, (val), +=)
+#endif
+#ifndef percpu_sub
+# define percpu_sub(var, val)           __percpu_generic_to_op(var, (val), -=)
+#endif
+#ifndef percpu_and
+# define percpu_and(var, val)           __percpu_generic_to_op(var, (val), &=)
+#endif
+#ifndef percpu_or
+# define percpu_or(var, val)            __percpu_generic_to_op(var, (val), |=)
+#endif
+#ifndef percpu_xor
+# define percpu_xor(var, val)           __percpu_generic_to_op(var, (val), ^=)
+#endif
 #endif /* _ASM_GENERIC_PERCPU_H_ */
diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h
index 79a7ff925bf8..4ce48e878530 100644
--- a/include/asm-generic/sections.h
+++ b/include/asm-generic/sections.h
@@ -9,7 +9,7 @@ extern char __bss_start[], __bss_stop[];
 extern char __init_begin[], __init_end[];
 extern char _sinittext[], _einittext[];
 extern char _end[];
-extern char __per_cpu_start[], __per_cpu_end[];
+extern char __per_cpu_load[], __per_cpu_start[], __per_cpu_end[];
 extern char __kprobes_text_start[], __kprobes_text_end[];
 extern char __initdata_begin[], __initdata_end[];
 extern char __start_rodata[], __end_rodata[];
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index c61fab1dd2f8..53e21f36a802 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -430,12 +430,47 @@
        *(.initcall7.init)                                              \
        *(.initcall7s.init)
-#define PERCPU(align)                                                   \
+/**
-        . = ALIGN(align);                                               \
+ * PERCPU_VADDR - define output section for percpu area
-        VMLINUX_SYMBOL(__per_cpu_start) = .;                            \
+ * @vaddr: explicit base address (optional)
-        .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {          \
+ * @phdr: destination PHDR (optional)
+ *
+ * Macro which expands to output section for percpu area.  If @vaddr
+ * is not blank, it specifies explicit base address and all percpu
+ * symbols will be offset from the given address.  If blank, @vaddr
+ * always equals @laddr + LOAD_OFFSET.
+ *
+ * @phdr defines the output PHDR to use if not blank.  Be warned that
+ * output PHDR is sticky.  If @phdr is specified, the next output
+ * section in the linker script will go there too.  @phdr should have
+ * a leading colon.
+ *
+ * This macro defines three symbols, __per_cpu_load, __per_cpu_start
+ * and __per_cpu_end.  The first one is the vaddr of loaded percpu
+ * init data.  __per_cpu_start equals @vaddr and __per_cpu_end is the
+ * end offset.
+ */
+#define PERCPU_VADDR(vaddr, phdr)                                       \
+        VMLINUX_SYMBOL(__per_cpu_load) = .;                             \
+        .data.percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load)          \
+                                - LOAD_OFFSET) {                        \
+                VMLINUX_SYMBOL(__per_cpu_start) = .;                    \
+                *(.data.percpu.first)                                   \
                *(.data.percpu.page_aligned)                            \
                *(.data.percpu)                                         \
                *(.data.percpu.shared_aligned)                          \
-        }                                                               \
+                VMLINUX_SYMBOL(__per_cpu_end) = .;                      \
-        VMLINUX_SYMBOL(__per_cpu_end) = .;
+        } phdr                                                          \
+        . = VMLINUX_SYMBOL(__per_cpu_load) + SIZEOF(.data.percpu);
+/**
+ * PERCPU - define output section for percpu area, simple version
+ * @align: required alignment
+ *
+ * Align to @align and outputs output section for percpu area.  This
+ * macro doesn't maniuplate @vaddr or @phdr and __per_cpu_load and
+ * __per_cpu_start will be identical.
+ */
+#define PERCPU(align)                                                   \
+        . = ALIGN(align);                                               \
+        PERCPU_VADDR( , )
diff --git a/include/linux/magic.h b/include/linux/magic.h
index 439f6f3cb0c4..561a5ff92c88 100644
--- a/include/linux/magic.h
+++ b/include/linux/magic.h
@@ -47,4 +47,5 @@
 #define FUTEXFS_SUPER_MAGIC     0xBAD1DEA
 #define INOTIFYFS_SUPER_MAGIC   0x2BAD1DEA
+#define STACK_END_MAGIC         0x57AC6E9D
 #endif /* __LINUX_MAGIC_H__ */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9f2a3751873a..0e24202b5a4e 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -9,34 +9,39 @@
 #include <asm/percpu.h>
 #ifdef CONFIG_SMP
-#define DEFINE_PER_CPU(type, name)                                      \
+#define PER_CPU_BASE_SECTION ".data.percpu"
-        __attribute__((__section__(".data.percpu")))                    \
-        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
 #ifdef MODULE
-#define SHARED_ALIGNED_SECTION ".data.percpu"
+#define PER_CPU_SHARED_ALIGNED_SECTION ""
 #else
-#define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned"
+#define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
 #endif
+#define PER_CPU_FIRST_SECTION ".first"
-#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
+#else
-        __attribute__((__section__(SHARED_ALIGNED_SECTION)))            \
-        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name             \
+#define PER_CPU_BASE_SECTION ".data"
-        ____cacheline_aligned_in_smp
+#define PER_CPU_SHARED_ALIGNED_SECTION ""
+#define PER_CPU_FIRST_SECTION ""
+#endif
-#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                 \
+#define DEFINE_PER_CPU_SECTION(type, name, section)                     \
-        __attribute__((__section__(".data.percpu.page_aligned")))       \
+        __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
-#else
 #define DEFINE_PER_CPU(type, name)                                      \
-        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
+        DEFINE_PER_CPU_SECTION(type, name, "")
-#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                     \
+#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
-        DEFINE_PER_CPU(type, name)
+        DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
+        ____cacheline_aligned_in_smp
-#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)               \
+#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                         \
-        DEFINE_PER_CPU(type, name)
+        DEFINE_PER_CPU_SECTION(type, name, ".page_aligned")
-#endif
+#define DEFINE_PER_CPU_FIRST(type, name)                                \
+        DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 4cae9b81a1f8..a85b0cec7d12 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1157,10 +1157,9 @@ struct task_struct {
        pid_t pid;
        pid_t tgid;
-#ifdef CONFIG_CC_STACKPROTECTOR
        /* Canary value for the -fstack-protector gcc feature */
        unsigned long stack_canary;
-#endif
        /* 
         * pointers to (original) parent process, youngest child, younger sibling,
         * older sibling, respectively.  (p->father can be replaced with 
@@ -2066,6 +2065,19 @@ static inline int object_is_on_stack(void *obj)
 extern void thread_info_cache_init(void);
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long stack_not_used(struct task_struct *p)
+{
+        unsigned long *n = end_of_stack(p);
+        do {    /* Skip over canary */
+                n++;
+        } while (!*n);
+        return (unsigned long)n - (unsigned long)end_of_stack(p);
+}
+#endif
 /* set thread flags in other task's structures
 * - see asm/thread_info.h for TIF_xxxx flags available
 */
diff --git a/include/linux/stackprotector.h b/include/linux/stackprotector.h
new file mode 100644
index 000000000000..6f3e54c704c0
--- /dev/null
+++ b/include/linux/stackprotector.h
@@ -0,0 +1,16 @@
+#ifndef _LINUX_STACKPROTECTOR_H
+#define _LINUX_STACKPROTECTOR_H 1
+#include <linux/compiler.h>
+#include <linux/sched.h>
+#include <linux/random.h>
+#ifdef CONFIG_CC_STACKPROTECTOR
+# include <asm/stackprotector.h>
+#else
+static inline void boot_init_stack_canary(void)
+{
+}
+#endif
+#endif
diff --git a/init/main.c b/init/main.c
index 844209453c02..bfe4fb0c9842 100644
--- a/init/main.c
+++ b/init/main.c
@@ -14,6 +14,7 @@
 #include <linux/proc_fs.h>
 #include <linux/kernel.h>
 #include <linux/syscalls.h>
+#include <linux/stackprotector.h>
 #include <linux/string.h>
 #include <linux/ctype.h>
 #include <linux/delay.h>
@@ -539,6 +540,12 @@ asmlinkage void __init start_kernel(void)
         */
        lockdep_init();
        debug_objects_early_init();
+        /*
+         * Set up the the initial canary ASAP:
+         */
+        boot_init_stack_canary();
        cgroup_init_early();
        local_irq_disable();
diff --git a/kernel/exit.c b/kernel/exit.c
index c7740fa3252c..2a803c28df9e 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -977,12 +977,9 @@ static void check_stack_usage(void)
 {
        static DEFINE_SPINLOCK(low_water_lock);
        static int lowest_to_date = THREAD_SIZE;
-        unsigned long *n = end_of_stack(current);
        unsigned long free;
-        while (*n == 0)
+        free = stack_not_used(current);
-                n++;
-        free = (unsigned long)n - (unsigned long)end_of_stack(current);
        if (free >= lowest_to_date)
                return;
diff --git a/kernel/fork.c b/kernel/fork.c
index 1d68f1255dd8..4a9b318dad0d 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -61,6 +61,7 @@
 #include <linux/proc_fs.h>
 #include <linux/blkdev.h>
 #include <trace/sched.h>
+#include <linux/magic.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -212,6 +213,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
 {
        struct task_struct *tsk;
        struct thread_info *ti;
+        unsigned long *stackend;
        int err;
        prepare_to_copy(orig);
@@ -237,6 +240,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
                goto out;
        setup_thread_stack(tsk, orig);
+        stackend = end_of_stack(tsk);
+        *stackend = STACK_END_MAGIC;    /* for overflow detection */
 #ifdef CONFIG_CC_STACKPROTECTOR
        tsk->stack_canary = get_random_int();
diff --git a/kernel/panic.c b/kernel/panic.c
index 2a2ff36ff44d..33cab3de1763 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -74,6 +74,9 @@ NORET_TYPE void panic(const char * fmt, ...)
        vsnprintf(buf, sizeof(buf), fmt, args);
        va_end(args);
        printk(KERN_EMERG "Kernel panic - not syncing: %s\n",buf);
+#ifdef CONFIG_DEBUG_BUGVERBOSE
+        dump_stack();
+#endif
        bust_spinlocks(0);
        /*
@@ -355,15 +358,22 @@ EXPORT_SYMBOL(warn_slowpath);
 #endif
 #ifdef CONFIG_CC_STACKPROTECTOR
+#ifndef GCC_HAS_SP
+#warning You have selected the CONFIG_CC_STACKPROTECTOR option, but the gcc used does not support this.
+#endif
 /*
 * Called when gcc's -fstack-protector feature is used, and
 * gcc detects corruption of the on-stack canary value
 */
 void __stack_chk_fail(void)
 {
-        panic("stack-protector: Kernel stack is corrupted");
+        panic("stack-protector: Kernel stack is corrupted in: %p\n",
+                __builtin_return_address(0));
 }
 EXPORT_SYMBOL(__stack_chk_fail);
 #endif
 core_param(panic, panic_timeout, int, 0644);
diff --git a/kernel/sched.c b/kernel/sched.c
index 8be2c13b50d0..1d2909067040 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -5939,12 +5939,7 @@ void sched_show_task(struct task_struct *p)
                printk(KERN_CONT " %016lx ", thread_saved_pc(p));
 #endif
 #ifdef CONFIG_DEBUG_STACK_USAGE
-        {
+        free = stack_not_used(p);
-                unsigned long *n = end_of_stack(p);
-                while (!*n)
-                        n++;
-                free = (unsigned long)n - (unsigned long)end_of_stack(p);
-        }
 #endif
        printk(KERN_CONT "%5lu %5d %6d\n", free,
                task_pid_nr(p), task_pid_nr(p->real_parent));