Merge branch 'oprofile/core' into oprofile/x86

Conflicts:
	arch/x86/oprofile/op_model_amd.c

Signed-off-by: Robert Richter <robert.richter@amd.com>

115 files changed, 2945 insertions, 3671 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 9815221976a7..7ab9db88ab6a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -25,6 +25,7 @@ config X86
         select HAVE_IDE
         select HAVE_OPROFILE
         select HAVE_PERF_EVENTS if (!M386 && !M486)
+        select HAVE_IRQ_WORK
         select HAVE_IOREMAP_PROT
         select HAVE_KPROBES
         select ARCH_WANT_OPTIONAL_GPIOLIB
@@ -62,6 +63,10 @@ config X86
         select HAVE_USER_RETURN_NOTIFIER
         select HAVE_ARCH_JUMP_LABEL
         select HAVE_TEXT_POKE_SMP
+        select HAVE_GENERIC_HARDIRQS
+        select HAVE_SPARSE_IRQ
+        select GENERIC_IRQ_PROBE
+        select GENERIC_PENDING_IRQ if SMP
 
 config INSTRUCTION_DECODER
         def_bool (KPROBES || PERF_EVENTS)
@@ -203,20 +208,6 @@ config HAVE_INTEL_TXT
         def_bool y
         depends on EXPERIMENTAL && DMAR && ACPI
 
-# Use the generic interrupt handling code in kernel/irq/:
-config GENERIC_HARDIRQS
-        def_bool y
-
-config GENERIC_HARDIRQS_NO__DO_IRQ
-       def_bool y
-
-config GENERIC_IRQ_PROBE
-        def_bool y
-
-config GENERIC_PENDING_IRQ
-        def_bool y
-        depends on GENERIC_HARDIRQS && SMP
-
 config USE_GENERIC_SMP_HELPERS
         def_bool y
         depends on SMP
@@ -299,23 +290,6 @@ config X86_X2APIC
 
           If you don't know what to do here, say N.
 
-config SPARSE_IRQ
-        bool "Support sparse irq numbering"
-        depends on PCI_MSI || HT_IRQ
-        ---help---
-          This enables support for sparse irqs. This is useful for distro
-          kernels that want to define a high CONFIG_NR_CPUS value but still
-          want to have low kernel memory footprint on smaller machines.
-
-          ( Sparse IRQs can also be beneficial on NUMA boxes, as they spread
-            out the irq_desc[] array in a more NUMA-friendly way. )
-
-          If you don't know what to do here, say N.
-
-config NUMA_IRQ_DESC
-        def_bool y
-        depends on SPARSE_IRQ && NUMA
-
 config X86_MPPARSE
         bool "Enable MPS table" if ACPI
         default y
@@ -520,25 +494,6 @@ if PARAVIRT_GUEST
 
 source "arch/x86/xen/Kconfig"
 
-config VMI
-        bool "VMI Guest support (DEPRECATED)"
-        select PARAVIRT
-        depends on X86_32
-        ---help---
-          VMI provides a paravirtualized interface to the VMware ESX server
-          (it could be used by other hypervisors in theory too, but is not
-          at the moment), by linking the kernel to a GPL-ed ROM module
-          provided by the hypervisor.
-
-          As of September 2009, VMware has started a phased retirement
-          of this feature from VMware's products. Please see
-          feature-removal-schedule.txt for details.  If you are
-          planning to enable this option, please note that you cannot
-          live migrate a VMI enabled VM to a future VMware product,
-          which doesn't support VMI. So if you expect your kernel to
-          seamlessly migrate to newer VMware products, keep this
-          disabled.
-
 config KVM_CLOCK
         bool "KVM paravirtualized clock"
         select PARAVIRT
@@ -673,7 +628,7 @@ config GART_IOMMU
         bool "GART IOMMU support" if EMBEDDED
         default y
         select SWIOTLB
-        depends on X86_64 && PCI && K8_NB
+        depends on X86_64 && PCI && AMD_NB
         ---help---
           Support for full DMA access of devices with 32bit memory access only
           on systems with more than 3GB. This is usually needed for USB,
@@ -798,6 +753,17 @@ config SCHED_MC
           making when dealing with multi-core CPU chips at a cost of slightly
           increased overhead in some places. If unsure say N here.
 
+config IRQ_TIME_ACCOUNTING
+        bool "Fine granularity task level IRQ time accounting"
+        default n
+        ---help---
+          Select this option to enable fine granularity task irq time
+          accounting. This is done by reading a timestamp on each
+          transitions between softirq and hardirq state, so there can be a
+          small performance impact.
+
+          If in doubt, say N here.
+
 source "kernel/Kconfig.preempt"
 
 config X86_UP_APIC
@@ -1151,6 +1117,9 @@ config X86_PAE
 config ARCH_PHYS_ADDR_T_64BIT
         def_bool X86_64 || X86_PAE
 
+config ARCH_DMA_ADDR_T_64BIT
+        def_bool X86_64 || HIGHMEM64G
+
 config DIRECT_GBPAGES
         bool "Enable 1GB pages for kernel pagetables" if EMBEDDED
         default y
@@ -1329,25 +1298,34 @@ config X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK
           Set whether the default state of memory_corruption_check is
           on or off.
 
-config X86_RESERVE_LOW_64K
-        bool "Reserve low 64K of RAM on AMI/Phoenix BIOSen"
-        default y
+config X86_RESERVE_LOW
+        int "Amount of low memory, in kilobytes, to reserve for the BIOS"
+        default 64
+        range 4 640
         ---help---
-          Reserve the first 64K of physical RAM on BIOSes that are known
-          to potentially corrupt that memory range. A numbers of BIOSes are
-          known to utilize this area during suspend/resume, so it must not
-          be used by the kernel.
+          Specify the amount of low memory to reserve for the BIOS.
+
+          The first page contains BIOS data structures that the kernel
+          must not use, so that page must always be reserved.
 
-          Set this to N if you are absolutely sure that you trust the BIOS
-          to get all its memory reservations and usages right.
+          By default we reserve the first 64K of physical RAM, as a
+          number of BIOSes are known to corrupt that memory range
+          during events such as suspend/resume or monitor cable
+          insertion, so it must not be used by the kernel.
 
-          If you have doubts about the BIOS (e.g. suspend/resume does not
-          work or there's kernel crashes after certain hardware hotplug
-          events) and it's not AMI or Phoenix, then you might want to enable
-          X86_CHECK_BIOS_CORRUPTION=y to allow the kernel to check typical
-          corruption patterns.
+          You can set this to 4 if you are absolutely sure that you
+          trust the BIOS to get all its memory reservations and usages
+          right.  If you know your BIOS have problems beyond the
+          default 64K area, you can set this to 640 to avoid using the
+          entire low memory range.
 
-          Say Y if unsure.
+          If you have doubts about the BIOS (e.g. suspend/resume does
+          not work or there's kernel crashes after certain hardware
+          hotplug events) then you might want to enable
+          X86_CHECK_BIOS_CORRUPTION=y to allow the kernel to check
+          typical corruption patterns.
+
+          Leave this to the default value of 64 if you are unsure.
 
 config MATH_EMULATION
         bool
@@ -1903,7 +1881,7 @@ config PCI_GODIRECT
         bool "Direct"
 
 config PCI_GOOLPC
-        bool "OLPC"
+        bool "OLPC XO-1"
         depends on OLPC
 
 config PCI_GOANY
@@ -2064,14 +2042,21 @@ config SCx200HR_TIMER
 config OLPC
         bool "One Laptop Per Child support"
         select GPIOLIB
+        select OLPC_OPENFIRMWARE
         ---help---
           Add support for detecting the unique features of the OLPC
           XO hardware.
 
+config OLPC_XO1
+        tristate "OLPC XO-1 support"
+        depends on OLPC && PCI
+        ---help---
+          Add support for non-essential features of the OLPC XO-1 laptop.
+
 config OLPC_OPENFIRMWARE
         bool "Support for OLPC's Open Firmware"
         depends on !X86_64 && !X86_PAE
-        default y if OLPC
+        default n
         help
           This option adds support for the implementation of Open Firmware
           that is used on the OLPC XO-1 Children's Machine.
@@ -2079,7 +2064,7 @@ config OLPC_OPENFIRMWARE
 
 endif # X86_32
 
-config K8_NB
+config AMD_NB
         def_bool y
         depends on CPU_SUP_AMD && PCI
 
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 75085080b63e..e5bb96b10f1a 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -43,6 +43,10 @@ config EARLY_PRINTK
           with klogd/syslogd or the X server. You should normally N here,
           unless you want to debug such a crash.
 
+config EARLY_PRINTK_MRST
+        bool "Early printk for MRST platform support"
+        depends on EARLY_PRINTK && X86_MRST
+
 config EARLY_PRINTK_DBGP
         bool "Early printk via EHCI debug port"
         depends on EARLY_PRINTK && PCI
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index e8c8881351b3..b02e509072a7 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -96,8 +96,12 @@ cfi := $(call as-instr,.cfi_startproc\n.cfi_rel_offset $(sp-y)$(comma)0\n.cfi_en
 # is .cfi_signal_frame supported too?
 cfi-sigframe := $(call as-instr,.cfi_startproc\n.cfi_signal_frame\n.cfi_endproc,-DCONFIG_AS_CFI_SIGNAL_FRAME=1)
 cfi-sections := $(call as-instr,.cfi_sections .debug_frame,-DCONFIG_AS_CFI_SECTIONS=1)
-KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections)
-KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections)
+
+# does binutils support specific instructions?
+asinstr := $(call as-instr,fxsaveq (%rax),-DCONFIG_AS_FXSAVEQ=1)
+
+KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr)
+KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr)
 
 LDFLAGS := -m elf_$(UTS_MACHINE)
 
diff --git a/arch/x86/ia32/ia32_aout.c b/arch/x86/ia32/ia32_aout.c
index 0350311906ae..2d93bdbc9ac0 100644
--- a/arch/x86/ia32/ia32_aout.c
+++ b/arch/x86/ia32/ia32_aout.c
@@ -34,7 +34,7 @@
 #include <asm/ia32.h>
 
 #undef WARN_OLD
-#undef CORE_DUMP /* probably broken */
+#undef CORE_DUMP /* definitely broken */
 
 static int load_aout_binary(struct linux_binprm *, struct pt_regs *regs);
 static int load_aout_library(struct file *);
@@ -131,21 +131,15 @@ static void set_brk(unsigned long start, unsigned long end)
  * macros to write out all the necessary info.
  */
 
-static int dump_write(struct file *file, const void *addr, int nr)
-{
-        return file->f_op->write(file, addr, nr, &file->f_pos) == nr;
-}
+#include <linux/coredump.h>
 
 #define DUMP_WRITE(addr, nr)                         \
         if (!dump_write(file, (void *)(addr), (nr))) \
                 goto end_coredump;
 
-#define DUMP_SEEK(offset)                                               \
-        if (file->f_op->llseek) {                                       \
-                if (file->f_op->llseek(file, (offset), 0) != (offset))  \
-                        goto end_coredump;                              \
-        } else                                                          \
-                file->f_pos = (offset)
+#define DUMP_SEEK(offset)               \
+        if (!dump_seek(file, offset))   \
+                goto end_coredump;
 
 #define START_DATA()    (u.u_tsize << PAGE_SHIFT)
 #define START_STACK(u)  (u.start_stack)
@@ -217,12 +211,6 @@ static int aout_core_dump(long signr, struct pt_regs *regs, struct file *file,
                 dump_size = dump.u_ssize << PAGE_SHIFT;
                 DUMP_WRITE(dump_start, dump_size);
         }
-        /*
-         * Finally dump the task struct.  Not be used by gdb, but
-         * could be useful
-         */
-        set_fs(KERNEL_DS);
-        DUMP_WRITE(current, sizeof(*current));
 end_coredump:
         set_fs(fs);
         return has_dumped;
diff --git a/arch/x86/include/asm/amd_iommu.h b/arch/x86/include/asm/amd_iommu.h
index 5af2982133b5..f16a2caca1e0 100644
--- a/arch/x86/include/asm/amd_iommu.h
+++ b/arch/x86/include/asm/amd_iommu.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
+ * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *         Leo Duran <leo.duran@amd.com>
  *
diff --git a/arch/x86/include/asm/amd_iommu_proto.h b/arch/x86/include/asm/amd_iommu_proto.h
index cb030374b90a..916bc8111a01 100644
--- a/arch/x86/include/asm/amd_iommu_proto.h
+++ b/arch/x86/include/asm/amd_iommu_proto.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009 Advanced Micro Devices, Inc.
+ * Copyright (C) 2009-2010 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h
index 08616180deaf..e3509fc303bf 100644
--- a/arch/x86/include/asm/amd_iommu_types.h
+++ b/arch/x86/include/asm/amd_iommu_types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
+ * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *         Leo Duran <leo.duran@amd.com>
  *
@@ -416,13 +416,22 @@ struct amd_iommu {
         struct dma_ops_domain *default_dom;
 
         /*
-         * This array is required to work around a potential BIOS bug.
-         * The BIOS may miss to restore parts of the PCI configuration
-         * space when the system resumes from S3. The result is that the
-         * IOMMU does not execute commands anymore which leads to system
-         * failure.
+         * We can't rely on the BIOS to restore all values on reinit, so we
+         * need to stash them
          */
-        u32 cache_cfg[4];
+
+        /* The iommu BAR */
+        u32 stored_addr_lo;
+        u32 stored_addr_hi;
+
+        /*
+         * Each iommu has 6 l1s, each of which is documented as having 0x12
+         * registers
+         */
+        u32 stored_l1[6][0x12];
+
+        /* The l2 indirect registers */
+        u32 stored_l2[0x83];
 };
 
 /*
diff --git a/arch/x86/include/asm/k8.h b/arch/x86/include/asm/amd_nb.h
index af00bd1d2089..c8517f81b21e 100644
--- a/arch/x86/include/asm/k8.h
+++ b/arch/x86/include/asm/amd_nb.h
@@ -1,5 +1,5 @@
-#ifndef _ASM_X86_K8_H
-#define _ASM_X86_K8_H
+#ifndef _ASM_X86_AMD_NB_H
+#define _ASM_X86_AMD_NB_H
 
 #include <linux/pci.h>
 
@@ -7,24 +7,27 @@ extern struct pci_device_id k8_nb_ids[];
 struct bootnode;
 
 extern int early_is_k8_nb(u32 value);
-extern struct pci_dev **k8_northbridges;
-extern int num_k8_northbridges;
 extern int cache_k8_northbridges(void);
 extern void k8_flush_garts(void);
 extern int k8_get_nodes(struct bootnode *nodes);
 extern int k8_numa_init(unsigned long start_pfn, unsigned long end_pfn);
 extern int k8_scan_nodes(void);
 
-#ifdef CONFIG_K8_NB
-extern int num_k8_northbridges;
+struct k8_northbridge_info {
+        u16 num;
+        u8 gart_supported;
+        struct pci_dev **nb_misc;
+};
+extern struct k8_northbridge_info k8_northbridges;
+
+#ifdef CONFIG_AMD_NB
 
 static inline struct pci_dev *node_to_k8_nb_misc(int node)
 {
-        return (node < num_k8_northbridges) ? k8_northbridges[node] : NULL;
+        return (node < k8_northbridges.num) ? k8_northbridges.nb_misc[node] : NULL;
 }
 
 #else
-#define num_k8_northbridges 0
 
 static inline struct pci_dev *node_to_k8_nb_misc(int node)
 {
@@ -33,4 +36,4 @@ static inline struct pci_dev *node_to_k8_nb_misc(int node)
 #endif
 
 
-#endif /* _ASM_X86_K8_H */
+#endif /* _ASM_X86_AMD_NB_H */
diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h
index a69b1ac9eaf8..2fefa501d3ba 100644
--- a/arch/x86/include/asm/apb_timer.h
+++ b/arch/x86/include/asm/apb_timer.h
@@ -54,7 +54,6 @@ extern struct clock_event_device *global_clock_event;
 extern unsigned long apbt_quick_calibrate(void);
 extern int arch_setup_apbt_irqs(int irq, int trigger, int mask, int cpu);
 extern void apbt_setup_secondary_clock(void);
-extern unsigned int boot_cpu_id;
 
 extern struct sfi_timer_table_entry *sfi_get_mtmr(int hint);
 extern void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr);
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index 1fa03e04ae44..286de34b0ed6 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -252,9 +252,7 @@ static inline int apic_is_clustered_box(void)
 }
 #endif
 
-extern u8 setup_APIC_eilvt_mce(u8 vector, u8 msg_type, u8 mask);
-extern u8 setup_APIC_eilvt_ibs(u8 vector, u8 msg_type, u8 mask);
-
+extern int setup_APIC_eilvt(u8 lvt_off, u8 vector, u8 msg_type, u8 mask);
 
 #else /* !CONFIG_X86_LOCAL_APIC */
 static inline void lapic_shutdown(void) { }
diff --git a/arch/x86/include/asm/apicdef.h b/arch/x86/include/asm/apicdef.h
index 7fe3b3060f08..a859ca461fb0 100644
--- a/arch/x86/include/asm/apicdef.h
+++ b/arch/x86/include/asm/apicdef.h
@@ -131,6 +131,7 @@
 #define APIC_EILVTn(n)  (0x500 + 0x10 * n)
 #define         APIC_EILVT_NR_AMD_K8    1       /* # of extended interrupts */
 #define         APIC_EILVT_NR_AMD_10H   4
+#define         APIC_EILVT_NR_MAX       APIC_EILVT_NR_AMD_10H
 #define         APIC_EILVT_LVTOFF(x)    (((x) >> 4) & 0xF)
 #define         APIC_EILVT_MSG_FIX      0x0
 #define         APIC_EILVT_MSG_SMI      0x2
diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h
index b185091bf19c..4fab24de26b1 100644
--- a/arch/x86/include/asm/cpu.h
+++ b/arch/x86/include/asm/cpu.h
@@ -32,6 +32,5 @@ extern void arch_unregister_cpu(int);
 
 DECLARE_PER_CPU(int, cpu_state);
 
-extern unsigned int boot_cpu_id;
 
 #endif /* _ASM_X86_CPU_H */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 3f76523589af..220e2ea08e80 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -152,10 +152,14 @@
 #define X86_FEATURE_3DNOWPREFETCH (6*32+ 8) /* 3DNow prefetch instructions */
 #define X86_FEATURE_OSVW        (6*32+ 9) /* OS Visible Workaround */
 #define X86_FEATURE_IBS         (6*32+10) /* Instruction Based Sampling */
-#define X86_FEATURE_SSE5        (6*32+11) /* SSE-5 */
+#define X86_FEATURE_XOP         (6*32+11) /* extended AVX instructions */
 #define X86_FEATURE_SKINIT      (6*32+12) /* SKINIT/STGI instructions */
 #define X86_FEATURE_WDT         (6*32+13) /* Watchdog timer */
+#define X86_FEATURE_LWP         (6*32+15) /* Light Weight Profiling */
+#define X86_FEATURE_FMA4        (6*32+16) /* 4 operands MAC instructions */
 #define X86_FEATURE_NODEID_MSR  (6*32+19) /* NodeId MSR */
+#define X86_FEATURE_TBM         (6*32+21) /* trailing bit manipulations */
+#define X86_FEATURE_TOPOEXT     (6*32+22) /* topology extensions CPUID leafs */
 
 /*
  * Auxiliary flags: Linux defined - For features scattered in various
@@ -180,6 +184,13 @@
 #define X86_FEATURE_LBRV        (8*32+ 6) /* AMD LBR Virtualization support */
 #define X86_FEATURE_SVML        (8*32+ 7) /* "svm_lock" AMD SVM locking MSR */
 #define X86_FEATURE_NRIPS       (8*32+ 8) /* "nrip_save" AMD SVM next_rip save */
+#define X86_FEATURE_TSCRATEMSR  (8*32+ 9) /* "tsc_scale" AMD TSC scaling support */
+#define X86_FEATURE_VMCBCLEAN   (8*32+10) /* "vmcb_clean" AMD VMCB clean bits support */
+#define X86_FEATURE_FLUSHBYASID (8*32+11) /* AMD flush-by-ASID support */
+#define X86_FEATURE_DECODEASSISTS (8*32+12) /* AMD Decode Assists support */
+#define X86_FEATURE_PAUSEFILTER (8*32+13) /* AMD filtered pause intercept */
+#define X86_FEATURE_PFTHRESHOLD (8*32+14) /* AMD pause filter threshold */
+
 
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
 #define X86_FEATURE_FSGSBASE    (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
diff --git a/arch/x86/include/asm/dwarf2.h b/arch/x86/include/asm/dwarf2.h
index 733f7e91e7a9..326099199318 100644
--- a/arch/x86/include/asm/dwarf2.h
+++ b/arch/x86/include/asm/dwarf2.h
@@ -89,6 +89,16 @@
         CFI_ADJUST_CFA_OFFSET -8
         .endm
 
+        .macro pushfq_cfi
+        pushfq
+        CFI_ADJUST_CFA_OFFSET 8
+        .endm
+
+        .macro popfq_cfi
+        popfq
+        CFI_ADJUST_CFA_OFFSET -8
+        .endm
+
         .macro movq_cfi reg offset=0
         movq %\reg, \offset(%rsp)
         CFI_REL_OFFSET \reg, \offset
@@ -109,6 +119,16 @@
         CFI_ADJUST_CFA_OFFSET -4
         .endm
 
+        .macro pushfl_cfi
+        pushfl
+        CFI_ADJUST_CFA_OFFSET 4
+        .endm
+
+        .macro popfl_cfi
+        popfl
+        CFI_ADJUST_CFA_OFFSET -4
+        .endm
+
         .macro movl_cfi reg offset=0
         movl %\reg, \offset(%esp)
         CFI_REL_OFFSET \reg, \offset
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index 8e8ec663a98f..b8e96a18676b 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR)
 BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR)
 BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
 
-#ifdef CONFIG_PERF_EVENTS
-BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR)
+#ifdef CONFIG_IRQ_WORK
+BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)
 #endif
 
 #ifdef CONFIG_X86_THERMAL_VECTOR
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index d07b44f7d1dc..4d293dced62f 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -214,5 +214,20 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr)
         BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START);
         return __virt_to_fix(vaddr);
 }
+
+/* Return an pointer with offset calculated */
+static inline unsigned long __set_fixmap_offset(enum fixed_addresses idx,
+                                phys_addr_t phys, pgprot_t flags)
+{
+        __set_fixmap(idx, phys, flags);
+        return fix_to_virt(idx) + (phys & (PAGE_SIZE - 1));
+}
+
+#define set_fixmap_offset(idx, phys)                    \
+        __set_fixmap_offset(idx, phys, PAGE_KERNEL)
+
+#define set_fixmap_offset_nocache(idx, phys)                    \
+        __set_fixmap_offset(idx, phys, PAGE_KERNEL_NOCACHE)
+
 #endif /* !__ASSEMBLY__ */
 #endif /* _ASM_X86_FIXMAP_H */
diff --git a/arch/x86/include/asm/gart.h b/arch/x86/include/asm/gart.h
index 4ac5b0f33fc1..bf357f9b25f0 100644
--- a/arch/x86/include/asm/gart.h
+++ b/arch/x86/include/asm/gart.h
@@ -17,6 +17,7 @@ extern int fix_aperture;
 #define GARTEN          (1<<0)
 #define DISGARTCPU      (1<<4)
 #define DISGARTIO       (1<<5)
+#define DISTLBWALKPRB   (1<<6)
 
 /* GART cache control register bits. */
 #define INVGART         (1<<0)
@@ -27,7 +28,6 @@ extern int fix_aperture;
 #define AMD64_GARTAPERTUREBASE  0x94
 #define AMD64_GARTTABLEBASE     0x98
 #define AMD64_GARTCACHECTL      0x9c
-#define AMD64_GARTEN            (1<<0)
 
 #ifdef CONFIG_GART_IOMMU
 extern int gart_iommu_aperture;
@@ -57,6 +57,19 @@ static inline void gart_iommu_hole_init(void)
 
 extern int agp_amd64_init(void);
 
+static inline void gart_set_size_and_enable(struct pci_dev *dev, u32 order)
+{
+        u32 ctl;
+
+        /*
+         * Don't enable translation but enable GART IO and CPU accesses.
+         * Also, set DISTLBWALKPRB since GART tables memory is UC.
+         */
+        ctl = DISTLBWALKPRB | order << 1;
+
+        pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl);
+}
+
 static inline void enable_gart_translation(struct pci_dev *dev, u64 addr)
 {
         u32 tmp, ctl;
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index aeab29aee617..55e4de613f0e 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -14,7 +14,7 @@ typedef struct {
 #endif
         unsigned int x86_platform_ipis; /* arch dependent */
         unsigned int apic_perf_irqs;
-        unsigned int apic_pending_irqs;
+        unsigned int apic_irq_work_irqs;
 #ifdef CONFIG_SMP
         unsigned int irq_resched_count;
         unsigned int irq_call_count;
diff --git a/arch/x86/include/asm/hpet.h b/arch/x86/include/asm/hpet.h
index 1d5c08a1bdfd..2c392d663dce 100644
--- a/arch/x86/include/asm/hpet.h
+++ b/arch/x86/include/asm/hpet.h
@@ -74,10 +74,12 @@ extern void hpet_disable(void);
 extern unsigned int hpet_readl(unsigned int a);
 extern void force_hpet_resume(void);
 
-extern void hpet_msi_unmask(unsigned int irq);
-extern void hpet_msi_mask(unsigned int irq);
-extern void hpet_msi_write(unsigned int irq, struct msi_msg *msg);
-extern void hpet_msi_read(unsigned int irq, struct msi_msg *msg);
+struct irq_data;
+extern void hpet_msi_unmask(struct irq_data *data);
+extern void hpet_msi_mask(struct irq_data *data);
+struct hpet_dev;
+extern void hpet_msi_write(struct hpet_dev *hdev, struct msi_msg *msg);
+extern void hpet_msi_read(struct hpet_dev *hdev, struct msi_msg *msg);
 
 #ifdef CONFIG_PCI_MSI
 extern int arch_setup_hpet_msi(unsigned int irq, unsigned int id);
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 46c0fe05f230..0274ec5a7e62 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -29,7 +29,7 @@
 extern void apic_timer_interrupt(void);
 extern void x86_platform_ipi(void);
 extern void error_interrupt(void);
-extern void perf_pending_interrupt(void);
+extern void irq_work_interrupt(void);
 
 extern void spurious_interrupt(void);
 extern void thermal_interrupt(void);
@@ -78,6 +78,13 @@ static inline void set_io_apic_irq_attr(struct io_apic_irq_attr *irq_attr,
         irq_attr->polarity      = polarity;
 }
 
+struct irq_2_iommu {
+        struct intel_iommu *iommu;
+        u16 irte_index;
+        u16 sub_handle;
+        u8  irte_mask;
+};
+
 /*
  * This is performance-critical, we want to do it O(1)
  *
@@ -89,15 +96,17 @@ struct irq_cfg {
         cpumask_var_t           old_domain;
         u8                      vector;
         u8                      move_in_progress : 1;
+#ifdef CONFIG_INTR_REMAP
+        struct irq_2_iommu      irq_2_iommu;
+#endif
 };
 
-extern struct irq_cfg *irq_cfg(unsigned int);
 extern int assign_irq_vector(int, struct irq_cfg *, const struct cpumask *);
 extern void send_cleanup_vector(struct irq_cfg *);
 
-struct irq_desc;
-extern unsigned int set_desc_affinity(struct irq_desc *, const struct cpumask *,
-                                      unsigned int *dest_id);
+struct irq_data;
+int __ioapic_set_affinity(struct irq_data *, const struct cpumask *,
+                          unsigned int *dest_id);
 extern int IO_APIC_get_PCI_irq_vector(int bus, int devfn, int pin, struct io_apic_irq_attr *irq_attr);
 extern void setup_ioapic_dest(void);
 
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index a73a8d5a5e69..4aa2bb3b242a 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -55,6 +55,12 @@ extern int save_i387_xstate_ia32(void __user *buf);
 extern int restore_i387_xstate_ia32(void __user *buf);
 #endif
 
+#ifdef CONFIG_MATH_EMULATION
+extern void finit_soft_fpu(struct i387_soft_struct *soft);
+#else
+static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
+#endif
+
 #define X87_FSW_ES (1 << 7)     /* Exception Summary */
 
 static __always_inline __pure bool use_xsaveopt(void)
@@ -67,6 +73,11 @@ static __always_inline __pure bool use_xsave(void)
         return static_cpu_has(X86_FEATURE_XSAVE);
 }
 
+static __always_inline __pure bool use_fxsr(void)
+{
+        return static_cpu_has(X86_FEATURE_FXSR);
+}
+
 extern void __sanitize_i387_state(struct task_struct *);
 
 static inline void sanitize_i387_state(struct task_struct *tsk)
@@ -77,19 +88,11 @@ static inline void sanitize_i387_state(struct task_struct *tsk)
 }
 
 #ifdef CONFIG_X86_64
-
-/* Ignore delayed exceptions from user space */
-static inline void tolerant_fwait(void)
-{
-        asm volatile("1: fwait\n"
-                     "2:\n"
-                     _ASM_EXTABLE(1b, 2b));
-}
-
 static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
 {
         int err;
 
+        /* See comment in fxsave() below. */
         asm volatile("1:  rex64/fxrstor (%[fx])\n\t"
                      "2:\n"
                      ".section .fixup,\"ax\"\n"
@@ -98,44 +101,10 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
                      ".previous\n"
                      _ASM_EXTABLE(1b, 3b)
                      : [err] "=r" (err)
-#if 0 /* See comment in fxsave() below. */
-                     : [fx] "r" (fx), "m" (*fx), "0" (0));
-#else
-                     : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0));
-#endif
+                     : [fx] "R" (fx), "m" (*fx), "0" (0));
         return err;
 }
 
-/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
-   is pending. Clear the x87 state here by setting it to fixed
-   values. The kernel data segment can be sometimes 0 and sometimes
-   new user value. Both should be ok.
-   Use the PDA as safe address because it should be already in L1. */
-static inline void fpu_clear(struct fpu *fpu)
-{
-        struct xsave_struct *xstate = &fpu->state->xsave;
-        struct i387_fxsave_struct *fx = &fpu->state->fxsave;
-
-        /*
-         * xsave header may indicate the init state of the FP.
-         */
-        if (use_xsave() &&
-            !(xstate->xsave_hdr.xstate_bv & XSTATE_FP))
-                return;
-
-        if (unlikely(fx->swd & X87_FSW_ES))
-                asm volatile("fnclex");
-        alternative_input(ASM_NOP8 ASM_NOP2,
-                          "    emms\n"          /* clear stack tags */
-                          "    fildl %%gs:0",   /* load to clear state */
-                          X86_FEATURE_FXSAVE_LEAK);
-}
-
-static inline void clear_fpu_state(struct task_struct *tsk)
-{
-        fpu_clear(&tsk->thread.fpu);
-}
-
 static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
 {
         int err;
@@ -149,6 +118,7 @@ static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
         if (unlikely(err))
                 return -EFAULT;
 
+        /* See comment in fxsave() below. */
         asm volatile("1:  rex64/fxsave (%[fx])\n\t"
                      "2:\n"
                      ".section .fixup,\"ax\"\n"
@@ -157,11 +127,7 @@ static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
                      ".previous\n"
                      _ASM_EXTABLE(1b, 3b)
                      : [err] "=r" (err), "=m" (*fx)
-#if 0 /* See comment in fxsave() below. */
-                     : [fx] "r" (fx), "0" (0));
-#else
-                     : [fx] "cdaSDb" (fx), "0" (0));
-#endif
+                     : [fx] "R" (fx), "0" (0));
         if (unlikely(err) &&
             __clear_user(fx, sizeof(struct i387_fxsave_struct)))
                 err = -EFAULT;
@@ -175,56 +141,29 @@ static inline void fpu_fxsave(struct fpu *fpu)
            uses any extended registers for addressing, a second REX prefix
            will be generated (to the assembler, rex64 followed by semicolon
            is a separate instruction), and hence the 64-bitness is lost. */
-#if 0
+
+#ifdef CONFIG_AS_FXSAVEQ
         /* Using "fxsaveq %0" would be the ideal choice, but is only supported
            starting with gas 2.16. */
         __asm__ __volatile__("fxsaveq %0"
                              : "=m" (fpu->state->fxsave));
-#elif 0
+#else
         /* Using, as a workaround, the properly prefixed form below isn't
            accepted by any binutils version so far released, complaining that
            the same type of prefix is used twice if an extended register is
-           needed for addressing (fix submitted to mainline 2005-11-21). */
-        __asm__ __volatile__("rex64/fxsave %0"
-                             : "=m" (fpu->state->fxsave));
-#else
-        /* This, however, we can work around by forcing the compiler to select
+           needed for addressing (fix submitted to mainline 2005-11-21).
+        asm volatile("rex64/fxsave %0"
+                     : "=m" (fpu->state->fxsave));
+           This, however, we can work around by forcing the compiler to select
            an addressing mode that doesn't require extended registers. */
-        __asm__ __volatile__("rex64/fxsave (%1)"
-                             : "=m" (fpu->state->fxsave)
-                             : "cdaSDb" (&fpu->state->fxsave));
+        asm volatile("rex64/fxsave (%[fx])"
+                     : "=m" (fpu->state->fxsave)
+                     : [fx] "R" (&fpu->state->fxsave));
 #endif
 }
 
-static inline void fpu_save_init(struct fpu *fpu)
-{
-        if (use_xsave())
-                fpu_xsave(fpu);
-        else
-                fpu_fxsave(fpu);
-
-        fpu_clear(fpu);
-}
-
-static inline void __save_init_fpu(struct task_struct *tsk)
-{
-        fpu_save_init(&tsk->thread.fpu);
-        task_thread_info(tsk)->status &= ~TS_USEDFPU;
-}
-
 #else  /* CONFIG_X86_32 */
 
-#ifdef CONFIG_MATH_EMULATION
-extern void finit_soft_fpu(struct i387_soft_struct *soft);
-#else
-static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
-#endif
-
-static inline void tolerant_fwait(void)
-{
-        asm volatile("fnclex ; fwait");
-}
-
 /* perform fxrstor iff the processor has extended states, otherwise frstor */
 static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
 {
@@ -241,6 +180,14 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
         return 0;
 }
 
+static inline void fpu_fxsave(struct fpu *fpu)
+{
+        asm volatile("fxsave %[fx]"
+                     : [fx] "=m" (fpu->state->fxsave));
+}
+
+#endif  /* CONFIG_X86_64 */
+
 /* We need a safe address that is cheap to find and that is already
    in L1 during context switch. The best choices are unfortunately
    different for UP and SMP */
@@ -256,47 +203,33 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
 static inline void fpu_save_init(struct fpu *fpu)
 {
         if (use_xsave()) {
-                struct xsave_struct *xstate = &fpu->state->xsave;
-                struct i387_fxsave_struct *fx = &fpu->state->fxsave;
-
                 fpu_xsave(fpu);
 
                 /*
                  * xsave header may indicate the init state of the FP.
                  */
-                if (!(xstate->xsave_hdr.xstate_bv & XSTATE_FP))
-                        goto end;
-
-                if (unlikely(fx->swd & X87_FSW_ES))
-                        asm volatile("fnclex");
-
-                /*
-                 * we can do a simple return here or be paranoid :)
-                 */
-                goto clear_state;
+                if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP))
+                        return;
+        } else if (use_fxsr()) {
+                fpu_fxsave(fpu);
+        } else {
+                asm volatile("fsave %[fx]; fwait"
+                             : [fx] "=m" (fpu->state->fsave));
+                return;
         }
 
-        /* Use more nops than strictly needed in case the compiler
-           varies code */
-        alternative_input(
-                "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
-                "fxsave %[fx]\n"
-                "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
-                X86_FEATURE_FXSR,
-                [fx] "m" (fpu->state->fxsave),
-                [fsw] "m" (fpu->state->fxsave.swd) : "memory");
-clear_state:
+        if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES))
+                asm volatile("fnclex");
+
         /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
            is pending.  Clear the x87 state here by setting it to fixed
            values. safe_address is a random variable that should be in L1 */
         alternative_input(
-                GENERIC_NOP8 GENERIC_NOP2,
+                ASM_NOP8 ASM_NOP2,
                 "emms\n\t"              /* clear stack tags */
-                "fildl %[addr]",        /* set F?P to defined value */
+                "fildl %P[addr]",       /* set F?P to defined value */
                 X86_FEATURE_FXSAVE_LEAK,
                 [addr] "m" (safe_address));
-end:
-        ;
 }
 
 static inline void __save_init_fpu(struct task_struct *tsk)
@@ -305,9 +238,6 @@ static inline void __save_init_fpu(struct task_struct *tsk)
         task_thread_info(tsk)->status &= ~TS_USEDFPU;
 }
 
-
-#endif  /* CONFIG_X86_64 */
-
 static inline int fpu_fxrstor_checking(struct fpu *fpu)
 {
         return fxrstor_checking(&fpu->state->fxsave);
@@ -344,7 +274,10 @@ static inline void __unlazy_fpu(struct task_struct *tsk)
 static inline void __clear_fpu(struct task_struct *tsk)
 {
         if (task_thread_info(tsk)->status & TS_USEDFPU) {
-                tolerant_fwait();
+                /* Ignore delayed exceptions from user space */
+                asm volatile("1: fwait\n"
+                             "2:\n"
+                             _ASM_EXTABLE(1b, 2b));
                 task_thread_info(tsk)->status &= ~TS_USEDFPU;
                 stts();
         }
@@ -405,19 +338,6 @@ static inline void irq_ts_restore(int TS_state)
                 stts();
 }
 
-#ifdef CONFIG_X86_64
-
-static inline void save_init_fpu(struct task_struct *tsk)
-{
-        __save_init_fpu(tsk);
-        stts();
-}
-
-#define unlazy_fpu      __unlazy_fpu
-#define clear_fpu       __clear_fpu
-
-#else  /* CONFIG_X86_32 */
-
 /*
  * These disable preemption on their own and are safe
  */
@@ -443,8 +363,6 @@ static inline void clear_fpu(struct task_struct *tsk)
         preempt_enable();
 }
 
-#endif  /* CONFIG_X86_64 */
-
 /*
  * i387 state interaction
  */
@@ -508,7 +426,4 @@ extern void fpu_finit(struct fpu *fpu);
 
 #endif /* __ASSEMBLY__ */
 
-#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5
-#define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5
-
 #endif /* _ASM_X86_I387_H */
diff --git a/arch/x86/include/asm/i8259.h b/arch/x86/include/asm/i8259.h
index 1655147646aa..a20365953bf8 100644
--- a/arch/x86/include/asm/i8259.h
+++ b/arch/x86/include/asm/i8259.h
@@ -55,6 +55,8 @@ extern struct irq_chip i8259A_chip;
 struct legacy_pic {
         int nr_legacy_irqs;
         struct irq_chip *chip;
+        void (*mask)(unsigned int irq);
+        void (*unmask)(unsigned int irq);
         void (*mask_all)(void);
         void (*restore_mask)(void);
         void (*init)(int auto_eoi);
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 30a3e9776123..6a45ec41ec26 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -206,6 +206,7 @@ static inline void __iomem *ioremap(resource_size_t offset, unsigned long size)
 
 extern void iounmap(volatile void __iomem *addr);
 
+extern void set_iounmap_nonlazy(void);
 
 #ifdef __KERNEL__
 
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 9cb2edb87c2f..c8be4566c3d2 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -170,12 +170,6 @@ extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
 
 extern void probe_nr_irqs_gsi(void);
 
-extern int setup_ioapic_entry(int apic, int irq,
-                              struct IO_APIC_route_entry *entry,
-                              unsigned int destination, int trigger,
-                              int polarity, int vector, int pin);
-extern void ioapic_write_entry(int apic, int pin,
-                               struct IO_APIC_route_entry e);
 extern void setup_ioapic_ids_from_mpc(void);
 
 struct mp_ioapic_gsi{
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index f275e2244505..1c23360fb2d8 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -3,4 +3,39 @@
 
 #define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8)
 
+#ifdef CONFIG_INTR_REMAP
+static inline void prepare_irte(struct irte *irte, int vector,
+                                unsigned int dest)
+{
+        memset(irte, 0, sizeof(*irte));
+
+        irte->present = 1;
+        irte->dst_mode = apic->irq_dest_mode;
+        /*
+         * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
+         * actual level or edge trigger will be setup in the IO-APIC
+         * RTE. This will help simplify level triggered irq migration.
+         * For more details, see the comments (in io_apic.c) explainig IO-APIC
+         * irq migration in the presence of interrupt-remapping.
+        */
+        irte->trigger_mode = 0;
+        irte->dlvry_mode = apic->irq_delivery_mode;
+        irte->vector = vector;
+        irte->dest_id = IRTE_DEST(dest);
+        irte->redir_hint = 1;
+}
+static inline bool irq_remapped(struct irq_cfg *cfg)
+{
+        return cfg->irq_2_iommu.iommu != NULL;
+}
+#else
+static void prepare_irte(struct irte *irte, int vector, unsigned int dest)
+{
+}
+static inline bool irq_remapped(struct irq_cfg *cfg)
+{
+        return false;
+}
+#endif
+
 #endif  /* _ASM_X86_IRQ_REMAPPING_H */
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index e2ca30092557..6af0894dafb4 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -114,9 +114,9 @@
 #define X86_PLATFORM_IPI_VECTOR         0xed
 
 /*
- * Performance monitoring pending work vector:
+ * IRQ work vector:
  */
-#define LOCAL_PENDING_VECTOR            0xec
+#define IRQ_WORK_VECTOR                 0xec
 
 #define UV_BAU_MESSAGE                  0xea
 
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 502e53f999cf..c52e2eb40a1e 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -652,20 +652,6 @@ static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
         return (struct kvm_mmu_page *)page_private(page);
 }
 
-static inline u16 kvm_read_fs(void)
-{
-        u16 seg;
-        asm("mov %%fs, %0" : "=g"(seg));
-        return seg;
-}
-
-static inline u16 kvm_read_gs(void)
-{
-        u16 seg;
-        asm("mov %%gs, %0" : "=g"(seg));
-        return seg;
-}
-
 static inline u16 kvm_read_ldt(void)
 {
         u16 ldt;
@@ -673,16 +659,6 @@ static inline u16 kvm_read_ldt(void)
         return ldt;
 }
 
-static inline void kvm_load_fs(u16 sel)
-{
-        asm("mov %0, %%fs" : : "rm"(sel));
-}
-
-static inline void kvm_load_gs(u16 sel)
-{
-        asm("mov %0, %%gs" : : "rm"(sel));
-}
-
 static inline void kvm_load_ldt(u16 sel)
 {
         asm("lldt %0" : : "rm"(sel));
diff --git a/arch/x86/include/asm/mrst.h b/arch/x86/include/asm/mrst.h
index 16350740edf6..4a711a684b17 100644
--- a/arch/x86/include/asm/mrst.h
+++ b/arch/x86/include/asm/mrst.h
@@ -10,6 +10,9 @@
  */
 #ifndef _ASM_X86_MRST_H
 #define _ASM_X86_MRST_H
+
+#include <linux/sfi.h>
+
 extern int pci_mrst_init(void);
 int __init sfi_parse_mrtc(struct sfi_table_header *table);
 
@@ -26,7 +29,7 @@ enum mrst_cpu_type {
 };
 
 extern enum mrst_cpu_type __mrst_cpu_chip;
-static enum mrst_cpu_type mrst_identify_cpu(void)
+static inline enum mrst_cpu_type mrst_identify_cpu(void)
 {
         return __mrst_cpu_chip;
 }
@@ -42,4 +45,9 @@ extern enum mrst_timer_options mrst_timer_options;
 #define SFI_MTMR_MAX_NUM 8
 #define SFI_MRTC_MAX    8
 
+extern struct console early_mrst_console;
+extern void mrst_early_console_init(void);
+
+extern struct console early_hsu_console;
+extern void hsu_early_console_init(void);
 #endif /* _ASM_X86_MRST_H */
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
new file mode 100644
index 000000000000..bcdff997668c
--- /dev/null
+++ b/arch/x86/include/asm/mwait.h
@@ -0,0 +1,15 @@
+#ifndef _ASM_X86_MWAIT_H
+#define _ASM_X86_MWAIT_H
+
+#define MWAIT_SUBSTATE_MASK             0xf
+#define MWAIT_CSTATE_MASK               0xf
+#define MWAIT_SUBSTATE_SIZE             4
+#define MWAIT_MAX_NUM_CSTATES           8
+
+#define CPUID_MWAIT_LEAF                5
+#define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1
+#define CPUID5_ECX_INTERRUPT_BREAK      0x2
+
+#define MWAIT_ECX_INTERRUPT_BREAK       0x1
+
+#endif /* _ASM_X86_MWAIT_H */
diff --git a/arch/x86/include/asm/olpc_ofw.h b/arch/x86/include/asm/olpc_ofw.h
index 08fde475cb3b..2a8478140bb3 100644
--- a/arch/x86/include/asm/olpc_ofw.h
+++ b/arch/x86/include/asm/olpc_ofw.h
@@ -21,10 +21,14 @@ extern void olpc_ofw_detect(void);
 /* install OFW's pde permanently into the kernel's pgtable */
 extern void setup_olpc_ofw_pgd(void);
 
+/* check if OFW was detected during boot */
+extern bool olpc_ofw_present(void);
+
 #else /* !CONFIG_OLPC_OPENFIRMWARE */
 
 static inline void olpc_ofw_detect(void) { }
 static inline void setup_olpc_ofw_pgd(void) { }
+static inline bool olpc_ofw_present(void) { return false; }
 
 #endif /* !CONFIG_OLPC_OPENFIRMWARE */
 
diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index a667f24c7254..1df66211fd1b 100644
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -8,7 +8,7 @@
 #define PAGE_SIZE       (_AC(1,UL) << PAGE_SHIFT)
 #define PAGE_MASK       (~(PAGE_SIZE-1))
 
-#define __PHYSICAL_MASK         ((phys_addr_t)(1ULL << __PHYSICAL_MASK_SHIFT) - 1)
+#define __PHYSICAL_MASK         ((phys_addr_t)((1ULL << __PHYSICAL_MASK_SHIFT) - 1))
 #define __VIRTUAL_MASK          ((1UL << __VIRTUAL_MASK_SHIFT) - 1)
 
 /* Cast PAGE_MASK to a signed type so that it is sign-extended if
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 5653f43d90e5..edecb4ed2210 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -416,11 +416,6 @@ static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
         PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
 }
 
-static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
-                                            unsigned long start, unsigned long count)
-{
-        PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
-}
 static inline void paravirt_release_pmd(unsigned long pfn)
 {
         PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index db9ef5532341..b82bac975250 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -255,7 +255,6 @@ struct pv_mmu_ops {
          */
         void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
         void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
-        void (*alloc_pmd_clone)(unsigned long pfn, unsigned long clonepfn, unsigned long start, unsigned long count);
         void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
         void (*release_pte)(unsigned long pfn);
         void (*release_pmd)(unsigned long pfn);
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index a34c785c5a63..ada823a13c7c 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -28,6 +28,8 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 extern spinlock_t pgd_lock;
 extern struct list_head pgd_list;
 
+extern struct mm_struct *pgd_page_get_mm(struct page *page);
+
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
 #else  /* !CONFIG_PARAVIRT */
@@ -603,6 +605,8 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm,
         pte_update(mm, addr, ptep);
 }
 
+#define flush_tlb_fix_spurious_fault(vma, address)
+
 /*
  * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
  *
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 076052cd62be..f96ac9bedf75 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -102,6 +102,8 @@ static inline void native_pgd_clear(pgd_t *pgd)
         native_set_pgd(pgd, native_make_pgd(0));
 }
 
+extern void sync_global_pgds(unsigned long start, unsigned long end);
+
 /*
  * Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 325b7bdbebaa..cae9c3cb95cf 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -110,6 +110,8 @@ struct cpuinfo_x86 {
         u16                     phys_proc_id;
         /* Core id: */
         u16                     cpu_core_id;
+        /* Compute unit id */
+        u8                      compute_unit_id;
         /* Index into per_cpu list: */
         u16                     cpu_index;
 #endif
@@ -602,7 +604,7 @@ extern unsigned long		mmu_cr4_features;
 
 static inline void set_in_cr4(unsigned long mask)
 {
-        unsigned cr4;
+        unsigned long cr4;
 
         mmu_cr4_features |= mask;
         cr4 = read_cr4();
@@ -612,7 +614,7 @@ static inline void set_in_cr4(unsigned long mask)
 
 static inline void clear_in_cr4(unsigned long mask)
 {
-        unsigned cr4;
+        unsigned long cr4;
 
         mmu_cr4_features &= ~mask;
         cr4 = read_cr4();
@@ -764,29 +766,6 @@ extern unsigned long		idle_halt;
 extern unsigned long            idle_nomwait;
 extern bool                     c1e_detected;
 
-/*
- * on systems with caches, caches must be flashed as the absolute
- * last instruction before going into a suspended halt.  Otherwise,
- * dirty data can linger in the cache and become stale on resume,
- * leading to strange errors.
- *
- * perform a variety of operations to guarantee that the compiler
- * will not reorder instructions.  wbinvd itself is serializing
- * so the processor will not reorder.
- *
- * Systems without cache can just go into halt.
- */
-static inline void wbinvd_halt(void)
-{
-        mb();
-        /* check for clflush to determine if wbinvd is legal */
-        if (cpu_has_clflush)
-                asm volatile("cli; wbinvd; 1: hlt; jmp 1b" : : : "memory");
-        else
-                while (1)
-                        halt();
-}
-
 extern void enable_sep_cpu(void);
 extern int sysenter_setup(void);
 
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index ef292c792d74..d6763b139a84 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -93,6 +93,11 @@ void *extend_brk(size_t size, size_t align);
                         : : "i" (sz));                                  \
         }
 
+/* Helper for reserving space for arrays of things */
+#define RESERVE_BRK_ARRAY(type, name, entries)          \
+        type *name;                                     \
+        RESERVE_BRK(name, sizeof(type) * entries)
+
 #ifdef __i386__
 
 void __init i386_start_kernel(void);
diff --git a/arch/x86/include/asm/vmi.h b/arch/x86/include/asm/vmi.h
deleted file mode 100644
index 61e08c0a2907..000000000000
--- a/arch/x86/include/asm/vmi.h
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * VMI interface definition
- *
- * Copyright (C) 2005, VMware, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * Maintained by: Zachary Amsden zach@vmware.com
- *
- */
-#include <linux/types.h>
-
-/*
- *---------------------------------------------------------------------
- *
- *  VMI Option ROM API
- *
- *---------------------------------------------------------------------
- */
-#define VMI_SIGNATURE 0x696d5663   /* "cVmi" */
-
-#define PCI_VENDOR_ID_VMWARE            0x15AD
-#define PCI_DEVICE_ID_VMWARE_VMI        0x0801
-
-/*
- * We use two version numbers for compatibility, with the major
- * number signifying interface breakages, and the minor number
- * interface extensions.
- */
-#define VMI_API_REV_MAJOR       3
-#define VMI_API_REV_MINOR       0
-
-#define VMI_CALL_CPUID                  0
-#define VMI_CALL_WRMSR                  1
-#define VMI_CALL_RDMSR                  2
-#define VMI_CALL_SetGDT                 3
-#define VMI_CALL_SetLDT                 4
-#define VMI_CALL_SetIDT                 5
-#define VMI_CALL_SetTR                  6
-#define VMI_CALL_GetGDT                 7
-#define VMI_CALL_GetLDT                 8
-#define VMI_CALL_GetIDT                 9
-#define VMI_CALL_GetTR                  10
-#define VMI_CALL_WriteGDTEntry          11
-#define VMI_CALL_WriteLDTEntry          12
-#define VMI_CALL_WriteIDTEntry          13
-#define VMI_CALL_UpdateKernelStack      14
-#define VMI_CALL_SetCR0                 15
-#define VMI_CALL_SetCR2                 16
-#define VMI_CALL_SetCR3                 17
-#define VMI_CALL_SetCR4                 18
-#define VMI_CALL_GetCR0                 19
-#define VMI_CALL_GetCR2                 20
-#define VMI_CALL_GetCR3                 21
-#define VMI_CALL_GetCR4                 22
-#define VMI_CALL_WBINVD                 23
-#define VMI_CALL_SetDR                  24
-#define VMI_CALL_GetDR                  25
-#define VMI_CALL_RDPMC                  26
-#define VMI_CALL_RDTSC                  27
-#define VMI_CALL_CLTS                   28
-#define VMI_CALL_EnableInterrupts       29
-#define VMI_CALL_DisableInterrupts      30
-#define VMI_CALL_GetInterruptMask       31
-#define VMI_CALL_SetInterruptMask       32
-#define VMI_CALL_IRET                   33
-#define VMI_CALL_SYSEXIT                34
-#define VMI_CALL_Halt                   35
-#define VMI_CALL_Reboot                 36
-#define VMI_CALL_Shutdown               37
-#define VMI_CALL_SetPxE                 38
-#define VMI_CALL_SetPxELong             39
-#define VMI_CALL_UpdatePxE              40
-#define VMI_CALL_UpdatePxELong          41
-#define VMI_CALL_MachineToPhysical      42
-#define VMI_CALL_PhysicalToMachine      43
-#define VMI_CALL_AllocatePage           44
-#define VMI_CALL_ReleasePage            45
-#define VMI_CALL_InvalPage              46
-#define VMI_CALL_FlushTLB               47
-#define VMI_CALL_SetLinearMapping       48
-
-#define VMI_CALL_SetIOPLMask            61
-#define VMI_CALL_SetInitialAPState      62
-#define VMI_CALL_APICWrite              63
-#define VMI_CALL_APICRead               64
-#define VMI_CALL_IODelay                65
-#define VMI_CALL_SetLazyMode            73
-
-/*
- *---------------------------------------------------------------------
- *
- * MMU operation flags
- *
- *---------------------------------------------------------------------
- */
-
-/* Flags used by VMI_{Allocate|Release}Page call */
-#define VMI_PAGE_PAE             0x10  /* Allocate PAE shadow */
-#define VMI_PAGE_CLONE           0x20  /* Clone from another shadow */
-#define VMI_PAGE_ZEROED          0x40  /* Page is pre-zeroed */
-
-
-/* Flags shared by Allocate|Release Page and PTE updates */
-#define VMI_PAGE_PT              0x01
-#define VMI_PAGE_PD              0x02
-#define VMI_PAGE_PDP             0x04
-#define VMI_PAGE_PML4            0x08
-
-#define VMI_PAGE_NORMAL          0x00 /* for debugging */
-
-/* Flags used by PTE updates */
-#define VMI_PAGE_CURRENT_AS      0x10 /* implies VMI_PAGE_VA_MASK is valid */
-#define VMI_PAGE_DEFER           0x20 /* may queue update until TLB inval */
-#define VMI_PAGE_VA_MASK         0xfffff000
-
-#ifdef CONFIG_X86_PAE
-#define VMI_PAGE_L1             (VMI_PAGE_PT | VMI_PAGE_PAE | VMI_PAGE_ZEROED)
-#define VMI_PAGE_L2             (VMI_PAGE_PD | VMI_PAGE_PAE | VMI_PAGE_ZEROED)
-#else
-#define VMI_PAGE_L1             (VMI_PAGE_PT | VMI_PAGE_ZEROED)
-#define VMI_PAGE_L2             (VMI_PAGE_PD | VMI_PAGE_ZEROED)
-#endif
-
-/* Flags used by VMI_FlushTLB call */
-#define VMI_FLUSH_TLB            0x01
-#define VMI_FLUSH_GLOBAL         0x02
-
-/*
- *---------------------------------------------------------------------
- *
- *  VMI relocation definitions for ROM call get_reloc
- *
- *---------------------------------------------------------------------
- */
-
-/* VMI Relocation types */
-#define VMI_RELOCATION_NONE     0
-#define VMI_RELOCATION_CALL_REL 1
-#define VMI_RELOCATION_JUMP_REL 2
-#define VMI_RELOCATION_NOP      3
-
-#ifndef __ASSEMBLY__
-struct vmi_relocation_info {
-        unsigned char           *eip;
-        unsigned char           type;
-        unsigned char           reserved[3];
-};
-#endif
-
-
-/*
- *---------------------------------------------------------------------
- *
- *  Generic ROM structures and definitions
- *
- *---------------------------------------------------------------------
- */
-
-#ifndef __ASSEMBLY__
-
-struct vrom_header {
-        u16     rom_signature;  /* option ROM signature */
-        u8      rom_length;     /* ROM length in 512 byte chunks */
-        u8      rom_entry[4];   /* 16-bit code entry point */
-        u8      rom_pad0;       /* 4-byte align pad */
-        u32     vrom_signature; /* VROM identification signature */
-        u8      api_version_min;/* Minor version of API */
-        u8      api_version_maj;/* Major version of API */
-        u8      jump_slots;     /* Number of jump slots */
-        u8      reserved1;      /* Reserved for expansion */
-        u32     virtual_top;    /* Hypervisor virtual address start */
-        u16     reserved2;      /* Reserved for expansion */
-        u16     license_offs;   /* Offset to License string */
-        u16     pci_header_offs;/* Offset to PCI OPROM header */
-        u16     pnp_header_offs;/* Offset to PnP OPROM header */
-        u32     rom_pad3;       /* PnP reserverd / VMI reserved */
-        u8      reserved[96];   /* Reserved for headers */
-        char    vmi_init[8];    /* VMI_Init jump point */
-        char    get_reloc[8];   /* VMI_GetRelocationInfo jump point */
-} __attribute__((packed));
-
-struct pnp_header {
-        char sig[4];
-        char rev;
-        char size;
-        short next;
-        short res;
-        long devID;
-        unsigned short manufacturer_offset;
-        unsigned short product_offset;
-} __attribute__((packed));
-
-struct pci_header {
-        char sig[4];
-        short vendorID;
-        short deviceID;
-        short vpdData;
-        short size;
-        char rev;
-        char class;
-        char subclass;
-        char interface;
-        short chunks;
-        char rom_version_min;
-        char rom_version_maj;
-        char codetype;
-        char lastRom;
-        short reserved;
-} __attribute__((packed));
-
-/* Function prototypes for bootstrapping */
-#ifdef CONFIG_VMI
-extern void vmi_init(void);
-extern void vmi_activate(void);
-extern void vmi_bringup(void);
-#else
-static inline void vmi_init(void) {}
-static inline void vmi_activate(void) {}
-static inline void vmi_bringup(void) {}
-#endif
-
-/* State needed to start an application processor in an SMP system. */
-struct vmi_ap_state {
-        u32 cr0;
-        u32 cr2;
-        u32 cr3;
-        u32 cr4;
-
-        u64 efer;
-
-        u32 eip;
-        u32 eflags;
-        u32 eax;
-        u32 ebx;
-        u32 ecx;
-        u32 edx;
-        u32 esp;
-        u32 ebp;
-        u32 esi;
-        u32 edi;
-        u16 cs;
-        u16 ss;
-        u16 ds;
-        u16 es;
-        u16 fs;
-        u16 gs;
-        u16 ldtr;
-
-        u16 gdtr_limit;
-        u32 gdtr_base;
-        u32 idtr_base;
-        u16 idtr_limit;
-};
-
-#endif
diff --git a/arch/x86/include/asm/vmi_time.h b/arch/x86/include/asm/vmi_time.h
deleted file mode 100644
index c6e0bee93e3c..000000000000
--- a/arch/x86/include/asm/vmi_time.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * VMI Time wrappers
- *
- * Copyright (C) 2006, VMware, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * Send feedback to dhecht@vmware.com
- *
- */
-
-#ifndef _ASM_X86_VMI_TIME_H
-#define _ASM_X86_VMI_TIME_H
-
-/*
- * Raw VMI call indices for timer functions
- */
-#define VMI_CALL_GetCycleFrequency      66
-#define VMI_CALL_GetCycleCounter        67
-#define VMI_CALL_SetAlarm               68
-#define VMI_CALL_CancelAlarm            69
-#define VMI_CALL_GetWallclockTime       70
-#define VMI_CALL_WallclockUpdated       71
-
-/* Cached VMI timer operations */
-extern struct vmi_timer_ops {
-        u64 (*get_cycle_frequency)(void);
-        u64 (*get_cycle_counter)(int);
-        u64 (*get_wallclock)(void);
-        int (*wallclock_updated)(void);
-        void (*set_alarm)(u32 flags, u64 expiry, u64 period);
-        void (*cancel_alarm)(u32 flags);
-} vmi_timer_ops;
-
-/* Prototypes */
-extern void __init vmi_time_init(void);
-extern unsigned long vmi_get_wallclock(void);
-extern int vmi_set_wallclock(unsigned long now);
-extern unsigned long long vmi_sched_clock(void);
-extern unsigned long vmi_tsc_khz(void);
-
-#ifdef CONFIG_X86_LOCAL_APIC
-extern void __devinit vmi_time_bsp_init(void);
-extern void __devinit vmi_time_ap_init(void);
-#endif
-
-/*
- * When run under a hypervisor, a vcpu is always in one of three states:
- * running, halted, or ready.  The vcpu is in the 'running' state if it
- * is executing.  When the vcpu executes the halt interface, the vcpu
- * enters the 'halted' state and remains halted until there is some work
- * pending for the vcpu (e.g. an alarm expires, host I/O completes on
- * behalf of virtual I/O).  At this point, the vcpu enters the 'ready'
- * state (waiting for the hypervisor to reschedule it).  Finally, at any
- * time when the vcpu is not in the 'running' state nor the 'halted'
- * state, it is in the 'ready' state.
- *
- * Real time is advances while the vcpu is 'running', 'ready', or
- * 'halted'.  Stolen time is the time in which the vcpu is in the
- * 'ready' state.  Available time is the remaining time -- the vcpu is
- * either 'running' or 'halted'.
- *
- * All three views of time are accessible through the VMI cycle
- * counters.
- */
-
-/* The cycle counters. */
-#define VMI_CYCLES_REAL         0
-#define VMI_CYCLES_AVAILABLE    1
-#define VMI_CYCLES_STOLEN       2
-
-/* The alarm interface 'flags' bits */
-#define VMI_ALARM_COUNTERS      2
-
-#define VMI_ALARM_COUNTER_MASK  0x000000ff
-
-#define VMI_ALARM_WIRED_IRQ0    0x00000000
-#define VMI_ALARM_WIRED_LVTT    0x00010000
-
-#define VMI_ALARM_IS_ONESHOT    0x00000000
-#define VMI_ALARM_IS_PERIODIC   0x00000100
-
-#define CONFIG_VMI_ALARM_HZ     100
-
-#endif /* _ASM_X86_VMI_TIME_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 9d3f485e5dd0..80a93dc99076 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -35,6 +35,7 @@ obj-y			:= process_$(BITS).o signal.o entry_$(BITS).o
 obj-y                   += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
 obj-y                   += time.o ioport.o ldt.o dumpstack.o
 obj-y                   += setup.o x86_init.o i8259.o irqinit.o jump_label.o
+obj-$(CONFIG_IRQ_WORK)  += irq_work.o
 obj-$(CONFIG_X86_VISWS) += visws_quirks.o
 obj-$(CONFIG_X86_32)    += probe_roms_32.o
 obj-$(CONFIG_X86_32)    += sys_i386_32.o i386_ksyms_32.o
@@ -85,15 +86,15 @@ obj-$(CONFIG_DOUBLEFAULT) 	+= doublefault_32.o
 obj-$(CONFIG_KGDB)              += kgdb.o
 obj-$(CONFIG_VM86)              += vm86_32.o
 obj-$(CONFIG_EARLY_PRINTK)      += early_printk.o
+obj-$(CONFIG_EARLY_PRINTK_MRST) += early_printk_mrst.o
 
 obj-$(CONFIG_HPET_TIMER)        += hpet.o
 obj-$(CONFIG_APB_TIMER)         += apb_timer.o
 
-obj-$(CONFIG_K8_NB)             += k8.o
+obj-$(CONFIG_AMD_NB)            += amd_nb.o
 obj-$(CONFIG_DEBUG_RODATA_TEST) += test_rodata.o
 obj-$(CONFIG_DEBUG_NX_TEST)     += test_nx.o
 
-obj-$(CONFIG_VMI)               += vmi_32.o vmiclock_32.o
 obj-$(CONFIG_KVM_GUEST)         += kvm.o
 obj-$(CONFIG_KVM_CLOCK)         += kvmclock.o
 obj-$(CONFIG_PARAVIRT)          += paravirt.o paravirt_patch_$(BITS).o
@@ -106,6 +107,7 @@ obj-$(CONFIG_SCx200)		+= scx200.o
 scx200-y                        += scx200_32.o
 
 obj-$(CONFIG_OLPC)              += olpc.o
+obj-$(CONFIG_OLPC_XO1)          += olpc-xo1.o
 obj-$(CONFIG_OLPC_OPENFIRMWARE) += olpc_ofw.o
 obj-$(CONFIG_X86_MRST)          += mrst.o
 
@@ -122,7 +124,6 @@ obj-$(CONFIG_SWIOTLB)			+= pci-swiotlb.o
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
         obj-$(CONFIG_X86_UV)            += tlb_uv.o bios_uv.o uv_irq.o uv_sysfs.o uv_time.o
-        obj-$(CONFIG_X86_PM_TIMER)      += pmtimer_64.o
         obj-$(CONFIG_AUDIT)             += audit_64.o
 
         obj-$(CONFIG_GART_IOMMU)        += pci-gart_64.o aperture_64.o
diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c
index fb16f17e59be..5812404a0d4c 100644
--- a/arch/x86/kernel/acpi/cstate.c
+++ b/arch/x86/kernel/acpi/cstate.c
@@ -13,6 +13,7 @@
 
 #include <acpi/processor.h>
 #include <asm/acpi.h>
+#include <asm/mwait.h>
 
 /*
  * Initialize bm_flags based on the CPU cache properties
@@ -65,16 +66,6 @@ static struct cstate_entry __percpu *cpu_cstate_entry;	/* per CPU ptr */
 
 static short mwait_supported[ACPI_PROCESSOR_MAX_POWER];
 
-#define MWAIT_SUBSTATE_MASK     (0xf)
-#define MWAIT_CSTATE_MASK       (0xf)
-#define MWAIT_SUBSTATE_SIZE     (4)
-
-#define CPUID_MWAIT_LEAF (5)
-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
-#define CPUID5_ECX_INTERRUPT_BREAK      (0x2)
-
-#define MWAIT_ECX_INTERRUPT_BREAK       (0x1)
-
 #define NATIVE_CSTATE_BEYOND_HALT       (2)
 
 static long acpi_processor_ffh_cstate_probe_cpu(void *_cx)
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 679b6450382b..d2fdb0826df2 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
+ * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *         Leo Duran <leo.duran@amd.com>
  *
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index 5a170cbbbed8..3cb482e123de 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
+ * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *         Leo Duran <leo.duran@amd.com>
  *
@@ -194,6 +194,39 @@ static inline unsigned long tbl_size(int entry_size)
         return 1UL << shift;
 }
 
+/* Access to l1 and l2 indexed register spaces */
+
+static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
+{
+        u32 val;
+
+        pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
+        pci_read_config_dword(iommu->dev, 0xfc, &val);
+        return val;
+}
+
+static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
+{
+        pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
+        pci_write_config_dword(iommu->dev, 0xfc, val);
+        pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
+}
+
+static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
+{
+        u32 val;
+
+        pci_write_config_dword(iommu->dev, 0xf0, address);
+        pci_read_config_dword(iommu->dev, 0xf4, &val);
+        return val;
+}
+
+static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
+{
+        pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
+        pci_write_config_dword(iommu->dev, 0xf4, val);
+}
+
 /****************************************************************************
  *
  * AMD IOMMU MMIO register space handling functions
@@ -619,6 +652,7 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
 {
         int cap_ptr = iommu->cap_ptr;
         u32 range, misc;
+        int i, j;
 
         pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
                               &iommu->cap);
@@ -633,12 +667,29 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
                                         MMIO_GET_LD(range));
         iommu->evt_msi_num = MMIO_MSI_NUM(misc);
 
-        if (is_rd890_iommu(iommu->dev)) {
-                pci_read_config_dword(iommu->dev, 0xf0, &iommu->cache_cfg[0]);
-                pci_read_config_dword(iommu->dev, 0xf4, &iommu->cache_cfg[1]);
-                pci_read_config_dword(iommu->dev, 0xf8, &iommu->cache_cfg[2]);
-                pci_read_config_dword(iommu->dev, 0xfc, &iommu->cache_cfg[3]);
-        }
+        if (!is_rd890_iommu(iommu->dev))
+                return;
+
+        /*
+         * Some rd890 systems may not be fully reconfigured by the BIOS, so
+         * it's necessary for us to store this information so it can be
+         * reprogrammed on resume
+         */
+
+        pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
+                              &iommu->stored_addr_lo);
+        pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
+                              &iommu->stored_addr_hi);
+
+        /* Low bit locks writes to configuration space */
+        iommu->stored_addr_lo &= ~1;
+
+        for (i = 0; i < 6; i++)
+                for (j = 0; j < 0x12; j++)
+                        iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
+
+        for (i = 0; i < 0x83; i++)
+                iommu->stored_l2[i] = iommu_read_l2(iommu, i);
 }
 
 /*
@@ -1127,14 +1178,53 @@ static void iommu_init_flags(struct amd_iommu *iommu)
         iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
 }
 
-static void iommu_apply_quirks(struct amd_iommu *iommu)
+static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
 {
-        if (is_rd890_iommu(iommu->dev)) {
-                pci_write_config_dword(iommu->dev, 0xf0, iommu->cache_cfg[0]);
-                pci_write_config_dword(iommu->dev, 0xf4, iommu->cache_cfg[1]);
-                pci_write_config_dword(iommu->dev, 0xf8, iommu->cache_cfg[2]);
-                pci_write_config_dword(iommu->dev, 0xfc, iommu->cache_cfg[3]);
-        }
+        int i, j;
+        u32 ioc_feature_control;
+        struct pci_dev *pdev = NULL;
+
+        /* RD890 BIOSes may not have completely reconfigured the iommu */
+        if (!is_rd890_iommu(iommu->dev))
+                return;
+
+        /*
+         * First, we need to ensure that the iommu is enabled. This is
+         * controlled by a register in the northbridge
+         */
+        pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0));
+
+        if (!pdev)
+                return;
+
+        /* Select Northbridge indirect register 0x75 and enable writing */
+        pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
+        pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
+
+        /* Enable the iommu */
+        if (!(ioc_feature_control & 0x1))
+                pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
+
+        pci_dev_put(pdev);
+
+        /* Restore the iommu BAR */
+        pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
+                               iommu->stored_addr_lo);
+        pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
+                               iommu->stored_addr_hi);
+
+        /* Restore the l1 indirect regs for each of the 6 l1s */
+        for (i = 0; i < 6; i++)
+                for (j = 0; j < 0x12; j++)
+                        iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
+
+        /* Restore the l2 indirect regs */
+        for (i = 0; i < 0x83; i++)
+                iommu_write_l2(iommu, i, iommu->stored_l2[i]);
+
+        /* Lock PCI setup registers */
+        pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
+                               iommu->stored_addr_lo | 1);
 }
 
 /*
@@ -1147,7 +1237,6 @@ static void enable_iommus(void)
 
         for_each_iommu(iommu) {
                 iommu_disable(iommu);
-                iommu_apply_quirks(iommu);
                 iommu_init_flags(iommu);
                 iommu_set_device_table(iommu);
                 iommu_enable_command_buffer(iommu);
@@ -1173,6 +1262,11 @@ static void disable_iommus(void)
 
 static int amd_iommu_resume(struct sys_device *dev)
 {
+        struct amd_iommu *iommu;
+
+        for_each_iommu(iommu)
+                iommu_apply_resume_quirks(iommu);
+
         /* re-load the hardware */
         enable_iommus();
 
diff --git a/arch/x86/kernel/k8.c b/arch/x86/kernel/amd_nb.c
index 0f7bc20cfcde..8f6463d8ed0d 100644
--- a/arch/x86/kernel/k8.c
+++ b/arch/x86/kernel/amd_nb.c
@@ -8,21 +8,19 @@
 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <asm/k8.h>
-
-int num_k8_northbridges;
-EXPORT_SYMBOL(num_k8_northbridges);
+#include <asm/amd_nb.h>
 
 static u32 *flush_words;
 
 struct pci_device_id k8_nb_ids[] = {
         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC) },
         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MISC) },
+        { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_MISC) },
         {}
 };
 EXPORT_SYMBOL(k8_nb_ids);
 
-struct pci_dev **k8_northbridges;
+struct k8_northbridge_info k8_northbridges;
 EXPORT_SYMBOL(k8_northbridges);
 
 static struct pci_dev *next_k8_northbridge(struct pci_dev *dev)
@@ -40,36 +38,45 @@ int cache_k8_northbridges(void)
         int i;
         struct pci_dev *dev;
 
-        if (num_k8_northbridges)
+        if (k8_northbridges.num)
                 return 0;
 
         dev = NULL;
         while ((dev = next_k8_northbridge(dev)) != NULL)
-                num_k8_northbridges++;
+                k8_northbridges.num++;
+
+        /* some CPU families (e.g. family 0x11) do not support GART */
+        if (boot_cpu_data.x86 == 0xf || boot_cpu_data.x86 == 0x10 ||
+            boot_cpu_data.x86 == 0x15)
+                k8_northbridges.gart_supported = 1;
 
-        k8_northbridges = kmalloc((num_k8_northbridges + 1) * sizeof(void *),
-                                  GFP_KERNEL);
-        if (!k8_northbridges)
+        k8_northbridges.nb_misc = kmalloc((k8_northbridges.num + 1) *
+                                          sizeof(void *), GFP_KERNEL);
+        if (!k8_northbridges.nb_misc)
                 return -ENOMEM;
 
-        if (!num_k8_northbridges) {
-                k8_northbridges[0] = NULL;
+        if (!k8_northbridges.num) {
+                k8_northbridges.nb_misc[0] = NULL;
                 return 0;
         }
 
-        flush_words = kmalloc(num_k8_northbridges * sizeof(u32), GFP_KERNEL);
-        if (!flush_words) {
-                kfree(k8_northbridges);
-                return -ENOMEM;
+        if (k8_northbridges.gart_supported) {
+                flush_words = kmalloc(k8_northbridges.num * sizeof(u32),
+                                      GFP_KERNEL);
+                if (!flush_words) {
+                        kfree(k8_northbridges.nb_misc);
+                        return -ENOMEM;
+                }
         }
 
         dev = NULL;
         i = 0;
         while ((dev = next_k8_northbridge(dev)) != NULL) {
-                k8_northbridges[i] = dev;
-                pci_read_config_dword(dev, 0x9c, &flush_words[i++]);
+                k8_northbridges.nb_misc[i] = dev;
+                if (k8_northbridges.gart_supported)
+                        pci_read_config_dword(dev, 0x9c, &flush_words[i++]);
         }
-        k8_northbridges[i] = NULL;
+        k8_northbridges.nb_misc[i] = NULL;
         return 0;
 }
 EXPORT_SYMBOL_GPL(cache_k8_northbridges);
@@ -93,22 +100,25 @@ void k8_flush_garts(void)
         unsigned long flags;
         static DEFINE_SPINLOCK(gart_lock);
 
+        if (!k8_northbridges.gart_supported)
+                return;
+
         /* Avoid races between AGP and IOMMU. In theory it's not needed
            but I'm not sure if the hardware won't lose flush requests
            when another is pending. This whole thing is so expensive anyways
            that it doesn't matter to serialize more. -AK */
         spin_lock_irqsave(&gart_lock, flags);
         flushed = 0;
-        for (i = 0; i < num_k8_northbridges; i++) {
-                pci_write_config_dword(k8_northbridges[i], 0x9c,
+        for (i = 0; i < k8_northbridges.num; i++) {
+                pci_write_config_dword(k8_northbridges.nb_misc[i], 0x9c,
                                        flush_words[i]|1);
                 flushed++;
         }
-        for (i = 0; i < num_k8_northbridges; i++) {
+        for (i = 0; i < k8_northbridges.num; i++) {
                 u32 w;
                 /* Make sure the hardware actually executed the flush*/
                 for (;;) {
-                        pci_read_config_dword(k8_northbridges[i],
+                        pci_read_config_dword(k8_northbridges.nb_misc[i],
                                               0x9c, &w);
                         if (!(w & 1))
                                 break;
diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c
index 8dd77800ff5d..92543c73cf8e 100644
--- a/arch/x86/kernel/apb_timer.c
+++ b/arch/x86/kernel/apb_timer.c
@@ -231,34 +231,6 @@ static void apbt_restart_clocksource(struct clocksource *cs)
         apbt_start_counter(phy_cs_timer_id);
 }
 
-/* Setup IRQ routing via IOAPIC */
-#ifdef CONFIG_SMP
-static void apbt_setup_irq(struct apbt_dev *adev)
-{
-        struct irq_chip *chip;
-        struct irq_desc *desc;
-
-        /* timer0 irq has been setup early */
-        if (adev->irq == 0)
-                return;
-        desc = irq_to_desc(adev->irq);
-        chip = get_irq_chip(adev->irq);
-        disable_irq(adev->irq);
-        desc->status |= IRQ_MOVE_PCNTXT;
-        irq_set_affinity(adev->irq, cpumask_of(adev->cpu));
-        /* APB timer irqs are set up as mp_irqs, timer is edge triggerred */
-        set_irq_chip_and_handler_name(adev->irq, chip, handle_edge_irq, "edge");
-        enable_irq(adev->irq);
-        if (system_state == SYSTEM_BOOTING)
-                if (request_irq(adev->irq, apbt_interrupt_handler,
-                                IRQF_TIMER | IRQF_DISABLED | IRQF_NOBALANCING,
-                                adev->name, adev)) {
-                        printk(KERN_ERR "Failed request IRQ for APBT%d\n",
-                               adev->num);
-                }
-}
-#endif
-
 static void apbt_enable_int(int n)
 {
         unsigned long ctrl = apbt_readl(n, APBTMR_N_CONTROL);
@@ -334,6 +306,27 @@ static int __init apbt_clockevent_register(void)
 }
 
 #ifdef CONFIG_SMP
+
+static void apbt_setup_irq(struct apbt_dev *adev)
+{
+        /* timer0 irq has been setup early */
+        if (adev->irq == 0)
+                return;
+
+        if (system_state == SYSTEM_BOOTING) {
+                irq_modify_status(adev->irq, 0, IRQ_MOVE_PCNTXT);
+                /* APB timer irqs are set up as mp_irqs, timer is edge type */
+                __set_irq_handler(adev->irq, handle_edge_irq, 0, "edge");
+                if (request_irq(adev->irq, apbt_interrupt_handler,
+                                IRQF_TIMER | IRQF_DISABLED | IRQF_NOBALANCING,
+                                adev->name, adev)) {
+                        printk(KERN_ERR "Failed request IRQ for APBT%d\n",
+                               adev->num);
+                }
+        } else
+                enable_irq(adev->irq);
+}
+
 /* Should be called with per cpu */
 void apbt_setup_secondary_clock(void)
 {
@@ -343,7 +336,7 @@ void apbt_setup_secondary_clock(void)
 
         /* Don't register boot CPU clockevent */
         cpu = smp_processor_id();
-        if (cpu == boot_cpu_id)
+        if (!cpu)
                 return;
         /*
          * We need to calculate the scaled math multiplication factor for
@@ -389,16 +382,17 @@ static int apbt_cpuhp_notify(struct notifier_block *n,
 
         switch (action & 0xf) {
         case CPU_DEAD:
+                disable_irq(adev->irq);
                 apbt_disable_int(cpu);
-                if (system_state == SYSTEM_RUNNING)
+                if (system_state == SYSTEM_RUNNING) {
                         pr_debug("skipping APBT CPU %lu offline\n", cpu);
-                else if (adev) {
+                } else if (adev) {
                         pr_debug("APBT clockevent for cpu %lu offline\n", cpu);
                         free_irq(adev->irq, adev);
                 }
                 break;
         default:
-                pr_debug(KERN_INFO "APBT notified %lu, no action\n", action);
+                pr_debug("APBT notified %lu, no action\n", action);
         }
         return NOTIFY_OK;
 }
@@ -552,7 +546,7 @@ bad_count:
                 pr_debug("APB CS going back %lx:%lx:%lx ",
                          t2, last_read, t2 - last_read);
 bad_count_x3:
-                pr_debug(KERN_INFO "tripple check enforced\n");
+                pr_debug("triple check enforced\n");
                 t0 = apbt_readl(phy_cs_timer_id,
                                 APBTMR_N_CURRENT_VALUE);
                 udelay(1);
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index a2e0caf26e17..377f5db3b8b4 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -27,7 +27,7 @@
 #include <asm/gart.h>
 #include <asm/pci-direct.h>
 #include <asm/dma.h>
-#include <asm/k8.h>
+#include <asm/amd_nb.h>
 #include <asm/x86_init.h>
 
 int gart_iommu_aperture;
@@ -307,7 +307,7 @@ void __init early_gart_iommu_check(void)
                                 continue;
 
                         ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
-                        aper_enabled = ctl & AMD64_GARTEN;
+                        aper_enabled = ctl & GARTEN;
                         aper_order = (ctl >> 1) & 7;
                         aper_size = (32 * 1024 * 1024) << aper_order;
                         aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
@@ -362,7 +362,7 @@ void __init early_gart_iommu_check(void)
                                 continue;
 
                         ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
-                        ctl &= ~AMD64_GARTEN;
+                        ctl &= ~GARTEN;
                         write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
                 }
         }
@@ -505,8 +505,13 @@ out:
 
         /* Fix up the north bridges */
         for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
-                int bus;
-                int dev_base, dev_limit;
+                int bus, dev_base, dev_limit;
+
+                /*
+                 * Don't enable translation yet but enable GART IO and CPU
+                 * accesses and set DISTLBWALKPRB since GART table memory is UC.
+                 */
+                u32 ctl = DISTLBWALKPRB | aper_order << 1;
 
                 bus = bus_dev_ranges[i].bus;
                 dev_base = bus_dev_ranges[i].dev_base;
@@ -515,10 +520,7 @@ out:
                         if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
                                 continue;
 
-                        /* Don't enable translation yet. That is done later.
-                           Assume this BIOS didn't initialise the GART so
-                           just overwrite all previous bits */
-                        write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1);
+                        write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
                         write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
                 }
         }
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index e3b534cda49a..850657d1b0ed 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -52,6 +52,7 @@
 #include <asm/mce.h>
 #include <asm/kvm_para.h>
 #include <asm/tsc.h>
+#include <asm/atomic.h>
 
 unsigned int num_processors;
 
@@ -370,38 +371,87 @@ static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
 }
 
 /*
- * Setup extended LVT, AMD specific (K8, family 10h)
+ * Setup extended LVT, AMD specific
  *
- * Vector mappings are hard coded. On K8 only offset 0 (APIC500) and
- * MCE interrupts are supported. Thus MCE offset must be set to 0.
+ * Software should use the LVT offsets the BIOS provides.  The offsets
+ * are determined by the subsystems using it like those for MCE
+ * threshold or IBS.  On K8 only offset 0 (APIC500) and MCE interrupts
+ * are supported. Beginning with family 10h at least 4 offsets are
+ * available.
  *
- * If mask=1, the LVT entry does not generate interrupts while mask=0
- * enables the vector. See also the BKDGs.
+ * Since the offsets must be consistent for all cores, we keep track
+ * of the LVT offsets in software and reserve the offset for the same
+ * vector also to be used on other cores. An offset is freed by
+ * setting the entry to APIC_EILVT_MASKED.
+ *
+ * If the BIOS is right, there should be no conflicts. Otherwise a
+ * "[Firmware Bug]: ..." error message is generated. However, if
+ * software does not properly determines the offsets, it is not
+ * necessarily a BIOS bug.
  */
 
-#define APIC_EILVT_LVTOFF_MCE 0
-#define APIC_EILVT_LVTOFF_IBS 1
+static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
 
-static void setup_APIC_eilvt(u8 lvt_off, u8 vector, u8 msg_type, u8 mask)
+static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
 {
-        unsigned long reg = (lvt_off << 4) + APIC_EILVTn(0);
-        unsigned int  v   = (mask << 16) | (msg_type << 8) | vector;
-
-        apic_write(reg, v);
+        return (old & APIC_EILVT_MASKED)
+                || (new == APIC_EILVT_MASKED)
+                || ((new & ~APIC_EILVT_MASKED) == old);
 }
 
-u8 setup_APIC_eilvt_mce(u8 vector, u8 msg_type, u8 mask)
+static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
 {
-        setup_APIC_eilvt(APIC_EILVT_LVTOFF_MCE, vector, msg_type, mask);
-        return APIC_EILVT_LVTOFF_MCE;
+        unsigned int rsvd;                      /* 0: uninitialized */
+
+        if (offset >= APIC_EILVT_NR_MAX)
+                return ~0;
+
+        rsvd = atomic_read(&eilvt_offsets[offset]) & ~APIC_EILVT_MASKED;
+        do {
+                if (rsvd &&
+                    !eilvt_entry_is_changeable(rsvd, new))
+                        /* may not change if vectors are different */
+                        return rsvd;
+                rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
+        } while (rsvd != new);
+
+        return new;
 }
 
-u8 setup_APIC_eilvt_ibs(u8 vector, u8 msg_type, u8 mask)
+/*
+ * If mask=1, the LVT entry does not generate interrupts while mask=0
+ * enables the vector. See also the BKDGs.
+ */
+
+int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
 {
-        setup_APIC_eilvt(APIC_EILVT_LVTOFF_IBS, vector, msg_type, mask);
-        return APIC_EILVT_LVTOFF_IBS;
+        unsigned long reg = APIC_EILVTn(offset);
+        unsigned int new, old, reserved;
+
+        new = (mask << 16) | (msg_type << 8) | vector;
+        old = apic_read(reg);
+        reserved = reserve_eilvt_offset(offset, new);
+
+        if (reserved != new) {
+                pr_err(FW_BUG "cpu %d, try to setup vector 0x%x, but "
+                       "vector 0x%x was already reserved by another core, "
+                       "APIC%lX=0x%x\n",
+                       smp_processor_id(), new, reserved, reg, old);
+                return -EINVAL;
+        }
+
+        if (!eilvt_entry_is_changeable(old, new)) {
+                pr_err(FW_BUG "cpu %d, try to setup vector 0x%x but "
+                       "register already in use, APIC%lX=0x%x\n",
+                       smp_processor_id(), new, reg, old);
+                return -EBUSY;
+        }
+
+        apic_write(reg, new);
+
+        return 0;
 }
-EXPORT_SYMBOL_GPL(setup_APIC_eilvt_ibs);
+EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
 
 /*
  * Program the next event, relative to now
@@ -1665,10 +1715,7 @@ int __init APIC_init_uniprocessor(void)
         }
 #endif
 
-#ifndef CONFIG_SMP
-        enable_IR_x2apic();
         default_setup_apic_routing();
-#endif
 
         verify_local_APIC();
         connect_bsp_APIC();
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 5c5b8f3dddb5..8ae808d110f4 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -131,13 +131,9 @@ struct irq_pin_list {
         struct irq_pin_list *next;
 };
 
-static struct irq_pin_list *get_one_free_irq_2_pin(int node)
+static struct irq_pin_list *alloc_irq_pin_list(int node)
 {
-        struct irq_pin_list *pin;
-
-        pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
-
-        return pin;
+        return kzalloc_node(sizeof(struct irq_pin_list), GFP_KERNEL, node);
 }
 
 /* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
@@ -150,10 +146,7 @@ static struct irq_cfg irq_cfgx[NR_IRQS];
 int __init arch_early_irq_init(void)
 {
         struct irq_cfg *cfg;
-        struct irq_desc *desc;
-        int count;
-        int node;
-        int i;
+        int count, node, i;
 
         if (!legacy_pic->nr_legacy_irqs) {
                 nr_irqs_gsi = 0;
@@ -162,13 +155,15 @@ int __init arch_early_irq_init(void)
 
         cfg = irq_cfgx;
         count = ARRAY_SIZE(irq_cfgx);
-        node= cpu_to_node(boot_cpu_id);
+        node = cpu_to_node(0);
+
+        /* Make sure the legacy interrupts are marked in the bitmap */
+        irq_reserve_irqs(0, legacy_pic->nr_legacy_irqs);
 
         for (i = 0; i < count; i++) {
-                desc = irq_to_desc(i);
-                desc->chip_data = &cfg[i];
-                zalloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node);
-                zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node);
+                set_irq_chip_data(i, &cfg[i]);
+                zalloc_cpumask_var_node(&cfg[i].domain, GFP_KERNEL, node);
+                zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_KERNEL, node);
                 /*
                  * For legacy IRQ's, start with assigning irq0 to irq15 to
                  * IRQ0_VECTOR to IRQ15_VECTOR on cpu 0.
@@ -183,170 +178,88 @@ int __init arch_early_irq_init(void)
 }
 
 #ifdef CONFIG_SPARSE_IRQ
-struct irq_cfg *irq_cfg(unsigned int irq)
+static struct irq_cfg *irq_cfg(unsigned int irq)
 {
-        struct irq_cfg *cfg = NULL;
-        struct irq_desc *desc;
-
-        desc = irq_to_desc(irq);
-        if (desc)
-                cfg = desc->chip_data;
-
-        return cfg;
+        return get_irq_chip_data(irq);
 }
 
-static struct irq_cfg *get_one_free_irq_cfg(int node)
+static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
 {
         struct irq_cfg *cfg;
 
-        cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
-        if (cfg) {
-                if (!zalloc_cpumask_var_node(&cfg->domain, GFP_ATOMIC, node)) {
-                        kfree(cfg);
-                        cfg = NULL;
-                } else if (!zalloc_cpumask_var_node(&cfg->old_domain,
-                                                          GFP_ATOMIC, node)) {
-                        free_cpumask_var(cfg->domain);
-                        kfree(cfg);
-                        cfg = NULL;
-                }
-        }
-
+        cfg = kzalloc_node(sizeof(*cfg), GFP_KERNEL, node);
+        if (!cfg)
+                return NULL;
+        if (!zalloc_cpumask_var_node(&cfg->domain, GFP_KERNEL, node))
+                goto out_cfg;
+        if (!zalloc_cpumask_var_node(&cfg->old_domain, GFP_KERNEL, node))
+                goto out_domain;
         return cfg;
+out_domain:
+        free_cpumask_var(cfg->domain);
+out_cfg:
+        kfree(cfg);
+        return NULL;
 }
 
-int arch_init_chip_data(struct irq_desc *desc, int node)
-{
-        struct irq_cfg *cfg;
-
-        cfg = desc->chip_data;
-        if (!cfg) {
-                desc->chip_data = get_one_free_irq_cfg(node);
-                if (!desc->chip_data) {
-                        printk(KERN_ERR "can not alloc irq_cfg\n");
-                        BUG_ON(1);
-                }
-        }
-
-        return 0;
-}
-
-/* for move_irq_desc */
-static void
-init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int node)
+static void free_irq_cfg(unsigned int at, struct irq_cfg *cfg)
 {
-        struct irq_pin_list *old_entry, *head, *tail, *entry;
-
-        cfg->irq_2_pin = NULL;
-        old_entry = old_cfg->irq_2_pin;
-        if (!old_entry)
-                return;
-
-        entry = get_one_free_irq_2_pin(node);
-        if (!entry)
+        if (!cfg)
                 return;
+        set_irq_chip_data(at, NULL);
+        free_cpumask_var(cfg->domain);
+        free_cpumask_var(cfg->old_domain);
+        kfree(cfg);
+}
 
-        entry->apic     = old_entry->apic;
-        entry->pin      = old_entry->pin;
-        head            = entry;
-        tail            = entry;
-        old_entry       = old_entry->next;
-        while (old_entry) {
-                entry = get_one_free_irq_2_pin(node);
-                if (!entry) {
-                        entry = head;
-                        while (entry) {
-                                head = entry->next;
-                                kfree(entry);
-                                entry = head;
-                        }
-                        /* still use the old one */
-                        return;
-                }
-                entry->apic     = old_entry->apic;
-                entry->pin      = old_entry->pin;
-                tail->next      = entry;
-                tail            = entry;
-                old_entry       = old_entry->next;
-        }
+#else
 
-        tail->next = NULL;
-        cfg->irq_2_pin = head;
+struct irq_cfg *irq_cfg(unsigned int irq)
+{
+        return irq < nr_irqs ? irq_cfgx + irq : NULL;
 }
 
-static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
+static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
 {
-        struct irq_pin_list *entry, *next;
-
-        if (old_cfg->irq_2_pin == cfg->irq_2_pin)
-                return;
+        return irq_cfgx + irq;
+}
 
-        entry = old_cfg->irq_2_pin;
+static inline void free_irq_cfg(unsigned int at, struct irq_cfg *cfg) { }
 
-        while (entry) {
-                next = entry->next;
-                kfree(entry);
-                entry = next;
-        }
-        old_cfg->irq_2_pin = NULL;
-}
+#endif
 
-void arch_init_copy_chip_data(struct irq_desc *old_desc,
-                                 struct irq_desc *desc, int node)
+static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
 {
+        int res = irq_alloc_desc_at(at, node);
         struct irq_cfg *cfg;
-        struct irq_cfg *old_cfg;
-
-        cfg = get_one_free_irq_cfg(node);
 
-        if (!cfg)
-                return;
-
-        desc->chip_data = cfg;
-
-        old_cfg = old_desc->chip_data;
-
-        cfg->vector = old_cfg->vector;
-        cfg->move_in_progress = old_cfg->move_in_progress;
-        cpumask_copy(cfg->domain, old_cfg->domain);
-        cpumask_copy(cfg->old_domain, old_cfg->old_domain);
-
-        init_copy_irq_2_pin(old_cfg, cfg, node);
-}
+        if (res < 0) {
+                if (res != -EEXIST)
+                        return NULL;
+                cfg = get_irq_chip_data(at);
+                if (cfg)
+                        return cfg;
+        }
 
-static void free_irq_cfg(struct irq_cfg *cfg)
-{
-        free_cpumask_var(cfg->domain);
-        free_cpumask_var(cfg->old_domain);
-        kfree(cfg);
+        cfg = alloc_irq_cfg(at, node);
+        if (cfg)
+                set_irq_chip_data(at, cfg);
+        else
+                irq_free_desc(at);
+        return cfg;
 }
 
-void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
+static int alloc_irq_from(unsigned int from, int node)
 {
-        struct irq_cfg *old_cfg, *cfg;
-
-        old_cfg = old_desc->chip_data;
-        cfg = desc->chip_data;
-
-        if (old_cfg == cfg)
-                return;
-
-        if (old_cfg) {
-                free_irq_2_pin(old_cfg, cfg);
-                free_irq_cfg(old_cfg);
-                old_desc->chip_data = NULL;
-        }
+        return irq_alloc_desc_from(from, node);
 }
-/* end for move_irq_desc */
 
-#else
-struct irq_cfg *irq_cfg(unsigned int irq)
+static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
 {
-        return irq < nr_irqs ? irq_cfgx + irq : NULL;
+        free_irq_cfg(at, cfg);
+        irq_free_desc(at);
 }
 
-#endif
-
 struct io_apic {
         unsigned int index;
         unsigned int unused[3];
@@ -451,7 +364,7 @@ __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
         io_apic_write(apic, 0x10 + 2*pin, eu.w1);
 }
 
-void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
+static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
 {
         unsigned long flags;
         raw_spin_lock_irqsave(&ioapic_lock, flags);
@@ -481,7 +394,7 @@ static void ioapic_mask_entry(int apic, int pin)
  * fast in the common case, and fast for shared ISA-space IRQs.
  */
 static int
-add_pin_to_irq_node_nopanic(struct irq_cfg *cfg, int node, int apic, int pin)
+__add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
 {
         struct irq_pin_list **last, *entry;
 
@@ -493,7 +406,7 @@ add_pin_to_irq_node_nopanic(struct irq_cfg *cfg, int node, int apic, int pin)
                 last = &entry->next;
         }
 
-        entry = get_one_free_irq_2_pin(node);
+        entry = alloc_irq_pin_list(node);
         if (!entry) {
                 printk(KERN_ERR "can not alloc irq_pin_list (%d,%d,%d)\n",
                                 node, apic, pin);
@@ -508,7 +421,7 @@ add_pin_to_irq_node_nopanic(struct irq_cfg *cfg, int node, int apic, int pin)
 
 static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
 {
-        if (add_pin_to_irq_node_nopanic(cfg, node, apic, pin))
+        if (__add_pin_to_irq_node(cfg, node, apic, pin))
                 panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
 }
 
@@ -571,11 +484,6 @@ static void __unmask_and_level_IO_APIC_irq(struct irq_pin_list *entry)
                              IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
 }
 
-static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
-{
-        io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
-}
-
 static void io_apic_sync(struct irq_pin_list *entry)
 {
         /*
@@ -587,44 +495,37 @@ static void io_apic_sync(struct irq_pin_list *entry)
         readl(&io_apic->data);
 }
 
-static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
+static void mask_ioapic(struct irq_cfg *cfg)
 {
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&ioapic_lock, flags);
         io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
+        raw_spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
-static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
+static void mask_ioapic_irq(struct irq_data *data)
 {
-        struct irq_cfg *cfg = desc->chip_data;
-        unsigned long flags;
-
-        BUG_ON(!cfg);
+        mask_ioapic(data->chip_data);
+}
 
-        raw_spin_lock_irqsave(&ioapic_lock, flags);
-        __mask_IO_APIC_irq(cfg);
-        raw_spin_unlock_irqrestore(&ioapic_lock, flags);
+static void __unmask_ioapic(struct irq_cfg *cfg)
+{
+        io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
 }
 
-static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
+static void unmask_ioapic(struct irq_cfg *cfg)
 {
-        struct irq_cfg *cfg = desc->chip_data;
         unsigned long flags;
 
         raw_spin_lock_irqsave(&ioapic_lock, flags);
-        __unmask_IO_APIC_irq(cfg);
+        __unmask_ioapic(cfg);
         raw_spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
-static void mask_IO_APIC_irq(unsigned int irq)
+static void unmask_ioapic_irq(struct irq_data *data)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-
-        mask_IO_APIC_irq_desc(desc);
-}
-static void unmask_IO_APIC_irq(unsigned int irq)
-{
-        struct irq_desc *desc = irq_to_desc(irq);
-
-        unmask_IO_APIC_irq_desc(desc);
+        unmask_ioapic(data->chip_data);
 }
 
 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
@@ -694,14 +595,14 @@ struct IO_APIC_route_entry **alloc_ioapic_entries(void)
         struct IO_APIC_route_entry **ioapic_entries;
 
         ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
-                                GFP_ATOMIC);
+                                GFP_KERNEL);
         if (!ioapic_entries)
                 return 0;
 
         for (apic = 0; apic < nr_ioapics; apic++) {
                 ioapic_entries[apic] =
                         kzalloc(sizeof(struct IO_APIC_route_entry) *
-                                nr_ioapic_registers[apic], GFP_ATOMIC);
+                                nr_ioapic_registers[apic], GFP_KERNEL);
                 if (!ioapic_entries[apic])
                         goto nomem;
         }
@@ -1259,7 +1160,6 @@ void __setup_vector_irq(int cpu)
         /* Initialize vector_irq on a new cpu */
         int irq, vector;
         struct irq_cfg *cfg;
-        struct irq_desc *desc;
 
         /*
          * vector_lock will make sure that we don't run into irq vector
@@ -1268,9 +1168,10 @@ void __setup_vector_irq(int cpu)
          */
         raw_spin_lock(&vector_lock);
         /* Mark the inuse vectors */
-        for_each_irq_desc(irq, desc) {
-                cfg = desc->chip_data;
-
+        for_each_active_irq(irq) {
+                cfg = get_irq_chip_data(irq);
+                if (!cfg)
+                        continue;
                 /*
                  * If it is a legacy IRQ handled by the legacy PIC, this cpu
                  * will be part of the irq_cfg's domain.
@@ -1327,17 +1228,17 @@ static inline int IO_APIC_irq_trigger(int irq)
 }
 #endif
 
-static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
+static void ioapic_register_intr(unsigned int irq, unsigned long trigger)
 {
 
         if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
             trigger == IOAPIC_LEVEL)
-                desc->status |= IRQ_LEVEL;
+                irq_set_status_flags(irq, IRQ_LEVEL);
         else
-                desc->status &= ~IRQ_LEVEL;
+                irq_clear_status_flags(irq, IRQ_LEVEL);
 
-        if (irq_remapped(irq)) {
-                desc->status |= IRQ_MOVE_PCNTXT;
+        if (irq_remapped(get_irq_chip_data(irq))) {
+                irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
                 if (trigger)
                         set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
                                                       handle_fasteoi_irq,
@@ -1358,10 +1259,10 @@ static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long t
                                               handle_edge_irq, "edge");
 }
 
-int setup_ioapic_entry(int apic_id, int irq,
-                       struct IO_APIC_route_entry *entry,
-                       unsigned int destination, int trigger,
-                       int polarity, int vector, int pin)
+static int setup_ioapic_entry(int apic_id, int irq,
+                              struct IO_APIC_route_entry *entry,
+                              unsigned int destination, int trigger,
+                              int polarity, int vector, int pin)
 {
         /*
          * add it to the IO-APIC irq-routing table:
@@ -1382,21 +1283,7 @@ int setup_ioapic_entry(int apic_id, int irq,
                 if (index < 0)
                         panic("Failed to allocate IRTE for ioapic %d\n", apic_id);
 
-                memset(&irte, 0, sizeof(irte));
-
-                irte.present = 1;
-                irte.dst_mode = apic->irq_dest_mode;
-                /*
-                 * Trigger mode in the IRTE will always be edge, and the
-                 * actual level or edge trigger will be setup in the IO-APIC
-                 * RTE. This will help simplify level triggered irq migration.
-                 * For more details, see the comments above explainig IO-APIC
-                 * irq migration in the presence of interrupt-remapping.
-                 */
-                irte.trigger_mode = 0;
-                irte.dlvry_mode = apic->irq_delivery_mode;
-                irte.vector = vector;
-                irte.dest_id = IRTE_DEST(destination);
+                prepare_irte(&irte, vector, destination);
 
                 /* Set source-id of interrupt request */
                 set_ioapic_sid(&irte, apic_id);
@@ -1431,18 +1318,14 @@ int setup_ioapic_entry(int apic_id, int irq,
         return 0;
 }
 
-static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq_desc *desc,
-                              int trigger, int polarity)
+static void setup_ioapic_irq(int apic_id, int pin, unsigned int irq,
+                             struct irq_cfg *cfg, int trigger, int polarity)
 {
-        struct irq_cfg *cfg;
         struct IO_APIC_route_entry entry;
         unsigned int dest;
 
         if (!IO_APIC_IRQ(irq))
                 return;
-
-        cfg = desc->chip_data;
-
         /*
          * For legacy irqs, cfg->domain starts with cpu 0 for legacy
          * controllers like 8259. Now that IO-APIC can handle this irq, update
@@ -1471,9 +1354,9 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq
                 return;
         }
 
-        ioapic_register_intr(irq, desc, trigger);
+        ioapic_register_intr(irq, trigger);
         if (irq < legacy_pic->nr_legacy_irqs)
-                legacy_pic->chip->mask(irq);
+                legacy_pic->mask(irq);
 
         ioapic_write_entry(apic_id, pin, entry);
 }
@@ -1484,11 +1367,9 @@ static struct {
 
 static void __init setup_IO_APIC_irqs(void)
 {
-        int apic_id, pin, idx, irq;
-        int notcon = 0;
-        struct irq_desc *desc;
+        int apic_id, pin, idx, irq, notcon = 0;
+        int node = cpu_to_node(0);
         struct irq_cfg *cfg;
-        int node = cpu_to_node(boot_cpu_id);
 
         apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
 
@@ -1525,19 +1406,17 @@ static void __init setup_IO_APIC_irqs(void)
                                 apic->multi_timer_check(apic_id, irq))
                         continue;
 
-                desc = irq_to_desc_alloc_node(irq, node);
-                if (!desc) {
-                        printk(KERN_INFO "can not get irq_desc for %d\n", irq);
+                cfg = alloc_irq_and_cfg_at(irq, node);
+                if (!cfg)
                         continue;
-                }
-                cfg = desc->chip_data;
+
                 add_pin_to_irq_node(cfg, node, apic_id, pin);
                 /*
                  * don't mark it in pin_programmed, so later acpi could
                  * set it correctly when irq < 16
                  */
-                setup_IO_APIC_irq(apic_id, pin, irq, desc,
-                                irq_trigger(idx), irq_polarity(idx));
+                setup_ioapic_irq(apic_id, pin, irq, cfg, irq_trigger(idx),
+                                  irq_polarity(idx));
         }
 
         if (notcon)
@@ -1552,9 +1431,7 @@ static void __init setup_IO_APIC_irqs(void)
  */
 void setup_IO_APIC_irq_extra(u32 gsi)
 {
-        int apic_id = 0, pin, idx, irq;
-        int node = cpu_to_node(boot_cpu_id);
-        struct irq_desc *desc;
+        int apic_id = 0, pin, idx, irq, node = cpu_to_node(0);
         struct irq_cfg *cfg;
 
         /*
@@ -1570,18 +1447,15 @@ void setup_IO_APIC_irq_extra(u32 gsi)
                 return;
 
         irq = pin_2_irq(idx, apic_id, pin);
-#ifdef CONFIG_SPARSE_IRQ
-        desc = irq_to_desc(irq);
-        if (desc)
+
+        /* Only handle the non legacy irqs on secondary ioapics */
+        if (apic_id == 0 || irq < NR_IRQS_LEGACY)
                 return;
-#endif
-        desc = irq_to_desc_alloc_node(irq, node);
-        if (!desc) {
-                printk(KERN_INFO "can not get irq_desc for %d\n", irq);
+
+        cfg = alloc_irq_and_cfg_at(irq, node);
+        if (!cfg)
                 return;
-        }
 
-        cfg = desc->chip_data;
         add_pin_to_irq_node(cfg, node, apic_id, pin);
 
         if (test_bit(pin, mp_ioapic_routing[apic_id].pin_programmed)) {
@@ -1591,7 +1465,7 @@ void setup_IO_APIC_irq_extra(u32 gsi)
         }
         set_bit(pin, mp_ioapic_routing[apic_id].pin_programmed);
 
-        setup_IO_APIC_irq(apic_id, pin, irq, desc,
+        setup_ioapic_irq(apic_id, pin, irq, cfg,
                         irq_trigger(idx), irq_polarity(idx));
 }
 
@@ -1642,7 +1516,6 @@ __apicdebuginit(void) print_IO_APIC(void)
         union IO_APIC_reg_03 reg_03;
         unsigned long flags;
         struct irq_cfg *cfg;
-        struct irq_desc *desc;
         unsigned int irq;
 
         printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
@@ -1729,10 +1602,10 @@ __apicdebuginit(void) print_IO_APIC(void)
         }
         }
         printk(KERN_DEBUG "IRQ to pin mappings:\n");
-        for_each_irq_desc(irq, desc) {
+        for_each_active_irq(irq) {
                 struct irq_pin_list *entry;
 
-                cfg = desc->chip_data;
+                cfg = get_irq_chip_data(irq);
                 if (!cfg)
                         continue;
                 entry = cfg->irq_2_pin;
@@ -2239,29 +2112,26 @@ static int __init timer_irq_works(void)
  * an edge even if it isn't on the 8259A...
  */
 
-static unsigned int startup_ioapic_irq(unsigned int irq)
+static unsigned int startup_ioapic_irq(struct irq_data *data)
 {
-        int was_pending = 0;
+        int was_pending = 0, irq = data->irq;
         unsigned long flags;
-        struct irq_cfg *cfg;
 
         raw_spin_lock_irqsave(&ioapic_lock, flags);
         if (irq < legacy_pic->nr_legacy_irqs) {
-                legacy_pic->chip->mask(irq);
+                legacy_pic->mask(irq);
                 if (legacy_pic->irq_pending(irq))
                         was_pending = 1;
         }
-        cfg = irq_cfg(irq);
-        __unmask_IO_APIC_irq(cfg);
+        __unmask_ioapic(data->chip_data);
         raw_spin_unlock_irqrestore(&ioapic_lock, flags);
 
         return was_pending;
 }
 
-static int ioapic_retrigger_irq(unsigned int irq)
+static int ioapic_retrigger_irq(struct irq_data *data)
 {
-
-        struct irq_cfg *cfg = irq_cfg(irq);
+        struct irq_cfg *cfg = data->chip_data;
         unsigned long flags;
 
         raw_spin_lock_irqsave(&vector_lock, flags);
@@ -2312,7 +2182,7 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq
                  * With interrupt-remapping, destination information comes
                  * from interrupt-remapping table entry.
                  */
-                if (!irq_remapped(irq))
+                if (!irq_remapped(cfg))
                         io_apic_write(apic, 0x11 + pin*2, dest);
                 reg = io_apic_read(apic, 0x10 + pin*2);
                 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
@@ -2322,65 +2192,46 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq
 }
 
 /*
- * Either sets desc->affinity to a valid value, and returns
+ * Either sets data->affinity to a valid value, and returns
  * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
- * leaves desc->affinity untouched.
+ * leaves data->affinity untouched.
  */
-unsigned int
-set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask,
-                  unsigned int *dest_id)
+int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
+                          unsigned int *dest_id)
 {
-        struct irq_cfg *cfg;
-        unsigned int irq;
+        struct irq_cfg *cfg = data->chip_data;
 
         if (!cpumask_intersects(mask, cpu_online_mask))
                 return -1;
 
-        irq = desc->irq;
-        cfg = desc->chip_data;
-        if (assign_irq_vector(irq, cfg, mask))
+        if (assign_irq_vector(data->irq, data->chip_data, mask))
                 return -1;
 
-        cpumask_copy(desc->affinity, mask);
+        cpumask_copy(data->affinity, mask);
 
-        *dest_id = apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
+        *dest_id = apic->cpu_mask_to_apicid_and(mask, cfg->domain);
         return 0;
 }
 
 static int
-set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
+ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
+                    bool force)
 {
-        struct irq_cfg *cfg;
+        unsigned int dest, irq = data->irq;
         unsigned long flags;
-        unsigned int dest;
-        unsigned int irq;
-        int ret = -1;
-
-        irq = desc->irq;
-        cfg = desc->chip_data;
+        int ret;
 
         raw_spin_lock_irqsave(&ioapic_lock, flags);
-        ret = set_desc_affinity(desc, mask, &dest);
+        ret = __ioapic_set_affinity(data, mask, &dest);
         if (!ret) {
                 /* Only the high 8 bits are valid. */
                 dest = SET_APIC_LOGICAL_ID(dest);
-                __target_IO_APIC_irq(irq, dest, cfg);
+                __target_IO_APIC_irq(irq, dest, data->chip_data);
         }
         raw_spin_unlock_irqrestore(&ioapic_lock, flags);
-
         return ret;
 }
 
-static int
-set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
-{
-        struct irq_desc *desc;
-
-        desc = irq_to_desc(irq);
-
-        return set_ioapic_affinity_irq_desc(desc, mask);
-}
-
 #ifdef CONFIG_INTR_REMAP
 
 /*
@@ -2395,24 +2246,21 @@ set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
  * the interrupt-remapping table entry.
  */
 static int
-migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
+ir_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
+                       bool force)
 {
-        struct irq_cfg *cfg;
+        struct irq_cfg *cfg = data->chip_data;
+        unsigned int dest, irq = data->irq;
         struct irte irte;
-        unsigned int dest;
-        unsigned int irq;
-        int ret = -1;
 
         if (!cpumask_intersects(mask, cpu_online_mask))
-                return ret;
+                return -EINVAL;
 
-        irq = desc->irq;
         if (get_irte(irq, &irte))
-                return ret;
+                return -EBUSY;
 
-        cfg = desc->chip_data;
         if (assign_irq_vector(irq, cfg, mask))
-                return ret;
+                return -EBUSY;
 
         dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
 
@@ -2427,29 +2275,14 @@ migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
         if (cfg->move_in_progress)
                 send_cleanup_vector(cfg);
 
-        cpumask_copy(desc->affinity, mask);
-
+        cpumask_copy(data->affinity, mask);
         return 0;
 }
 
-/*
- * Migrates the IRQ destination in the process context.
- */
-static int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
-                                            const struct cpumask *mask)
-{
-        return migrate_ioapic_irq_desc(desc, mask);
-}
-static int set_ir_ioapic_affinity_irq(unsigned int irq,
-                                       const struct cpumask *mask)
-{
-        struct irq_desc *desc = irq_to_desc(irq);
-
-        return set_ir_ioapic_affinity_irq_desc(desc, mask);
-}
 #else
-static inline int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
-                                                   const struct cpumask *mask)
+static inline int
+ir_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
+                       bool force)
 {
         return 0;
 }
@@ -2511,10 +2344,8 @@ unlock:
         irq_exit();
 }
 
-static void __irq_complete_move(struct irq_desc **descp, unsigned vector)
+static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
 {
-        struct irq_desc *desc = *descp;
-        struct irq_cfg *cfg = desc->chip_data;
         unsigned me;
 
         if (likely(!cfg->move_in_progress))
@@ -2526,31 +2357,28 @@ static void __irq_complete_move(struct irq_desc **descp, unsigned vector)
                 send_cleanup_vector(cfg);
 }
 
-static void irq_complete_move(struct irq_desc **descp)
+static void irq_complete_move(struct irq_cfg *cfg)
 {
-        __irq_complete_move(descp, ~get_irq_regs()->orig_ax);
+        __irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
 }
 
 void irq_force_complete_move(int irq)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg = desc->chip_data;
+        struct irq_cfg *cfg = get_irq_chip_data(irq);
 
         if (!cfg)
                 return;
 
-        __irq_complete_move(&desc, cfg->vector);
+        __irq_complete_move(cfg, cfg->vector);
 }
 #else
-static inline void irq_complete_move(struct irq_desc **descp) {}
+static inline void irq_complete_move(struct irq_cfg *cfg) { }
 #endif
 
-static void ack_apic_edge(unsigned int irq)
+static void ack_apic_edge(struct irq_data *data)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-
-        irq_complete_move(&desc);
-        move_native_irq(irq);
+        irq_complete_move(data->chip_data);
+        move_native_irq(data->irq);
         ack_APIC_irq();
 }
 
@@ -2572,10 +2400,12 @@ atomic_t irq_mis_count;
  * Otherwise, we simulate the EOI message manually by changing the trigger
  * mode to edge and then back to level, with RTE being masked during this.
 */
-static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
+static void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
 {
         struct irq_pin_list *entry;
+        unsigned long flags;
 
+        raw_spin_lock_irqsave(&ioapic_lock, flags);
         for_each_irq_pin(entry, cfg->irq_2_pin) {
                 if (mp_ioapics[entry->apic].apicver >= 0x20) {
                         /*
@@ -2584,7 +2414,7 @@ static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
                          * intr-remapping table entry. Hence for the io-apic
                          * EOI we use the pin number.
                          */
-                        if (irq_remapped(irq))
+                        if (irq_remapped(cfg))
                                 io_apic_eoi(entry->apic, entry->pin);
                         else
                                 io_apic_eoi(entry->apic, cfg->vector);
@@ -2593,36 +2423,22 @@ static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
                         __unmask_and_level_IO_APIC_irq(entry);
                 }
         }
-}
-
-static void eoi_ioapic_irq(struct irq_desc *desc)
-{
-        struct irq_cfg *cfg;
-        unsigned long flags;
-        unsigned int irq;
-
-        irq = desc->irq;
-        cfg = desc->chip_data;
-
-        raw_spin_lock_irqsave(&ioapic_lock, flags);
-        __eoi_ioapic_irq(irq, cfg);
         raw_spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
-static void ack_apic_level(unsigned int irq)
+static void ack_apic_level(struct irq_data *data)
 {
+        struct irq_cfg *cfg = data->chip_data;
+        int i, do_unmask_irq = 0, irq = data->irq;
         struct irq_desc *desc = irq_to_desc(irq);
         unsigned long v;
-        int i;
-        struct irq_cfg *cfg;
-        int do_unmask_irq = 0;
 
-        irq_complete_move(&desc);
+        irq_complete_move(cfg);
 #ifdef CONFIG_GENERIC_PENDING_IRQ
         /* If we are moving the irq we need to mask it */
         if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
                 do_unmask_irq = 1;
-                mask_IO_APIC_irq_desc(desc);
+                mask_ioapic(cfg);
         }
 #endif
 
@@ -2658,7 +2474,6 @@ static void ack_apic_level(unsigned int irq)
          * we use the above logic (mask+edge followed by unmask+level) from
          * Manfred Spraul to clear the remote IRR.
          */
-        cfg = desc->chip_data;
         i = cfg->vector;
         v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
 
@@ -2678,7 +2493,7 @@ static void ack_apic_level(unsigned int irq)
         if (!(v & (1 << (i & 0x1f)))) {
                 atomic_inc(&irq_mis_count);
 
-                eoi_ioapic_irq(desc);
+                eoi_ioapic_irq(irq, cfg);
         }
 
         /* Now we can move and renable the irq */
@@ -2709,61 +2524,57 @@ static void ack_apic_level(unsigned int irq)
                  * accurate and is causing problems then it is a hardware bug
                  * and you can go talk to the chipset vendor about it.
                  */
-                cfg = desc->chip_data;
                 if (!io_apic_level_ack_pending(cfg))
                         move_masked_irq(irq);
-                unmask_IO_APIC_irq_desc(desc);
+                unmask_ioapic(cfg);
         }
 }
 
 #ifdef CONFIG_INTR_REMAP
-static void ir_ack_apic_edge(unsigned int irq)
+static void ir_ack_apic_edge(struct irq_data *data)
 {
         ack_APIC_irq();
 }
 
-static void ir_ack_apic_level(unsigned int irq)
+static void ir_ack_apic_level(struct irq_data *data)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-
         ack_APIC_irq();
-        eoi_ioapic_irq(desc);
+        eoi_ioapic_irq(data->irq, data->chip_data);
 }
 #endif /* CONFIG_INTR_REMAP */
 
 static struct irq_chip ioapic_chip __read_mostly = {
-        .name           = "IO-APIC",
-        .startup        = startup_ioapic_irq,
-        .mask           = mask_IO_APIC_irq,
-        .unmask         = unmask_IO_APIC_irq,
-        .ack            = ack_apic_edge,
-        .eoi            = ack_apic_level,
+        .name                   = "IO-APIC",
+        .irq_startup            = startup_ioapic_irq,
+        .irq_mask               = mask_ioapic_irq,
+        .irq_unmask             = unmask_ioapic_irq,
+        .irq_ack                = ack_apic_edge,
+        .irq_eoi                = ack_apic_level,
 #ifdef CONFIG_SMP
-        .set_affinity   = set_ioapic_affinity_irq,
+        .irq_set_affinity       = ioapic_set_affinity,
 #endif
-        .retrigger      = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 static struct irq_chip ir_ioapic_chip __read_mostly = {
-        .name           = "IR-IO-APIC",
-        .startup        = startup_ioapic_irq,
-        .mask           = mask_IO_APIC_irq,
-        .unmask         = unmask_IO_APIC_irq,
+        .name                   = "IR-IO-APIC",
+        .irq_startup            = startup_ioapic_irq,
+        .irq_mask               = mask_ioapic_irq,
+        .irq_unmask             = unmask_ioapic_irq,
 #ifdef CONFIG_INTR_REMAP
-        .ack            = ir_ack_apic_edge,
-        .eoi            = ir_ack_apic_level,
+        .irq_ack                = ir_ack_apic_edge,
+        .irq_eoi                = ir_ack_apic_level,
 #ifdef CONFIG_SMP
-        .set_affinity   = set_ir_ioapic_affinity_irq,
+        .irq_set_affinity       = ir_ioapic_set_affinity,
 #endif
 #endif
-        .retrigger      = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 static inline void init_IO_APIC_traps(void)
 {
-        int irq;
-        struct irq_desc *desc;
         struct irq_cfg *cfg;
+        unsigned int irq;
 
         /*
          * NOTE! The local APIC isn't very good at handling
@@ -2776,8 +2587,8 @@ static inline void init_IO_APIC_traps(void)
          * Also, we've got to be careful not to trash gate
          * 0x80, because int 0x80 is hm, kind of importantish. ;)
          */
-        for_each_irq_desc(irq, desc) {
-                cfg = desc->chip_data;
+        for_each_active_irq(irq) {
+                cfg = get_irq_chip_data(irq);
                 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
                         /*
                          * Hmm.. We don't have an entry for this,
@@ -2788,7 +2599,7 @@ static inline void init_IO_APIC_traps(void)
                                 legacy_pic->make_irq(irq);
                         else
                                 /* Strange. Oh, well.. */
-                                desc->chip = &no_irq_chip;
+                                set_irq_chip(irq, &no_irq_chip);
                 }
         }
 }
@@ -2797,7 +2608,7 @@ static inline void init_IO_APIC_traps(void)
  * The local APIC irq-chip implementation:
  */
 
-static void mask_lapic_irq(unsigned int irq)
+static void mask_lapic_irq(struct irq_data *data)
 {
         unsigned long v;
 
@@ -2805,7 +2616,7 @@ static void mask_lapic_irq(unsigned int irq)
         apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
 }
 
-static void unmask_lapic_irq(unsigned int irq)
+static void unmask_lapic_irq(struct irq_data *data)
 {
         unsigned long v;
 
@@ -2813,21 +2624,21 @@ static void unmask_lapic_irq(unsigned int irq)
         apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
 }
 
-static void ack_lapic_irq(unsigned int irq)
+static void ack_lapic_irq(struct irq_data *data)
 {
         ack_APIC_irq();
 }
 
 static struct irq_chip lapic_chip __read_mostly = {
         .name           = "local-APIC",
-        .mask           = mask_lapic_irq,
-        .unmask         = unmask_lapic_irq,
-        .ack            = ack_lapic_irq,
+        .irq_mask       = mask_lapic_irq,
+        .irq_unmask     = unmask_lapic_irq,
+        .irq_ack        = ack_lapic_irq,
 };
 
-static void lapic_register_intr(int irq, struct irq_desc *desc)
+static void lapic_register_intr(int irq)
 {
-        desc->status &= ~IRQ_LEVEL;
+        irq_clear_status_flags(irq, IRQ_LEVEL);
         set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
                                       "edge");
 }
@@ -2930,9 +2741,8 @@ int timer_through_8259 __initdata;
  */
 static inline void __init check_timer(void)
 {
-        struct irq_desc *desc = irq_to_desc(0);
-        struct irq_cfg *cfg = desc->chip_data;
-        int node = cpu_to_node(boot_cpu_id);
+        struct irq_cfg *cfg = get_irq_chip_data(0);
+        int node = cpu_to_node(0);
         int apic1, pin1, apic2, pin2;
         unsigned long flags;
         int no_pin1 = 0;
@@ -2942,7 +2752,7 @@ static inline void __init check_timer(void)
         /*
          * get/set the timer IRQ vector:
          */
-        legacy_pic->chip->mask(0);
+        legacy_pic->mask(0);
         assign_irq_vector(0, cfg, apic->target_cpus());
 
         /*
@@ -3001,7 +2811,7 @@ static inline void __init check_timer(void)
                         add_pin_to_irq_node(cfg, node, apic1, pin1);
                         setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
                 } else {
-                        /* for edge trigger, setup_IO_APIC_irq already
+                        /* for edge trigger, setup_ioapic_irq already
                          * leave it unmasked.
                          * so only need to unmask if it is level-trigger
                          * do we really have level trigger timer?
@@ -3009,12 +2819,12 @@ static inline void __init check_timer(void)
                         int idx;
                         idx = find_irq_entry(apic1, pin1, mp_INT);
                         if (idx != -1 && irq_trigger(idx))
-                                unmask_IO_APIC_irq_desc(desc);
+                                unmask_ioapic(cfg);
                 }
                 if (timer_irq_works()) {
                         if (nmi_watchdog == NMI_IO_APIC) {
                                 setup_nmi();
-                                legacy_pic->chip->unmask(0);
+                                legacy_pic->unmask(0);
                         }
                         if (disable_timer_pin_1 > 0)
                                 clear_IO_APIC_pin(0, pin1);
@@ -3037,14 +2847,14 @@ static inline void __init check_timer(void)
                  */
                 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
                 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
-                legacy_pic->chip->unmask(0);
+                legacy_pic->unmask(0);
                 if (timer_irq_works()) {
                         apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
                         timer_through_8259 = 1;
                         if (nmi_watchdog == NMI_IO_APIC) {
-                                legacy_pic->chip->mask(0);
+                                legacy_pic->mask(0);
                                 setup_nmi();
-                                legacy_pic->chip->unmask(0);
+                                legacy_pic->unmask(0);
                         }
                         goto out;
                 }
@@ -3052,7 +2862,7 @@ static inline void __init check_timer(void)
                  * Cleanup, just in case ...
                  */
                 local_irq_disable();
-                legacy_pic->chip->mask(0);
+                legacy_pic->mask(0);
                 clear_IO_APIC_pin(apic2, pin2);
                 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
         }
@@ -3069,16 +2879,16 @@ static inline void __init check_timer(void)
         apic_printk(APIC_QUIET, KERN_INFO
                     "...trying to set up timer as Virtual Wire IRQ...\n");
 
-        lapic_register_intr(0, desc);
+        lapic_register_intr(0);
         apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);     /* Fixed mode */
-        legacy_pic->chip->unmask(0);
+        legacy_pic->unmask(0);
 
         if (timer_irq_works()) {
                 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
                 goto out;
         }
         local_irq_disable();
-        legacy_pic->chip->mask(0);
+        legacy_pic->mask(0);
         apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
         apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
 
@@ -3244,49 +3054,42 @@ device_initcall(ioapic_init_sysfs);
 /*
  * Dynamic irq allocate and deallocation
  */
-unsigned int create_irq_nr(unsigned int irq_want, int node)
+unsigned int create_irq_nr(unsigned int from, int node)
 {
-        /* Allocate an unused irq */
-        unsigned int irq;
-        unsigned int new;
+        struct irq_cfg *cfg;
         unsigned long flags;
-        struct irq_cfg *cfg_new = NULL;
-        struct irq_desc *desc_new = NULL;
-
-        irq = 0;
-        if (irq_want < nr_irqs_gsi)
-                irq_want = nr_irqs_gsi;
-
-        raw_spin_lock_irqsave(&vector_lock, flags);
-        for (new = irq_want; new < nr_irqs; new++) {
-                desc_new = irq_to_desc_alloc_node(new, node);
-                if (!desc_new) {
-                        printk(KERN_INFO "can not get irq_desc for %d\n", new);
-                        continue;
-                }
-                cfg_new = desc_new->chip_data;
-
-                if (cfg_new->vector != 0)
-                        continue;
+        unsigned int ret = 0;
+        int irq;
 
-                desc_new = move_irq_desc(desc_new, node);
-                cfg_new = desc_new->chip_data;
+        if (from < nr_irqs_gsi)
+                from = nr_irqs_gsi;
 
-                if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0)
-                        irq = new;
-                break;
+        irq = alloc_irq_from(from, node);
+        if (irq < 0)
+                return 0;
+        cfg = alloc_irq_cfg(irq, node);
+        if (!cfg) {
+                free_irq_at(irq, NULL);
+                return 0;
         }
-        raw_spin_unlock_irqrestore(&vector_lock, flags);
 
-        if (irq > 0)
-                dynamic_irq_init_keep_chip_data(irq);
+        raw_spin_lock_irqsave(&vector_lock, flags);
+        if (!__assign_irq_vector(irq, cfg, apic->target_cpus()))
+                ret = irq;
+        raw_spin_unlock_irqrestore(&vector_lock, flags);
 
-        return irq;
+        if (ret) {
+                set_irq_chip_data(irq, cfg);
+                irq_clear_status_flags(irq, IRQ_NOREQUEST);
+        } else {
+                free_irq_at(irq, cfg);
+        }
+        return ret;
 }
 
 int create_irq(void)
 {
-        int node = cpu_to_node(boot_cpu_id);
+        int node = cpu_to_node(0);
         unsigned int irq_want;
         int irq;
 
@@ -3301,14 +3104,17 @@ int create_irq(void)
 
 void destroy_irq(unsigned int irq)
 {
+        struct irq_cfg *cfg = get_irq_chip_data(irq);
         unsigned long flags;
 
-        dynamic_irq_cleanup_keep_chip_data(irq);
+        irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
 
-        free_irte(irq);
+        if (intr_remapping_enabled)
+                free_irte(irq);
         raw_spin_lock_irqsave(&vector_lock, flags);
-        __clear_irq_vector(irq, get_irq_chip_data(irq));
+        __clear_irq_vector(irq, cfg);
         raw_spin_unlock_irqrestore(&vector_lock, flags);
+        free_irq_at(irq, cfg);
 }
 
 /*
@@ -3332,7 +3138,7 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
 
         dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
 
-        if (irq_remapped(irq)) {
+        if (irq_remapped(get_irq_chip_data(irq))) {
                 struct irte irte;
                 int ir_index;
                 u16 sub_handle;
@@ -3340,14 +3146,7 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
                 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
                 BUG_ON(ir_index == -1);
 
-                memset (&irte, 0, sizeof(irte));
-
-                irte.present = 1;
-                irte.dst_mode = apic->irq_dest_mode;
-                irte.trigger_mode = 0; /* edge */
-                irte.dlvry_mode = apic->irq_delivery_mode;
-                irte.vector = cfg->vector;
-                irte.dest_id = IRTE_DEST(dest);
+                prepare_irte(&irte, cfg->vector, dest);
 
                 /* Set source-id of interrupt request */
                 if (pdev)
@@ -3392,26 +3191,24 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
 }
 
 #ifdef CONFIG_SMP
-static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
+static int
+msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg;
+        struct irq_cfg *cfg = data->chip_data;
         struct msi_msg msg;
         unsigned int dest;
 
-        if (set_desc_affinity(desc, mask, &dest))
+        if (__ioapic_set_affinity(data, mask, &dest))
                 return -1;
 
-        cfg = desc->chip_data;
-
-        get_cached_msi_msg_desc(desc, &msg);
+        __get_cached_msi_msg(data->msi_desc, &msg);
 
         msg.data &= ~MSI_DATA_VECTOR_MASK;
         msg.data |= MSI_DATA_VECTOR(cfg->vector);
         msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
         msg.address_lo |= MSI_ADDR_DEST_ID(dest);
 
-        write_msi_msg_desc(desc, &msg);
+        __write_msi_msg(data->msi_desc, &msg);
 
         return 0;
 }
@@ -3421,17 +3218,17 @@ static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
  * done in the process context using interrupt-remapping hardware.
  */
 static int
-ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
+ir_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
+                    bool force)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg = desc->chip_data;
-        unsigned int dest;
+        struct irq_cfg *cfg = data->chip_data;
+        unsigned int dest, irq = data->irq;
         struct irte irte;
 
         if (get_irte(irq, &irte))
                 return -1;
 
-        if (set_desc_affinity(desc, mask, &dest))
+        if (__ioapic_set_affinity(data, mask, &dest))
                 return -1;
 
         irte.vector = cfg->vector;
@@ -3461,27 +3258,27 @@ ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
  * which implement the MSI or MSI-X Capability Structure.
  */
 static struct irq_chip msi_chip = {
-        .name           = "PCI-MSI",
-        .unmask         = unmask_msi_irq,
-        .mask           = mask_msi_irq,
-        .ack            = ack_apic_edge,
+        .name                   = "PCI-MSI",
+        .irq_unmask             = unmask_msi_irq,
+        .irq_mask               = mask_msi_irq,
+        .irq_ack                = ack_apic_edge,
 #ifdef CONFIG_SMP
-        .set_affinity   = set_msi_irq_affinity,
+        .irq_set_affinity       = msi_set_affinity,
 #endif
-        .retrigger      = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 static struct irq_chip msi_ir_chip = {
-        .name           = "IR-PCI-MSI",
-        .unmask         = unmask_msi_irq,
-        .mask           = mask_msi_irq,
+        .name                   = "IR-PCI-MSI",
+        .irq_unmask             = unmask_msi_irq,
+        .irq_mask               = mask_msi_irq,
 #ifdef CONFIG_INTR_REMAP
-        .ack            = ir_ack_apic_edge,
+        .irq_ack                = ir_ack_apic_edge,
 #ifdef CONFIG_SMP
-        .set_affinity   = ir_set_msi_irq_affinity,
+        .irq_set_affinity       = ir_msi_set_affinity,
 #endif
 #endif
-        .retrigger      = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 /*
@@ -3513,8 +3310,8 @@ static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
 
 static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
 {
-        int ret;
         struct msi_msg msg;
+        int ret;
 
         ret = msi_compose_msg(dev, irq, &msg, -1);
         if (ret < 0)
@@ -3523,12 +3320,8 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
         set_irq_msi(irq, msidesc);
         write_msi_msg(irq, &msg);
 
-        if (irq_remapped(irq)) {
-                struct irq_desc *desc = irq_to_desc(irq);
-                /*
-                 * irq migration in process context
-                 */
-                desc->status |= IRQ_MOVE_PCNTXT;
+        if (irq_remapped(get_irq_chip_data(irq))) {
+                irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
                 set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
         } else
                 set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
@@ -3540,13 +3333,10 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
 
 int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
 {
-        unsigned int irq;
-        int ret, sub_handle;
+        int node, ret, sub_handle, index = 0;
+        unsigned int irq, irq_want;
         struct msi_desc *msidesc;
-        unsigned int irq_want;
         struct intel_iommu *iommu = NULL;
-        int index = 0;
-        int node;
 
         /* x86 doesn't support multiple MSI yet */
         if (type == PCI_CAP_ID_MSI && nvec > 1)
@@ -3606,18 +3396,17 @@ void arch_teardown_msi_irq(unsigned int irq)
 
 #if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP)
 #ifdef CONFIG_SMP
-static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
+static int
+dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
+                      bool force)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg;
+        struct irq_cfg *cfg = data->chip_data;
+        unsigned int dest, irq = data->irq;
         struct msi_msg msg;
-        unsigned int dest;
 
-        if (set_desc_affinity(desc, mask, &dest))
+        if (__ioapic_set_affinity(data, mask, &dest))
                 return -1;
 
-        cfg = desc->chip_data;
-
         dmar_msi_read(irq, &msg);
 
         msg.data &= ~MSI_DATA_VECTOR_MASK;
@@ -3633,14 +3422,14 @@ static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
 #endif /* CONFIG_SMP */
 
 static struct irq_chip dmar_msi_type = {
-        .name = "DMAR_MSI",
-        .unmask = dmar_msi_unmask,
-        .mask = dmar_msi_mask,
-        .ack = ack_apic_edge,
+        .name                   = "DMAR_MSI",
+        .irq_unmask             = dmar_msi_unmask,
+        .irq_mask               = dmar_msi_mask,
+        .irq_ack                = ack_apic_edge,
 #ifdef CONFIG_SMP
-        .set_affinity = dmar_msi_set_affinity,
+        .irq_set_affinity       = dmar_msi_set_affinity,
 #endif
-        .retrigger = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 int arch_setup_dmar_msi(unsigned int irq)
@@ -3661,26 +3450,24 @@ int arch_setup_dmar_msi(unsigned int irq)
 #ifdef CONFIG_HPET_TIMER
 
 #ifdef CONFIG_SMP
-static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
+static int hpet_msi_set_affinity(struct irq_data *data,
+                                 const struct cpumask *mask, bool force)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg;
+        struct irq_cfg *cfg = data->chip_data;
         struct msi_msg msg;
         unsigned int dest;
 
-        if (set_desc_affinity(desc, mask, &dest))
+        if (__ioapic_set_affinity(data, mask, &dest))
                 return -1;
 
-        cfg = desc->chip_data;
-
-        hpet_msi_read(irq, &msg);
+        hpet_msi_read(data->handler_data, &msg);
 
         msg.data &= ~MSI_DATA_VECTOR_MASK;
         msg.data |= MSI_DATA_VECTOR(cfg->vector);
         msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
         msg.address_lo |= MSI_ADDR_DEST_ID(dest);
 
-        hpet_msi_write(irq, &msg);
+        hpet_msi_write(data->handler_data, &msg);
 
         return 0;
 }
@@ -3688,34 +3475,33 @@ static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
 #endif /* CONFIG_SMP */
 
 static struct irq_chip ir_hpet_msi_type = {
-        .name = "IR-HPET_MSI",
-        .unmask = hpet_msi_unmask,
-        .mask = hpet_msi_mask,
+        .name                   = "IR-HPET_MSI",
+        .irq_unmask             = hpet_msi_unmask,
+        .irq_mask               = hpet_msi_mask,
 #ifdef CONFIG_INTR_REMAP
-        .ack = ir_ack_apic_edge,
+        .irq_ack                = ir_ack_apic_edge,
 #ifdef CONFIG_SMP
-        .set_affinity = ir_set_msi_irq_affinity,
+        .irq_set_affinity       = ir_msi_set_affinity,
 #endif
 #endif
-        .retrigger = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 static struct irq_chip hpet_msi_type = {
         .name = "HPET_MSI",
-        .unmask = hpet_msi_unmask,
-        .mask = hpet_msi_mask,
-        .ack = ack_apic_edge,
+        .irq_unmask = hpet_msi_unmask,
+        .irq_mask = hpet_msi_mask,
+        .irq_ack = ack_apic_edge,
 #ifdef CONFIG_SMP
-        .set_affinity = hpet_msi_set_affinity,
+        .irq_set_affinity = hpet_msi_set_affinity,
 #endif
-        .retrigger = ioapic_retrigger_irq,
+        .irq_retrigger = ioapic_retrigger_irq,
 };
 
 int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
 {
-        int ret;
         struct msi_msg msg;
-        struct irq_desc *desc = irq_to_desc(irq);
+        int ret;
 
         if (intr_remapping_enabled) {
                 struct intel_iommu *iommu = map_hpet_to_ir(id);
@@ -3733,9 +3519,9 @@ int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
         if (ret < 0)
                 return ret;
 
-        hpet_msi_write(irq, &msg);
-        desc->status |= IRQ_MOVE_PCNTXT;
-        if (irq_remapped(irq))
+        hpet_msi_write(get_irq_data(irq), &msg);
+        irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
+        if (irq_remapped(get_irq_chip_data(irq)))
                 set_irq_chip_and_handler_name(irq, &ir_hpet_msi_type,
                                               handle_edge_irq, "edge");
         else
@@ -3768,33 +3554,30 @@ static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
         write_ht_irq_msg(irq, &msg);
 }
 
-static int set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
+static int
+ht_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg;
+        struct irq_cfg *cfg = data->chip_data;
         unsigned int dest;
 
-        if (set_desc_affinity(desc, mask, &dest))
+        if (__ioapic_set_affinity(data, mask, &dest))
                 return -1;
 
-        cfg = desc->chip_data;
-
-        target_ht_irq(irq, dest, cfg->vector);
-
+        target_ht_irq(data->irq, dest, cfg->vector);
         return 0;
 }
 
 #endif
 
 static struct irq_chip ht_irq_chip = {
-        .name           = "PCI-HT",
-        .mask           = mask_ht_irq,
-        .unmask         = unmask_ht_irq,
-        .ack            = ack_apic_edge,
+        .name                   = "PCI-HT",
+        .irq_mask               = mask_ht_irq,
+        .irq_unmask             = unmask_ht_irq,
+        .irq_ack                = ack_apic_edge,
 #ifdef CONFIG_SMP
-        .set_affinity   = set_ht_irq_affinity,
+        .irq_set_affinity       = ht_set_affinity,
 #endif
-        .retrigger      = ioapic_retrigger_irq,
+        .irq_retrigger          = ioapic_retrigger_irq,
 };
 
 int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
@@ -3885,14 +3668,13 @@ int __init arch_probe_nr_irqs(void)
         if (nr < nr_irqs)
                 nr_irqs = nr;
 
-        return 0;
+        return NR_IRQS_LEGACY;
 }
 #endif
 
 static int __io_apic_set_pci_routing(struct device *dev, int irq,
                                 struct io_apic_irq_attr *irq_attr)
 {
-        struct irq_desc *desc;
         struct irq_cfg *cfg;
         int node;
         int ioapic, pin;
@@ -3908,13 +3690,11 @@ static int __io_apic_set_pci_routing(struct device *dev, int irq,
         if (dev)
                 node = dev_to_node(dev);
         else
-                node = cpu_to_node(boot_cpu_id);
+                node = cpu_to_node(0);
 
-        desc = irq_to_desc_alloc_node(irq, node);
-        if (!desc) {
-                printk(KERN_INFO "can not get irq_desc %d\n", irq);
+        cfg = alloc_irq_and_cfg_at(irq, node);
+        if (!cfg)
                 return 0;
-        }
 
         pin = irq_attr->ioapic_pin;
         trigger = irq_attr->trigger;
@@ -3924,15 +3704,14 @@ static int __io_apic_set_pci_routing(struct device *dev, int irq,
          * IRQs < 16 are already in the irq_2_pin[] map
          */
         if (irq >= legacy_pic->nr_legacy_irqs) {
-                cfg = desc->chip_data;
-                if (add_pin_to_irq_node_nopanic(cfg, node, ioapic, pin)) {
+                if (__add_pin_to_irq_node(cfg, node, ioapic, pin)) {
                         printk(KERN_INFO "can not add pin %d for irq %d\n",
                                 pin, irq);
                         return 0;
                 }
         }
 
-        setup_IO_APIC_irq(ioapic, pin, irq, desc, trigger, polarity);
+        setup_ioapic_irq(ioapic, pin, irq, cfg, trigger, polarity);
 
         return 0;
 }
@@ -4125,14 +3904,14 @@ void __init setup_ioapic_dest(void)
                  */
                 if (desc->status &
                     (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
-                        mask = desc->affinity;
+                        mask = desc->irq_data.affinity;
                 else
                         mask = apic->target_cpus();
 
                 if (intr_remapping_enabled)
-                        set_ir_ioapic_affinity_irq_desc(desc, mask);
+                        ir_ioapic_set_affinity(&desc->irq_data, mask, false);
                 else
-                        set_ioapic_affinity_irq_desc(desc, mask);
+                        ioapic_set_affinity(&desc->irq_data, mask, false);
         }
 
 }
@@ -4316,19 +4095,18 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
 void __init pre_init_apic_IRQ0(void)
 {
         struct irq_cfg *cfg;
-        struct irq_desc *desc;
 
         printk(KERN_INFO "Early APIC setup for system timer0\n");
 #ifndef CONFIG_SMP
         phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid);
 #endif
-        desc = irq_to_desc_alloc_node(0, 0);
+        /* Make sure the irq descriptor is set up */
+        cfg = alloc_irq_and_cfg_at(0, 0);
 
         setup_local_APIC();
 
-        cfg = irq_cfg(0);
         add_pin_to_irq_node(cfg, 0, 0, 0);
         set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
 
-        setup_IO_APIC_irq(0, 0, 0, desc, 0, 0);
+        setup_ioapic_irq(0, 0, 0, cfg, 0, 0);
 }
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c
index a43f71cb30f8..c90041ccb742 100644
--- a/arch/x86/kernel/apic/nmi.c
+++ b/arch/x86/kernel/apic/nmi.c
@@ -178,7 +178,7 @@ int __init check_nmi_watchdog(void)
 error:
         if (nmi_watchdog == NMI_IO_APIC) {
                 if (!timer_through_8259)
-                        legacy_pic->chip->mask(0);
+                        legacy_pic->mask(0);
                 on_each_cpu(__acpi_nmi_disable, NULL, 1);
         }
 
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index 83e9be4778e2..f9e4e6a54073 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -54,6 +54,9 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
  */
 void __init default_setup_apic_routing(void)
 {
+
+        enable_IR_x2apic();
+
 #ifdef CONFIG_X86_X2APIC
         if (x2apic_mode
 #ifdef CONFIG_X86_UV
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index ba5f62f45f01..9e093f8fe78c 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -148,7 +148,7 @@ static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
         /* calling is from identify_secondary_cpu() ? */
-        if (c->cpu_index == boot_cpu_id)
+        if (!c->cpu_index)
                 return;
 
         /*
@@ -253,37 +253,51 @@ static int __cpuinit nearby_node(int apicid)
 #endif
 
 /*
- * Fixup core topology information for AMD multi-node processors.
- * Assumption: Number of cores in each internal node is the same.
+ * Fixup core topology information for
+ * (1) AMD multi-node processors
+ *     Assumption: Number of cores in each internal node is the same.
+ * (2) AMD processors supporting compute units
  */
 #ifdef CONFIG_X86_HT
-static void __cpuinit amd_fixup_dcm(struct cpuinfo_x86 *c)
+static void __cpuinit amd_get_topology(struct cpuinfo_x86 *c)
 {
-        unsigned long long value;
-        u32 nodes, cores_per_node;
+        u32 nodes;
+        u8 node_id;
         int cpu = smp_processor_id();
 
-        if (!cpu_has(c, X86_FEATURE_NODEID_MSR))
-                return;
+        /* get information required for multi-node processors */
+        if (cpu_has(c, X86_FEATURE_TOPOEXT)) {
+                u32 eax, ebx, ecx, edx;
 
-        /* fixup topology information only once for a core */
-        if (cpu_has(c, X86_FEATURE_AMD_DCM))
-                return;
+                cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
+                nodes = ((ecx >> 8) & 7) + 1;
+                node_id = ecx & 7;
 
-        rdmsrl(MSR_FAM10H_NODE_ID, value);
+                /* get compute unit information */
+                smp_num_siblings = ((ebx >> 8) & 3) + 1;
+                c->compute_unit_id = ebx & 0xff;
+        } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
+                u64 value;
 
-        nodes = ((value >> 3) & 7) + 1;
-        if (nodes == 1)
+                rdmsrl(MSR_FAM10H_NODE_ID, value);
+                nodes = ((value >> 3) & 7) + 1;
+                node_id = value & 7;
+        } else
                 return;
 
-        set_cpu_cap(c, X86_FEATURE_AMD_DCM);
-        cores_per_node = c->x86_max_cores / nodes;
+        /* fixup multi-node processor information */
+        if (nodes > 1) {
+                u32 cores_per_node;
+
+                set_cpu_cap(c, X86_FEATURE_AMD_DCM);
+                cores_per_node = c->x86_max_cores / nodes;
 
-        /* store NodeID, use llc_shared_map to store sibling info */
-        per_cpu(cpu_llc_id, cpu) = value & 7;
+                /* store NodeID, use llc_shared_map to store sibling info */
+                per_cpu(cpu_llc_id, cpu) = node_id;
 
-        /* fixup core id to be in range from 0 to (cores_per_node - 1) */
-        c->cpu_core_id = c->cpu_core_id % cores_per_node;
+                /* core id to be in range from 0 to (cores_per_node - 1) */
+                c->cpu_core_id = c->cpu_core_id % cores_per_node;
+        }
 }
 #endif
 
@@ -304,9 +318,7 @@ static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
         c->phys_proc_id = c->initial_apicid >> bits;
         /* use socket ID also for last level cache */
         per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
-        /* fixup topology information on multi-node processors */
-        if ((c->x86 == 0x10) && (c->x86_model == 9))
-                amd_fixup_dcm(c);
+        amd_get_topology(c);
 #endif
 }
 
@@ -412,6 +424,23 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
                         set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
         }
 #endif
+
+        /* We need to do the following only once */
+        if (c != &boot_cpu_data)
+                return;
+
+        if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
+
+                if (c->x86 > 0x10 ||
+                    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
+                        u64 val;
+
+                        rdmsrl(MSR_K7_HWCR, val);
+                        if (!(val & BIT(24)))
+                                printk(KERN_WARNING FW_BUG "TSC doesn't count "
+                                        "with P0 frequency!\n");
+                }
+        }
 }
 
 static void __cpuinit init_amd(struct cpuinfo_x86 *c)
@@ -523,7 +552,7 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
 #endif
 
         if (c->extended_cpuid_level >= 0x80000006) {
-                if ((c->x86 >= 0x0f) && (cpuid_edx(0x80000006) & 0xf000))
+                if (cpuid_edx(0x80000006) & 0xf000)
                         num_cache_leaves = 4;
                 else
                         num_cache_leaves = 3;
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index f2f9ac7da25c..4b68bda30938 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -665,7 +665,7 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
                 this_cpu->c_early_init(c);
 
 #ifdef CONFIG_SMP
-        c->cpu_index = boot_cpu_id;
+        c->cpu_index = 0;
 #endif
         filter_cpuid_features(c, false);
 }
@@ -704,16 +704,21 @@ void __init early_cpu_init(void)
 }
 
 /*
- * The NOPL instruction is supposed to exist on all CPUs with
- * family >= 6; unfortunately, that's not true in practice because
- * of early VIA chips and (more importantly) broken virtualizers that
- * are not easy to detect.  In the latter case it doesn't even *fail*
- * reliably, so probing for it doesn't even work.  Disable it completely
+ * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
+ * unfortunately, that's not true in practice because of early VIA
+ * chips and (more importantly) broken virtualizers that are not easy
+ * to detect. In the latter case it doesn't even *fail* reliably, so
+ * probing for it doesn't even work. Disable it completely on 32-bit
  * unless we can find a reliable way to detect all the broken cases.
+ * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
  */
 static void __cpuinit detect_nopl(struct cpuinfo_x86 *c)
 {
+#ifdef CONFIG_X86_32
         clear_cpu_cap(c, X86_FEATURE_NOPL);
+#else
+        set_cpu_cap(c, X86_FEATURE_NOPL);
+#endif
 }
 
 static void __cpuinit generic_identify(struct cpuinfo_x86 *c)
@@ -1264,13 +1269,6 @@ void __cpuinit cpu_init(void)
         clear_all_debug_regs();
         dbg_restore_debug_regs();
 
-        /*
-         * Force FPU initialization:
-         */
-        current_thread_info()->status = 0;
-        clear_used_math();
-        mxcsr_feature_mask_init();
-
         fpu_init();
         xsave_init();
 }
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index f668bb1f7d43..e765633f210e 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -32,6 +32,7 @@ struct cpu_dev {
 extern const struct cpu_dev *const __x86_cpu_dev_start[],
                             *const __x86_cpu_dev_end[];
 
+extern void get_cpu_cap(struct cpuinfo_x86 *c);
 extern void cpu_detect_cache_sizes(struct cpuinfo_x86 *c);
 extern void get_cpu_cap(struct cpuinfo_x86 *c);
 
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index b4389441efbb..695f17731e23 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -170,7 +170,7 @@ static void __cpuinit intel_smp_check(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
         /* calling is from identify_secondary_cpu() ? */
-        if (c->cpu_index == boot_cpu_id)
+        if (!c->cpu_index)
                 return;
 
         /*
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 898c2f4eab88..12cd823c8d03 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -17,7 +17,7 @@
 
 #include <asm/processor.h>
 #include <linux/smp.h>
-#include <asm/k8.h>
+#include <asm/amd_nb.h>
 #include <asm/smp.h>
 
 #define LVL_1_INST      1
@@ -306,7 +306,7 @@ struct _cache_attr {
         ssize_t (*store)(struct _cpuid4_info *, const char *, size_t count);
 };
 
-#ifdef CONFIG_CPU_SUP_AMD
+#ifdef CONFIG_AMD_NB
 
 /*
  * L3 cache descriptors
@@ -369,7 +369,7 @@ static void __cpuinit amd_check_l3_disable(struct _cpuid4_info_regs *this_leaf,
                         return;
 
         /* not in virtualized environments */
-        if (num_k8_northbridges == 0)
+        if (k8_northbridges.num == 0)
                 return;
 
         /*
@@ -377,7 +377,7 @@ static void __cpuinit amd_check_l3_disable(struct _cpuid4_info_regs *this_leaf,
          * never freed but this is done only on shutdown so it doesn't matter.
          */
         if (!l3_caches) {
-                int size = num_k8_northbridges * sizeof(struct amd_l3_cache *);
+                int size = k8_northbridges.num * sizeof(struct amd_l3_cache *);
 
                 l3_caches = kzalloc(size, GFP_ATOMIC);
                 if (!l3_caches)
@@ -556,12 +556,12 @@ static struct _cache_attr cache_disable_0 = __ATTR(cache_disable_0, 0644,
 static struct _cache_attr cache_disable_1 = __ATTR(cache_disable_1, 0644,
                 show_cache_disable_1, store_cache_disable_1);
 
-#else   /* CONFIG_CPU_SUP_AMD */
+#else   /* CONFIG_AMD_NB */
 static void __cpuinit
 amd_check_l3_disable(struct _cpuid4_info_regs *this_leaf, int index)
 {
 };
-#endif /* CONFIG_CPU_SUP_AMD */
+#endif /* CONFIG_AMD_NB */
 
 static int
 __cpuinit cpuid4_cache_lookup_regs(int index,
@@ -1000,7 +1000,7 @@ static struct attribute *default_attrs[] = {
 
 static struct attribute *default_l3_attrs[] = {
         DEFAULT_SYSFS_CACHE_ATTRS,
-#ifdef CONFIG_CPU_SUP_AMD
+#ifdef CONFIG_AMD_NB
         &cache_disable_0.attr,
         &cache_disable_1.attr,
 #endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 5e975298fa81..80c482382d5c 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -131,7 +131,8 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c)
         u32 low = 0, high = 0, address = 0;
         unsigned int bank, block;
         struct thresh_restart tr;
-        u8 lvt_off;
+        int lvt_off = -1;
+        u8 offset;
 
         for (bank = 0; bank < NR_BANKS; ++bank) {
                 for (block = 0; block < NR_BLOCKS; ++block) {
@@ -141,6 +142,7 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c)
                                 address = (low & MASK_BLKPTR_LO) >> 21;
                                 if (!address)
                                         break;
+
                                 address += MCG_XBLK_ADDR;
                         } else
                                 ++address;
@@ -148,12 +150,8 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c)
                         if (rdmsr_safe(address, &low, &high))
                                 break;
 
-                        if (!(high & MASK_VALID_HI)) {
-                                if (block)
-                                        continue;
-                                else
-                                        break;
-                        }
+                        if (!(high & MASK_VALID_HI))
+                                continue;
 
                         if (!(high & MASK_CNTP_HI)  ||
                              (high & MASK_LOCKED_HI))
@@ -165,8 +163,28 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c)
                         if (shared_bank[bank] && c->cpu_core_id)
                                 break;
 #endif
-                        lvt_off = setup_APIC_eilvt_mce(THRESHOLD_APIC_VECTOR,
-                                                       APIC_EILVT_MSG_FIX, 0);
+                        offset = (high & MASK_LVTOFF_HI) >> 20;
+                        if (lvt_off < 0) {
+                                if (setup_APIC_eilvt(offset,
+                                                     THRESHOLD_APIC_VECTOR,
+                                                     APIC_EILVT_MSG_FIX, 0)) {
+                                        pr_err(FW_BUG "cpu %d, failed to "
+                                               "setup threshold interrupt "
+                                               "for bank %d, block %d "
+                                               "(MSR%08X=0x%x%08x)",
+                                               smp_processor_id(), bank, block,
+                                               address, high, low);
+                                        continue;
+                                }
+                                lvt_off = offset;
+                        } else if (lvt_off != offset) {
+                                pr_err(FW_BUG "cpu %d, invalid threshold "
+                                       "interrupt offset %d for bank %d,"
+                                       "block %d (MSR%08X=0x%x%08x)",
+                                       smp_processor_id(), lvt_off, bank,
+                                       block, address, high, low);
+                                continue;
+                        }
 
                         high &= ~MASK_LVTOFF_HI;
                         high |= lvt_off << 20;
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index d9368eeda309..4b683267eca5 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -216,7 +216,7 @@ static __cpuinit int thermal_throttle_add_dev(struct sys_device *sys_dev,
                 err = sysfs_add_file_to_group(&sys_dev->kobj,
                                               &attr_core_power_limit_count.attr,
                                               thermal_attr_group.name);
-        if (cpu_has(c, X86_FEATURE_PTS))
+        if (cpu_has(c, X86_FEATURE_PTS)) {
                 err = sysfs_add_file_to_group(&sys_dev->kobj,
                                               &attr_package_throttle_count.attr,
                                               thermal_attr_group.name);
@@ -224,6 +224,7 @@ static __cpuinit int thermal_throttle_add_dev(struct sys_device *sys_dev,
                         err = sysfs_add_file_to_group(&sys_dev->kobj,
                                         &attr_package_power_limit_count.attr,
                                         thermal_attr_group.name);
+        }
 
         return err;
 }
@@ -349,7 +350,7 @@ static void intel_thermal_interrupt(void)
 
 static void unexpected_thermal_interrupt(void)
 {
-        printk(KERN_ERR "CPU%d: Unexpected LVT TMR interrupt!\n",
+        printk(KERN_ERR "CPU%d: Unexpected LVT thermal interrupt!\n",
                         smp_processor_id());
         add_taint(TAINT_MACHINE_CHECK);
 }
diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c
index c5f59d071425..ac140c7be396 100644
--- a/arch/x86/kernel/cpu/mtrr/cleanup.c
+++ b/arch/x86/kernel/cpu/mtrr/cleanup.c
@@ -827,7 +827,7 @@ int __init amd_special_default_mtrr(void)
 
         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
                 return 0;
-        if (boot_cpu_data.x86 < 0xf || boot_cpu_data.x86 > 0x11)
+        if (boot_cpu_data.x86 < 0xf)
                 return 0;
         /* In case some hypervisor doesn't pass SYSCFG through: */
         if (rdmsr_safe(MSR_K8_SYSCFG, &l, &h) < 0)
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 7d28d7d03885..9f27228ceffd 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -64,18 +64,59 @@ static inline void k8_check_syscfg_dram_mod_en(void)
         }
 }
 
+/* Get the size of contiguous MTRR range */
+static u64 get_mtrr_size(u64 mask)
+{
+        u64 size;
+
+        mask >>= PAGE_SHIFT;
+        mask |= size_or_mask;
+        size = -mask;
+        size <<= PAGE_SHIFT;
+        return size;
+}
+
 /*
- * Returns the effective MTRR type for the region
- * Error returns:
- * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
- * - 0xFF - when MTRR is not enabled
+ * Check and return the effective type for MTRR-MTRR type overlap.
+ * Returns 1 if the effective type is UNCACHEABLE, else returns 0
  */
-u8 mtrr_type_lookup(u64 start, u64 end)
+static int check_type_overlap(u8 *prev, u8 *curr)
+{
+        if (*prev == MTRR_TYPE_UNCACHABLE || *curr == MTRR_TYPE_UNCACHABLE) {
+                *prev = MTRR_TYPE_UNCACHABLE;
+                *curr = MTRR_TYPE_UNCACHABLE;
+                return 1;
+        }
+
+        if ((*prev == MTRR_TYPE_WRBACK && *curr == MTRR_TYPE_WRTHROUGH) ||
+            (*prev == MTRR_TYPE_WRTHROUGH && *curr == MTRR_TYPE_WRBACK)) {
+                *prev = MTRR_TYPE_WRTHROUGH;
+                *curr = MTRR_TYPE_WRTHROUGH;
+        }
+
+        if (*prev != *curr) {
+                *prev = MTRR_TYPE_UNCACHABLE;
+                *curr = MTRR_TYPE_UNCACHABLE;
+                return 1;
+        }
+
+        return 0;
+}
+
+/*
+ * Error/Semi-error returns:
+ * 0xFF - when MTRR is not enabled
+ * *repeat == 1 implies [start:end] spanned across MTRR range and type returned
+ *              corresponds only to [start:*partial_end].
+ *              Caller has to lookup again for [*partial_end:end].
+ */
+static u8 __mtrr_type_lookup(u64 start, u64 end, u64 *partial_end, int *repeat)
 {
         int i;
         u64 base, mask;
         u8 prev_match, curr_match;
 
+        *repeat = 0;
         if (!mtrr_state_set)
                 return 0xFF;
 
@@ -126,8 +167,34 @@ u8 mtrr_type_lookup(u64 start, u64 end)
 
                 start_state = ((start & mask) == (base & mask));
                 end_state = ((end & mask) == (base & mask));
-                if (start_state != end_state)
-                        return 0xFE;
+
+                if (start_state != end_state) {
+                        /*
+                         * We have start:end spanning across an MTRR.
+                         * We split the region into
+                         * either
+                         * (start:mtrr_end) (mtrr_end:end)
+                         * or
+                         * (start:mtrr_start) (mtrr_start:end)
+                         * depending on kind of overlap.
+                         * Return the type for first region and a pointer to
+                         * the start of second region so that caller will
+                         * lookup again on the second region.
+                         * Note: This way we handle multiple overlaps as well.
+                         */
+                        if (start_state)
+                                *partial_end = base + get_mtrr_size(mask);
+                        else
+                                *partial_end = base;
+
+                        if (unlikely(*partial_end <= start)) {
+                                WARN_ON(1);
+                                *partial_end = start + PAGE_SIZE;
+                        }
+
+                        end = *partial_end - 1; /* end is inclusive */
+                        *repeat = 1;
+                }
 
                 if ((start & mask) != (base & mask))
                         continue;
@@ -138,21 +205,8 @@ u8 mtrr_type_lookup(u64 start, u64 end)
                         continue;
                 }
 
-                if (prev_match == MTRR_TYPE_UNCACHABLE ||
-                    curr_match == MTRR_TYPE_UNCACHABLE) {
-                        return MTRR_TYPE_UNCACHABLE;
-                }
-
-                if ((prev_match == MTRR_TYPE_WRBACK &&
-                     curr_match == MTRR_TYPE_WRTHROUGH) ||
-                    (prev_match == MTRR_TYPE_WRTHROUGH &&
-                     curr_match == MTRR_TYPE_WRBACK)) {
-                        prev_match = MTRR_TYPE_WRTHROUGH;
-                        curr_match = MTRR_TYPE_WRTHROUGH;
-                }
-
-                if (prev_match != curr_match)
-                        return MTRR_TYPE_UNCACHABLE;
+                if (check_type_overlap(&prev_match, &curr_match))
+                        return curr_match;
         }
 
         if (mtrr_tom2) {
@@ -166,6 +220,36 @@ u8 mtrr_type_lookup(u64 start, u64 end)
         return mtrr_state.def_type;
 }
 
+/*
+ * Returns the effective MTRR type for the region
+ * Error return:
+ * 0xFF - when MTRR is not enabled
+ */
+u8 mtrr_type_lookup(u64 start, u64 end)
+{
+        u8 type, prev_type;
+        int repeat;
+        u64 partial_end;
+
+        type = __mtrr_type_lookup(start, end, &partial_end, &repeat);
+
+        /*
+         * Common path is with repeat = 0.
+         * However, we can have cases where [start:end] spans across some
+         * MTRR range. Do repeated lookups for that case here.
+         */
+        while (repeat) {
+                prev_type = type;
+                start = partial_end;
+                type = __mtrr_type_lookup(start, end, &partial_end, &repeat);
+
+                if (check_type_overlap(&prev_type, &type))
+                        return type;
+        }
+
+        return type;
+}
+
 /* Get the MSR pair relating to a var range */
 static void
 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index e2513f26ba8b..fe73c1844a9a 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1196,25 +1196,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
         return handled;
 }
 
-void smp_perf_pending_interrupt(struct pt_regs *regs)
-{
-        irq_enter();
-        ack_APIC_irq();
-        inc_irq_stat(apic_pending_irqs);
-        perf_event_do_pending();
-        irq_exit();
-}
-
-void set_perf_event_pending(void)
-{
-#ifdef CONFIG_X86_LOCAL_APIC
-        if (!x86_pmu.apic || !x86_pmu_initialized())
-                return;
-
-        apic->send_IPI_self(LOCAL_PENDING_VECTOR);
-#endif
-}
-
 void perf_events_lapic_init(void)
 {
         if (!x86_pmu.apic || !x86_pmu_initialized())
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index c2897b7b4a3b..46d58448c3af 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -52,7 +52,7 @@ static __initconst const u64 amd_hw_cache_event_ids
  [ C(DTLB) ] = {
         [ C(OP_READ) ] = {
                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
-                [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
+                [ C(RESULT_MISS)   ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */
         },
         [ C(OP_WRITE) ] = {
                 [ C(RESULT_ACCESS) ] = 0,
@@ -66,7 +66,7 @@ static __initconst const u64 amd_hw_cache_event_ids
  [ C(ITLB) ] = {
         [ C(OP_READ) ] = {
                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
-                [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
+                [ C(RESULT_MISS)   ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */
         },
         [ C(OP_WRITE) ] = {
                 [ C(RESULT_ACCESS) ] = -1,
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index fb329e9f8494..d9f4ff8fcd69 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -700,11 +700,10 @@ static void probe_nmi_watchdog(void)
 {
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_AMD:
-                if (boot_cpu_data.x86 != 6 && boot_cpu_data.x86 != 15 &&
-                    boot_cpu_data.x86 != 16 && boot_cpu_data.x86 != 17)
-                        return;
-                wd_ops = &k7_wd_ops;
-                break;
+                if (boot_cpu_data.x86 == 6 ||
+                    (boot_cpu_data.x86 >= 0xf && boot_cpu_data.x86 <= 0x15))
+                        wd_ops = &k7_wd_ops;
+                return;
         case X86_VENDOR_INTEL:
                 /* Work around where perfctr1 doesn't have a working enable
                  * bit as described in the following errata:
diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index d49079515122..c7f64e6f537a 100644
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -44,6 +44,12 @@ void __cpuinit init_scattered_cpuid_features(struct cpuinfo_x86 *c)
                 { X86_FEATURE_LBRV,             CR_EDX, 1, 0x8000000a, 0 },
                 { X86_FEATURE_SVML,             CR_EDX, 2, 0x8000000a, 0 },
                 { X86_FEATURE_NRIPS,            CR_EDX, 3, 0x8000000a, 0 },
+                { X86_FEATURE_TSCRATEMSR,       CR_EDX, 4, 0x8000000a, 0 },
+                { X86_FEATURE_VMCBCLEAN,        CR_EDX, 5, 0x8000000a, 0 },
+                { X86_FEATURE_FLUSHBYASID,      CR_EDX, 6, 0x8000000a, 0 },
+                { X86_FEATURE_DECODEASSISTS,    CR_EDX, 7, 0x8000000a, 0 },
+                { X86_FEATURE_PAUSEFILTER,      CR_EDX,10, 0x8000000a, 0 },
+                { X86_FEATURE_PFTHRESHOLD,      CR_EDX,12, 0x8000000a, 0 },
                 { 0, 0, 0, 0, 0 }
         };
 
diff --git a/arch/x86/kernel/crash_dump_64.c b/arch/x86/kernel/crash_dump_64.c
index 045b36cada65..994828899e09 100644
--- a/arch/x86/kernel/crash_dump_64.c
+++ b/arch/x86/kernel/crash_dump_64.c
@@ -34,7 +34,7 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf,
         if (!csize)
                 return 0;
 
-        vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
+        vaddr = ioremap_cache(pfn << PAGE_SHIFT, PAGE_SIZE);
         if (!vaddr)
                 return -ENOMEM;
 
@@ -46,6 +46,7 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf,
         } else
                 memcpy(buf, vaddr + offset, csize);
 
+        set_iounmap_nonlazy();
         iounmap(vaddr);
         return csize;
 }
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index ebdb85cf2686..76b8cd953dee 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -97,7 +97,6 @@ static void __init nvidia_bugs(int num, int slot, int func)
 }
 
 #if defined(CONFIG_ACPI) && defined(CONFIG_X86_IO_APIC)
-#if defined(CONFIG_ACPI) && defined(CONFIG_X86_IO_APIC)
 static u32 __init ati_ixp4x0_rev(int num, int slot, int func)
 {
         u32 d;
@@ -115,7 +114,6 @@ static u32 __init ati_ixp4x0_rev(int num, int slot, int func)
         d &= 0xff;
         return d;
 }
-#endif
 
 static void __init ati_bugs(int num, int slot, int func)
 {
diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c
index fa99bae75ace..4572f25f9325 100644
--- a/arch/x86/kernel/early_printk.c
+++ b/arch/x86/kernel/early_printk.c
@@ -14,6 +14,7 @@
 #include <xen/hvc-console.h>
 #include <asm/pci-direct.h>
 #include <asm/fixmap.h>
+#include <asm/mrst.h>
 #include <asm/pgtable.h>
 #include <linux/usb/ehci_def.h>
 
@@ -239,6 +240,18 @@ static int __init setup_early_printk(char *buf)
                 if (!strncmp(buf, "xen", 3))
                         early_console_register(&xenboot_console, keep);
 #endif
+#ifdef CONFIG_X86_MRST_EARLY_PRINTK
+                if (!strncmp(buf, "mrst", 4)) {
+                        mrst_early_console_init();
+                        early_console_register(&early_mrst_console, keep);
+                }
+
+                if (!strncmp(buf, "hsu", 3)) {
+                        hsu_early_console_init();
+                        early_console_register(&early_hsu_console, keep);
+                }
+
+#endif
                 buf++;
         }
         return 0;
diff --git a/arch/x86/kernel/early_printk_mrst.c b/arch/x86/kernel/early_printk_mrst.c
new file mode 100644
index 000000000000..65df603622b2
--- /dev/null
+++ b/arch/x86/kernel/early_printk_mrst.c
@@ -0,0 +1,319 @@
+/*
+ * early_printk_mrst.c - early consoles for Intel MID platforms
+ *
+ * Copyright (c) 2008-2010, Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+/*
+ * This file implements two early consoles named mrst and hsu.
+ * mrst is based on Maxim3110 spi-uart device, it exists in both
+ * Moorestown and Medfield platforms, while hsu is based on a High
+ * Speed UART device which only exists in the Medfield platform
+ */
+
+#include <linux/serial_reg.h>
+#include <linux/serial_mfd.h>
+#include <linux/kmsg_dump.h>
+#include <linux/console.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/io.h>
+
+#include <asm/fixmap.h>
+#include <asm/pgtable.h>
+#include <asm/mrst.h>
+
+#define MRST_SPI_TIMEOUT                0x200000
+#define MRST_REGBASE_SPI0               0xff128000
+#define MRST_REGBASE_SPI1               0xff128400
+#define MRST_CLK_SPI0_REG               0xff11d86c
+
+/* Bit fields in CTRLR0 */
+#define SPI_DFS_OFFSET                  0
+
+#define SPI_FRF_OFFSET                  4
+#define SPI_FRF_SPI                     0x0
+#define SPI_FRF_SSP                     0x1
+#define SPI_FRF_MICROWIRE               0x2
+#define SPI_FRF_RESV                    0x3
+
+#define SPI_MODE_OFFSET                 6
+#define SPI_SCPH_OFFSET                 6
+#define SPI_SCOL_OFFSET                 7
+#define SPI_TMOD_OFFSET                 8
+#define SPI_TMOD_TR                     0x0             /* xmit & recv */
+#define SPI_TMOD_TO                     0x1             /* xmit only */
+#define SPI_TMOD_RO                     0x2             /* recv only */
+#define SPI_TMOD_EPROMREAD              0x3             /* eeprom read mode */
+
+#define SPI_SLVOE_OFFSET                10
+#define SPI_SRL_OFFSET                  11
+#define SPI_CFS_OFFSET                  12
+
+/* Bit fields in SR, 7 bits */
+#define SR_MASK                         0x7f            /* cover 7 bits */
+#define SR_BUSY                         (1 << 0)
+#define SR_TF_NOT_FULL                  (1 << 1)
+#define SR_TF_EMPT                      (1 << 2)
+#define SR_RF_NOT_EMPT                  (1 << 3)
+#define SR_RF_FULL                      (1 << 4)
+#define SR_TX_ERR                       (1 << 5)
+#define SR_DCOL                         (1 << 6)
+
+struct dw_spi_reg {
+        u32     ctrl0;
+        u32     ctrl1;
+        u32     ssienr;
+        u32     mwcr;
+        u32     ser;
+        u32     baudr;
+        u32     txfltr;
+        u32     rxfltr;
+        u32     txflr;
+        u32     rxflr;
+        u32     sr;
+        u32     imr;
+        u32     isr;
+        u32     risr;
+        u32     txoicr;
+        u32     rxoicr;
+        u32     rxuicr;
+        u32     msticr;
+        u32     icr;
+        u32     dmacr;
+        u32     dmatdlr;
+        u32     dmardlr;
+        u32     idr;
+        u32     version;
+
+        /* Currently operates as 32 bits, though only the low 16 bits matter */
+        u32     dr;
+} __packed;
+
+#define dw_readl(dw, name)              __raw_readl(&(dw)->name)
+#define dw_writel(dw, name, val)        __raw_writel((val), &(dw)->name)
+
+/* Default use SPI0 register for mrst, we will detect Penwell and use SPI1 */
+static unsigned long mrst_spi_paddr = MRST_REGBASE_SPI0;
+
+static u32 *pclk_spi0;
+/* Always contains an accessable address, start with 0 */
+static struct dw_spi_reg *pspi;
+
+static struct kmsg_dumper dw_dumper;
+static int dumper_registered;
+
+static void dw_kmsg_dump(struct kmsg_dumper *dumper,
+                        enum kmsg_dump_reason reason,
+                        const char *s1, unsigned long l1,
+                        const char *s2, unsigned long l2)
+{
+        int i;
+
+        /* When run to this, we'd better re-init the HW */
+        mrst_early_console_init();
+
+        for (i = 0; i < l1; i++)
+                early_mrst_console.write(&early_mrst_console, s1 + i, 1);
+        for (i = 0; i < l2; i++)
+                early_mrst_console.write(&early_mrst_console, s2 + i, 1);
+}
+
+/* Set the ratio rate to 115200, 8n1, IRQ disabled */
+static void max3110_write_config(void)
+{
+        u16 config;
+
+        config = 0xc001;
+        dw_writel(pspi, dr, config);
+}
+
+/* Translate char to a eligible word and send to max3110 */
+static void max3110_write_data(char c)
+{
+        u16 data;
+
+        data = 0x8000 | c;
+        dw_writel(pspi, dr, data);
+}
+
+void mrst_early_console_init(void)
+{
+        u32 ctrlr0 = 0;
+        u32 spi0_cdiv;
+        u32 freq; /* Freqency info only need be searched once */
+
+        /* Base clk is 100 MHz, the actual clk = 100M / (clk_divider + 1) */
+        pclk_spi0 = (void *)set_fixmap_offset_nocache(FIX_EARLYCON_MEM_BASE,
+                                                        MRST_CLK_SPI0_REG);
+        spi0_cdiv = ((*pclk_spi0) & 0xe00) >> 9;
+        freq = 100000000 / (spi0_cdiv + 1);
+
+        if (mrst_identify_cpu() == MRST_CPU_CHIP_PENWELL)
+                mrst_spi_paddr = MRST_REGBASE_SPI1;
+
+        pspi = (void *)set_fixmap_offset_nocache(FIX_EARLYCON_MEM_BASE,
+                                                mrst_spi_paddr);
+
+        /* Disable SPI controller */
+        dw_writel(pspi, ssienr, 0);
+
+        /* Set control param, 8 bits, transmit only mode */
+        ctrlr0 = dw_readl(pspi, ctrl0);
+
+        ctrlr0 &= 0xfcc0;
+        ctrlr0 |= 0xf | (SPI_FRF_SPI << SPI_FRF_OFFSET)
+                      | (SPI_TMOD_TO << SPI_TMOD_OFFSET);
+        dw_writel(pspi, ctrl0, ctrlr0);
+
+        /*
+         * Change the spi0 clk to comply with 115200 bps, use 100000 to
+         * calculate the clk dividor to make the clock a little slower
+         * than real baud rate.
+         */
+        dw_writel(pspi, baudr, freq/100000);
+
+        /* Disable all INT for early phase */
+        dw_writel(pspi, imr, 0x0);
+
+        /* Set the cs to spi-uart */
+        dw_writel(pspi, ser, 0x2);
+
+        /* Enable the HW, the last step for HW init */
+        dw_writel(pspi, ssienr, 0x1);
+
+        /* Set the default configuration */
+        max3110_write_config();
+
+        /* Register the kmsg dumper */
+        if (!dumper_registered) {
+                dw_dumper.dump = dw_kmsg_dump;
+                kmsg_dump_register(&dw_dumper);
+                dumper_registered = 1;
+        }
+}
+
+/* Slave select should be called in the read/write function */
+static void early_mrst_spi_putc(char c)
+{
+        unsigned int timeout;
+        u32 sr;
+
+        timeout = MRST_SPI_TIMEOUT;
+        /* Early putc needs to make sure the TX FIFO is not full */
+        while (--timeout) {
+                sr = dw_readl(pspi, sr);
+                if (!(sr & SR_TF_NOT_FULL))
+                        cpu_relax();
+                else
+                        break;
+        }
+
+        if (!timeout)
+                pr_warning("MRST earlycon: timed out\n");
+        else
+                max3110_write_data(c);
+}
+
+/* Early SPI only uses polling mode */
+static void early_mrst_spi_write(struct console *con, const char *str, unsigned n)
+{
+        int i;
+
+        for (i = 0; i < n && *str; i++) {
+                if (*str == '\n')
+                        early_mrst_spi_putc('\r');
+                early_mrst_spi_putc(*str);
+                str++;
+        }
+}
+
+struct console early_mrst_console = {
+        .name =         "earlymrst",
+        .write =        early_mrst_spi_write,
+        .flags =        CON_PRINTBUFFER,
+        .index =        -1,
+};
+
+/*
+ * Following is the early console based on Medfield HSU (High
+ * Speed UART) device.
+ */
+#define HSU_PORT2_PADDR         0xffa28180
+
+static void __iomem *phsu;
+
+void hsu_early_console_init(void)
+{
+        u8 lcr;
+
+        phsu = (void *)set_fixmap_offset_nocache(FIX_EARLYCON_MEM_BASE,
+                                                        HSU_PORT2_PADDR);
+
+        /* Disable FIFO */
+        writeb(0x0, phsu + UART_FCR);
+
+        /* Set to default 115200 bps, 8n1 */
+        lcr = readb(phsu + UART_LCR);
+        writeb((0x80 | lcr), phsu + UART_LCR);
+        writeb(0x18, phsu + UART_DLL);
+        writeb(lcr,  phsu + UART_LCR);
+        writel(0x3600, phsu + UART_MUL*4);
+
+        writeb(0x8, phsu + UART_MCR);
+        writeb(0x7, phsu + UART_FCR);
+        writeb(0x3, phsu + UART_LCR);
+
+        /* Clear IRQ status */
+        readb(phsu + UART_LSR);
+        readb(phsu + UART_RX);
+        readb(phsu + UART_IIR);
+        readb(phsu + UART_MSR);
+
+        /* Enable FIFO */
+        writeb(0x7, phsu + UART_FCR);
+}
+
+#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE)
+
+static void early_hsu_putc(char ch)
+{
+        unsigned int timeout = 10000; /* 10ms */
+        u8 status;
+
+        while (--timeout) {
+                status = readb(phsu + UART_LSR);
+                if (status & BOTH_EMPTY)
+                        break;
+                udelay(1);
+        }
+
+        /* Only write the char when there was no timeout */
+        if (timeout)
+                writeb(ch, phsu + UART_TX);
+}
+
+static void early_hsu_write(struct console *con, const char *str, unsigned n)
+{
+        int i;
+
+        for (i = 0; i < n && *str; i++) {
+                if (*str == '\n')
+                        early_hsu_putc('\r');
+                early_hsu_putc(*str);
+                str++;
+        }
+}
+
+struct console early_hsu_console = {
+        .name =         "earlyhsu",
+        .write =        early_hsu_write,
+        .flags =        CON_PRINTBUFFER,
+        .index =        -1,
+};
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 227d00920d2f..9fb188d7bc76 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -115,8 +115,7 @@
 
  /* unfortunately push/pop can't be no-op */
 .macro PUSH_GS
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
 .endm
 .macro POP_GS pop=0
         addl $(4 + \pop), %esp
@@ -140,14 +139,12 @@
 #else   /* CONFIG_X86_32_LAZY_GS */
 
 .macro PUSH_GS
-        pushl %gs
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %gs
         /*CFI_REL_OFFSET gs, 0*/
 .endm
 
 .macro POP_GS pop=0
-98:     popl %gs
-        CFI_ADJUST_CFA_OFFSET -4
+98:     popl_cfi %gs
         /*CFI_RESTORE gs*/
   .if \pop <> 0
         add $\pop, %esp
@@ -195,35 +192,25 @@
 .macro SAVE_ALL
         cld
         PUSH_GS
-        pushl %fs
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %fs
         /*CFI_REL_OFFSET fs, 0;*/
-        pushl %es
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %es
         /*CFI_REL_OFFSET es, 0;*/
-        pushl %ds
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ds
         /*CFI_REL_OFFSET ds, 0;*/
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         CFI_REL_OFFSET eax, 0
-        pushl %ebp
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ebp
         CFI_REL_OFFSET ebp, 0
-        pushl %edi
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %edi
         CFI_REL_OFFSET edi, 0
-        pushl %esi
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %esi
         CFI_REL_OFFSET esi, 0
-        pushl %edx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %edx
         CFI_REL_OFFSET edx, 0
-        pushl %ecx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ecx
         CFI_REL_OFFSET ecx, 0
-        pushl %ebx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ebx
         CFI_REL_OFFSET ebx, 0
         movl $(__USER_DS), %edx
         movl %edx, %ds
@@ -234,39 +221,29 @@
 .endm
 
 .macro RESTORE_INT_REGS
-        popl %ebx
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %ebx
         CFI_RESTORE ebx
-        popl %ecx
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %ecx
         CFI_RESTORE ecx
-        popl %edx
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %edx
         CFI_RESTORE edx
-        popl %esi
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %esi
         CFI_RESTORE esi
-        popl %edi
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %edi
         CFI_RESTORE edi
-        popl %ebp
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %ebp
         CFI_RESTORE ebp
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %eax
         CFI_RESTORE eax
 .endm
 
 .macro RESTORE_REGS pop=0
         RESTORE_INT_REGS
-1:      popl %ds
-        CFI_ADJUST_CFA_OFFSET -4
+1:      popl_cfi %ds
         /*CFI_RESTORE ds;*/
-2:      popl %es
-        CFI_ADJUST_CFA_OFFSET -4
+2:      popl_cfi %es
         /*CFI_RESTORE es;*/
-3:      popl %fs
-        CFI_ADJUST_CFA_OFFSET -4
+3:      popl_cfi %fs
         /*CFI_RESTORE fs;*/
         POP_GS \pop
 .pushsection .fixup, "ax"
@@ -320,16 +297,12 @@
 
 ENTRY(ret_from_fork)
         CFI_STARTPROC
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         call schedule_tail
         GET_THREAD_INFO(%ebp)
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
-        pushl $0x0202                   # Reset kernel eflags
-        CFI_ADJUST_CFA_OFFSET 4
-        popfl
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %eax
+        pushl_cfi $0x0202               # Reset kernel eflags
+        popfl_cfi
         jmp syscall_exit
         CFI_ENDPROC
 END(ret_from_fork)
@@ -409,29 +382,23 @@ sysenter_past_esp:
          * enough kernel state to call TRACE_IRQS_OFF can be called - but
          * we immediately enable interrupts at that point anyway.
          */
-        pushl $(__USER_DS)
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $(__USER_DS)
         /*CFI_REL_OFFSET ss, 0*/
-        pushl %ebp
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ebp
         CFI_REL_OFFSET esp, 0
-        pushfl
+        pushfl_cfi
         orl $X86_EFLAGS_IF, (%esp)
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $(__USER_CS)
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $(__USER_CS)
         /*CFI_REL_OFFSET cs, 0*/
         /*
          * Push current_thread_info()->sysenter_return to the stack.
          * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
          * pushed above; +8 corresponds to copy_thread's esp0 setting.
          */
-        pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
         CFI_REL_OFFSET eip, 0
 
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         SAVE_ALL
         ENABLE_INTERRUPTS(CLBR_NONE)
 
@@ -486,8 +453,7 @@ sysenter_audit:
         movl %eax,%edx                  /* 2nd arg: syscall number */
         movl $AUDIT_ARCH_I386,%eax      /* 1st arg: audit arch */
         call audit_syscall_entry
-        pushl %ebx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ebx
         movl PT_EAX(%esp),%eax          /* reload syscall number */
         jmp sysenter_do_call
 
@@ -529,8 +495,7 @@ ENDPROC(ia32_sysenter_target)
         # system call handler stub
 ENTRY(system_call)
         RING0_INT_FRAME                 # can't unwind into user space anyway
-        pushl %eax                      # save orig_eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax                  # save orig_eax
         SAVE_ALL
         GET_THREAD_INFO(%ebp)
                                         # system call tracing in operation / emulation
@@ -566,7 +531,6 @@ restore_all_notrace:
         je ldt_ss                       # returning to user-space with LDT SS
 restore_nocheck:
         RESTORE_REGS 4                  # skip orig_eax/error_code
-        CFI_ADJUST_CFA_OFFSET -4
 irq_return:
         INTERRUPT_RETURN
 .section .fixup,"ax"
@@ -619,10 +583,8 @@ ldt_ss:
         shr $16, %edx
         mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
         mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
-        pushl $__ESPFIX_SS
-        CFI_ADJUST_CFA_OFFSET 4
-        push %eax                       /* new kernel esp */
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $__ESPFIX_SS
+        pushl_cfi %eax                  /* new kernel esp */
         /* Disable interrupts, but do not irqtrace this section: we
          * will soon execute iret and the tracer was already set to
          * the irqstate after the iret */
@@ -666,11 +628,9 @@ work_notifysig:				# deal with pending signals and
 
         ALIGN
 work_notifysig_v86:
-        pushl %ecx                      # save ti_flags for do_notify_resume
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ecx                  # save ti_flags for do_notify_resume
         call save_v86_state             # %eax contains pt_regs pointer
-        popl %ecx
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %ecx
         movl %eax, %esp
 #else
         movl %esp, %eax
@@ -750,14 +710,18 @@ ptregs_##name: \
 #define PTREGSCALL3(name) \
         ALIGN; \
 ptregs_##name: \
+        CFI_STARTPROC; \
         leal 4(%esp),%eax; \
-        pushl %eax; \
+        pushl_cfi %eax; \
         movl PT_EDX(%eax),%ecx; \
         movl PT_ECX(%eax),%edx; \
         movl PT_EBX(%eax),%eax; \
         call sys_##name; \
         addl $4,%esp; \
-        ret
+        CFI_ADJUST_CFA_OFFSET -4; \
+        ret; \
+        CFI_ENDPROC; \
+ENDPROC(ptregs_##name)
 
 PTREGSCALL1(iopl)
 PTREGSCALL0(fork)
@@ -772,15 +736,19 @@ PTREGSCALL1(vm86old)
 /* Clone is an oddball.  The 4th arg is in %edi */
         ALIGN;
 ptregs_clone:
+        CFI_STARTPROC
         leal 4(%esp),%eax
-        pushl %eax
-        pushl PT_EDI(%eax)
+        pushl_cfi %eax
+        pushl_cfi PT_EDI(%eax)
         movl PT_EDX(%eax),%ecx
         movl PT_ECX(%eax),%edx
         movl PT_EBX(%eax),%eax
         call sys_clone
         addl $8,%esp
+        CFI_ADJUST_CFA_OFFSET -8
         ret
+        CFI_ENDPROC
+ENDPROC(ptregs_clone)
 
 .macro FIXUP_ESPFIX_STACK
 /*
@@ -795,10 +763,8 @@ ptregs_clone:
         mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
         shl $16, %eax
         addl %esp, %eax                 /* the adjusted stack pointer */
-        pushl $__KERNEL_DS
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $__KERNEL_DS
+        pushl_cfi %eax
         lss (%esp), %esp                /* switch to the normal stack segment */
         CFI_ADJUST_CFA_OFFSET -8
 .endm
@@ -835,8 +801,7 @@ vector=FIRST_EXTERNAL_VECTOR
       .if vector <> FIRST_EXTERNAL_VECTOR
         CFI_ADJUST_CFA_OFFSET -4
       .endif
-1:      pushl $(~vector+0x80)   /* Note: always in signed byte range */
-        CFI_ADJUST_CFA_OFFSET 4
+1:      pushl_cfi $(~vector+0x80)       /* Note: always in signed byte range */
       .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
         jmp 2f
       .endif
@@ -876,8 +841,7 @@ ENDPROC(common_interrupt)
 #define BUILD_INTERRUPT3(name, nr, fn)  \
 ENTRY(name)                             \
         RING0_INT_FRAME;                \
-        pushl $~(nr);                   \
-        CFI_ADJUST_CFA_OFFSET 4;        \
+        pushl_cfi $~(nr);               \
         SAVE_ALL;                       \
         TRACE_IRQS_OFF                  \
         movl %esp,%eax;                 \
@@ -893,21 +857,18 @@ ENDPROC(name)
 
 ENTRY(coprocessor_error)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_coprocessor_error
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi $do_coprocessor_error
         jmp error_code
         CFI_ENDPROC
 END(coprocessor_error)
 
 ENTRY(simd_coprocessor_error)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
 #ifdef CONFIG_X86_INVD_BUG
         /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
-661:    pushl $do_general_protection
+661:    pushl_cfi $do_general_protection
 662:
 .section .altinstructions,"a"
         .balign 4
@@ -922,19 +883,16 @@ ENTRY(simd_coprocessor_error)
 664:
 .previous
 #else
-        pushl $do_simd_coprocessor_error
+        pushl_cfi $do_simd_coprocessor_error
 #endif
-        CFI_ADJUST_CFA_OFFSET 4
         jmp error_code
         CFI_ENDPROC
 END(simd_coprocessor_error)
 
 ENTRY(device_not_available)
         RING0_INT_FRAME
-        pushl $-1                       # mark this as an int
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_device_not_available
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $-1                   # mark this as an int
+        pushl_cfi $do_device_not_available
         jmp error_code
         CFI_ENDPROC
 END(device_not_available)
@@ -956,82 +914,68 @@ END(native_irq_enable_sysexit)
 
 ENTRY(overflow)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_overflow
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi $do_overflow
         jmp error_code
         CFI_ENDPROC
 END(overflow)
 
 ENTRY(bounds)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_bounds
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi $do_bounds
         jmp error_code
         CFI_ENDPROC
 END(bounds)
 
 ENTRY(invalid_op)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_invalid_op
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi $do_invalid_op
         jmp error_code
         CFI_ENDPROC
 END(invalid_op)
 
 ENTRY(coprocessor_segment_overrun)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_coprocessor_segment_overrun
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi $do_coprocessor_segment_overrun
         jmp error_code
         CFI_ENDPROC
 END(coprocessor_segment_overrun)
 
 ENTRY(invalid_TSS)
         RING0_EC_FRAME
-        pushl $do_invalid_TSS
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $do_invalid_TSS
         jmp error_code
         CFI_ENDPROC
 END(invalid_TSS)
 
 ENTRY(segment_not_present)
         RING0_EC_FRAME
-        pushl $do_segment_not_present
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $do_segment_not_present
         jmp error_code
         CFI_ENDPROC
 END(segment_not_present)
 
 ENTRY(stack_segment)
         RING0_EC_FRAME
-        pushl $do_stack_segment
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $do_stack_segment
         jmp error_code
         CFI_ENDPROC
 END(stack_segment)
 
 ENTRY(alignment_check)
         RING0_EC_FRAME
-        pushl $do_alignment_check
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $do_alignment_check
         jmp error_code
         CFI_ENDPROC
 END(alignment_check)
 
 ENTRY(divide_error)
         RING0_INT_FRAME
-        pushl $0                        # no error code
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_divide_error
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0                    # no error code
+        pushl_cfi $do_divide_error
         jmp error_code
         CFI_ENDPROC
 END(divide_error)
@@ -1039,10 +983,8 @@ END(divide_error)
 #ifdef CONFIG_X86_MCE
 ENTRY(machine_check)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl machine_check_vector
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi machine_check_vector
         jmp error_code
         CFI_ENDPROC
 END(machine_check)
@@ -1050,10 +992,8 @@ END(machine_check)
 
 ENTRY(spurious_interrupt_bug)
         RING0_INT_FRAME
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $do_spurious_interrupt_bug
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
+        pushl_cfi $do_spurious_interrupt_bug
         jmp error_code
         CFI_ENDPROC
 END(spurious_interrupt_bug)
@@ -1084,8 +1024,7 @@ ENTRY(xen_sysenter_target)
 
 ENTRY(xen_hypervisor_callback)
         CFI_STARTPROC
-        pushl $0
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $0
         SAVE_ALL
         TRACE_IRQS_OFF
 
@@ -1121,23 +1060,20 @@ ENDPROC(xen_hypervisor_callback)
 # We distinguish between categories by maintaining a status value in EAX.
 ENTRY(xen_failsafe_callback)
         CFI_STARTPROC
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         movl $1,%eax
 1:      mov 4(%esp),%ds
 2:      mov 8(%esp),%es
 3:      mov 12(%esp),%fs
 4:      mov 16(%esp),%gs
         testl %eax,%eax
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %eax
         lea 16(%esp),%esp
         CFI_ADJUST_CFA_OFFSET -16
         jz 5f
         addl $16,%esp
         jmp iret_exc            # EAX != 0 => Category 2 (Bad IRET)
-5:      pushl $0                # EAX == 0 => Category 1 (Bad segment)
-        CFI_ADJUST_CFA_OFFSET 4
+5:      pushl_cfi $0            # EAX == 0 => Category 1 (Bad segment)
         SAVE_ALL
         jmp ret_from_exception
         CFI_ENDPROC
@@ -1287,40 +1223,29 @@ syscall_table_size=(.-sys_call_table)
 
 ENTRY(page_fault)
         RING0_EC_FRAME
-        pushl $do_page_fault
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $do_page_fault
         ALIGN
 error_code:
         /* the function address is in %gs's slot on the stack */
-        pushl %fs
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %fs
         /*CFI_REL_OFFSET fs, 0*/
-        pushl %es
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %es
         /*CFI_REL_OFFSET es, 0*/
-        pushl %ds
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ds
         /*CFI_REL_OFFSET ds, 0*/
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         CFI_REL_OFFSET eax, 0
-        pushl %ebp
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ebp
         CFI_REL_OFFSET ebp, 0
-        pushl %edi
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %edi
         CFI_REL_OFFSET edi, 0
-        pushl %esi
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %esi
         CFI_REL_OFFSET esi, 0
-        pushl %edx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %edx
         CFI_REL_OFFSET edx, 0
-        pushl %ecx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ecx
         CFI_REL_OFFSET ecx, 0
-        pushl %ebx
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ebx
         CFI_REL_OFFSET ebx, 0
         cld
         movl $(__KERNEL_PERCPU), %ecx
@@ -1362,12 +1287,9 @@ END(page_fault)
         movl TSS_sysenter_sp0 + \offset(%esp), %esp
         CFI_DEF_CFA esp, 0
         CFI_UNDEFINED eip
-        pushfl
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $__KERNEL_CS
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl $sysenter_past_esp
-        CFI_ADJUST_CFA_OFFSET 4
+        pushfl_cfi
+        pushl_cfi $__KERNEL_CS
+        pushl_cfi $sysenter_past_esp
         CFI_REL_OFFSET eip, 0
 .endm
 
@@ -1377,8 +1299,7 @@ ENTRY(debug)
         jne debug_stack_correct
         FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
 debug_stack_correct:
-        pushl $-1                       # mark this as an int
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $-1                   # mark this as an int
         SAVE_ALL
         TRACE_IRQS_OFF
         xorl %edx,%edx                  # error code 0
@@ -1398,32 +1319,27 @@ END(debug)
  */
 ENTRY(nmi)
         RING0_INT_FRAME
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         movl %ss, %eax
         cmpw $__ESPFIX_SS, %ax
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %eax
         je nmi_espfix_stack
         cmpl $ia32_sysenter_target,(%esp)
         je nmi_stack_fixup
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         movl %esp,%eax
         /* Do not access memory above the end of our stack page,
          * it might not exist.
          */
         andl $(THREAD_SIZE-1),%eax
         cmpl $(THREAD_SIZE-20),%eax
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
+        popl_cfi %eax
         jae nmi_stack_correct
         cmpl $ia32_sysenter_target,12(%esp)
         je nmi_debug_stack_check
 nmi_stack_correct:
         /* We have a RING0_INT_FRAME here */
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         SAVE_ALL
         xorl %edx,%edx          # zero error code
         movl %esp,%eax          # pt_regs pointer
@@ -1452,18 +1368,14 @@ nmi_espfix_stack:
          *
          * create the pointer to lss back
          */
-        pushl %ss
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl %esp
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %ss
+        pushl_cfi %esp
         addl $4, (%esp)
         /* copy the iret frame of 12 bytes */
         .rept 3
-        pushl 16(%esp)
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi 16(%esp)
         .endr
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi %eax
         SAVE_ALL
         FIXUP_ESPFIX_STACK              # %eax == %esp
         xorl %edx,%edx                  # zero error code
@@ -1477,8 +1389,7 @@ END(nmi)
 
 ENTRY(int3)
         RING0_INT_FRAME
-        pushl $-1                       # mark this as an int
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $-1                   # mark this as an int
         SAVE_ALL
         TRACE_IRQS_OFF
         xorl %edx,%edx          # zero error code
@@ -1490,8 +1401,7 @@ END(int3)
 
 ENTRY(general_protection)
         RING0_EC_FRAME
-        pushl $do_general_protection
-        CFI_ADJUST_CFA_OFFSET 4
+        pushl_cfi $do_general_protection
         jmp error_code
         CFI_ENDPROC
 END(general_protection)
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 17be5ec7cbba..a7ae7fd1010f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -213,23 +213,17 @@ ENDPROC(native_usergs_sysret64)
         .macro FAKE_STACK_FRAME child_rip
         /* push in order ss, rsp, eflags, cs, rip */
         xorl %eax, %eax
-        pushq $__KERNEL_DS /* ss */
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi $__KERNEL_DS /* ss */
         /*CFI_REL_OFFSET        ss,0*/
-        pushq %rax /* rsp */
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi %rax /* rsp */
         CFI_REL_OFFSET  rsp,0
-        pushq $X86_EFLAGS_IF /* eflags - interrupts on */
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi $X86_EFLAGS_IF /* eflags - interrupts on */
         /*CFI_REL_OFFSET        rflags,0*/
-        pushq $__KERNEL_CS /* cs */
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi $__KERNEL_CS /* cs */
         /*CFI_REL_OFFSET        cs,0*/
-        pushq \child_rip /* rip */
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi \child_rip /* rip */
         CFI_REL_OFFSET  rip,0
-        pushq   %rax /* orig rax */
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi %rax /* orig rax */
         .endm
 
         .macro UNFAKE_STACK_FRAME
@@ -398,10 +392,8 @@ ENTRY(ret_from_fork)
 
         LOCK ; btr $TIF_FORK,TI_flags(%r8)
 
-        push kernel_eflags(%rip)
-        CFI_ADJUST_CFA_OFFSET 8
-        popf                                    # reset kernel eflags
-        CFI_ADJUST_CFA_OFFSET -8
+        pushq_cfi kernel_eflags(%rip)
+        popfq_cfi                               # reset kernel eflags
 
         call schedule_tail                      # rdi: 'prev' task parameter
 
@@ -521,11 +513,9 @@ sysret_careful:
         jnc sysret_signal
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
-        pushq %rdi
-        CFI_ADJUST_CFA_OFFSET 8
+        pushq_cfi %rdi
         call schedule
-        popq  %rdi
-        CFI_ADJUST_CFA_OFFSET -8
+        popq_cfi %rdi
         jmp sysret_check
 
         /* Handle a signal */
@@ -634,11 +624,9 @@ int_careful:
         jnc  int_very_careful
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
-        pushq %rdi
-        CFI_ADJUST_CFA_OFFSET 8
+        pushq_cfi %rdi
         call schedule
-        popq %rdi
-        CFI_ADJUST_CFA_OFFSET -8
+        popq_cfi %rdi
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp int_with_check
@@ -652,12 +640,10 @@ int_check_syscall_exit_work:
         /* Check for syscall exit trace */
         testl $_TIF_WORK_SYSCALL_EXIT,%edx
         jz int_signal
-        pushq %rdi
-        CFI_ADJUST_CFA_OFFSET 8
+        pushq_cfi %rdi
         leaq 8(%rsp),%rdi       # &ptregs -> arg1
         call syscall_trace_leave
-        popq %rdi
-        CFI_ADJUST_CFA_OFFSET -8
+        popq_cfi %rdi
         andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
         jmp int_restore_rest
 
@@ -714,9 +700,8 @@ END(ptregscall_common)
 
 ENTRY(stub_execve)
         CFI_STARTPROC
-        popq %r11
-        CFI_ADJUST_CFA_OFFSET -8
-        CFI_REGISTER rip, r11
+        addq $8, %rsp
+        PARTIAL_FRAME 0
         SAVE_REST
         FIXUP_TOP_OF_STACK %r11
         movq %rsp, %rcx
@@ -735,7 +720,7 @@ END(stub_execve)
 ENTRY(stub_rt_sigreturn)
         CFI_STARTPROC
         addq $8, %rsp
-        CFI_ADJUST_CFA_OFFSET   -8
+        PARTIAL_FRAME 0
         SAVE_REST
         movq %rsp,%rdi
         FIXUP_TOP_OF_STACK %r11
@@ -766,8 +751,7 @@ vector=FIRST_EXTERNAL_VECTOR
       .if vector <> FIRST_EXTERNAL_VECTOR
         CFI_ADJUST_CFA_OFFSET -8
       .endif
-1:      pushq $(~vector+0x80)   /* Note: always in signed byte range */
-        CFI_ADJUST_CFA_OFFSET 8
+1:      pushq_cfi $(~vector+0x80)       /* Note: always in signed byte range */
       .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
         jmp 2f
       .endif
@@ -796,8 +780,8 @@ END(interrupt)
 
 /* 0(%rsp): ~(interrupt number) */
         .macro interrupt func
-        subq $10*8, %rsp
-        CFI_ADJUST_CFA_OFFSET 10*8
+        subq $ORIG_RAX-ARGOFFSET+8, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8
         call save_args
         PARTIAL_FRAME 0
         call \func
@@ -822,6 +806,7 @@ ret_from_intr:
         TRACE_IRQS_OFF
         decl PER_CPU_VAR(irq_count)
         leaveq
+        CFI_RESTORE             rbp
         CFI_DEF_CFA_REGISTER    rsp
         CFI_ADJUST_CFA_OFFSET   -8
 exit_intr:
@@ -903,11 +888,9 @@ retint_careful:
         jnc   retint_signal
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
-        pushq %rdi
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi %rdi
         call  schedule
-        popq %rdi
-        CFI_ADJUST_CFA_OFFSET   -8
+        popq_cfi %rdi
         GET_THREAD_INFO(%rcx)
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
@@ -956,8 +939,7 @@ END(common_interrupt)
 .macro apicinterrupt num sym do_sym
 ENTRY(\sym)
         INTR_FRAME
-        pushq $~(\num)
-        CFI_ADJUST_CFA_OFFSET 8
+        pushq_cfi $~(\num)
         interrupt \do_sym
         jmp ret_from_intr
         CFI_ENDPROC
@@ -1023,9 +1005,9 @@ apicinterrupt ERROR_APIC_VECTOR \
 apicinterrupt SPURIOUS_APIC_VECTOR \
         spurious_interrupt smp_spurious_interrupt
 
-#ifdef CONFIG_PERF_EVENTS
-apicinterrupt LOCAL_PENDING_VECTOR \
-        perf_pending_interrupt smp_perf_pending_interrupt
+#ifdef CONFIG_IRQ_WORK
+apicinterrupt IRQ_WORK_VECTOR \
+        irq_work_interrupt smp_irq_work_interrupt
 #endif
 
 /*
@@ -1036,8 +1018,8 @@ ENTRY(\sym)
         INTR_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
         pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
-        subq $15*8,%rsp
-        CFI_ADJUST_CFA_OFFSET 15*8
+        subq $ORIG_RAX-R15, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
         call error_entry
         DEFAULT_FRAME 0
         movq %rsp,%rdi          /* pt_regs pointer */
@@ -1052,9 +1034,9 @@ END(\sym)
 ENTRY(\sym)
         INTR_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $-1               /* ORIG_RAX: no syscall to restart */
-        CFI_ADJUST_CFA_OFFSET 8
-        subq $15*8, %rsp
+        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
+        subq $ORIG_RAX-R15, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
         call save_paranoid
         TRACE_IRQS_OFF
         movq %rsp,%rdi          /* pt_regs pointer */
@@ -1070,9 +1052,9 @@ END(\sym)
 ENTRY(\sym)
         INTR_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $-1               /* ORIG_RAX: no syscall to restart */
-        CFI_ADJUST_CFA_OFFSET 8
-        subq $15*8, %rsp
+        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
+        subq $ORIG_RAX-R15, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
         call save_paranoid
         TRACE_IRQS_OFF
         movq %rsp,%rdi          /* pt_regs pointer */
@@ -1089,8 +1071,8 @@ END(\sym)
 ENTRY(\sym)
         XCPT_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
-        subq $15*8,%rsp
-        CFI_ADJUST_CFA_OFFSET 15*8
+        subq $ORIG_RAX-R15, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
         call error_entry
         DEFAULT_FRAME 0
         movq %rsp,%rdi                  /* pt_regs pointer */
@@ -1107,8 +1089,8 @@ END(\sym)
 ENTRY(\sym)
         XCPT_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
-        subq $15*8,%rsp
-        CFI_ADJUST_CFA_OFFSET 15*8
+        subq $ORIG_RAX-R15, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
         call save_paranoid
         DEFAULT_FRAME 0
         TRACE_IRQS_OFF
@@ -1139,16 +1121,14 @@ zeroentry simd_coprocessor_error do_simd_coprocessor_error
         /* edi:  new selector */
 ENTRY(native_load_gs_index)
         CFI_STARTPROC
-        pushf
-        CFI_ADJUST_CFA_OFFSET 8
+        pushfq_cfi
         DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
         SWAPGS
 gs_change:
         movl %edi,%gs
 2:      mfence          /* workaround */
         SWAPGS
-        popf
-        CFI_ADJUST_CFA_OFFSET -8
+        popfq_cfi
         ret
         CFI_ENDPROC
 END(native_load_gs_index)
@@ -1215,8 +1195,7 @@ END(kernel_execve)
 /* Call softirq on interrupt stack. Interrupts are off. */
 ENTRY(call_softirq)
         CFI_STARTPROC
-        push %rbp
-        CFI_ADJUST_CFA_OFFSET   8
+        pushq_cfi %rbp
         CFI_REL_OFFSET rbp,0
         mov  %rsp,%rbp
         CFI_DEF_CFA_REGISTER rbp
@@ -1225,6 +1204,7 @@ ENTRY(call_softirq)
         push  %rbp                      # backlink for old unwinder
         call __do_softirq
         leaveq
+        CFI_RESTORE             rbp
         CFI_DEF_CFA_REGISTER    rsp
         CFI_ADJUST_CFA_OFFSET   -8
         decl PER_CPU_VAR(irq_count)
@@ -1368,7 +1348,7 @@ paranoidzeroentry machine_check *machine_check_vector(%rip)
 
         /* ebx: no swapgs flag */
 ENTRY(paranoid_exit)
-        INTR_FRAME
+        DEFAULT_FRAME
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         testl %ebx,%ebx                         /* swapgs needed? */
@@ -1445,7 +1425,6 @@ error_swapgs:
 error_sti:
         TRACE_IRQS_OFF
         ret
-        CFI_ENDPROC
 
 /*
  * There are two places in the kernel that can potentially fault with
@@ -1470,6 +1449,7 @@ bstep_iret:
         /* Fix truncated RIP */
         movq %rcx,RIP+8(%rsp)
         jmp error_swapgs
+        CFI_ENDPROC
 END(error_entry)
 
 
@@ -1498,8 +1478,8 @@ ENTRY(nmi)
         INTR_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
         pushq_cfi $-1
-        subq $15*8, %rsp
-        CFI_ADJUST_CFA_OFFSET 15*8
+        subq $ORIG_RAX-R15, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
         call save_paranoid
         DEFAULT_FRAME 0
         /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index 7494999141b3..efaf906daf93 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -440,9 +440,9 @@ static int hpet_legacy_next_event(unsigned long delta,
 static DEFINE_PER_CPU(struct hpet_dev *, cpu_hpet_dev);
 static struct hpet_dev  *hpet_devs;
 
-void hpet_msi_unmask(unsigned int irq)
+void hpet_msi_unmask(struct irq_data *data)
 {
-        struct hpet_dev *hdev = get_irq_data(irq);
+        struct hpet_dev *hdev = data->handler_data;
         unsigned int cfg;
 
         /* unmask it */
@@ -451,10 +451,10 @@ void hpet_msi_unmask(unsigned int irq)
         hpet_writel(cfg, HPET_Tn_CFG(hdev->num));
 }
 
-void hpet_msi_mask(unsigned int irq)
+void hpet_msi_mask(struct irq_data *data)
 {
+        struct hpet_dev *hdev = data->handler_data;
         unsigned int cfg;
-        struct hpet_dev *hdev = get_irq_data(irq);
 
         /* mask it */
         cfg = hpet_readl(HPET_Tn_CFG(hdev->num));
@@ -462,18 +462,14 @@ void hpet_msi_mask(unsigned int irq)
         hpet_writel(cfg, HPET_Tn_CFG(hdev->num));
 }
 
-void hpet_msi_write(unsigned int irq, struct msi_msg *msg)
+void hpet_msi_write(struct hpet_dev *hdev, struct msi_msg *msg)
 {
-        struct hpet_dev *hdev = get_irq_data(irq);
-
         hpet_writel(msg->data, HPET_Tn_ROUTE(hdev->num));
         hpet_writel(msg->address_lo, HPET_Tn_ROUTE(hdev->num) + 4);
 }
 
-void hpet_msi_read(unsigned int irq, struct msi_msg *msg)
+void hpet_msi_read(struct hpet_dev *hdev, struct msi_msg *msg)
 {
-        struct hpet_dev *hdev = get_irq_data(irq);
-
         msg->data = hpet_readl(HPET_Tn_ROUTE(hdev->num));
         msg->address_lo = hpet_readl(HPET_Tn_ROUTE(hdev->num) + 4);
         msg->address_hi = 0;
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index a46cb3522c0c..58bb239a2fd7 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -68,19 +68,22 @@ static void __cpuinit init_thread_xstate(void)
          */
 
         if (!HAVE_HWFP) {
+                /*
+                 * Disable xsave as we do not support it if i387
+                 * emulation is enabled.
+                 */
+                setup_clear_cpu_cap(X86_FEATURE_XSAVE);
+                setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
                 xstate_size = sizeof(struct i387_soft_struct);
                 return;
         }
 
         if (cpu_has_fxsr)
                 xstate_size = sizeof(struct i387_fxsave_struct);
-#ifdef CONFIG_X86_32
         else
                 xstate_size = sizeof(struct i387_fsave_struct);
-#endif
 }
 
-#ifdef CONFIG_X86_64
 /*
  * Called at bootup to set up the initial FPU state that is later cloned
  * into all processes.
@@ -88,12 +91,21 @@ static void __cpuinit init_thread_xstate(void)
 
 void __cpuinit fpu_init(void)
 {
-        unsigned long oldcr0 = read_cr0();
-
-        set_in_cr4(X86_CR4_OSFXSR);
-        set_in_cr4(X86_CR4_OSXMMEXCPT);
+        unsigned long cr0;
+        unsigned long cr4_mask = 0;
 
-        write_cr0(oldcr0 & ~(X86_CR0_TS|X86_CR0_EM)); /* clear TS and EM */
+        if (cpu_has_fxsr)
+                cr4_mask |= X86_CR4_OSFXSR;
+        if (cpu_has_xmm)
+                cr4_mask |= X86_CR4_OSXMMEXCPT;
+        if (cr4_mask)
+                set_in_cr4(cr4_mask);
+
+        cr0 = read_cr0();
+        cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
+        if (!HAVE_HWFP)
+                cr0 |= X86_CR0_EM;
+        write_cr0(cr0);
 
         if (!smp_processor_id())
                 init_thread_xstate();
@@ -104,24 +116,12 @@ void __cpuinit fpu_init(void)
         clear_used_math();
 }
 
-#else   /* CONFIG_X86_64 */
-
-void __cpuinit fpu_init(void)
-{
-        if (!smp_processor_id())
-                init_thread_xstate();
-}
-
-#endif  /* CONFIG_X86_32 */
-
 void fpu_finit(struct fpu *fpu)
 {
-#ifdef CONFIG_X86_32
         if (!HAVE_HWFP) {
                 finit_soft_fpu(&fpu->state->soft);
                 return;
         }
-#endif
 
         if (cpu_has_fxsr) {
                 struct i387_fxsave_struct *fx = &fpu->state->fxsave;
@@ -386,19 +386,17 @@ convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
 #ifdef CONFIG_X86_64
         env->fip = fxsave->rip;
         env->foo = fxsave->rdp;
+        /*
+         * should be actually ds/cs at fpu exception time, but
+         * that information is not available in 64bit mode.
+         */
+        env->fcs = task_pt_regs(tsk)->cs;
         if (tsk == current) {
-                /*
-                 * should be actually ds/cs at fpu exception time, but
-                 * that information is not available in 64bit mode.
-                 */
-                asm("mov %%ds, %[fos]" : [fos] "=r" (env->fos));
-                asm("mov %%cs, %[fcs]" : [fcs] "=r" (env->fcs));
+                savesegment(ds, env->fos);
         } else {
-                struct pt_regs *regs = task_pt_regs(tsk);
-
-                env->fos = 0xffff0000 | tsk->thread.ds;
-                env->fcs = regs->cs;
+                env->fos = tsk->thread.ds;
         }
+        env->fos |= 0xffff0000;
 #else
         env->fip = fxsave->fip;
         env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c
index cafa7c80ac95..20757cb2efa3 100644
--- a/arch/x86/kernel/i8259.c
+++ b/arch/x86/kernel/i8259.c
@@ -29,24 +29,10 @@
  * plus some generic x86 specific things if generic specifics makes
  * any sense at all.
  */
+static void init_8259A(int auto_eoi);
 
 static int i8259A_auto_eoi;
 DEFINE_RAW_SPINLOCK(i8259A_lock);
-static void mask_and_ack_8259A(unsigned int);
-static void mask_8259A(void);
-static void unmask_8259A(void);
-static void disable_8259A_irq(unsigned int irq);
-static void enable_8259A_irq(unsigned int irq);
-static void init_8259A(int auto_eoi);
-static int i8259A_irq_pending(unsigned int irq);
-
-struct irq_chip i8259A_chip = {
-        .name           = "XT-PIC",
-        .mask           = disable_8259A_irq,
-        .disable        = disable_8259A_irq,
-        .unmask         = enable_8259A_irq,
-        .mask_ack       = mask_and_ack_8259A,
-};
 
 /*
  * 8259A PIC functions to handle ISA devices:
@@ -68,7 +54,7 @@ unsigned int cached_irq_mask = 0xffff;
  */
 unsigned long io_apic_irqs;
 
-static void disable_8259A_irq(unsigned int irq)
+static void mask_8259A_irq(unsigned int irq)
 {
         unsigned int mask = 1 << irq;
         unsigned long flags;
@@ -82,7 +68,12 @@ static void disable_8259A_irq(unsigned int irq)
         raw_spin_unlock_irqrestore(&i8259A_lock, flags);
 }
 
-static void enable_8259A_irq(unsigned int irq)
+static void disable_8259A_irq(struct irq_data *data)
+{
+        mask_8259A_irq(data->irq);
+}
+
+static void unmask_8259A_irq(unsigned int irq)
 {
         unsigned int mask = ~(1 << irq);
         unsigned long flags;
@@ -96,6 +87,11 @@ static void enable_8259A_irq(unsigned int irq)
         raw_spin_unlock_irqrestore(&i8259A_lock, flags);
 }
 
+static void enable_8259A_irq(struct irq_data *data)
+{
+        unmask_8259A_irq(data->irq);
+}
+
 static int i8259A_irq_pending(unsigned int irq)
 {
         unsigned int mask = 1<<irq;
@@ -117,7 +113,7 @@ static void make_8259A_irq(unsigned int irq)
         disable_irq_nosync(irq);
         io_apic_irqs &= ~(1<<irq);
         set_irq_chip_and_handler_name(irq, &i8259A_chip, handle_level_irq,
-                                      "XT");
+                                      i8259A_chip.name);
         enable_irq(irq);
 }
 
@@ -150,8 +146,9 @@ static inline int i8259A_irq_real(unsigned int irq)
  * first, _then_ send the EOI, and the order of EOI
  * to the two 8259s is important!
  */
-static void mask_and_ack_8259A(unsigned int irq)
+static void mask_and_ack_8259A(struct irq_data *data)
 {
+        unsigned int irq = data->irq;
         unsigned int irqmask = 1 << irq;
         unsigned long flags;
 
@@ -223,6 +220,14 @@ spurious_8259A_irq:
         }
 }
 
+struct irq_chip i8259A_chip = {
+        .name           = "XT-PIC",
+        .irq_mask       = disable_8259A_irq,
+        .irq_disable    = disable_8259A_irq,
+        .irq_unmask     = enable_8259A_irq,
+        .irq_mask_ack   = mask_and_ack_8259A,
+};
+
 static char irq_trigger[2];
 /**
  * ELCR registers (0x4d0, 0x4d1) control edge/level of IRQ
@@ -342,9 +347,9 @@ static void init_8259A(int auto_eoi)
                  * In AEOI mode we just have to mask the interrupt
                  * when acking.
                  */
-                i8259A_chip.mask_ack = disable_8259A_irq;
+                i8259A_chip.irq_mask_ack = disable_8259A_irq;
         else
-                i8259A_chip.mask_ack = mask_and_ack_8259A;
+                i8259A_chip.irq_mask_ack = mask_and_ack_8259A;
 
         udelay(100);            /* wait for 8259A to initialize */
 
@@ -363,14 +368,6 @@ static void init_8259A(int auto_eoi)
 static void legacy_pic_noop(void) { };
 static void legacy_pic_uint_noop(unsigned int unused) { };
 static void legacy_pic_int_noop(int unused) { };
-
-static struct irq_chip dummy_pic_chip  = {
-        .name = "dummy pic",
-        .mask = legacy_pic_uint_noop,
-        .unmask = legacy_pic_uint_noop,
-        .disable = legacy_pic_uint_noop,
-        .mask_ack = legacy_pic_uint_noop,
-};
 static int legacy_pic_irq_pending_noop(unsigned int irq)
 {
         return 0;
@@ -378,7 +375,9 @@ static int legacy_pic_irq_pending_noop(unsigned int irq)
 
 struct legacy_pic null_legacy_pic = {
         .nr_legacy_irqs = 0,
-        .chip = &dummy_pic_chip,
+        .chip = &dummy_irq_chip,
+        .mask = legacy_pic_uint_noop,
+        .unmask = legacy_pic_uint_noop,
         .mask_all = legacy_pic_noop,
         .restore_mask = legacy_pic_noop,
         .init = legacy_pic_int_noop,
@@ -389,7 +388,9 @@ struct legacy_pic null_legacy_pic = {
 struct legacy_pic default_legacy_pic = {
         .nr_legacy_irqs = NR_IRQS_LEGACY,
         .chip  = &i8259A_chip,
-        .mask_all  = mask_8259A,
+        .mask = mask_8259A_irq,
+        .unmask = unmask_8259A_irq,
+        .mask_all = mask_8259A,
         .restore_mask = unmask_8259A,
         .init = init_8259A,
         .irq_pending = i8259A_irq_pending,
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 91fd0c70a18a..83ec0175f986 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -67,10 +67,10 @@ static int show_other_interrupts(struct seq_file *p, int prec)
         for_each_online_cpu(j)
                 seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
         seq_printf(p, "  Performance monitoring interrupts\n");
-        seq_printf(p, "%*s: ", prec, "PND");
+        seq_printf(p, "%*s: ", prec, "IWI");
         for_each_online_cpu(j)
-                seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs);
-        seq_printf(p, "  Performance pending work\n");
+                seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
+        seq_printf(p, "  IRQ work interrupts\n");
 #endif
         if (x86_platform_ipi_callback) {
                 seq_printf(p, "%*s: ", prec, "PLT");
@@ -159,7 +159,7 @@ int show_interrupts(struct seq_file *p, void *v)
         seq_printf(p, "%*d: ", prec, i);
         for_each_online_cpu(j)
                 seq_printf(p, "%10u ", kstat_irqs_cpu(i, j));
-        seq_printf(p, " %8s", desc->chip->name);
+        seq_printf(p, " %8s", desc->irq_data.chip->name);
         seq_printf(p, "-%-8s", desc->name);
 
         if (action) {
@@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
         sum += irq_stats(cpu)->apic_timer_irqs;
         sum += irq_stats(cpu)->irq_spurious_count;
         sum += irq_stats(cpu)->apic_perf_irqs;
-        sum += irq_stats(cpu)->apic_pending_irqs;
+        sum += irq_stats(cpu)->apic_irq_work_irqs;
 #endif
         if (x86_platform_ipi_callback)
                 sum += irq_stats(cpu)->x86_platform_ipis;
@@ -282,6 +282,7 @@ void fixup_irqs(void)
         unsigned int irq, vector;
         static int warned;
         struct irq_desc *desc;
+        struct irq_data *data;
 
         for_each_irq_desc(irq, desc) {
                 int break_affinity = 0;
@@ -296,7 +297,8 @@ void fixup_irqs(void)
                 /* interrupt's are disabled at this point */
                 raw_spin_lock(&desc->lock);
 
-                affinity = desc->affinity;
+                data = &desc->irq_data;
+                affinity = data->affinity;
                 if (!irq_has_action(irq) ||
                     cpumask_equal(affinity, cpu_online_mask)) {
                         raw_spin_unlock(&desc->lock);
@@ -315,16 +317,16 @@ void fixup_irqs(void)
                         affinity = cpu_all_mask;
                 }
 
-                if (!(desc->status & IRQ_MOVE_PCNTXT) && desc->chip->mask)
-                        desc->chip->mask(irq);
+                if (!(desc->status & IRQ_MOVE_PCNTXT) && data->chip->irq_mask)
+                        data->chip->irq_mask(data);
 
-                if (desc->chip->set_affinity)
-                        desc->chip->set_affinity(irq, affinity);
+                if (data->chip->irq_set_affinity)
+                        data->chip->irq_set_affinity(data, affinity, true);
                 else if (!(warned++))
                         set_affinity = 0;
 
-                if (!(desc->status & IRQ_MOVE_PCNTXT) && desc->chip->unmask)
-                        desc->chip->unmask(irq);
+                if (!(desc->status & IRQ_MOVE_PCNTXT) && data->chip->irq_unmask)
+                        data->chip->irq_unmask(data);
 
                 raw_spin_unlock(&desc->lock);
 
@@ -355,10 +357,10 @@ void fixup_irqs(void)
                 if (irr  & (1 << (vector % 32))) {
                         irq = __get_cpu_var(vector_irq)[vector];
 
-                        desc = irq_to_desc(irq);
+                        data = irq_get_irq_data(irq);
                         raw_spin_lock(&desc->lock);
-                        if (desc->chip->retrigger)
-                                desc->chip->retrigger(irq);
+                        if (data->chip->irq_retrigger)
+                                data->chip->irq_retrigger(data);
                         raw_spin_unlock(&desc->lock);
                 }
         }
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c
new file mode 100644
index 000000000000..ca8f703a1e70
--- /dev/null
+++ b/arch/x86/kernel/irq_work.c
@@ -0,0 +1,30 @@
+/*
+ * x86 specific code for irq_work
+ *
+ * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/irq_work.h>
+#include <linux/hardirq.h>
+#include <asm/apic.h>
+
+void smp_irq_work_interrupt(struct pt_regs *regs)
+{
+        irq_enter();
+        ack_APIC_irq();
+        inc_irq_stat(apic_irq_work_irqs);
+        irq_work_run();
+        irq_exit();
+}
+
+void arch_irq_work_raise(void)
+{
+#ifdef CONFIG_X86_LOCAL_APIC
+        if (!cpu_has_apic)
+                return;
+
+        apic->send_IPI_self(IRQ_WORK_VECTOR);
+        apic_wait_icr_idle();
+#endif
+}
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 990ae7cfc578..c752e973958d 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -100,6 +100,8 @@ int vector_used_by_percpu_irq(unsigned int vector)
 
 void __init init_ISA_irqs(void)
 {
+        struct irq_chip *chip = legacy_pic->chip;
+        const char *name = chip->name;
         int i;
 
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_LOCAL_APIC)
@@ -107,19 +109,8 @@ void __init init_ISA_irqs(void)
 #endif
         legacy_pic->init(0);
 
-        /*
-         * 16 old-style INTA-cycle interrupts:
-         */
-        for (i = 0; i < legacy_pic->nr_legacy_irqs; i++) {
-                struct irq_desc *desc = irq_to_desc(i);
-
-                desc->status = IRQ_DISABLED;
-                desc->action = NULL;
-                desc->depth = 1;
-
-                set_irq_chip_and_handler_name(i, &i8259A_chip,
-                                              handle_level_irq, "XT");
-        }
+        for (i = 0; i < legacy_pic->nr_legacy_irqs; i++)
+                set_irq_chip_and_handler_name(i, chip, handle_level_irq, name);
 }
 
 void __init init_IRQ(void)
@@ -224,9 +215,9 @@ static void __init apic_intr_init(void)
         alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
         alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
 
-        /* Performance monitoring interrupts: */
-# ifdef CONFIG_PERF_EVENTS
-        alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
+        /* IRQ work interrupts: */
+# ifdef CONFIG_IRQ_WORK
+        alloc_intr_gate(IRQ_WORK_VECTOR, irq_work_interrupt);
 # endif
 
 #endif
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 035c8c529181..b3ea9db39db6 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -36,7 +36,7 @@ static int init_one_level2_page(struct kimage *image, pgd_t *pgd,
                 if (!page)
                         goto out;
                 pud = (pud_t *)page_address(page);
-                memset(pud, 0, PAGE_SIZE);
+                clear_page(pud);
                 set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
         }
         pud = pud_offset(pgd, addr);
@@ -45,7 +45,7 @@ static int init_one_level2_page(struct kimage *image, pgd_t *pgd,
                 if (!page)
                         goto out;
                 pmd = (pmd_t *)page_address(page);
-                memset(pmd, 0, PAGE_SIZE);
+                clear_page(pmd);
                 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
         }
         pmd = pmd_offset(pud, addr);
diff --git a/arch/x86/kernel/olpc-xo1.c b/arch/x86/kernel/olpc-xo1.c
new file mode 100644
index 000000000000..f5442c03abc3
--- /dev/null
+++ b/arch/x86/kernel/olpc-xo1.c
@@ -0,0 +1,140 @@
+/*
+ * Support for features of the OLPC XO-1 laptop
+ *
+ * Copyright (C) 2010 One Laptop per Child
+ * Copyright (C) 2006 Red Hat, Inc.
+ * Copyright (C) 2006 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+
+#include <asm/io.h>
+#include <asm/olpc.h>
+
+#define DRV_NAME "olpc-xo1"
+
+#define PMS_BAR         4
+#define ACPI_BAR        5
+
+/* PMC registers (PMS block) */
+#define PM_SCLK         0x10
+#define PM_IN_SLPCTL    0x20
+#define PM_WKXD         0x34
+#define PM_WKD          0x30
+#define PM_SSC          0x54
+
+/* PM registers (ACPI block) */
+#define PM1_CNT         0x08
+#define PM_GPE0_STS     0x18
+
+static unsigned long acpi_base;
+static unsigned long pms_base;
+
+static void xo1_power_off(void)
+{
+        printk(KERN_INFO "OLPC XO-1 power off sequence...\n");
+
+        /* Enable all of these controls with 0 delay */
+        outl(0x40000000, pms_base + PM_SCLK);
+        outl(0x40000000, pms_base + PM_IN_SLPCTL);
+        outl(0x40000000, pms_base + PM_WKXD);
+        outl(0x40000000, pms_base + PM_WKD);
+
+        /* Clear status bits (possibly unnecessary) */
+        outl(0x0002ffff, pms_base  + PM_SSC);
+        outl(0xffffffff, acpi_base + PM_GPE0_STS);
+
+        /* Write SLP_EN bit to start the machinery */
+        outl(0x00002000, acpi_base + PM1_CNT);
+}
+
+/* Read the base addresses from the PCI BAR info */
+static int __devinit setup_bases(struct pci_dev *pdev)
+{
+        int r;
+
+        r = pci_enable_device_io(pdev);
+        if (r) {
+                dev_err(&pdev->dev, "can't enable device IO\n");
+                return r;
+        }
+
+        r = pci_request_region(pdev, ACPI_BAR, DRV_NAME);
+        if (r) {
+                dev_err(&pdev->dev, "can't alloc PCI BAR #%d\n", ACPI_BAR);
+                return r;
+        }
+
+        r = pci_request_region(pdev, PMS_BAR, DRV_NAME);
+        if (r) {
+                dev_err(&pdev->dev, "can't alloc PCI BAR #%d\n", PMS_BAR);
+                pci_release_region(pdev, ACPI_BAR);
+                return r;
+        }
+
+        acpi_base = pci_resource_start(pdev, ACPI_BAR);
+        pms_base = pci_resource_start(pdev, PMS_BAR);
+
+        return 0;
+}
+
+static int __devinit olpc_xo1_probe(struct platform_device *pdev)
+{
+        struct pci_dev *pcidev;
+        int r;
+
+        pcidev = pci_get_device(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CS5536_ISA,
+                                NULL);
+        if (!pdev)
+                return -ENODEV;
+
+        r = setup_bases(pcidev);
+        if (r)
+                return r;
+
+        pm_power_off = xo1_power_off;
+
+        printk(KERN_INFO "OLPC XO-1 support registered\n");
+        return 0;
+}
+
+static int __devexit olpc_xo1_remove(struct platform_device *pdev)
+{
+        pm_power_off = NULL;
+        return 0;
+}
+
+static struct platform_driver olpc_xo1_driver = {
+        .driver = {
+                .name = DRV_NAME,
+                .owner = THIS_MODULE,
+        },
+        .probe = olpc_xo1_probe,
+        .remove = __devexit_p(olpc_xo1_remove),
+};
+
+static int __init olpc_xo1_init(void)
+{
+        return platform_driver_register(&olpc_xo1_driver);
+}
+
+static void __exit olpc_xo1_exit(void)
+{
+        platform_driver_unregister(&olpc_xo1_driver);
+}
+
+MODULE_AUTHOR("Daniel Drake <dsd@laptop.org>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:olpc-xo1");
+
+module_init(olpc_xo1_init);
+module_exit(olpc_xo1_exit);
diff --git a/arch/x86/kernel/olpc.c b/arch/x86/kernel/olpc.c
index 0e0cdde519be..edaf3fe8dc5e 100644
--- a/arch/x86/kernel/olpc.c
+++ b/arch/x86/kernel/olpc.c
@@ -17,6 +17,7 @@
 #include <linux/spinlock.h>
 #include <linux/io.h>
 #include <linux/string.h>
+#include <linux/platform_device.h>
 
 #include <asm/geode.h>
 #include <asm/setup.h>
@@ -114,6 +115,7 @@ int olpc_ec_cmd(unsigned char cmd, unsigned char *inbuf, size_t inlen,
         unsigned long flags;
         int ret = -EIO;
         int i;
+        int restarts = 0;
 
         spin_lock_irqsave(&ec_lock, flags);
 
@@ -169,7 +171,9 @@ restart:
                         if (wait_on_obf(0x6c, 1)) {
                                 printk(KERN_ERR "olpc-ec:  timeout waiting for"
                                                 " EC to provide data!\n");
-                                goto restart;
+                                if (restarts++ < 10)
+                                        goto restart;
+                                goto err;
                         }
                         outbuf[i] = inb(0x68);
                         pr_devel("olpc-ec:  received 0x%x\n", outbuf[i]);
@@ -183,8 +187,21 @@ err:
 }
 EXPORT_SYMBOL_GPL(olpc_ec_cmd);
 
-#ifdef CONFIG_OLPC_OPENFIRMWARE
-static void __init platform_detect(void)
+static bool __init check_ofw_architecture(void)
+{
+        size_t propsize;
+        char olpc_arch[5];
+        const void *args[] = { NULL, "architecture", olpc_arch, (void *)5 };
+        void *res[] = { &propsize };
+
+        if (olpc_ofw("getprop", args, res)) {
+                printk(KERN_ERR "ofw: getprop call failed!\n");
+                return false;
+        }
+        return propsize == 5 && strncmp("OLPC", olpc_arch, 5) == 0;
+}
+
+static u32 __init get_board_revision(void)
 {
         size_t propsize;
         __be32 rev;
@@ -193,45 +210,43 @@ static void __init platform_detect(void)
 
         if (olpc_ofw("getprop", args, res) || propsize != 4) {
                 printk(KERN_ERR "ofw: getprop call failed!\n");
-                rev = cpu_to_be32(0);
+                return cpu_to_be32(0);
         }
-        olpc_platform_info.boardrev = be32_to_cpu(rev);
+        return be32_to_cpu(rev);
 }
-#else
-static void __init platform_detect(void)
+
+static bool __init platform_detect(void)
 {
-        /* stopgap until OFW support is added to the kernel */
-        olpc_platform_info.boardrev = olpc_board(0xc2);
+        if (!check_ofw_architecture())
+                return false;
+        olpc_platform_info.flags |= OLPC_F_PRESENT;
+        olpc_platform_info.boardrev = get_board_revision();
+        return true;
 }
-#endif
 
-static int __init olpc_init(void)
+static int __init add_xo1_platform_devices(void)
 {
-        unsigned char *romsig;
+        struct platform_device *pdev;
 
-        /* The ioremap check is dangerous; limit what we run it on */
-        if (!is_geode() || cs5535_has_vsa2())
-                return 0;
+        pdev = platform_device_register_simple("xo1-rfkill", -1, NULL, 0);
+        if (IS_ERR(pdev))
+                return PTR_ERR(pdev);
 
-        spin_lock_init(&ec_lock);
+        pdev = platform_device_register_simple("olpc-xo1", -1, NULL, 0);
+        if (IS_ERR(pdev))
+                return PTR_ERR(pdev);
 
-        romsig = ioremap(0xffffffc0, 16);
-        if (!romsig)
-                return 0;
+        return 0;
+}
 
-        if (strncmp(romsig, "CL1   Q", 7))
-                goto unmap;
-        if (strncmp(romsig+6, romsig+13, 3)) {
-                printk(KERN_INFO "OLPC BIOS signature looks invalid.  "
-                                "Assuming not OLPC\n");
-                goto unmap;
-        }
+static int __init olpc_init(void)
+{
+        int r = 0;
 
-        printk(KERN_INFO "OLPC board with OpenFirmware %.16s\n", romsig);
-        olpc_platform_info.flags |= OLPC_F_PRESENT;
+        if (!olpc_ofw_present() || !platform_detect())
+                return 0;
 
-        /* get the platform revision */
-        platform_detect();
+        spin_lock_init(&ec_lock);
 
         /* assume B1 and above models always have a DCON */
         if (olpc_board_at_least(olpc_board(0xb1)))
@@ -242,8 +257,10 @@ static int __init olpc_init(void)
                         (unsigned char *) &olpc_platform_info.ecver, 1);
 
 #ifdef CONFIG_PCI_OLPC
-        /* If the VSA exists let it emulate PCI, if not emulate in kernel */
-        if (!cs5535_has_vsa2())
+        /* If the VSA exists let it emulate PCI, if not emulate in kernel.
+         * XO-1 only. */
+        if (olpc_platform_info.boardrev < olpc_board_pre(0xd0) &&
+                        !cs5535_has_vsa2())
                 x86_init.pci.arch_init = pci_olpc_init;
 #endif
 
@@ -252,8 +269,12 @@ static int __init olpc_init(void)
                         olpc_platform_info.boardrev >> 4,
                         olpc_platform_info.ecver);
 
-unmap:
-        iounmap(romsig);
+        if (olpc_platform_info.boardrev < olpc_board_pre(0xd0)) { /* XO-1 */
+                r = add_xo1_platform_devices();
+                if (r)
+                        return r;
+        }
+
         return 0;
 }
 
diff --git a/arch/x86/kernel/olpc_ofw.c b/arch/x86/kernel/olpc_ofw.c
index 3218aa71ab5e..787320464379 100644
--- a/arch/x86/kernel/olpc_ofw.c
+++ b/arch/x86/kernel/olpc_ofw.c
@@ -74,6 +74,12 @@ int __olpc_ofw(const char *name, int nr_args, const void **args, int nr_res,
 }
 EXPORT_SYMBOL_GPL(__olpc_ofw);
 
+bool olpc_ofw_present(void)
+{
+        return olpc_ofw_cif != NULL;
+}
+EXPORT_SYMBOL_GPL(olpc_ofw_present);
+
 /* OFW cif _should_ be above this address */
 #define OFW_MIN 0xff000000
 
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 1db183ed7c01..c5b250011fd4 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -413,7 +413,6 @@ struct pv_mmu_ops pv_mmu_ops = {
 
         .alloc_pte = paravirt_nop,
         .alloc_pmd = paravirt_nop,
-        .alloc_pmd_clone = paravirt_nop,
         .alloc_pud = paravirt_nop,
         .release_pte = paravirt_nop,
         .release_pmd = paravirt_nop,
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index 0f7f130caa67..c562207b1b3d 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -39,7 +39,7 @@
 #include <asm/cacheflush.h>
 #include <asm/swiotlb.h>
 #include <asm/dma.h>
-#include <asm/k8.h>
+#include <asm/amd_nb.h>
 #include <asm/x86_init.h>
 
 static unsigned long iommu_bus_base;    /* GART remapping area (physical) */
@@ -560,8 +560,11 @@ static void enable_gart_translations(void)
 {
         int i;
 
-        for (i = 0; i < num_k8_northbridges; i++) {
-                struct pci_dev *dev = k8_northbridges[i];
+        if (!k8_northbridges.gart_supported)
+                return;
+
+        for (i = 0; i < k8_northbridges.num; i++) {
+                struct pci_dev *dev = k8_northbridges.nb_misc[i];
 
                 enable_gart_translation(dev, __pa(agp_gatt_table));
         }
@@ -592,16 +595,19 @@ static void gart_fixup_northbridges(struct sys_device *dev)
         if (!fix_up_north_bridges)
                 return;
 
+        if (!k8_northbridges.gart_supported)
+                return;
+
         pr_info("PCI-DMA: Restoring GART aperture settings\n");
 
-        for (i = 0; i < num_k8_northbridges; i++) {
-                struct pci_dev *dev = k8_northbridges[i];
+        for (i = 0; i < k8_northbridges.num; i++) {
+                struct pci_dev *dev = k8_northbridges.nb_misc[i];
 
                 /*
                  * Don't enable translations just yet.  That is the next
                  * step.  Restore the pre-suspend aperture settings.
                  */
-                pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, aperture_order << 1);
+                gart_set_size_and_enable(dev, aperture_order);
                 pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, aperture_alloc >> 25);
         }
 }
@@ -649,8 +655,8 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
 
         aper_size = aper_base = info->aper_size = 0;
         dev = NULL;
-        for (i = 0; i < num_k8_northbridges; i++) {
-                dev = k8_northbridges[i];
+        for (i = 0; i < k8_northbridges.num; i++) {
+                dev = k8_northbridges.nb_misc[i];
                 new_aper_base = read_aperture(dev, &new_aper_size);
                 if (!new_aper_base)
                         goto nommu;
@@ -718,10 +724,13 @@ static void gart_iommu_shutdown(void)
         if (!no_agp)
                 return;
 
-        for (i = 0; i < num_k8_northbridges; i++) {
+        if (!k8_northbridges.gart_supported)
+                return;
+
+        for (i = 0; i < k8_northbridges.num; i++) {
                 u32 ctl;
 
-                dev = k8_northbridges[i];
+                dev = k8_northbridges.nb_misc[i];
                 pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl);
 
                 ctl &= ~GARTEN;
@@ -739,7 +748,7 @@ int __init gart_iommu_init(void)
         unsigned long scratch;
         long i;
 
-        if (num_k8_northbridges == 0)
+        if (!k8_northbridges.gart_supported)
                 return 0;
 
 #ifndef CONFIG_AGP_AMD64
diff --git a/arch/x86/kernel/pmtimer_64.c b/arch/x86/kernel/pmtimer_64.c
deleted file mode 100644
index b112406f1996..000000000000
--- a/arch/x86/kernel/pmtimer_64.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/* Ported over from i386 by AK, original copyright was:
- *
- * (C) Dominik Brodowski <linux@brodo.de> 2003
- *
- * Driver to use the Power Management Timer (PMTMR) available in some
- * southbridges as primary timing source for the Linux kernel.
- *
- * Based on parts of linux/drivers/acpi/hardware/hwtimer.c, timer_pit.c,
- * timer_hpet.c, and on Arjan van de Ven's implementation for 2.4.
- *
- * This file is licensed under the GPL v2.
- *
- * Dropped all the hardware bug workarounds for now. Hopefully they
- * are not needed on 64bit chipsets.
- */
-
-#include <linux/jiffies.h>
-#include <linux/kernel.h>
-#include <linux/time.h>
-#include <linux/init.h>
-#include <linux/cpumask.h>
-#include <linux/acpi_pmtmr.h>
-
-#include <asm/io.h>
-#include <asm/proto.h>
-#include <asm/msr.h>
-#include <asm/vsyscall.h>
-
-static inline u32 cyc2us(u32 cycles)
-{
-        /* The Power Management Timer ticks at 3.579545 ticks per microsecond.
-         * 1 / PM_TIMER_FREQUENCY == 0.27936511 =~ 286/1024 [error: 0.024%]
-         *
-         * Even with HZ = 100, delta is at maximum 35796 ticks, so it can
-         * easily be multiplied with 286 (=0x11E) without having to fear
-         * u32 overflows.
-         */
-        cycles *= 286;
-        return (cycles >> 10);
-}
-
-static unsigned pmtimer_wait_tick(void)
-{
-        u32 a, b;
-        for (a = b = inl(pmtmr_ioport) & ACPI_PM_MASK;
-             a == b;
-             b = inl(pmtmr_ioport) & ACPI_PM_MASK)
-                cpu_relax();
-        return b;
-}
-
-/* note: wait time is rounded up to one tick */
-void pmtimer_wait(unsigned us)
-{
-        u32 a, b;
-        a = pmtimer_wait_tick();
-        do {
-                b = inl(pmtmr_ioport);
-                cpu_relax();
-        } while (cyc2us(b - a) < us);
-}
-
-static int __init nopmtimer_setup(char *s)
-{
-        pmtmr_ioport = 0;
-        return 1;
-}
-
-__setup("nopmtimer", nopmtimer_setup);
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 3d9ea531ddd1..b3d7a3a04f38 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -424,7 +424,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         load_TLS(next, cpu);
 
         /* Must be after DS reload */
-        unlazy_fpu(prev_p);
+        __unlazy_fpu(prev_p);
 
         /* Make sure cpu is ready for new context */
         if (preload_fpu)
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index e3af342fe83a..7a4cf14223ba 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -84,7 +84,7 @@ static int __init reboot_setup(char *str)
                         }
                                 /* we will leave sorting out the final value
                                    when we are ready to reboot, since we might not
-                                   have set up boot_cpu_id or smp_num_cpu */
+                                   have detected BSP APIC ID or smp_num_cpu */
                         break;
 #endif /* CONFIG_SMP */
 
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 00e167870f71..a59f6a6df5e2 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -83,7 +83,6 @@
 #include <asm/dmi.h>
 #include <asm/io_apic.h>
 #include <asm/ist.h>
-#include <asm/vmi.h>
 #include <asm/setup_arch.h>
 #include <asm/bios_ebda.h>
 #include <asm/cacheflush.h>
@@ -107,7 +106,7 @@
 #include <asm/percpu.h>
 #include <asm/topology.h>
 #include <asm/apicdef.h>
-#include <asm/k8.h>
+#include <asm/amd_nb.h>
 #ifdef CONFIG_X86_64
 #include <asm/numa_64.h>
 #endif
@@ -126,7 +125,6 @@ unsigned long max_pfn_mapped;
 RESERVE_BRK(dmi_alloc, 65536);
 #endif
 
-unsigned int boot_cpu_id __read_mostly;
 
 static __initdata unsigned long _brk_start = (unsigned long)__brk_base;
 unsigned long _brk_end = (unsigned long)__brk_base;
@@ -619,79 +617,7 @@ static __init void reserve_ibft_region(void)
                 reserve_early_overlap_ok(addr, addr + size, "ibft");
 }
 
-#ifdef CONFIG_X86_RESERVE_LOW_64K
-static int __init dmi_low_memory_corruption(const struct dmi_system_id *d)
-{
-        printk(KERN_NOTICE
-                "%s detected: BIOS may corrupt low RAM, working around it.\n",
-                d->ident);
-
-        e820_update_range(0, 0x10000, E820_RAM, E820_RESERVED);
-        sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
-
-        return 0;
-}
-#endif
-
-/* List of systems that have known low memory corruption BIOS problems */
-static struct dmi_system_id __initdata bad_bios_dmi_table[] = {
-#ifdef CONFIG_X86_RESERVE_LOW_64K
-        {
-                .callback = dmi_low_memory_corruption,
-                .ident = "AMI BIOS",
-                .matches = {
-                        DMI_MATCH(DMI_BIOS_VENDOR, "American Megatrends Inc."),
-                },
-        },
-        {
-                .callback = dmi_low_memory_corruption,
-                .ident = "Phoenix BIOS",
-                .matches = {
-                        DMI_MATCH(DMI_BIOS_VENDOR, "Phoenix Technologies"),
-                },
-        },
-        {
-                .callback = dmi_low_memory_corruption,
-                .ident = "Phoenix/MSC BIOS",
-                .matches = {
-                        DMI_MATCH(DMI_BIOS_VENDOR, "Phoenix/MSC"),
-                },
-        },
-        /*
-         * AMI BIOS with low memory corruption was found on Intel DG45ID and
-         * DG45FC boards.
-         * It has a different DMI_BIOS_VENDOR = "Intel Corp.", for now we will
-         * match only DMI_BOARD_NAME and see if there is more bad products
-         * with this vendor.
-         */
-        {
-                .callback = dmi_low_memory_corruption,
-                .ident = "AMI BIOS",
-                .matches = {
-                        DMI_MATCH(DMI_BOARD_NAME, "DG45ID"),
-                },
-        },
-        {
-                .callback = dmi_low_memory_corruption,
-                .ident = "AMI BIOS",
-                .matches = {
-                        DMI_MATCH(DMI_BOARD_NAME, "DG45FC"),
-                },
-        },
-        /*
-         * The Dell Inspiron Mini 1012 has DMI_BIOS_VENDOR = "Dell Inc.", so
-         * match on the product name.
-         */
-        {
-                .callback = dmi_low_memory_corruption,
-                .ident = "Phoenix BIOS",
-                .matches = {
-                        DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 1012"),
-                },
-        },
-#endif
-        {}
-};
+static unsigned reserve_low = CONFIG_X86_RESERVE_LOW << 10;
 
 static void __init trim_bios_range(void)
 {
@@ -699,8 +625,14 @@ static void __init trim_bios_range(void)
          * A special case is the first 4Kb of memory;
          * This is a BIOS owned area, not kernel ram, but generally
          * not listed as such in the E820 table.
+         *
+         * This typically reserves additional memory (64KiB by default)
+         * since some BIOSes are known to corrupt low memory.  See the
+         * Kconfig help text for X86_RESERVE_LOW.
          */
-        e820_update_range(0, PAGE_SIZE, E820_RAM, E820_RESERVED);
+        e820_update_range(0, ALIGN(reserve_low, PAGE_SIZE),
+                          E820_RAM, E820_RESERVED);
+
         /*
          * special case: Some BIOSen report the PC BIOS
          * area (640->1Mb) as ram even though it is not.
@@ -710,6 +642,28 @@ static void __init trim_bios_range(void)
         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 }
 
+static int __init parse_reservelow(char *p)
+{
+        unsigned long long size;
+
+        if (!p)
+                return -EINVAL;
+
+        size = memparse(p, &p);
+
+        if (size < 4096)
+                size = 4096;
+
+        if (size > 640*1024)
+                size = 640*1024;
+
+        reserve_low = size;
+
+        return 0;
+}
+
+early_param("reservelow", parse_reservelow);
+
 /*
  * Determine if we were loaded by an EFI loader.  If so, then we have also been
  * passed the efi memmap, systab, etc., so we should use these data structures
@@ -736,10 +690,10 @@ void __init setup_arch(char **cmdline_p)
         printk(KERN_INFO "Command line: %s\n", boot_command_line);
 #endif
 
-        /* VMI may relocate the fixmap; do this before touching ioremap area */
-        vmi_init();
-
-        /* OFW also may relocate the fixmap */
+        /*
+         * If we have OLPC OFW, we might end up relocating the fixmap due to
+         * reserve_top(), so do this before touching the ioremap area.
+         */
         olpc_ofw_detect();
 
         early_trap_init();
@@ -840,9 +794,6 @@ void __init setup_arch(char **cmdline_p)
 
         x86_report_nx();
 
-        /* Must be before kernel pagetables are setup */
-        vmi_activate();
-
         /* after early param, so could get panic from serial */
         reserve_early_setup_data();
 
@@ -865,8 +816,6 @@ void __init setup_arch(char **cmdline_p)
 
         dmi_scan_machine();
 
-        dmi_check_system(bad_bios_dmi_table);
-
         /*
          * VMware detection requires dmi to be available, so this
          * needs to be done after dmi_scan_machine, for the BP.
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index a60df9ae6454..2335c15c93a4 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -253,7 +253,7 @@ void __init setup_per_cpu_areas(void)
                  * Up to this point, the boot CPU has been using .init.data
                  * area.  Reload any changed state for the boot CPU.
                  */
-                if (cpu == boot_cpu_id)
+                if (!cpu)
                         switch_to_new_gdt(cpu);
         }
 
diff --git a/arch/x86/kernel/sfi.c b/arch/x86/kernel/sfi.c
index cb22acf3ed09..dd4c281ffe57 100644
--- a/arch/x86/kernel/sfi.c
+++ b/arch/x86/kernel/sfi.c
@@ -34,7 +34,7 @@
 #ifdef CONFIG_X86_LOCAL_APIC
 static unsigned long sfi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE;
 
-void __init mp_sfi_register_lapic_address(unsigned long address)
+static void __init mp_sfi_register_lapic_address(unsigned long address)
 {
         mp_lapic_addr = address;
 
@@ -46,7 +46,7 @@ void __init mp_sfi_register_lapic_address(unsigned long address)
 }
 
 /* All CPUs enumerated by SFI must be present and enabled */
-void __cpuinit mp_sfi_register_lapic(u8 id)
+static void __cpuinit mp_sfi_register_lapic(u8 id)
 {
         if (MAX_APICS - id <= 0) {
                 pr_warning("Processor #%d invalid (max %d)\n",
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 8b3bfc4dd708..dfb50890b5b7 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -62,7 +62,7 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/mtrr.h>
-#include <asm/vmi.h>
+#include <asm/mwait.h>
 #include <asm/apic.h>
 #include <asm/setup.h>
 #include <asm/uv/uv.h>
@@ -311,7 +311,6 @@ notrace static void __cpuinit start_secondary(void *unused)
         __flush_tlb_all();
 #endif
 
-        vmi_bringup();
         cpu_init();
         preempt_disable();
         smp_callin();
@@ -324,9 +323,9 @@ notrace static void __cpuinit start_secondary(void *unused)
         check_tsc_sync_target();
 
         if (nmi_watchdog == NMI_IO_APIC) {
-                legacy_pic->chip->mask(0);
+                legacy_pic->mask(0);
                 enable_NMI_through_LVT0();
-                legacy_pic->chip->unmask(0);
+                legacy_pic->unmask(0);
         }
 
         /* This must be done before setting cpu_online_mask */
@@ -397,6 +396,19 @@ void __cpuinit smp_store_cpu_info(int id)
                 identify_secondary_cpu(c);
 }
 
+static void __cpuinit link_thread_siblings(int cpu1, int cpu2)
+{
+        struct cpuinfo_x86 *c1 = &cpu_data(cpu1);
+        struct cpuinfo_x86 *c2 = &cpu_data(cpu2);
+
+        cpumask_set_cpu(cpu1, cpu_sibling_mask(cpu2));
+        cpumask_set_cpu(cpu2, cpu_sibling_mask(cpu1));
+        cpumask_set_cpu(cpu1, cpu_core_mask(cpu2));
+        cpumask_set_cpu(cpu2, cpu_core_mask(cpu1));
+        cpumask_set_cpu(cpu1, c2->llc_shared_map);
+        cpumask_set_cpu(cpu2, c1->llc_shared_map);
+}
+
 
 void __cpuinit set_cpu_sibling_map(int cpu)
 {
@@ -409,14 +421,13 @@ void __cpuinit set_cpu_sibling_map(int cpu)
                 for_each_cpu(i, cpu_sibling_setup_mask) {
                         struct cpuinfo_x86 *o = &cpu_data(i);
 
-                        if (c->phys_proc_id == o->phys_proc_id &&
-                            c->cpu_core_id == o->cpu_core_id) {
-                                cpumask_set_cpu(i, cpu_sibling_mask(cpu));
-                                cpumask_set_cpu(cpu, cpu_sibling_mask(i));
-                                cpumask_set_cpu(i, cpu_core_mask(cpu));
-                                cpumask_set_cpu(cpu, cpu_core_mask(i));
-                                cpumask_set_cpu(i, c->llc_shared_map);
-                                cpumask_set_cpu(cpu, o->llc_shared_map);
+                        if (cpu_has(c, X86_FEATURE_TOPOEXT)) {
+                                if (c->phys_proc_id == o->phys_proc_id &&
+                                    c->compute_unit_id == o->compute_unit_id)
+                                        link_thread_siblings(cpu, i);
+                        } else if (c->phys_proc_id == o->phys_proc_id &&
+                                   c->cpu_core_id == o->cpu_core_id) {
+                                link_thread_siblings(cpu, i);
                         }
                 }
         } else {
@@ -1109,8 +1120,6 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
         }
         set_cpu_sibling_map(0);
 
-        enable_IR_x2apic();
-        default_setup_apic_routing();
 
         if (smp_sanity_check(max_cpus) < 0) {
                 printk(KERN_INFO "SMP disabled\n");
@@ -1118,6 +1127,8 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
                 goto out;
         }
 
+        default_setup_apic_routing();
+
         preempt_disable();
         if (read_apic_id() != boot_cpu_physical_apicid) {
                 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
@@ -1383,11 +1394,88 @@ void play_dead_common(void)
         local_irq_disable();
 }
 
+/*
+ * We need to flush the caches before going to sleep, lest we have
+ * dirty data in our caches when we come back up.
+ */
+static inline void mwait_play_dead(void)
+{
+        unsigned int eax, ebx, ecx, edx;
+        unsigned int highest_cstate = 0;
+        unsigned int highest_subcstate = 0;
+        int i;
+        void *mwait_ptr;
+
+        if (!cpu_has(&current_cpu_data, X86_FEATURE_MWAIT))
+                return;
+        if (!cpu_has(&current_cpu_data, X86_FEATURE_CLFLSH))
+                return;
+        if (current_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
+                return;
+
+        eax = CPUID_MWAIT_LEAF;
+        ecx = 0;
+        native_cpuid(&eax, &ebx, &ecx, &edx);
+
+        /*
+         * eax will be 0 if EDX enumeration is not valid.
+         * Initialized below to cstate, sub_cstate value when EDX is valid.
+         */
+        if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
+                eax = 0;
+        } else {
+                edx >>= MWAIT_SUBSTATE_SIZE;
+                for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
+                        if (edx & MWAIT_SUBSTATE_MASK) {
+                                highest_cstate = i;
+                                highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
+                        }
+                }
+                eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
+                        (highest_subcstate - 1);
+        }
+
+        /*
+         * This should be a memory location in a cache line which is
+         * unlikely to be touched by other processors.  The actual
+         * content is immaterial as it is not actually modified in any way.
+         */
+        mwait_ptr = &current_thread_info()->flags;
+
+        wbinvd();
+
+        while (1) {
+                /*
+                 * The CLFLUSH is a workaround for erratum AAI65 for
+                 * the Xeon 7400 series.  It's not clear it is actually
+                 * needed, but it should be harmless in either case.
+                 * The WBINVD is insufficient due to the spurious-wakeup
+                 * case where we return around the loop.
+                 */
+                clflush(mwait_ptr);
+                __monitor(mwait_ptr, 0, 0);
+                mb();
+                __mwait(eax, 0);
+        }
+}
+
+static inline void hlt_play_dead(void)
+{
+        if (current_cpu_data.x86 >= 4)
+                wbinvd();
+
+        while (1) {
+                native_halt();
+        }
+}
+
 void native_play_dead(void)
 {
         play_dead_common();
         tboot_shutdown(TB_SHUTDOWN_WFS);
-        wbinvd_halt();
+
+        mwait_play_dead();      /* Only returns on failure */
+        hlt_play_dead();
 }
 
 #else /* ... !CONFIG_HOTPLUG_CPU */
diff --git a/arch/x86/kernel/sys_i386_32.c b/arch/x86/kernel/sys_i386_32.c
index d5e06624e34a..0b0cb5fede19 100644
--- a/arch/x86/kernel/sys_i386_32.c
+++ b/arch/x86/kernel/sys_i386_32.c
@@ -33,8 +33,8 @@ int kernel_execve(const char *filename,
                   const char *const envp[])
 {
         long __res;
-        asm volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx"
+        asm volatile ("int $0x80"
         : "=a" (__res)
-        : "0" (__NR_execve), "ri" (filename), "c" (argv), "d" (envp) : "memory");
+        : "0" (__NR_execve), "b" (filename), "c" (argv), "d" (envp) : "memory");
         return __res;
 }
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 60788dee0f8a..d43968503dd2 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -776,21 +776,10 @@ asmlinkage void math_state_restore(void)
 }
 EXPORT_SYMBOL_GPL(math_state_restore);
 
-#ifndef CONFIG_MATH_EMULATION
-void math_emulate(struct math_emu_info *info)
-{
-        printk(KERN_EMERG
-                "math-emulation not enabled and no coprocessor found.\n");
-        printk(KERN_EMERG "killing %s.\n", current->comm);
-        force_sig(SIGFPE, current);
-        schedule();
-}
-#endif /* CONFIG_MATH_EMULATION */
-
 dotraplinkage void __kprobes
 do_device_not_available(struct pt_regs *regs, long error_code)
 {
-#ifdef CONFIG_X86_32
+#ifdef CONFIG_MATH_EMULATION
         if (read_cr0() & X86_CR0_EM) {
                 struct math_emu_info info = { };
 
@@ -798,12 +787,12 @@ do_device_not_available(struct pt_regs *regs, long error_code)
 
                 info.regs = regs;
                 math_emulate(&info);
-        } else {
-                math_state_restore(); /* interrupts still off */
-                conditional_sti(regs);
+                return;
         }
-#else
-        math_state_restore();
+#endif
+        math_state_restore(); /* interrupts still off */
+#ifdef CONFIG_X86_32
+        conditional_sti(regs);
 #endif
 }
 
@@ -881,18 +870,6 @@ void __init trap_init(void)
 #endif
 
 #ifdef CONFIG_X86_32
-        if (cpu_has_fxsr) {
-                printk(KERN_INFO "Enabling fast FPU save and restore... ");
-                set_in_cr4(X86_CR4_OSFXSR);
-                printk("done.\n");
-        }
-        if (cpu_has_xmm) {
-                printk(KERN_INFO
-                        "Enabling unmasked SIMD FPU exception support... ");
-                set_in_cr4(X86_CR4_OSXMMEXCPT);
-                printk("done.\n");
-        }
-
         set_system_trap_gate(SYSCALL_VECTOR, &system_call);
         set_bit(SYSCALL_VECTOR, used_vectors);
 #endif
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 26a863a9c2a8..0c40d8b72416 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -104,10 +104,14 @@ int __init notsc_setup(char *str)
 
 __setup("notsc", notsc_setup);
 
+static int no_sched_irq_time;
+
 static int __init tsc_setup(char *str)
 {
         if (!strcmp(str, "reliable"))
                 tsc_clocksource_reliable = 1;
+        if (!strncmp(str, "noirqtime", 9))
+                no_sched_irq_time = 1;
         return 1;
 }
 
@@ -801,6 +805,7 @@ void mark_tsc_unstable(char *reason)
         if (!tsc_unstable) {
                 tsc_unstable = 1;
                 sched_clock_stable = 0;
+                disable_sched_clock_irqtime();
                 printk(KERN_INFO "Marking TSC unstable due to %s\n", reason);
                 /* Change only the rating, when not registered */
                 if (clocksource_tsc.mult)
@@ -892,60 +897,6 @@ static void __init init_tsc_clocksource(void)
         clocksource_register_khz(&clocksource_tsc, tsc_khz);
 }
 
-#ifdef CONFIG_X86_64
-/*
- * calibrate_cpu is used on systems with fixed rate TSCs to determine
- * processor frequency
- */
-#define TICK_COUNT 100000000
-static unsigned long __init calibrate_cpu(void)
-{
-        int tsc_start, tsc_now;
-        int i, no_ctr_free;
-        unsigned long evntsel3 = 0, pmc3 = 0, pmc_now = 0;
-        unsigned long flags;
-
-        for (i = 0; i < 4; i++)
-                if (avail_to_resrv_perfctr_nmi_bit(i))
-                        break;
-        no_ctr_free = (i == 4);
-        if (no_ctr_free) {
-                WARN(1, KERN_WARNING "Warning: AMD perfctrs busy ... "
-                     "cpu_khz value may be incorrect.\n");
-                i = 3;
-                rdmsrl(MSR_K7_EVNTSEL3, evntsel3);
-                wrmsrl(MSR_K7_EVNTSEL3, 0);
-                rdmsrl(MSR_K7_PERFCTR3, pmc3);
-        } else {
-                reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i);
-                reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
-        }
-        local_irq_save(flags);
-        /* start measuring cycles, incrementing from 0 */
-        wrmsrl(MSR_K7_PERFCTR0 + i, 0);
-        wrmsrl(MSR_K7_EVNTSEL0 + i, 1 << 22 | 3 << 16 | 0x76);
-        rdtscl(tsc_start);
-        do {
-                rdmsrl(MSR_K7_PERFCTR0 + i, pmc_now);
-                tsc_now = get_cycles();
-        } while ((tsc_now - tsc_start) < TICK_COUNT);
-
-        local_irq_restore(flags);
-        if (no_ctr_free) {
-                wrmsrl(MSR_K7_EVNTSEL3, 0);
-                wrmsrl(MSR_K7_PERFCTR3, pmc3);
-                wrmsrl(MSR_K7_EVNTSEL3, evntsel3);
-        } else {
-                release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
-                release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
-        }
-
-        return pmc_now * tsc_khz / (tsc_now - tsc_start);
-}
-#else
-static inline unsigned long calibrate_cpu(void) { return cpu_khz; }
-#endif
-
 void __init tsc_init(void)
 {
         u64 lpj;
@@ -964,10 +915,6 @@ void __init tsc_init(void)
                 return;
         }
 
-        if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) &&
-                        (boot_cpu_data.x86_vendor == X86_VENDOR_AMD))
-                cpu_khz = calibrate_cpu();
-
         printk("Detected %lu.%03lu MHz processor.\n",
                         (unsigned long)cpu_khz / 1000,
                         (unsigned long)cpu_khz % 1000);
@@ -987,6 +934,9 @@ void __init tsc_init(void)
         /* now allow native_sched_clock() to use rdtsc */
         tsc_disabled = 0;
 
+        if (!no_sched_irq_time)
+                enable_sched_clock_irqtime();
+
         lpj = ((u64)tsc_khz * 1000);
         do_div(lpj, HZ);
         lpj_fine = lpj;
diff --git a/arch/x86/kernel/uv_irq.c b/arch/x86/kernel/uv_irq.c
index 1132129db792..7b24460917d5 100644
--- a/arch/x86/kernel/uv_irq.c
+++ b/arch/x86/kernel/uv_irq.c
@@ -28,34 +28,21 @@ struct uv_irq_2_mmr_pnode{
 static spinlock_t               uv_irq_lock;
 static struct rb_root           uv_irq_root;
 
-static int uv_set_irq_affinity(unsigned int, const struct cpumask *);
+static int uv_set_irq_affinity(struct irq_data *, const struct cpumask *, bool);
 
-static void uv_noop(unsigned int irq)
-{
-}
-
-static unsigned int uv_noop_ret(unsigned int irq)
-{
-        return 0;
-}
+static void uv_noop(struct irq_data *data) { }
 
-static void uv_ack_apic(unsigned int irq)
+static void uv_ack_apic(struct irq_data *data)
 {
         ack_APIC_irq();
 }
 
 static struct irq_chip uv_irq_chip = {
-        .name           = "UV-CORE",
-        .startup        = uv_noop_ret,
-        .shutdown       = uv_noop,
-        .enable         = uv_noop,
-        .disable        = uv_noop,
-        .ack            = uv_noop,
-        .mask           = uv_noop,
-        .unmask         = uv_noop,
-        .eoi            = uv_ack_apic,
-        .end            = uv_noop,
-        .set_affinity   = uv_set_irq_affinity,
+        .name                   = "UV-CORE",
+        .irq_mask               = uv_noop,
+        .irq_unmask             = uv_noop,
+        .irq_eoi                = uv_ack_apic,
+        .irq_set_affinity       = uv_set_irq_affinity,
 };
 
 /*
@@ -144,26 +131,22 @@ arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
                        unsigned long mmr_offset, int limit)
 {
         const struct cpumask *eligible_cpu = cpumask_of(cpu);
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg;
-        int mmr_pnode;
+        struct irq_cfg *cfg = get_irq_chip_data(irq);
         unsigned long mmr_value;
         struct uv_IO_APIC_route_entry *entry;
-        int err;
+        int mmr_pnode, err;
 
         BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) !=
                         sizeof(unsigned long));
 
-        cfg = irq_cfg(irq);
-
         err = assign_irq_vector(irq, cfg, eligible_cpu);
         if (err != 0)
                 return err;
 
         if (limit == UV_AFFINITY_CPU)
-                desc->status |= IRQ_NO_BALANCING;
+                irq_set_status_flags(irq, IRQ_NO_BALANCING);
         else
-                desc->status |= IRQ_MOVE_PCNTXT;
+                irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
 
         set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
                                       irq_name);
@@ -206,17 +189,17 @@ static void arch_disable_uv_irq(int mmr_pnode, unsigned long mmr_offset)
         uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
 }
 
-static int uv_set_irq_affinity(unsigned int irq, const struct cpumask *mask)
+static int
+uv_set_irq_affinity(struct irq_data *data, const struct cpumask *mask,
+                    bool force)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-        struct irq_cfg *cfg = desc->chip_data;
+        struct irq_cfg *cfg = data->chip_data;
         unsigned int dest;
-        unsigned long mmr_value;
+        unsigned long mmr_value, mmr_offset;
         struct uv_IO_APIC_route_entry *entry;
-        unsigned long mmr_offset;
         int mmr_pnode;
 
-        if (set_desc_affinity(desc, mask, &dest))
+        if (__ioapic_set_affinity(data, mask, &dest))
                 return -1;
 
         mmr_value = 0;
@@ -231,7 +214,7 @@ static int uv_set_irq_affinity(unsigned int irq, const struct cpumask *mask)
         entry->dest             = dest;
 
         /* Get previously stored MMR and pnode of hub sourcing interrupts */
-        if (uv_irq_2_mmr_info(irq, &mmr_offset, &mmr_pnode))
+        if (uv_irq_2_mmr_info(data->irq, &mmr_offset, &mmr_pnode))
                 return -1;
 
         uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
diff --git a/arch/x86/kernel/visws_quirks.c b/arch/x86/kernel/visws_quirks.c
index e680ea52db9b..3371bd053b89 100644
--- a/arch/x86/kernel/visws_quirks.c
+++ b/arch/x86/kernel/visws_quirks.c
@@ -66,10 +66,7 @@ static void __init visws_time_init(void)
 }
 
 /* Replaces the default init_ISA_irqs in the generic setup */
-static void __init visws_pre_intr_init(void)
-{
-        init_VISWS_APIC_irqs();
-}
+static void __init visws_pre_intr_init(void);
 
 /* Quirk for machine specific memory setup. */
 
@@ -429,67 +426,34 @@ static int is_co_apic(unsigned int irq)
 /*
  * This is the SGI Cobalt (IO-)APIC:
  */
-
-static void enable_cobalt_irq(unsigned int irq)
+static void enable_cobalt_irq(struct irq_data *data)
 {
-        co_apic_set(is_co_apic(irq), irq);
+        co_apic_set(is_co_apic(data->irq), data->irq);
 }
 
-static void disable_cobalt_irq(unsigned int irq)
+static void disable_cobalt_irq(struct irq_data *data)
 {
-        int entry = is_co_apic(irq);
+        int entry = is_co_apic(data->irq);
 
         co_apic_write(CO_APIC_LO(entry), CO_APIC_MASK);
         co_apic_read(CO_APIC_LO(entry));
 }
 
-/*
- * "irq" really just serves to identify the device.  Here is where we
- * map this to the Cobalt APIC entry where it's physically wired.
- * This is called via request_irq -> setup_irq -> irq_desc->startup()
- */
-static unsigned int startup_cobalt_irq(unsigned int irq)
+static void ack_cobalt_irq(struct irq_data *data)
 {
         unsigned long flags;
-        struct irq_desc *desc = irq_to_desc(irq);
 
         spin_lock_irqsave(&cobalt_lock, flags);
-        if ((desc->status & (IRQ_DISABLED | IRQ_INPROGRESS | IRQ_WAITING)))
-                desc->status &= ~(IRQ_DISABLED | IRQ_INPROGRESS | IRQ_WAITING);
-        enable_cobalt_irq(irq);
-        spin_unlock_irqrestore(&cobalt_lock, flags);
-        return 0;
-}
-
-static void ack_cobalt_irq(unsigned int irq)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&cobalt_lock, flags);
-        disable_cobalt_irq(irq);
+        disable_cobalt_irq(data);
         apic_write(APIC_EOI, APIC_EIO_ACK);
         spin_unlock_irqrestore(&cobalt_lock, flags);
 }
 
-static void end_cobalt_irq(unsigned int irq)
-{
-        unsigned long flags;
-        struct irq_desc *desc = irq_to_desc(irq);
-
-        spin_lock_irqsave(&cobalt_lock, flags);
-        if (!(desc->status & (IRQ_DISABLED | IRQ_INPROGRESS)))
-                enable_cobalt_irq(irq);
-        spin_unlock_irqrestore(&cobalt_lock, flags);
-}
-
 static struct irq_chip cobalt_irq_type = {
-        .name =         "Cobalt-APIC",
-        .startup =      startup_cobalt_irq,
-        .shutdown =     disable_cobalt_irq,
-        .enable =       enable_cobalt_irq,
-        .disable =      disable_cobalt_irq,
-        .ack =          ack_cobalt_irq,
-        .end =          end_cobalt_irq,
+        .name           = "Cobalt-APIC",
+        .irq_enable     = enable_cobalt_irq,
+        .irq_disable    = disable_cobalt_irq,
+        .irq_ack        = ack_cobalt_irq,
 };
 
 
@@ -503,35 +467,34 @@ static struct irq_chip cobalt_irq_type = {
  * interrupt controller type, and through a special virtual interrupt-
  * controller. Device drivers only see the virtual interrupt sources.
  */
-static unsigned int startup_piix4_master_irq(unsigned int irq)
+static unsigned int startup_piix4_master_irq(struct irq_data *data)
 {
         legacy_pic->init(0);
-
-        return startup_cobalt_irq(irq);
+        enable_cobalt_irq(data);
 }
 
-static void end_piix4_master_irq(unsigned int irq)
+static void end_piix4_master_irq(struct irq_data *data)
 {
         unsigned long flags;
 
         spin_lock_irqsave(&cobalt_lock, flags);
-        enable_cobalt_irq(irq);
+        enable_cobalt_irq(data);
         spin_unlock_irqrestore(&cobalt_lock, flags);
 }
 
 static struct irq_chip piix4_master_irq_type = {
-        .name =         "PIIX4-master",
-        .startup =      startup_piix4_master_irq,
-        .ack =          ack_cobalt_irq,
-        .end =          end_piix4_master_irq,
+        .name           = "PIIX4-master",
+        .irq_startup    = startup_piix4_master_irq,
+        .irq_ack        = ack_cobalt_irq,
 };
 
+static void pii4_mask(struct irq_data *data) { }
 
 static struct irq_chip piix4_virtual_irq_type = {
-        .name =         "PIIX4-virtual",
+        .name           = "PIIX4-virtual",
+        .mask           = pii4_mask,
 };
 
-
 /*
  * PIIX4-8259 master/virtual functions to handle interrupt requests
  * from legacy devices: floppy, parallel, serial, rtc.
@@ -549,9 +512,8 @@ static struct irq_chip piix4_virtual_irq_type = {
  */
 static irqreturn_t piix4_master_intr(int irq, void *dev_id)
 {
-        int realirq;
-        struct irq_desc *desc;
         unsigned long flags;
+        int realirq;
 
         raw_spin_lock_irqsave(&i8259A_lock, flags);
 
@@ -592,18 +554,10 @@ static irqreturn_t piix4_master_intr(int irq, void *dev_id)
 
         raw_spin_unlock_irqrestore(&i8259A_lock, flags);
 
-        desc = irq_to_desc(realirq);
-
         /*
          * handle this 'virtual interrupt' as a Cobalt one now.
          */
-        kstat_incr_irqs_this_cpu(realirq, desc);
-
-        if (likely(desc->action != NULL))
-                handle_IRQ_event(realirq, desc->action);
-
-        if (!(desc->status & IRQ_DISABLED))
-                legacy_pic->chip->unmask(realirq);
+        generic_handle_irq(realirq);
 
         return IRQ_HANDLED;
 
@@ -624,41 +578,35 @@ static struct irqaction cascade_action = {
 
 static inline void set_piix4_virtual_irq_type(void)
 {
-        piix4_virtual_irq_type.shutdown = i8259A_chip.mask;
         piix4_virtual_irq_type.enable = i8259A_chip.unmask;
         piix4_virtual_irq_type.disable = i8259A_chip.mask;
+        piix4_virtual_irq_type.unmask = i8259A_chip.unmask;
 }
 
-void init_VISWS_APIC_irqs(void)
+static void __init visws_pre_intr_init(void)
 {
         int i;
 
-        for (i = 0; i < CO_IRQ_APIC0 + CO_APIC_LAST + 1; i++) {
-                struct irq_desc *desc = irq_to_desc(i);
-
-                desc->status = IRQ_DISABLED;
-                desc->action = 0;
-                desc->depth = 1;
+        set_piix4_virtual_irq_type();
 
-                if (i == 0) {
-                        desc->chip = &cobalt_irq_type;
-                }
-                else if (i == CO_IRQ_IDE0) {
-                        desc->chip = &cobalt_irq_type;
-                }
-                else if (i == CO_IRQ_IDE1) {
-                        desc->chip = &cobalt_irq_type;
-                }
-                else if (i == CO_IRQ_8259) {
-                        desc->chip = &piix4_master_irq_type;
-                }
-                else if (i < CO_IRQ_APIC0) {
-                        set_piix4_virtual_irq_type();
-                        desc->chip = &piix4_virtual_irq_type;
-                }
-                else if (IS_CO_APIC(i)) {
-                        desc->chip = &cobalt_irq_type;
-                }
+        for (i = 0; i < CO_IRQ_APIC0 + CO_APIC_LAST + 1; i++) {
+                struct irq_chip *chip = NULL;
+
+                if (i == 0)
+                        chip = &cobalt_irq_type;
+                else if (i == CO_IRQ_IDE0)
+                        chip = &cobalt_irq_type;
+                else if (i == CO_IRQ_IDE1)
+                        >chip = &cobalt_irq_type;
+                else if (i == CO_IRQ_8259)
+                        chip = &piix4_master_irq_type;
+                else if (i < CO_IRQ_APIC0)
+                        chip = &piix4_virtual_irq_type;
+                else if (IS_CO_APIC(i))
+                        chip = &cobalt_irq_type;
+
+                if (chip)
+                        set_irq_chip(i, chip);
         }
 
         setup_irq(CO_IRQ_8259, &master_action);
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
deleted file mode 100644
index ce9fbacb7526..000000000000
--- a/arch/x86/kernel/vmi_32.c
+++ /dev/null
@@ -1,893 +0,0 @@
-/*
- * VMI specific paravirt-ops implementation
- *
- * Copyright (C) 2005, VMware, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * Send feedback to zach@vmware.com
- *
- */
-
-#include <linux/module.h>
-#include <linux/cpu.h>
-#include <linux/bootmem.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/sched.h>
-#include <linux/gfp.h>
-#include <asm/vmi.h>
-#include <asm/io.h>
-#include <asm/fixmap.h>
-#include <asm/apicdef.h>
-#include <asm/apic.h>
-#include <asm/pgalloc.h>
-#include <asm/processor.h>
-#include <asm/timer.h>
-#include <asm/vmi_time.h>
-#include <asm/kmap_types.h>
-#include <asm/setup.h>
-
-/* Convenient for calling VMI functions indirectly in the ROM */
-typedef u32 __attribute__((regparm(1))) (VROMFUNC)(void);
-typedef u64 __attribute__((regparm(2))) (VROMLONGFUNC)(int);
-
-#define call_vrom_func(rom,func) \
-   (((VROMFUNC *)(rom->func))())
-
-#define call_vrom_long_func(rom,func,arg) \
-   (((VROMLONGFUNC *)(rom->func)) (arg))
-
-static struct vrom_header *vmi_rom;
-static int disable_pge;
-static int disable_pse;
-static int disable_sep;
-static int disable_tsc;
-static int disable_mtrr;
-static int disable_noidle;
-static int disable_vmi_timer;
-
-/* Cached VMI operations */
-static struct {
-        void (*cpuid)(void /* non-c */);
-        void (*_set_ldt)(u32 selector);
-        void (*set_tr)(u32 selector);
-        void (*write_idt_entry)(struct desc_struct *, int, u32, u32);
-        void (*write_gdt_entry)(struct desc_struct *, int, u32, u32);
-        void (*write_ldt_entry)(struct desc_struct *, int, u32, u32);
-        void (*set_kernel_stack)(u32 selector, u32 sp0);
-        void (*allocate_page)(u32, u32, u32, u32, u32);
-        void (*release_page)(u32, u32);
-        void (*set_pte)(pte_t, pte_t *, unsigned);
-        void (*update_pte)(pte_t *, unsigned);
-        void (*set_linear_mapping)(int, void *, u32, u32);
-        void (*_flush_tlb)(int);
-        void (*set_initial_ap_state)(int, int);
-        void (*halt)(void);
-        void (*set_lazy_mode)(int mode);
-} vmi_ops;
-
-/* Cached VMI operations */
-struct vmi_timer_ops vmi_timer_ops;
-
-/*
- * VMI patching routines.
- */
-#define MNEM_CALL 0xe8
-#define MNEM_JMP  0xe9
-#define MNEM_RET  0xc3
-
-#define IRQ_PATCH_INT_MASK 0
-#define IRQ_PATCH_DISABLE  5
-
-static inline void patch_offset(void *insnbuf,
-                                unsigned long ip, unsigned long dest)
-{
-        *(unsigned long *)(insnbuf+1) = dest-ip-5;
-}
-
-static unsigned patch_internal(int call, unsigned len, void *insnbuf,
-                               unsigned long ip)
-{
-        u64 reloc;
-        struct vmi_relocation_info *const rel = (struct vmi_relocation_info *)&reloc;
-        reloc = call_vrom_long_func(vmi_rom, get_reloc, call);
-        switch(rel->type) {
-                case VMI_RELOCATION_CALL_REL:
-                        BUG_ON(len < 5);
-                        *(char *)insnbuf = MNEM_CALL;
-                        patch_offset(insnbuf, ip, (unsigned long)rel->eip);
-                        return 5;
-
-                case VMI_RELOCATION_JUMP_REL:
-                        BUG_ON(len < 5);
-                        *(char *)insnbuf = MNEM_JMP;
-                        patch_offset(insnbuf, ip, (unsigned long)rel->eip);
-                        return 5;
-
-                case VMI_RELOCATION_NOP:
-                        /* obliterate the whole thing */
-                        return 0;
-
-                case VMI_RELOCATION_NONE:
-                        /* leave native code in place */
-                        break;
-
-                default:
-                        BUG();
-        }
-        return len;
-}
-
-/*
- * Apply patch if appropriate, return length of new instruction
- * sequence.  The callee does nop padding for us.
- */
-static unsigned vmi_patch(u8 type, u16 clobbers, void *insns,
-                          unsigned long ip, unsigned len)
-{
-        switch (type) {
-                case PARAVIRT_PATCH(pv_irq_ops.irq_disable):
-                        return patch_internal(VMI_CALL_DisableInterrupts, len,
-                                              insns, ip);
-                case PARAVIRT_PATCH(pv_irq_ops.irq_enable):
-                        return patch_internal(VMI_CALL_EnableInterrupts, len,
-                                              insns, ip);
-                case PARAVIRT_PATCH(pv_irq_ops.restore_fl):
-                        return patch_internal(VMI_CALL_SetInterruptMask, len,
-                                              insns, ip);
-                case PARAVIRT_PATCH(pv_irq_ops.save_fl):
-                        return patch_internal(VMI_CALL_GetInterruptMask, len,
-                                              insns, ip);
-                case PARAVIRT_PATCH(pv_cpu_ops.iret):
-                        return patch_internal(VMI_CALL_IRET, len, insns, ip);
-                case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit):
-                        return patch_internal(VMI_CALL_SYSEXIT, len, insns, ip);
-                default:
-                        break;
-        }
-        return len;
-}
-
-/* CPUID has non-C semantics, and paravirt-ops API doesn't match hardware ISA */
-static void vmi_cpuid(unsigned int *ax, unsigned int *bx,
-                               unsigned int *cx, unsigned int *dx)
-{
-        int override = 0;
-        if (*ax == 1)
-                override = 1;
-        asm volatile ("call *%6"
-                      : "=a" (*ax),
-                        "=b" (*bx),
-                        "=c" (*cx),
-                        "=d" (*dx)
-                      : "0" (*ax), "2" (*cx), "r" (vmi_ops.cpuid));
-        if (override) {
-                if (disable_pse)
-                        *dx &= ~X86_FEATURE_PSE;
-                if (disable_pge)
-                        *dx &= ~X86_FEATURE_PGE;
-                if (disable_sep)
-                        *dx &= ~X86_FEATURE_SEP;
-                if (disable_tsc)
-                        *dx &= ~X86_FEATURE_TSC;
-                if (disable_mtrr)
-                        *dx &= ~X86_FEATURE_MTRR;
-        }
-}
-
-static inline void vmi_maybe_load_tls(struct desc_struct *gdt, int nr, struct desc_struct *new)
-{
-        if (gdt[nr].a != new->a || gdt[nr].b != new->b)
-                write_gdt_entry(gdt, nr, new, 0);
-}
-
-static void vmi_load_tls(struct thread_struct *t, unsigned int cpu)
-{
-        struct desc_struct *gdt = get_cpu_gdt_table(cpu);
-        vmi_maybe_load_tls(gdt, GDT_ENTRY_TLS_MIN + 0, &t->tls_array[0]);
-        vmi_maybe_load_tls(gdt, GDT_ENTRY_TLS_MIN + 1, &t->tls_array[1]);
-        vmi_maybe_load_tls(gdt, GDT_ENTRY_TLS_MIN + 2, &t->tls_array[2]);
-}
-
-static void vmi_set_ldt(const void *addr, unsigned entries)
-{
-        unsigned cpu = smp_processor_id();
-        struct desc_struct desc;
-
-        pack_descriptor(&desc, (unsigned long)addr,
-                        entries * sizeof(struct desc_struct) - 1,
-                        DESC_LDT, 0);
-        write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_LDT, &desc, DESC_LDT);
-        vmi_ops._set_ldt(entries ? GDT_ENTRY_LDT*sizeof(struct desc_struct) : 0);
-}
-
-static void vmi_set_tr(void)
-{
-        vmi_ops.set_tr(GDT_ENTRY_TSS*sizeof(struct desc_struct));
-}
-
-static void vmi_write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
-{
-        u32 *idt_entry = (u32 *)g;
-        vmi_ops.write_idt_entry(dt, entry, idt_entry[0], idt_entry[1]);
-}
-
-static void vmi_write_gdt_entry(struct desc_struct *dt, int entry,
-                                const void *desc, int type)
-{
-        u32 *gdt_entry = (u32 *)desc;
-        vmi_ops.write_gdt_entry(dt, entry, gdt_entry[0], gdt_entry[1]);
-}
-
-static void vmi_write_ldt_entry(struct desc_struct *dt, int entry,
-                                const void *desc)
-{
-        u32 *ldt_entry = (u32 *)desc;
-        vmi_ops.write_ldt_entry(dt, entry, ldt_entry[0], ldt_entry[1]);
-}
-
-static void vmi_load_sp0(struct tss_struct *tss,
-                                   struct thread_struct *thread)
-{
-        tss->x86_tss.sp0 = thread->sp0;
-
-        /* This can only happen when SEP is enabled, no need to test "SEP"arately */
-        if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) {
-                tss->x86_tss.ss1 = thread->sysenter_cs;
-                wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
-        }
-        vmi_ops.set_kernel_stack(__KERNEL_DS, tss->x86_tss.sp0);
-}
-
-static void vmi_flush_tlb_user(void)
-{
-        vmi_ops._flush_tlb(VMI_FLUSH_TLB);
-}
-
-static void vmi_flush_tlb_kernel(void)
-{
-        vmi_ops._flush_tlb(VMI_FLUSH_TLB | VMI_FLUSH_GLOBAL);
-}
-
-/* Stub to do nothing at all; used for delays and unimplemented calls */
-static void vmi_nop(void)
-{
-}
-
-static void vmi_allocate_pte(struct mm_struct *mm, unsigned long pfn)
-{
-        vmi_ops.allocate_page(pfn, VMI_PAGE_L1, 0, 0, 0);
-}
-
-static void vmi_allocate_pmd(struct mm_struct *mm, unsigned long pfn)
-{
-        /*
-         * This call comes in very early, before mem_map is setup.
-         * It is called only for swapper_pg_dir, which already has
-         * data on it.
-         */
-        vmi_ops.allocate_page(pfn, VMI_PAGE_L2, 0, 0, 0);
-}
-
-static void vmi_allocate_pmd_clone(unsigned long pfn, unsigned long clonepfn, unsigned long start, unsigned long count)
-{
-        vmi_ops.allocate_page(pfn, VMI_PAGE_L2 | VMI_PAGE_CLONE, clonepfn, start, count);
-}
-
-static void vmi_release_pte(unsigned long pfn)
-{
-        vmi_ops.release_page(pfn, VMI_PAGE_L1);
-}
-
-static void vmi_release_pmd(unsigned long pfn)
-{
-        vmi_ops.release_page(pfn, VMI_PAGE_L2);
-}
-
-/*
- * We use the pgd_free hook for releasing the pgd page:
- */
-static void vmi_pgd_free(struct mm_struct *mm, pgd_t *pgd)
-{
-        unsigned long pfn = __pa(pgd) >> PAGE_SHIFT;
-
-        vmi_ops.release_page(pfn, VMI_PAGE_L2);
-}
-
-/*
- * Helper macros for MMU update flags.  We can defer updates until a flush
- * or page invalidation only if the update is to the current address space
- * (otherwise, there is no flush).  We must check against init_mm, since
- * this could be a kernel update, which usually passes init_mm, although
- * sometimes this check can be skipped if we know the particular function
- * is only called on user mode PTEs.  We could change the kernel to pass
- * current->active_mm here, but in particular, I was unsure if changing
- * mm/highmem.c to do this would still be correct on other architectures.
- */
-#define is_current_as(mm, mustbeuser) ((mm) == current->active_mm ||    \
-                                       (!mustbeuser && (mm) == &init_mm))
-#define vmi_flags_addr(mm, addr, level, user)                           \
-        ((level) | (is_current_as(mm, user) ?                           \
-                (VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0))
-#define vmi_flags_addr_defer(mm, addr, level, user)                     \
-        ((level) | (is_current_as(mm, user) ?                           \
-                (VMI_PAGE_DEFER | VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0))
-
-static void vmi_update_pte(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
-{
-        vmi_ops.update_pte(ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
-}
-
-static void vmi_update_pte_defer(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
-{
-        vmi_ops.update_pte(ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 0));
-}
-
-static void vmi_set_pte(pte_t *ptep, pte_t pte)
-{
-        /* XXX because of set_pmd_pte, this can be called on PT or PD layers */
-        vmi_ops.set_pte(pte, ptep, VMI_PAGE_PT);
-}
-
-static void vmi_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte)
-{
-        vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
-}
-
-static void vmi_set_pmd(pmd_t *pmdp, pmd_t pmdval)
-{
-#ifdef CONFIG_X86_PAE
-        const pte_t pte = { .pte = pmdval.pmd };
-#else
-        const pte_t pte = { pmdval.pud.pgd.pgd };
-#endif
-        vmi_ops.set_pte(pte, (pte_t *)pmdp, VMI_PAGE_PD);
-}
-
-#ifdef CONFIG_X86_PAE
-
-static void vmi_set_pte_atomic(pte_t *ptep, pte_t pteval)
-{
-        /*
-         * XXX This is called from set_pmd_pte, but at both PT
-         * and PD layers so the VMI_PAGE_PT flag is wrong.  But
-         * it is only called for large page mapping changes,
-         * the Xen backend, doesn't support large pages, and the
-         * ESX backend doesn't depend on the flag.
-         */
-        set_64bit((unsigned long long *)ptep,pte_val(pteval));
-        vmi_ops.update_pte(ptep, VMI_PAGE_PT);
-}
-
-static void vmi_set_pud(pud_t *pudp, pud_t pudval)
-{
-        /* Um, eww */
-        const pte_t pte = { .pte = pudval.pgd.pgd };
-        vmi_ops.set_pte(pte, (pte_t *)pudp, VMI_PAGE_PDP);
-}
-
-static void vmi_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
-{
-        const pte_t pte = { .pte = 0 };
-        vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
-}
-
-static void vmi_pmd_clear(pmd_t *pmd)
-{
-        const pte_t pte = { .pte = 0 };
-        vmi_ops.set_pte(pte, (pte_t *)pmd, VMI_PAGE_PD);
-}
-#endif
-
-#ifdef CONFIG_SMP
-static void __devinit
-vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip,
-                     unsigned long start_esp)
-{
-        struct vmi_ap_state ap;
-
-        /* Default everything to zero.  This is fine for most GPRs. */
-        memset(&ap, 0, sizeof(struct vmi_ap_state));
-
-        ap.gdtr_limit = GDT_SIZE - 1;
-        ap.gdtr_base = (unsigned long) get_cpu_gdt_table(phys_apicid);
-
-        ap.idtr_limit = IDT_ENTRIES * 8 - 1;
-        ap.idtr_base = (unsigned long) idt_table;
-
-        ap.ldtr = 0;
-
-        ap.cs = __KERNEL_CS;
-        ap.eip = (unsigned long) start_eip;
-        ap.ss = __KERNEL_DS;
-        ap.esp = (unsigned long) start_esp;
-
-        ap.ds = __USER_DS;
-        ap.es = __USER_DS;
-        ap.fs = __KERNEL_PERCPU;
-        ap.gs = __KERNEL_STACK_CANARY;
-
-        ap.eflags = 0;
-
-#ifdef CONFIG_X86_PAE
-        /* efer should match BSP efer. */
-        if (cpu_has_nx) {
-                unsigned l, h;
-                rdmsr(MSR_EFER, l, h);
-                ap.efer = (unsigned long long) h << 32 | l;
-        }
-#endif
-
-        ap.cr3 = __pa(swapper_pg_dir);
-        /* Protected mode, paging, AM, WP, NE, MP. */
-        ap.cr0 = 0x80050023;
-        ap.cr4 = mmu_cr4_features;
-        vmi_ops.set_initial_ap_state((u32)&ap, phys_apicid);
-}
-#endif
-
-static void vmi_start_context_switch(struct task_struct *prev)
-{
-        paravirt_start_context_switch(prev);
-        vmi_ops.set_lazy_mode(2);
-}
-
-static void vmi_end_context_switch(struct task_struct *next)
-{
-        vmi_ops.set_lazy_mode(0);
-        paravirt_end_context_switch(next);
-}
-
-static void vmi_enter_lazy_mmu(void)
-{
-        paravirt_enter_lazy_mmu();
-        vmi_ops.set_lazy_mode(1);
-}
-
-static void vmi_leave_lazy_mmu(void)
-{
-        vmi_ops.set_lazy_mode(0);
-        paravirt_leave_lazy_mmu();
-}
-
-static inline int __init check_vmi_rom(struct vrom_header *rom)
-{
-        struct pci_header *pci;
-        struct pnp_header *pnp;
-        const char *manufacturer = "UNKNOWN";
-        const char *product = "UNKNOWN";
-        const char *license = "unspecified";
-
-        if (rom->rom_signature != 0xaa55)
-                return 0;
-        if (rom->vrom_signature != VMI_SIGNATURE)
-                return 0;
-        if (rom->api_version_maj != VMI_API_REV_MAJOR ||
-            rom->api_version_min+1 < VMI_API_REV_MINOR+1) {
-                printk(KERN_WARNING "VMI: Found mismatched rom version %d.%d\n",
-                                rom->api_version_maj,
-                                rom->api_version_min);
-                return 0;
-        }
-
-        /*
-         * Relying on the VMI_SIGNATURE field is not 100% safe, so check
-         * the PCI header and device type to make sure this is really a
-         * VMI device.
-         */
-        if (!rom->pci_header_offs) {
-                printk(KERN_WARNING "VMI: ROM does not contain PCI header.\n");
-                return 0;
-        }
-
-        pci = (struct pci_header *)((char *)rom+rom->pci_header_offs);
-        if (pci->vendorID != PCI_VENDOR_ID_VMWARE ||
-            pci->deviceID != PCI_DEVICE_ID_VMWARE_VMI) {
-                /* Allow it to run... anyways, but warn */
-                printk(KERN_WARNING "VMI: ROM from unknown manufacturer\n");
-        }
-
-        if (rom->pnp_header_offs) {
-                pnp = (struct pnp_header *)((char *)rom+rom->pnp_header_offs);
-                if (pnp->manufacturer_offset)
-                        manufacturer = (const char *)rom+pnp->manufacturer_offset;
-                if (pnp->product_offset)
-                        product = (const char *)rom+pnp->product_offset;
-        }
-
-        if (rom->license_offs)
-                license = (char *)rom+rom->license_offs;
-
-        printk(KERN_INFO "VMI: Found %s %s, API version %d.%d, ROM version %d.%d\n",
-                manufacturer, product,
-                rom->api_version_maj, rom->api_version_min,
-                pci->rom_version_maj, pci->rom_version_min);
-
-        /* Don't allow BSD/MIT here for now because we don't want to end up
-           with any binary only shim layers */
-        if (strcmp(license, "GPL") && strcmp(license, "GPL v2")) {
-                printk(KERN_WARNING "VMI: Non GPL license `%s' found for ROM. Not used.\n",
-                        license);
-                return 0;
-        }
-
-        return 1;
-}
-
-/*
- * Probe for the VMI option ROM
- */
-static inline int __init probe_vmi_rom(void)
-{
-        unsigned long base;
-
-        /* VMI ROM is in option ROM area, check signature */
-        for (base = 0xC0000; base < 0xE0000; base += 2048) {
-                struct vrom_header *romstart;
-                romstart = (struct vrom_header *)isa_bus_to_virt(base);
-                if (check_vmi_rom(romstart)) {
-                        vmi_rom = romstart;
-                        return 1;
-                }
-        }
-        return 0;
-}
-
-/*
- * VMI setup common to all processors
- */
-void vmi_bringup(void)
-{
-        /* We must establish the lowmem mapping for MMU ops to work */
-        if (vmi_ops.set_linear_mapping)
-                vmi_ops.set_linear_mapping(0, (void *)__PAGE_OFFSET, MAXMEM_PFN, 0);
-}
-
-/*
- * Return a pointer to a VMI function or NULL if unimplemented
- */
-static void *vmi_get_function(int vmicall)
-{
-        u64 reloc;
-        const struct vmi_relocation_info *rel = (struct vmi_relocation_info *)&reloc;
-        reloc = call_vrom_long_func(vmi_rom, get_reloc, vmicall);
-        BUG_ON(rel->type == VMI_RELOCATION_JUMP_REL);
-        if (rel->type == VMI_RELOCATION_CALL_REL)
-                return (void *)rel->eip;
-        else
-                return NULL;
-}
-
-/*
- * Helper macro for making the VMI paravirt-ops fill code readable.
- * For unimplemented operations, fall back to default, unless nop
- * is returned by the ROM.
- */
-#define para_fill(opname, vmicall)                              \
-do {                                                            \
-        reloc = call_vrom_long_func(vmi_rom, get_reloc,         \
-                                    VMI_CALL_##vmicall);        \
-        if (rel->type == VMI_RELOCATION_CALL_REL)               \
-                opname = (void *)rel->eip;                      \
-        else if (rel->type == VMI_RELOCATION_NOP)               \
-                opname = (void *)vmi_nop;                       \
-        else if (rel->type != VMI_RELOCATION_NONE)              \
-                printk(KERN_WARNING "VMI: Unknown relocation "  \
-                                    "type %d for " #vmicall"\n",\
-                                        rel->type);             \
-} while (0)
-
-/*
- * Helper macro for making the VMI paravirt-ops fill code readable.
- * For cached operations which do not match the VMI ROM ABI and must
- * go through a tranlation stub.  Ignore NOPs, since it is not clear
- * a NOP * VMI function corresponds to a NOP paravirt-op when the
- * functions are not in 1-1 correspondence.
- */
-#define para_wrap(opname, wrapper, cache, vmicall)              \
-do {                                                            \
-        reloc = call_vrom_long_func(vmi_rom, get_reloc,         \
-                                    VMI_CALL_##vmicall);        \
-        BUG_ON(rel->type == VMI_RELOCATION_JUMP_REL);           \
-        if (rel->type == VMI_RELOCATION_CALL_REL) {             \
-                opname = wrapper;                               \
-                vmi_ops.cache = (void *)rel->eip;               \
-        }                                                       \
-} while (0)
-
-/*
- * Activate the VMI interface and switch into paravirtualized mode
- */
-static inline int __init activate_vmi(void)
-{
-        short kernel_cs;
-        u64 reloc;
-        const struct vmi_relocation_info *rel = (struct vmi_relocation_info *)&reloc;
-
-        /*
-         * Prevent page tables from being allocated in highmem, even if
-         * CONFIG_HIGHPTE is enabled.
-         */
-        __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
-
-        if (call_vrom_func(vmi_rom, vmi_init) != 0) {
-                printk(KERN_ERR "VMI ROM failed to initialize!");
-                return 0;
-        }
-        savesegment(cs, kernel_cs);
-
-        pv_info.paravirt_enabled = 1;
-        pv_info.kernel_rpl = kernel_cs & SEGMENT_RPL_MASK;
-        pv_info.name = "vmi [deprecated]";
-
-        pv_init_ops.patch = vmi_patch;
-
-        /*
-         * Many of these operations are ABI compatible with VMI.
-         * This means we can fill in the paravirt-ops with direct
-         * pointers into the VMI ROM.  If the calling convention for
-         * these operations changes, this code needs to be updated.
-         *
-         * Exceptions
-         *  CPUID paravirt-op uses pointers, not the native ISA
-         *  halt has no VMI equivalent; all VMI halts are "safe"
-         *  no MSR support yet - just trap and emulate.  VMI uses the
-         *    same ABI as the native ISA, but Linux wants exceptions
-         *    from bogus MSR read / write handled
-         *  rdpmc is not yet used in Linux
-         */
-
-        /* CPUID is special, so very special it gets wrapped like a present */
-        para_wrap(pv_cpu_ops.cpuid, vmi_cpuid, cpuid, CPUID);
-
-        para_fill(pv_cpu_ops.clts, CLTS);
-        para_fill(pv_cpu_ops.get_debugreg, GetDR);
-        para_fill(pv_cpu_ops.set_debugreg, SetDR);
-        para_fill(pv_cpu_ops.read_cr0, GetCR0);
-        para_fill(pv_mmu_ops.read_cr2, GetCR2);
-        para_fill(pv_mmu_ops.read_cr3, GetCR3);
-        para_fill(pv_cpu_ops.read_cr4, GetCR4);
-        para_fill(pv_cpu_ops.write_cr0, SetCR0);
-        para_fill(pv_mmu_ops.write_cr2, SetCR2);
-        para_fill(pv_mmu_ops.write_cr3, SetCR3);
-        para_fill(pv_cpu_ops.write_cr4, SetCR4);
-
-        para_fill(pv_irq_ops.save_fl.func, GetInterruptMask);
-        para_fill(pv_irq_ops.restore_fl.func, SetInterruptMask);
-        para_fill(pv_irq_ops.irq_disable.func, DisableInterrupts);
-        para_fill(pv_irq_ops.irq_enable.func, EnableInterrupts);
-
-        para_fill(pv_cpu_ops.wbinvd, WBINVD);
-        para_fill(pv_cpu_ops.read_tsc, RDTSC);
-
-        /* The following we emulate with trap and emulate for now */
-        /* paravirt_ops.read_msr = vmi_rdmsr */
-        /* paravirt_ops.write_msr = vmi_wrmsr */
-        /* paravirt_ops.rdpmc = vmi_rdpmc */
-
-        /* TR interface doesn't pass TR value, wrap */
-        para_wrap(pv_cpu_ops.load_tr_desc, vmi_set_tr, set_tr, SetTR);
-
-        /* LDT is special, too */
-        para_wrap(pv_cpu_ops.set_ldt, vmi_set_ldt, _set_ldt, SetLDT);
-
-        para_fill(pv_cpu_ops.load_gdt, SetGDT);
-        para_fill(pv_cpu_ops.load_idt, SetIDT);
-        para_fill(pv_cpu_ops.store_gdt, GetGDT);
-        para_fill(pv_cpu_ops.store_idt, GetIDT);
-        para_fill(pv_cpu_ops.store_tr, GetTR);
-        pv_cpu_ops.load_tls = vmi_load_tls;
-        para_wrap(pv_cpu_ops.write_ldt_entry, vmi_write_ldt_entry,
-                  write_ldt_entry, WriteLDTEntry);
-        para_wrap(pv_cpu_ops.write_gdt_entry, vmi_write_gdt_entry,
-                  write_gdt_entry, WriteGDTEntry);
-        para_wrap(pv_cpu_ops.write_idt_entry, vmi_write_idt_entry,
-                  write_idt_entry, WriteIDTEntry);
-        para_wrap(pv_cpu_ops.load_sp0, vmi_load_sp0, set_kernel_stack, UpdateKernelStack);
-        para_fill(pv_cpu_ops.set_iopl_mask, SetIOPLMask);
-        para_fill(pv_cpu_ops.io_delay, IODelay);
-
-        para_wrap(pv_cpu_ops.start_context_switch, vmi_start_context_switch,
-                  set_lazy_mode, SetLazyMode);
-        para_wrap(pv_cpu_ops.end_context_switch, vmi_end_context_switch,
-                  set_lazy_mode, SetLazyMode);
-
-        para_wrap(pv_mmu_ops.lazy_mode.enter, vmi_enter_lazy_mmu,
-                  set_lazy_mode, SetLazyMode);
-        para_wrap(pv_mmu_ops.lazy_mode.leave, vmi_leave_lazy_mmu,
-                  set_lazy_mode, SetLazyMode);
-
-        /* user and kernel flush are just handled with different flags to FlushTLB */
-        para_wrap(pv_mmu_ops.flush_tlb_user, vmi_flush_tlb_user, _flush_tlb, FlushTLB);
-        para_wrap(pv_mmu_ops.flush_tlb_kernel, vmi_flush_tlb_kernel, _flush_tlb, FlushTLB);
-        para_fill(pv_mmu_ops.flush_tlb_single, InvalPage);
-
-        /*
-         * Until a standard flag format can be agreed on, we need to
-         * implement these as wrappers in Linux.  Get the VMI ROM
-         * function pointers for the two backend calls.
-         */
-#ifdef CONFIG_X86_PAE
-        vmi_ops.set_pte = vmi_get_function(VMI_CALL_SetPxELong);
-        vmi_ops.update_pte = vmi_get_function(VMI_CALL_UpdatePxELong);
-#else
-        vmi_ops.set_pte = vmi_get_function(VMI_CALL_SetPxE);
-        vmi_ops.update_pte = vmi_get_function(VMI_CALL_UpdatePxE);
-#endif
-
-        if (vmi_ops.set_pte) {
-                pv_mmu_ops.set_pte = vmi_set_pte;
-                pv_mmu_ops.set_pte_at = vmi_set_pte_at;
-                pv_mmu_ops.set_pmd = vmi_set_pmd;
-#ifdef CONFIG_X86_PAE
-                pv_mmu_ops.set_pte_atomic = vmi_set_pte_atomic;
-                pv_mmu_ops.set_pud = vmi_set_pud;
-                pv_mmu_ops.pte_clear = vmi_pte_clear;
-                pv_mmu_ops.pmd_clear = vmi_pmd_clear;
-#endif
-        }
-
-        if (vmi_ops.update_pte) {
-                pv_mmu_ops.pte_update = vmi_update_pte;
-                pv_mmu_ops.pte_update_defer = vmi_update_pte_defer;
-        }
-
-        vmi_ops.allocate_page = vmi_get_function(VMI_CALL_AllocatePage);
-        if (vmi_ops.allocate_page) {
-                pv_mmu_ops.alloc_pte = vmi_allocate_pte;
-                pv_mmu_ops.alloc_pmd = vmi_allocate_pmd;
-                pv_mmu_ops.alloc_pmd_clone = vmi_allocate_pmd_clone;
-        }
-
-        vmi_ops.release_page = vmi_get_function(VMI_CALL_ReleasePage);
-        if (vmi_ops.release_page) {
-                pv_mmu_ops.release_pte = vmi_release_pte;
-                pv_mmu_ops.release_pmd = vmi_release_pmd;
-                pv_mmu_ops.pgd_free = vmi_pgd_free;
-        }
-
-        /* Set linear is needed in all cases */
-        vmi_ops.set_linear_mapping = vmi_get_function(VMI_CALL_SetLinearMapping);
-
-        /*
-         * These MUST always be patched.  Don't support indirect jumps
-         * through these operations, as the VMI interface may use either
-         * a jump or a call to get to these operations, depending on
-         * the backend.  They are performance critical anyway, so requiring
-         * a patch is not a big problem.
-         */
-        pv_cpu_ops.irq_enable_sysexit = (void *)0xfeedbab0;
-        pv_cpu_ops.iret = (void *)0xbadbab0;
-
-#ifdef CONFIG_SMP
-        para_wrap(pv_apic_ops.startup_ipi_hook, vmi_startup_ipi_hook, set_initial_ap_state, SetInitialAPState);
-#endif
-
-#ifdef CONFIG_X86_LOCAL_APIC
-       para_fill(apic->read, APICRead);
-       para_fill(apic->write, APICWrite);
-#endif
-
-        /*
-         * Check for VMI timer functionality by probing for a cycle frequency method
-         */
-        reloc = call_vrom_long_func(vmi_rom, get_reloc, VMI_CALL_GetCycleFrequency);
-        if (!disable_vmi_timer && rel->type != VMI_RELOCATION_NONE) {
-                vmi_timer_ops.get_cycle_frequency = (void *)rel->eip;
-                vmi_timer_ops.get_cycle_counter =
-                        vmi_get_function(VMI_CALL_GetCycleCounter);
-                vmi_timer_ops.get_wallclock =
-                        vmi_get_function(VMI_CALL_GetWallclockTime);
-                vmi_timer_ops.wallclock_updated =
-                        vmi_get_function(VMI_CALL_WallclockUpdated);
-                vmi_timer_ops.set_alarm = vmi_get_function(VMI_CALL_SetAlarm);
-                vmi_timer_ops.cancel_alarm =
-                         vmi_get_function(VMI_CALL_CancelAlarm);
-                x86_init.timers.timer_init = vmi_time_init;
-#ifdef CONFIG_X86_LOCAL_APIC
-                x86_init.timers.setup_percpu_clockev = vmi_time_bsp_init;
-                x86_cpuinit.setup_percpu_clockev = vmi_time_ap_init;
-#endif
-                pv_time_ops.sched_clock = vmi_sched_clock;
-                x86_platform.calibrate_tsc = vmi_tsc_khz;
-                x86_platform.get_wallclock = vmi_get_wallclock;
-                x86_platform.set_wallclock = vmi_set_wallclock;
-
-                /* We have true wallclock functions; disable CMOS clock sync */
-                no_sync_cmos_clock = 1;
-        } else {
-                disable_noidle = 1;
-                disable_vmi_timer = 1;
-        }
-
-        para_fill(pv_irq_ops.safe_halt, Halt);
-
-        /*
-         * Alternative instruction rewriting doesn't happen soon enough
-         * to convert VMI_IRET to a call instead of a jump; so we have
-         * to do this before IRQs get reenabled.  Fortunately, it is
-         * idempotent.
-         */
-        apply_paravirt(__parainstructions, __parainstructions_end);
-
-        vmi_bringup();
-
-        return 1;
-}
-
-#undef para_fill
-
-void __init vmi_init(void)
-{
-        if (!vmi_rom)
-                probe_vmi_rom();
-        else
-                check_vmi_rom(vmi_rom);
-
-        /* In case probing for or validating the ROM failed, basil */
-        if (!vmi_rom)
-                return;
-
-        reserve_top_address(-vmi_rom->virtual_top);
-
-#ifdef CONFIG_X86_IO_APIC
-        /* This is virtual hardware; timer routing is wired correctly */
-        no_timer_check = 1;
-#endif
-}
-
-void __init vmi_activate(void)
-{
-        unsigned long flags;
-
-        if (!vmi_rom)
-                return;
-
-        local_irq_save(flags);
-        activate_vmi();
-        local_irq_restore(flags & X86_EFLAGS_IF);
-}
-
-static int __init parse_vmi(char *arg)
-{
-        if (!arg)
-                return -EINVAL;
-
-        if (!strcmp(arg, "disable_pge")) {
-                clear_cpu_cap(&boot_cpu_data, X86_FEATURE_PGE);
-                disable_pge = 1;
-        } else if (!strcmp(arg, "disable_pse")) {
-                clear_cpu_cap(&boot_cpu_data, X86_FEATURE_PSE);
-                disable_pse = 1;
-        } else if (!strcmp(arg, "disable_sep")) {
-                clear_cpu_cap(&boot_cpu_data, X86_FEATURE_SEP);
-                disable_sep = 1;
-        } else if (!strcmp(arg, "disable_tsc")) {
-                clear_cpu_cap(&boot_cpu_data, X86_FEATURE_TSC);
-                disable_tsc = 1;
-        } else if (!strcmp(arg, "disable_mtrr")) {
-                clear_cpu_cap(&boot_cpu_data, X86_FEATURE_MTRR);
-                disable_mtrr = 1;
-        } else if (!strcmp(arg, "disable_timer")) {
-                disable_vmi_timer = 1;
-                disable_noidle = 1;
-        } else if (!strcmp(arg, "disable_noidle"))
-                disable_noidle = 1;
-        return 0;
-}
-
-early_param("vmi", parse_vmi);
diff --git a/arch/x86/kernel/vmiclock_32.c b/arch/x86/kernel/vmiclock_32.c
deleted file mode 100644
index 5e1ff66ecd73..000000000000
--- a/arch/x86/kernel/vmiclock_32.c
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * VMI paravirtual timer support routines.
- *
- * Copyright (C) 2007, VMware, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- */
-
-#include <linux/smp.h>
-#include <linux/interrupt.h>
-#include <linux/cpumask.h>
-#include <linux/clocksource.h>
-#include <linux/clockchips.h>
-
-#include <asm/vmi.h>
-#include <asm/vmi_time.h>
-#include <asm/apicdef.h>
-#include <asm/apic.h>
-#include <asm/timer.h>
-#include <asm/i8253.h>
-#include <asm/irq_vectors.h>
-
-#define VMI_ONESHOT  (VMI_ALARM_IS_ONESHOT  | VMI_CYCLES_REAL | vmi_get_alarm_wiring())
-#define VMI_PERIODIC (VMI_ALARM_IS_PERIODIC | VMI_CYCLES_REAL | vmi_get_alarm_wiring())
-
-static DEFINE_PER_CPU(struct clock_event_device, local_events);
-
-static inline u32 vmi_counter(u32 flags)
-{
-        /* Given VMI_ONESHOT or VMI_PERIODIC, return the corresponding
-         * cycle counter. */
-        return flags & VMI_ALARM_COUNTER_MASK;
-}
-
-/* paravirt_ops.get_wallclock = vmi_get_wallclock */
-unsigned long vmi_get_wallclock(void)
-{
-        unsigned long long wallclock;
-        wallclock = vmi_timer_ops.get_wallclock(); // nsec
-        (void)do_div(wallclock, 1000000000);       // sec
-
-        return wallclock;
-}
-
-/* paravirt_ops.set_wallclock = vmi_set_wallclock */
-int vmi_set_wallclock(unsigned long now)
-{
-        return 0;
-}
-
-/* paravirt_ops.sched_clock = vmi_sched_clock */
-unsigned long long vmi_sched_clock(void)
-{
-        return cycles_2_ns(vmi_timer_ops.get_cycle_counter(VMI_CYCLES_AVAILABLE));
-}
-
-/* x86_platform.calibrate_tsc = vmi_tsc_khz */
-unsigned long vmi_tsc_khz(void)
-{
-        unsigned long long khz;
-        khz = vmi_timer_ops.get_cycle_frequency();
-        (void)do_div(khz, 1000);
-        return khz;
-}
-
-static inline unsigned int vmi_get_timer_vector(void)
-{
-        return IRQ0_VECTOR;
-}
-
-/** vmi clockchip */
-#ifdef CONFIG_X86_LOCAL_APIC
-static unsigned int startup_timer_irq(unsigned int irq)
-{
-        unsigned long val = apic_read(APIC_LVTT);
-        apic_write(APIC_LVTT, vmi_get_timer_vector());
-
-        return (val & APIC_SEND_PENDING);
-}
-
-static void mask_timer_irq(unsigned int irq)
-{
-        unsigned long val = apic_read(APIC_LVTT);
-        apic_write(APIC_LVTT, val | APIC_LVT_MASKED);
-}
-
-static void unmask_timer_irq(unsigned int irq)
-{
-        unsigned long val = apic_read(APIC_LVTT);
-        apic_write(APIC_LVTT, val & ~APIC_LVT_MASKED);
-}
-
-static void ack_timer_irq(unsigned int irq)
-{
-        ack_APIC_irq();
-}
-
-static struct irq_chip vmi_chip __read_mostly = {
-        .name           = "VMI-LOCAL",
-        .startup        = startup_timer_irq,
-        .mask           = mask_timer_irq,
-        .unmask         = unmask_timer_irq,
-        .ack            = ack_timer_irq
-};
-#endif
-
-/** vmi clockevent */
-#define VMI_ALARM_WIRED_IRQ0    0x00000000
-#define VMI_ALARM_WIRED_LVTT    0x00010000
-static int vmi_wiring = VMI_ALARM_WIRED_IRQ0;
-
-static inline int vmi_get_alarm_wiring(void)
-{
-        return vmi_wiring;
-}
-
-static void vmi_timer_set_mode(enum clock_event_mode mode,
-                               struct clock_event_device *evt)
-{
-        cycle_t now, cycles_per_hz;
-        BUG_ON(!irqs_disabled());
-
-        switch (mode) {
-        case CLOCK_EVT_MODE_ONESHOT:
-        case CLOCK_EVT_MODE_RESUME:
-                break;
-        case CLOCK_EVT_MODE_PERIODIC:
-                cycles_per_hz = vmi_timer_ops.get_cycle_frequency();
-                (void)do_div(cycles_per_hz, HZ);
-                now = vmi_timer_ops.get_cycle_counter(vmi_counter(VMI_PERIODIC));
-                vmi_timer_ops.set_alarm(VMI_PERIODIC, now, cycles_per_hz);
-                break;
-        case CLOCK_EVT_MODE_UNUSED:
-        case CLOCK_EVT_MODE_SHUTDOWN:
-                switch (evt->mode) {
-                case CLOCK_EVT_MODE_ONESHOT:
-                        vmi_timer_ops.cancel_alarm(VMI_ONESHOT);
-                        break;
-                case CLOCK_EVT_MODE_PERIODIC:
-                        vmi_timer_ops.cancel_alarm(VMI_PERIODIC);
-                        break;
-                default:
-                        break;
-                }
-                break;
-        default:
-                break;
-        }
-}
-
-static int vmi_timer_next_event(unsigned long delta,
-                                struct clock_event_device *evt)
-{
-        /* Unfortunately, set_next_event interface only passes relative
-         * expiry, but we want absolute expiry.  It'd be better if were
-         * were passed an absolute expiry, since a bunch of time may
-         * have been stolen between the time the delta is computed and
-         * when we set the alarm below. */
-        cycle_t now = vmi_timer_ops.get_cycle_counter(vmi_counter(VMI_ONESHOT));
-
-        BUG_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
-        vmi_timer_ops.set_alarm(VMI_ONESHOT, now + delta, 0);
-        return 0;
-}
-
-static struct clock_event_device vmi_clockevent = {
-        .name           = "vmi-timer",
-        .features       = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
-        .shift          = 22,
-        .set_mode       = vmi_timer_set_mode,
-        .set_next_event = vmi_timer_next_event,
-        .rating         = 1000,
-        .irq            = 0,
-};
-
-static irqreturn_t vmi_timer_interrupt(int irq, void *dev_id)
-{
-        struct clock_event_device *evt = &__get_cpu_var(local_events);
-        evt->event_handler(evt);
-        return IRQ_HANDLED;
-}
-
-static struct irqaction vmi_clock_action  = {
-        .name           = "vmi-timer",
-        .handler        = vmi_timer_interrupt,
-        .flags          = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_TIMER,
-};
-
-static void __devinit vmi_time_init_clockevent(void)
-{
-        cycle_t cycles_per_msec;
-        struct clock_event_device *evt;
-
-        int cpu = smp_processor_id();
-        evt = &__get_cpu_var(local_events);
-
-        /* Use cycles_per_msec since div_sc params are 32-bits. */
-        cycles_per_msec = vmi_timer_ops.get_cycle_frequency();
-        (void)do_div(cycles_per_msec, 1000);
-
-        memcpy(evt, &vmi_clockevent, sizeof(*evt));
-        /* Must pick .shift such that .mult fits in 32-bits.  Choosing
-         * .shift to be 22 allows 2^(32-22) cycles per nano-seconds
-         * before overflow. */
-        evt->mult = div_sc(cycles_per_msec, NSEC_PER_MSEC, evt->shift);
-        /* Upper bound is clockevent's use of ulong for cycle deltas. */
-        evt->max_delta_ns = clockevent_delta2ns(ULONG_MAX, evt);
-        evt->min_delta_ns = clockevent_delta2ns(1, evt);
-        evt->cpumask = cpumask_of(cpu);
-
-        printk(KERN_WARNING "vmi: registering clock event %s. mult=%u shift=%u\n",
-               evt->name, evt->mult, evt->shift);
-        clockevents_register_device(evt);
-}
-
-void __init vmi_time_init(void)
-{
-        unsigned int cpu;
-        /* Disable PIT: BIOSes start PIT CH0 with 18.2hz peridic. */
-        outb_pit(0x3a, PIT_MODE); /* binary, mode 5, LSB/MSB, ch 0 */
-
-        vmi_time_init_clockevent();
-        setup_irq(0, &vmi_clock_action);
-        for_each_possible_cpu(cpu)
-                per_cpu(vector_irq, cpu)[vmi_get_timer_vector()] = 0;
-}
-
-#ifdef CONFIG_X86_LOCAL_APIC
-void __devinit vmi_time_bsp_init(void)
-{
-        /*
-         * On APIC systems, we want local timers to fire on each cpu.  We do
-         * this by programming LVTT to deliver timer events to the IRQ handler
-         * for IRQ-0, since we can't re-use the APIC local timer handler
-         * without interfering with that code.
-         */
-        clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
-        local_irq_disable();
-#ifdef CONFIG_SMP
-        /*
-         * XXX handle_percpu_irq only defined for SMP; we need to switch over
-         * to using it, since this is a local interrupt, which each CPU must
-         * handle individually without locking out or dropping simultaneous
-         * local timers on other CPUs.  We also don't want to trigger the
-         * quirk workaround code for interrupts which gets invoked from
-         * handle_percpu_irq via eoi, so we use our own IRQ chip.
-         */
-        set_irq_chip_and_handler_name(0, &vmi_chip, handle_percpu_irq, "lvtt");
-#else
-        set_irq_chip_and_handler_name(0, &vmi_chip, handle_edge_irq, "lvtt");
-#endif
-        vmi_wiring = VMI_ALARM_WIRED_LVTT;
-        apic_write(APIC_LVTT, vmi_get_timer_vector());
-        local_irq_enable();
-        clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
-}
-
-void __devinit vmi_time_ap_init(void)
-{
-        vmi_time_init_clockevent();
-        apic_write(APIC_LVTT, vmi_get_timer_vector());
-}
-#endif
-
-/** vmi clocksource */
-static struct clocksource clocksource_vmi;
-
-static cycle_t read_real_cycles(struct clocksource *cs)
-{
-        cycle_t ret = (cycle_t)vmi_timer_ops.get_cycle_counter(VMI_CYCLES_REAL);
-        return max(ret, clocksource_vmi.cycle_last);
-}
-
-static struct clocksource clocksource_vmi = {
-        .name                   = "vmi-timer",
-        .rating                 = 450,
-        .read                   = read_real_cycles,
-        .mask                   = CLOCKSOURCE_MASK(64),
-        .mult                   = 0, /* to be set */
-        .shift                  = 22,
-        .flags                  = CLOCK_SOURCE_IS_CONTINUOUS,
-};
-
-static int __init init_vmi_clocksource(void)
-{
-        cycle_t cycles_per_msec;
-
-        if (!vmi_timer_ops.get_cycle_frequency)
-                return 0;
-        /* Use khz2mult rather than hz2mult since hz arg is only 32-bits. */
-        cycles_per_msec = vmi_timer_ops.get_cycle_frequency();
-        (void)do_div(cycles_per_msec, 1000);
-
-        /* Note that clocksource.{mult, shift} converts in the opposite direction
-         * as clockevents.  */
-        clocksource_vmi.mult = clocksource_khz2mult(cycles_per_msec,
-                                                    clocksource_vmi.shift);
-
-        printk(KERN_WARNING "vmi: registering clock source khz=%lld\n", cycles_per_msec);
-        return clocksource_register(&clocksource_vmi);
-
-}
-module_init(init_vmi_clocksource);
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 77d8c0f4817d..22b06f7660f4 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1056,14 +1056,13 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
 
         vcpu->arch.apic = apic;
 
-        apic->regs_page = alloc_page(GFP_KERNEL);
+        apic->regs_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
         if (apic->regs_page == NULL) {
                 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
                        vcpu->vcpu_id);
                 goto nomem_free_apic;
         }
         apic->regs = page_address(apic->regs_page);
-        memset(apic->regs, 0, PAGE_SIZE);
         apic->vcpu = vcpu;
 
         hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index bc5b9b8d4a33..8a3f9f64f86f 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -766,7 +766,6 @@ static void init_vmcb(struct vcpu_svm *svm)
 
         control->iopm_base_pa = iopm_base;
         control->msrpm_base_pa = __pa(svm->msrpm);
-        control->tsc_offset = 0;
         control->int_ctl = V_INTR_MASKING_MASK;
 
         init_seg(&save->es);
@@ -902,6 +901,7 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
         svm->asid_generation = 0;
         init_vmcb(svm);
+        svm->vmcb->control.tsc_offset = 0-native_read_tsc();
 
         err = fx_init(&svm->vcpu);
         if (err)
@@ -3163,8 +3163,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
         sync_lapic_to_cr8(vcpu);
 
         save_host_msrs(vcpu);
-        fs_selector = kvm_read_fs();
-        gs_selector = kvm_read_gs();
+        savesegment(fs, fs_selector);
+        savesegment(gs, gs_selector);
         ldt_selector = kvm_read_ldt();
         svm->vmcb->save.cr2 = vcpu->arch.cr2;
         /* required for live migration with NPT */
@@ -3251,10 +3251,15 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
 
-        kvm_load_fs(fs_selector);
-        kvm_load_gs(gs_selector);
-        kvm_load_ldt(ldt_selector);
         load_host_msrs(vcpu);
+        loadsegment(fs, fs_selector);
+#ifdef CONFIG_X86_64
+        load_gs_index(gs_selector);
+        wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
+#else
+        loadsegment(gs, gs_selector);
+#endif
+        kvm_load_ldt(ldt_selector);
 
         reload_tss(vcpu);
 
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 49b25eee25ac..7bddfab12013 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -803,7 +803,7 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu)
          */
         vmx->host_state.ldt_sel = kvm_read_ldt();
         vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
-        vmx->host_state.fs_sel = kvm_read_fs();
+        savesegment(fs, vmx->host_state.fs_sel);
         if (!(vmx->host_state.fs_sel & 7)) {
                 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
                 vmx->host_state.fs_reload_needed = 0;
@@ -811,7 +811,7 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu)
                 vmcs_write16(HOST_FS_SELECTOR, 0);
                 vmx->host_state.fs_reload_needed = 1;
         }
-        vmx->host_state.gs_sel = kvm_read_gs();
+        savesegment(gs, vmx->host_state.gs_sel);
         if (!(vmx->host_state.gs_sel & 7))
                 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
         else {
@@ -841,27 +841,21 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu)
 
 static void __vmx_load_host_state(struct vcpu_vmx *vmx)
 {
-        unsigned long flags;
-
         if (!vmx->host_state.loaded)
                 return;
 
         ++vmx->vcpu.stat.host_state_reload;
         vmx->host_state.loaded = 0;
         if (vmx->host_state.fs_reload_needed)
-                kvm_load_fs(vmx->host_state.fs_sel);
+                loadsegment(fs, vmx->host_state.fs_sel);
         if (vmx->host_state.gs_ldt_reload_needed) {
                 kvm_load_ldt(vmx->host_state.ldt_sel);
-                /*
-                 * If we have to reload gs, we must take care to
-                 * preserve our gs base.
-                 */
-                local_irq_save(flags);
-                kvm_load_gs(vmx->host_state.gs_sel);
 #ifdef CONFIG_X86_64
-                wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
+                load_gs_index(vmx->host_state.gs_sel);
+                wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
+#else
+                loadsegment(gs, vmx->host_state.gs_sel);
 #endif
-                local_irq_restore(flags);
         }
         reload_tss();
 #ifdef CONFIG_X86_64
@@ -2589,8 +2583,8 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
-        vmcs_write16(HOST_FS_SELECTOR, kvm_read_fs());    /* 22.2.4 */
-        vmcs_write16(HOST_GS_SELECTOR, kvm_read_gs());    /* 22.2.4 */
+        vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
+        vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
 #ifdef CONFIG_X86_64
         rdmsrl(MSR_FS_BASE, a);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3a09c625d526..6c2ecf0a806d 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1991,13 +1991,14 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
                 0 /* Reserved, DCA */ | F(XMM4_1) |
                 F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
-                0 /* Reserved, AES */ | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX);
+                0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
+                F(F16C);
         /* cpuid 0x80000001.ecx */
         const u32 kvm_supported_word6_x86_features =
                 F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
                 F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
-                F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
-                0 /* SKINIT */ | 0 /* WDT */;
+                F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(XOP) |
+                0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM);
 
         /* all calls to cpuid_count() should be made on the same cpu */
         get_cpu();
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index 9d5f55848455..73b1e1a1f489 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -791,22 +791,22 @@ static void lguest_flush_tlb_kernel(void)
  * simple as setting a bit.  We don't actually "ack" interrupts as such, we
  * just mask and unmask them.  I wonder if we should be cleverer?
  */
-static void disable_lguest_irq(unsigned int irq)
+static void disable_lguest_irq(struct irq_data *data)
 {
-        set_bit(irq, lguest_data.blocked_interrupts);
+        set_bit(data->irq, lguest_data.blocked_interrupts);
 }
 
-static void enable_lguest_irq(unsigned int irq)
+static void enable_lguest_irq(struct irq_data *data)
 {
-        clear_bit(irq, lguest_data.blocked_interrupts);
+        clear_bit(data->irq, lguest_data.blocked_interrupts);
 }
 
 /* This structure describes the lguest IRQ controller. */
 static struct irq_chip lguest_irq_controller = {
         .name           = "lguest",
-        .mask           = disable_lguest_irq,
-        .mask_ack       = disable_lguest_irq,
-        .unmask         = enable_lguest_irq,
+        .irq_mask       = disable_lguest_irq,
+        .irq_mask_ack   = disable_lguest_irq,
+        .irq_unmask     = enable_lguest_irq,
 };
 
 /*
@@ -838,12 +838,12 @@ static void __init lguest_init_IRQ(void)
  * rather than set them in lguest_init_IRQ we are called here every time an
  * lguest device needs an interrupt.
  *
- * FIXME: irq_to_desc_alloc_node() can fail due to lack of memory, we should
+ * FIXME: irq_alloc_desc_at() can fail due to lack of memory, we should
  * pass that up!
  */
 void lguest_setup_irq(unsigned int irq)
 {
-        irq_to_desc_alloc_node(irq, 0);
+        irq_alloc_desc_at(irq, 0);
         set_irq_chip_and_handler_name(irq, &lguest_irq_controller,
                                       handle_level_irq, "level");
 }
diff --git a/arch/x86/lib/memcpy_32.c b/arch/x86/lib/memcpy_32.c
index 5415a9d06f53..b908a59eccf5 100644
--- a/arch/x86/lib/memcpy_32.c
+++ b/arch/x86/lib/memcpy_32.c
@@ -22,22 +22,187 @@ EXPORT_SYMBOL(memset);
 
 void *memmove(void *dest, const void *src, size_t n)
 {
-        int d0, d1, d2;
-
-        if (dest < src) {
-                memcpy(dest, src, n);
-        } else {
-                __asm__ __volatile__(
-                        "std\n\t"
-                        "rep\n\t"
-                        "movsb\n\t"
-                        "cld"
-                        : "=&c" (d0), "=&S" (d1), "=&D" (d2)
-                        :"0" (n),
-                         "1" (n-1+src),
-                         "2" (n-1+dest)
-                        :"memory");
-        }
-        return dest;
+        int d0,d1,d2,d3,d4,d5;
+        char *ret = dest;
+
+        __asm__ __volatile__(
+                /* Handle more 16bytes in loop */
+                "cmp $0x10, %0\n\t"
+                "jb     1f\n\t"
+
+                /* Decide forward/backward copy mode */
+                "cmp %2, %1\n\t"
+                "jb     2f\n\t"
+
+                /*
+                 * movs instruction have many startup latency
+                 * so we handle small size by general register.
+                 */
+                "cmp  $680, %0\n\t"
+                "jb 3f\n\t"
+                /*
+                 * movs instruction is only good for aligned case.
+                 */
+                "mov %1, %3\n\t"
+                "xor %2, %3\n\t"
+                "and $0xff, %3\n\t"
+                "jz 4f\n\t"
+                "3:\n\t"
+                "sub $0x10, %0\n\t"
+
+                /*
+                 * We gobble 16byts forward in each loop.
+                 */
+                "3:\n\t"
+                "sub $0x10, %0\n\t"
+                "mov 0*4(%1), %3\n\t"
+                "mov 1*4(%1), %4\n\t"
+                "mov  %3, 0*4(%2)\n\t"
+                "mov  %4, 1*4(%2)\n\t"
+                "mov 2*4(%1), %3\n\t"
+                "mov 3*4(%1), %4\n\t"
+                "mov  %3, 2*4(%2)\n\t"
+                "mov  %4, 3*4(%2)\n\t"
+                "lea  0x10(%1), %1\n\t"
+                "lea  0x10(%2), %2\n\t"
+                "jae 3b\n\t"
+                "add $0x10, %0\n\t"
+                "jmp 1f\n\t"
+
+                /*
+                 * Handle data forward by movs.
+                 */
+                ".p2align 4\n\t"
+                "4:\n\t"
+                "mov -4(%1, %0), %3\n\t"
+                "lea -4(%2, %0), %4\n\t"
+                "shr $2, %0\n\t"
+                "rep movsl\n\t"
+                "mov %3, (%4)\n\t"
+                "jmp 11f\n\t"
+                /*
+                 * Handle data backward by movs.
+                 */
+                ".p2align 4\n\t"
+                "6:\n\t"
+                "mov (%1), %3\n\t"
+                "mov %2, %4\n\t"
+                "lea -4(%1, %0), %1\n\t"
+                "lea -4(%2, %0), %2\n\t"
+                "shr $2, %0\n\t"
+                "std\n\t"
+                "rep movsl\n\t"
+                "mov %3,(%4)\n\t"
+                "cld\n\t"
+                "jmp 11f\n\t"
+
+                /*
+                 * Start to prepare for backward copy.
+                 */
+                ".p2align 4\n\t"
+                "2:\n\t"
+                "cmp  $680, %0\n\t"
+                "jb 5f\n\t"
+                "mov %1, %3\n\t"
+                "xor %2, %3\n\t"
+                "and $0xff, %3\n\t"
+                "jz 6b\n\t"
+
+                /*
+                 * Calculate copy position to tail.
+                 */
+                "5:\n\t"
+                "add %0, %1\n\t"
+                "add %0, %2\n\t"
+                "sub $0x10, %0\n\t"
+
+                /*
+                 * We gobble 16byts backward in each loop.
+                 */
+                "7:\n\t"
+                "sub $0x10, %0\n\t"
+
+                "mov -1*4(%1), %3\n\t"
+                "mov -2*4(%1), %4\n\t"
+                "mov  %3, -1*4(%2)\n\t"
+                "mov  %4, -2*4(%2)\n\t"
+                "mov -3*4(%1), %3\n\t"
+                "mov -4*4(%1), %4\n\t"
+                "mov  %3, -3*4(%2)\n\t"
+                "mov  %4, -4*4(%2)\n\t"
+                "lea  -0x10(%1), %1\n\t"
+                "lea  -0x10(%2), %2\n\t"
+                "jae 7b\n\t"
+                /*
+                 * Calculate copy position to head.
+                 */
+                "add $0x10, %0\n\t"
+                "sub %0, %1\n\t"
+                "sub %0, %2\n\t"
+
+                /*
+                 * Move data from 8 bytes to 15 bytes.
+                 */
+                ".p2align 4\n\t"
+                "1:\n\t"
+                "cmp $8, %0\n\t"
+                "jb 8f\n\t"
+                "mov 0*4(%1), %3\n\t"
+                "mov 1*4(%1), %4\n\t"
+                "mov -2*4(%1, %0), %5\n\t"
+                "mov -1*4(%1, %0), %1\n\t"
+
+                "mov  %3, 0*4(%2)\n\t"
+                "mov  %4, 1*4(%2)\n\t"
+                "mov  %5, -2*4(%2, %0)\n\t"
+                "mov  %1, -1*4(%2, %0)\n\t"
+                "jmp 11f\n\t"
+
+                /*
+                 * Move data from 4 bytes to 7 bytes.
+                 */
+                ".p2align 4\n\t"
+                "8:\n\t"
+                "cmp $4, %0\n\t"
+                "jb 9f\n\t"
+                "mov 0*4(%1), %3\n\t"
+                "mov -1*4(%1, %0), %4\n\t"
+                "mov  %3, 0*4(%2)\n\t"
+                "mov  %4, -1*4(%2, %0)\n\t"
+                "jmp 11f\n\t"
+
+                /*
+                 * Move data from 2 bytes to 3 bytes.
+                 */
+                ".p2align 4\n\t"
+                "9:\n\t"
+                "cmp $2, %0\n\t"
+                "jb 10f\n\t"
+                "movw 0*2(%1), %%dx\n\t"
+                "movw -1*2(%1, %0), %%bx\n\t"
+                "movw %%dx, 0*2(%2)\n\t"
+                "movw %%bx, -1*2(%2, %0)\n\t"
+                "jmp 11f\n\t"
+
+                /*
+                 * Move data for 1 byte.
+                 */
+                ".p2align 4\n\t"
+                "10:\n\t"
+                "cmp $1, %0\n\t"
+                "jb 11f\n\t"
+                "movb (%1), %%cl\n\t"
+                "movb %%cl, (%2)\n\t"
+                ".p2align 4\n\t"
+                "11:"
+                : "=&c" (d0), "=&S" (d1), "=&D" (d2),
+                  "=r" (d3),"=r" (d4), "=r"(d5)
+                :"0" (n),
+                 "1" (src),
+                 "2" (dest)
+                :"memory");
+
+        return ret;
+
 }
 EXPORT_SYMBOL(memmove);
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index bcbcd1e0f7d5..75ef61e35e38 100644
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -40,84 +40,132 @@
 ENTRY(__memcpy)
 ENTRY(memcpy)
         CFI_STARTPROC
+        movq %rdi, %rax
 
         /*
-         * Put the number of full 64-byte blocks into %ecx.
-         * Tail portion is handled at the end:
+         * Use 32bit CMP here to avoid long NOP padding.
          */
-        movq %rdi, %rax
-        movl %edx, %ecx
-        shrl   $6, %ecx
-        jz .Lhandle_tail
+        cmp  $0x20, %edx
+        jb .Lhandle_tail
 
-        .p2align 4
-.Lloop_64:
         /*
-         * We decrement the loop index here - and the zero-flag is
-         * checked at the end of the loop (instructions inbetween do
-         * not change the zero flag):
+         * We check whether memory false dependece could occur,
+         * then jump to corresponding copy mode.
          */
-        decl %ecx
+        cmp  %dil, %sil
+        jl .Lcopy_backward
+        subl $0x20, %edx
+.Lcopy_forward_loop:
+        subq $0x20,     %rdx
 
         /*
-         * Move in blocks of 4x16 bytes:
+         * Move in blocks of 4x8 bytes:
          */
-        movq 0*8(%rsi),         %r11
-        movq 1*8(%rsi),         %r8
-        movq %r11,              0*8(%rdi)
-        movq %r8,               1*8(%rdi)
-
-        movq 2*8(%rsi),         %r9
-        movq 3*8(%rsi),         %r10
-        movq %r9,               2*8(%rdi)
-        movq %r10,              3*8(%rdi)
-
-        movq 4*8(%rsi),         %r11
-        movq 5*8(%rsi),         %r8
-        movq %r11,              4*8(%rdi)
-        movq %r8,               5*8(%rdi)
-
-        movq 6*8(%rsi),         %r9
-        movq 7*8(%rsi),         %r10
-        movq %r9,               6*8(%rdi)
-        movq %r10,              7*8(%rdi)
-
-        leaq 64(%rsi), %rsi
-        leaq 64(%rdi), %rdi
-
-        jnz  .Lloop_64
+        movq 0*8(%rsi), %r8
+        movq 1*8(%rsi), %r9
+        movq 2*8(%rsi), %r10
+        movq 3*8(%rsi), %r11
+        leaq 4*8(%rsi), %rsi
+
+        movq %r8,       0*8(%rdi)
+        movq %r9,       1*8(%rdi)
+        movq %r10,      2*8(%rdi)
+        movq %r11,      3*8(%rdi)
+        leaq 4*8(%rdi), %rdi
+        jae  .Lcopy_forward_loop
+        addq $0x20,     %rdx
+        jmp  .Lhandle_tail
+
+.Lcopy_backward:
+        /*
+         * Calculate copy position to tail.
+         */
+        addq %rdx,      %rsi
+        addq %rdx,      %rdi
+        subq $0x20,     %rdx
+        /*
+         * At most 3 ALU operations in one cycle,
+         * so append NOPS in the same 16bytes trunk.
+         */
+        .p2align 4
+.Lcopy_backward_loop:
+        subq $0x20,     %rdx
+        movq -1*8(%rsi),        %r8
+        movq -2*8(%rsi),        %r9
+        movq -3*8(%rsi),        %r10
+        movq -4*8(%rsi),        %r11
+        leaq -4*8(%rsi),        %rsi
+        movq %r8,               -1*8(%rdi)
+        movq %r9,               -2*8(%rdi)
+        movq %r10,              -3*8(%rdi)
+        movq %r11,              -4*8(%rdi)
+        leaq -4*8(%rdi),        %rdi
+        jae  .Lcopy_backward_loop
 
+        /*
+         * Calculate copy position to head.
+         */
+        addq $0x20,     %rdx
+        subq %rdx,      %rsi
+        subq %rdx,      %rdi
 .Lhandle_tail:
-        movl %edx, %ecx
-        andl  $63, %ecx
-        shrl   $3, %ecx
-        jz   .Lhandle_7
+        cmpq $16,       %rdx
+        jb   .Lless_16bytes
 
+        /*
+         * Move data from 16 bytes to 31 bytes.
+         */
+        movq 0*8(%rsi), %r8
+        movq 1*8(%rsi), %r9
+        movq -2*8(%rsi, %rdx),  %r10
+        movq -1*8(%rsi, %rdx),  %r11
+        movq %r8,       0*8(%rdi)
+        movq %r9,       1*8(%rdi)
+        movq %r10,      -2*8(%rdi, %rdx)
+        movq %r11,      -1*8(%rdi, %rdx)
+        retq
         .p2align 4
-.Lloop_8:
-        decl %ecx
-        movq (%rsi),            %r8
-        movq %r8,               (%rdi)
-        leaq 8(%rdi),           %rdi
-        leaq 8(%rsi),           %rsi
-        jnz  .Lloop_8
-
-.Lhandle_7:
-        movl %edx, %ecx
-        andl $7, %ecx
-        jz .Lend
+.Lless_16bytes:
+        cmpq $8,        %rdx
+        jb   .Lless_8bytes
+        /*
+         * Move data from 8 bytes to 15 bytes.
+         */
+        movq 0*8(%rsi), %r8
+        movq -1*8(%rsi, %rdx),  %r9
+        movq %r8,       0*8(%rdi)
+        movq %r9,       -1*8(%rdi, %rdx)
+        retq
+        .p2align 4
+.Lless_8bytes:
+        cmpq $4,        %rdx
+        jb   .Lless_3bytes
 
+        /*
+         * Move data from 4 bytes to 7 bytes.
+         */
+        movl (%rsi), %ecx
+        movl -4(%rsi, %rdx), %r8d
+        movl %ecx, (%rdi)
+        movl %r8d, -4(%rdi, %rdx)
+        retq
         .p2align 4
+.Lless_3bytes:
+        cmpl $0, %edx
+        je .Lend
+        /*
+         * Move data from 1 bytes to 3 bytes.
+         */
 .Lloop_1:
         movb (%rsi), %r8b
         movb %r8b, (%rdi)
         incq %rdi
         incq %rsi
-        decl %ecx
+        decl %edx
         jnz .Lloop_1
 
 .Lend:
-        ret
+        retq
         CFI_ENDPROC
 ENDPROC(memcpy)
 ENDPROC(__memcpy)
diff --git a/arch/x86/lib/memmove_64.c b/arch/x86/lib/memmove_64.c
index 0a33909bf122..6d0f0ec41b34 100644
--- a/arch/x86/lib/memmove_64.c
+++ b/arch/x86/lib/memmove_64.c
@@ -8,14 +8,185 @@
 #undef memmove
 void *memmove(void *dest, const void *src, size_t count)
 {
-        if (dest < src) {
-                return memcpy(dest, src, count);
-        } else {
-                char *p = dest + count;
-                const char *s = src + count;
-                while (count--)
-                        *--p = *--s;
-        }
-        return dest;
+        unsigned long d0,d1,d2,d3,d4,d5,d6,d7;
+        char *ret;
+
+        __asm__ __volatile__(
+                /* Handle more 32bytes in loop */
+                "mov %2, %3\n\t"
+                "cmp $0x20, %0\n\t"
+                "jb     1f\n\t"
+
+                /* Decide forward/backward copy mode */
+                "cmp %2, %1\n\t"
+                "jb     2f\n\t"
+
+                /*
+                 * movsq instruction have many startup latency
+                 * so we handle small size by general register.
+                 */
+                "cmp  $680, %0\n\t"
+                "jb 3f\n\t"
+                /*
+                 * movsq instruction is only good for aligned case.
+                 */
+                "cmpb %%dil, %%sil\n\t"
+                "je 4f\n\t"
+                "3:\n\t"
+                "sub $0x20, %0\n\t"
+                /*
+                 * We gobble 32byts forward in each loop.
+                 */
+                "5:\n\t"
+                "sub $0x20, %0\n\t"
+                "movq 0*8(%1), %4\n\t"
+                "movq 1*8(%1), %5\n\t"
+                "movq 2*8(%1), %6\n\t"
+                "movq 3*8(%1), %7\n\t"
+                "leaq 4*8(%1), %1\n\t"
+
+                "movq %4, 0*8(%2)\n\t"
+                "movq %5, 1*8(%2)\n\t"
+                "movq %6, 2*8(%2)\n\t"
+                "movq %7, 3*8(%2)\n\t"
+                "leaq 4*8(%2), %2\n\t"
+                "jae 5b\n\t"
+                "addq $0x20, %0\n\t"
+                "jmp 1f\n\t"
+                /*
+                 * Handle data forward by movsq.
+                 */
+                ".p2align 4\n\t"
+                "4:\n\t"
+                "movq %0, %8\n\t"
+                "movq -8(%1, %0), %4\n\t"
+                "lea -8(%2, %0), %5\n\t"
+                "shrq $3, %8\n\t"
+                "rep movsq\n\t"
+                "movq %4, (%5)\n\t"
+                "jmp 13f\n\t"
+                /*
+                 * Handle data backward by movsq.
+                 */
+                ".p2align 4\n\t"
+                "7:\n\t"
+                "movq %0, %8\n\t"
+                "movq (%1), %4\n\t"
+                "movq %2, %5\n\t"
+                "leaq -8(%1, %0), %1\n\t"
+                "leaq -8(%2, %0), %2\n\t"
+                "shrq $3, %8\n\t"
+                "std\n\t"
+                "rep movsq\n\t"
+                "cld\n\t"
+                "movq %4, (%5)\n\t"
+                "jmp 13f\n\t"
+
+                /*
+                 * Start to prepare for backward copy.
+                 */
+                ".p2align 4\n\t"
+                "2:\n\t"
+                "cmp $680, %0\n\t"
+                "jb 6f \n\t"
+                "cmp %%dil, %%sil\n\t"
+                "je 7b \n\t"
+                "6:\n\t"
+                /*
+                 * Calculate copy position to tail.
+                 */
+                "addq %0, %1\n\t"
+                "addq %0, %2\n\t"
+                "subq $0x20, %0\n\t"
+                /*
+                 * We gobble 32byts backward in each loop.
+                 */
+                "8:\n\t"
+                "subq $0x20, %0\n\t"
+                "movq -1*8(%1), %4\n\t"
+                "movq -2*8(%1), %5\n\t"
+                "movq -3*8(%1), %6\n\t"
+                "movq -4*8(%1), %7\n\t"
+                "leaq -4*8(%1), %1\n\t"
+
+                "movq %4, -1*8(%2)\n\t"
+                "movq %5, -2*8(%2)\n\t"
+                "movq %6, -3*8(%2)\n\t"
+                "movq %7, -4*8(%2)\n\t"
+                "leaq -4*8(%2), %2\n\t"
+                "jae 8b\n\t"
+                /*
+                 * Calculate copy position to head.
+                 */
+                "addq $0x20, %0\n\t"
+                "subq %0, %1\n\t"
+                "subq %0, %2\n\t"
+                "1:\n\t"
+                "cmpq $16, %0\n\t"
+                "jb 9f\n\t"
+                /*
+                 * Move data from 16 bytes to 31 bytes.
+                 */
+                "movq 0*8(%1), %4\n\t"
+                "movq 1*8(%1), %5\n\t"
+                "movq -2*8(%1, %0), %6\n\t"
+                "movq -1*8(%1, %0), %7\n\t"
+                "movq %4, 0*8(%2)\n\t"
+                "movq %5, 1*8(%2)\n\t"
+                "movq %6, -2*8(%2, %0)\n\t"
+                "movq %7, -1*8(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                ".p2align 4\n\t"
+                "9:\n\t"
+                "cmpq $8, %0\n\t"
+                "jb 10f\n\t"
+                /*
+                 * Move data from 8 bytes to 15 bytes.
+                 */
+                "movq 0*8(%1), %4\n\t"
+                "movq -1*8(%1, %0), %5\n\t"
+                "movq %4, 0*8(%2)\n\t"
+                "movq %5, -1*8(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                "10:\n\t"
+                "cmpq $4, %0\n\t"
+                "jb 11f\n\t"
+                /*
+                 * Move data from 4 bytes to 7 bytes.
+                 */
+                "movl (%1), %4d\n\t"
+                "movl -4(%1, %0), %5d\n\t"
+                "movl %4d, (%2)\n\t"
+                "movl %5d, -4(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                "11:\n\t"
+                "cmp $2, %0\n\t"
+                "jb 12f\n\t"
+                /*
+                 * Move data from 2 bytes to 3 bytes.
+                 */
+                "movw (%1), %4w\n\t"
+                "movw -2(%1, %0), %5w\n\t"
+                "movw %4w, (%2)\n\t"
+                "movw %5w, -2(%2, %0)\n\t"
+                "jmp 13f\n\t"
+                "12:\n\t"
+                "cmp $1, %0\n\t"
+                "jb 13f\n\t"
+                /*
+                 * Move data for 1 byte.
+                 */
+                "movb (%1), %4b\n\t"
+                "movb %4b, (%2)\n\t"
+                "13:\n\t"
+                : "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) ,
+                  "=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7)
+                :"0" (count),
+                 "1" (src),
+                 "2" (dest)
+                :"memory");
+
+                return ret;
+
 }
 EXPORT_SYMBOL(memmove);
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index a24c6cfdccc4..79b0b372d2d0 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -229,7 +229,16 @@ void vmalloc_sync_all(void)
 
                 spin_lock_irqsave(&pgd_lock, flags);
                 list_for_each_entry(page, &pgd_list, lru) {
-                        if (!vmalloc_sync_one(page_address(page), address))
+                        spinlock_t *pgt_lock;
+                        pmd_t *ret;
+
+                        pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
+
+                        spin_lock(pgt_lock);
+                        ret = vmalloc_sync_one(page_address(page), address);
+                        spin_unlock(pgt_lock);
+
+                        if (!ret)
                                 break;
                 }
                 spin_unlock_irqrestore(&pgd_lock, flags);
@@ -328,29 +337,7 @@ out:
 
 void vmalloc_sync_all(void)
 {
-        unsigned long address;
-
-        for (address = VMALLOC_START & PGDIR_MASK; address <= VMALLOC_END;
-             address += PGDIR_SIZE) {
-
-                const pgd_t *pgd_ref = pgd_offset_k(address);
-                unsigned long flags;
-                struct page *page;
-
-                if (pgd_none(*pgd_ref))
-                        continue;
-
-                spin_lock_irqsave(&pgd_lock, flags);
-                list_for_each_entry(page, &pgd_list, lru) {
-                        pgd_t *pgd;
-                        pgd = (pgd_t *)page_address(page) + pgd_index(address);
-                        if (pgd_none(*pgd))
-                                set_pgd(pgd, *pgd_ref);
-                        else
-                                BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
-                }
-                spin_unlock_irqrestore(&pgd_lock, flags);
-        }
+        sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
 }
 
 /*
@@ -898,8 +885,14 @@ spurious_fault(unsigned long error_code, unsigned long address)
         if (pmd_large(*pmd))
                 return spurious_fault_check(error_code, (pte_t *) pmd);
 
+        /*
+         * Note: don't use pte_present() here, since it returns true
+         * if the _PAGE_PROTNONE bit is set.  However, this aliases the
+         * _PAGE_GLOBAL bit, which for kernel pages give false positives
+         * when CONFIG_DEBUG_PAGEALLOC is used.
+         */
         pte = pte_offset_kernel(pmd, address);
-        if (!pte_present(*pte))
+        if (!(pte_flags(*pte) & _PAGE_PRESENT))
                 return 0;
 
         ret = spurious_fault_check(error_code, pte);
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index bca79091b9d6..558f2d332076 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -67,7 +67,7 @@ static __init void *alloc_low_page(void)
                 panic("alloc_low_page: ran out of memory");
 
         adr = __va(pfn * PAGE_SIZE);
-        memset(adr, 0, PAGE_SIZE);
+        clear_page(adr);
         return adr;
 }
 
@@ -558,7 +558,7 @@ char swsusp_pg_dir[PAGE_SIZE]
 
 static inline void save_pg_dir(void)
 {
-        memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
+        copy_page(swsusp_pg_dir, swapper_pg_dir);
 }
 #else /* !CONFIG_ACPI_SLEEP */
 static inline void save_pg_dir(void)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 9a6674689a20..c55f900fbf89 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -98,6 +98,43 @@ static int __init nonx32_setup(char *str)
 __setup("noexec32=", nonx32_setup);
 
 /*
+ * When memory was added/removed make sure all the processes MM have
+ * suitable PGD entries in the local PGD level page.
+ */
+void sync_global_pgds(unsigned long start, unsigned long end)
+{
+        unsigned long address;
+
+        for (address = start; address <= end; address += PGDIR_SIZE) {
+                const pgd_t *pgd_ref = pgd_offset_k(address);
+                unsigned long flags;
+                struct page *page;
+
+                if (pgd_none(*pgd_ref))
+                        continue;
+
+                spin_lock_irqsave(&pgd_lock, flags);
+                list_for_each_entry(page, &pgd_list, lru) {
+                        pgd_t *pgd;
+                        spinlock_t *pgt_lock;
+
+                        pgd = (pgd_t *)page_address(page) + pgd_index(address);
+                        pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
+                        spin_lock(pgt_lock);
+
+                        if (pgd_none(*pgd))
+                                set_pgd(pgd, *pgd_ref);
+                        else
+                                BUG_ON(pgd_page_vaddr(*pgd)
+                                       != pgd_page_vaddr(*pgd_ref));
+
+                        spin_unlock(pgt_lock);
+                }
+                spin_unlock_irqrestore(&pgd_lock, flags);
+        }
+}
+
+/*
  * NOTE: This function is marked __ref because it calls __init function
  * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0.
  */
@@ -293,7 +330,7 @@ static __ref void *alloc_low_page(unsigned long *phys)
                 panic("alloc_low_page: ran out of memory");
 
         adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE);
-        memset(adr, 0, PAGE_SIZE);
+        clear_page(adr);
         *phys  = pfn * PAGE_SIZE;
         return adr;
 }
@@ -534,11 +571,13 @@ kernel_physical_mapping_init(unsigned long start,
                              unsigned long end,
                              unsigned long page_size_mask)
 {
-
+        bool pgd_changed = false;
         unsigned long next, last_map_addr = end;
+        unsigned long addr;
 
         start = (unsigned long)__va(start);
         end = (unsigned long)__va(end);
+        addr = start;
 
         for (; start < end; start = next) {
                 pgd_t *pgd = pgd_offset_k(start);
@@ -563,7 +602,12 @@ kernel_physical_mapping_init(unsigned long start,
                 spin_lock(&init_mm.page_table_lock);
                 pgd_populate(&init_mm, pgd, __va(pud_phys));
                 spin_unlock(&init_mm.page_table_lock);
+                pgd_changed = true;
         }
+
+        if (pgd_changed)
+                sync_global_pgds(addr, end);
+
         __flush_tlb_all();
 
         return last_map_addr;
@@ -1003,6 +1047,7 @@ vmemmap_populate(struct page *start_page, unsigned long size, int node)
                 }
 
         }
+        sync_global_pgds((unsigned long)start_page, end);
         return 0;
 }
 
diff --git a/arch/x86/mm/k8topology_64.c b/arch/x86/mm/k8topology_64.c
index 970ed579d4e4..52d54bfc1ebb 100644
--- a/arch/x86/mm/k8topology_64.c
+++ b/arch/x86/mm/k8topology_64.c
@@ -22,7 +22,7 @@
 #include <asm/numa.h>
 #include <asm/mpspec.h>
 #include <asm/apic.h>
-#include <asm/k8.h>
+#include <asm/amd_nb.h>
 
 static struct bootnode __initdata nodes[8];
 static nodemask_t __initdata nodes_parsed = NODE_MASK_NONE;
@@ -54,8 +54,8 @@ static __init int find_northbridge(void)
 static __init void early_get_boot_cpu_id(void)
 {
         /*
-         * need to get boot_cpu_id so can use that to create apicid_to_node
-         * in k8_scan_nodes()
+         * need to get the APIC ID of the BSP so can use that to
+         * create apicid_to_node in k8_scan_nodes()
          */
 #ifdef CONFIG_X86_MPPARSE
         /*
@@ -212,7 +212,7 @@ int __init k8_scan_nodes(void)
         bits = boot_cpu_data.x86_coreid_bits;
         cores = (1<<bits);
         apicid_base = 0;
-        /* need to get boot_cpu_id early for system with apicid lifting */
+        /* get the APIC ID of the BSP early for systems with apicid lifting */
         early_get_boot_cpu_id();
         if (boot_cpu_physical_apicid > 0) {
                 pr_info("BSP APIC ID: %02x\n", boot_cpu_physical_apicid);
diff --git a/arch/x86/mm/kmemcheck/opcode.c b/arch/x86/mm/kmemcheck/opcode.c
index 63c19e27aa6f..324aa3f07237 100644
--- a/arch/x86/mm/kmemcheck/opcode.c
+++ b/arch/x86/mm/kmemcheck/opcode.c
@@ -9,7 +9,7 @@ static bool opcode_is_prefix(uint8_t b)
                 b == 0xf0 || b == 0xf2 || b == 0xf3
                 /* Group 2 */
                 || b == 0x2e || b == 0x36 || b == 0x3e || b == 0x26
-                || b == 0x64 || b == 0x65 || b == 0x2e || b == 0x3e
+                || b == 0x64 || b == 0x65
                 /* Group 3 */
                 || b == 0x66
                 /* Group 4 */
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index a7bcc23ef96c..4962f1aeda6f 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -18,7 +18,7 @@
 #include <asm/dma.h>
 #include <asm/numa.h>
 #include <asm/acpi.h>
-#include <asm/k8.h>
+#include <asm/amd_nb.h>
 
 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
 EXPORT_SYMBOL(node_data);
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 5c4ee422590e..8be8c7d7bc89 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -87,7 +87,19 @@ static inline void pgd_list_del(pgd_t *pgd)
 #define UNSHARED_PTRS_PER_PGD                           \
         (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
 
-static void pgd_ctor(pgd_t *pgd)
+
+static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
+{
+        BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
+        virt_to_page(pgd)->index = (pgoff_t)mm;
+}
+
+struct mm_struct *pgd_page_get_mm(struct page *page)
+{
+        return (struct mm_struct *)page->index;
+}
+
+static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
 {
         /* If the pgd points to a shared pagetable level (either the
            ptes in non-PAE, or shared PMD in PAE), then just copy the
@@ -98,15 +110,13 @@ static void pgd_ctor(pgd_t *pgd)
                 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
                                 swapper_pg_dir + KERNEL_PGD_BOUNDARY,
                                 KERNEL_PGD_PTRS);
-                paravirt_alloc_pmd_clone(__pa(pgd) >> PAGE_SHIFT,
-                                         __pa(swapper_pg_dir) >> PAGE_SHIFT,
-                                         KERNEL_PGD_BOUNDARY,
-                                         KERNEL_PGD_PTRS);
         }
 
         /* list required to sync kernel mapping updates */
-        if (!SHARED_KERNEL_PMD)
+        if (!SHARED_KERNEL_PMD) {
+                pgd_set_mm(pgd, mm);
                 pgd_list_add(pgd);
+        }
 }
 
 static void pgd_dtor(pgd_t *pgd)
@@ -272,7 +282,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
          */
         spin_lock_irqsave(&pgd_lock, flags);
 
-        pgd_ctor(pgd);
+        pgd_ctor(mm, pgd);
         pgd_prepopulate_pmd(mm, pgd, pmds);
 
         spin_unlock_irqrestore(&pgd_lock, flags);
diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c
index f9897f7a9ef1..9c0d0d399c30 100644
--- a/arch/x86/mm/srat_64.c
+++ b/arch/x86/mm/srat_64.c
@@ -420,9 +420,11 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end)
                 return -1;
         }
 
-        for_each_node_mask(i, nodes_parsed)
-                e820_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
-                                                nodes[i].end >> PAGE_SHIFT);
+        for (i = 0; i < num_node_memblks; i++)
+                e820_register_active_regions(memblk_nodeid[i],
+                                node_memblk_range[i].start >> PAGE_SHIFT,
+                                node_memblk_range[i].end >> PAGE_SHIFT);
+
         /* for out of order entries in SRAT */
         sort_node_map();
         if (!nodes_cover_memory(nodes)) {
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index c03f14ab6667..49358481c733 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -5,6 +5,7 @@
 #include <linux/smp.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/cpu.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
@@ -52,6 +53,8 @@ union smp_flush_state {
    want false sharing in the per cpu data segment. */
 static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
 
+static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
+
 /*
  * We cannot call mmdrop() because we are in interrupt context,
  * instead update mm->cpu_vm_mask.
@@ -173,7 +176,7 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
         union smp_flush_state *f;
 
         /* Caller has disabled preemption */
-        sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
+        sender = this_cpu_read(tlb_vector_offset);
         f = &flush_state[sender];
 
         /*
@@ -218,6 +221,47 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
         flush_tlb_others_ipi(cpumask, mm, va);
 }
 
+static void __cpuinit calculate_tlb_offset(void)
+{
+        int cpu, node, nr_node_vecs;
+        /*
+         * we are changing tlb_vector_offset for each CPU in runtime, but this
+         * will not cause inconsistency, as the write is atomic under X86. we
+         * might see more lock contentions in a short time, but after all CPU's
+         * tlb_vector_offset are changed, everything should go normal
+         *
+         * Note: if NUM_INVALIDATE_TLB_VECTORS % nr_online_nodes !=0, we might
+         * waste some vectors.
+         **/
+        if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
+                nr_node_vecs = 1;
+        else
+                nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;
+
+        for_each_online_node(node) {
+                int node_offset = (node % NUM_INVALIDATE_TLB_VECTORS) *
+                        nr_node_vecs;
+                int cpu_offset = 0;
+                for_each_cpu(cpu, cpumask_of_node(node)) {
+                        per_cpu(tlb_vector_offset, cpu) = node_offset +
+                                cpu_offset;
+                        cpu_offset++;
+                        cpu_offset = cpu_offset % nr_node_vecs;
+                }
+        }
+}
+
+static int tlb_cpuhp_notify(struct notifier_block *n,
+                unsigned long action, void *hcpu)
+{
+        switch (action & 0xf) {
+        case CPU_ONLINE:
+        case CPU_DEAD:
+                calculate_tlb_offset();
+        }
+        return NOTIFY_OK;
+}
+
 static int __cpuinit init_smp_flush(void)
 {
         int i;
@@ -225,6 +269,8 @@ static int __cpuinit init_smp_flush(void)
         for (i = 0; i < ARRAY_SIZE(flush_state); i++)
                 raw_spin_lock_init(&flush_state[i].tlbstate_lock);
 
+        calculate_tlb_offset();
+        hotcpu_notifier(tlb_cpuhp_notify, 0);
         return 0;
 }
 core_initcall(init_smp_flush);
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c
index 65f0a1eb6b86..8d17db266bbf 100644
--- a/arch/x86/oprofile/op_model_amd.c
+++ b/arch/x86/oprofile/op_model_amd.c
@@ -71,7 +71,7 @@ static struct ibs_state ibs_state;
  * IBS cpuid feature detection
  */
 
-#define IBS_CPUID_FEATURES      0x8000001b
+#define IBS_CPUID_FEATURES              0x8000001b
 
 /*
  * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
@@ -314,6 +314,74 @@ static void op_amd_stop_ibs(void)
                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
 }
 
+static inline int eilvt_is_available(int offset)
+{
+        /* check if we may assign a vector */
+        return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
+}
+
+static inline int ibs_eilvt_valid(void)
+{
+        u64 val;
+        int offset;
+
+        rdmsrl(MSR_AMD64_IBSCTL, val);
+        if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
+                pr_err(FW_BUG "cpu %d, invalid IBS "
+                       "interrupt offset %d (MSR%08X=0x%016llx)",
+                       smp_processor_id(), offset,
+                       MSR_AMD64_IBSCTL, val);
+                return 0;
+        }
+
+        offset = val & IBSCTL_LVT_OFFSET_MASK;
+
+        if (eilvt_is_available(offset))
+                return !0;
+
+        pr_err(FW_BUG "cpu %d, IBS interrupt offset %d "
+               "not available (MSR%08X=0x%016llx)",
+               smp_processor_id(), offset,
+               MSR_AMD64_IBSCTL, val);
+
+        return 0;
+}
+
+static inline int get_ibs_offset(void)
+{
+        u64 val;
+
+        rdmsrl(MSR_AMD64_IBSCTL, val);
+        if (!(val & IBSCTL_LVT_OFFSET_VALID))
+                return -EINVAL;
+
+        return val & IBSCTL_LVT_OFFSET_MASK;
+}
+
+static void setup_APIC_ibs(void)
+{
+        int offset;
+
+        offset = get_ibs_offset();
+        if (offset < 0)
+                goto failed;
+
+        if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
+                return;
+failed:
+        pr_warn("oprofile: IBS APIC setup failed on cpu #%d\n",
+                smp_processor_id());
+}
+
+static void clear_APIC_ibs(void)
+{
+        int offset;
+
+        offset = get_ibs_offset();
+        if (offset >= 0)
+                setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
+}
+
 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 
 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
@@ -424,13 +492,13 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
         }
 
         if (ibs_caps)
-                setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
+                setup_APIC_ibs();
 }
 
 static void op_amd_cpu_shutdown(void)
 {
         if (ibs_caps)
-                setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
+                clear_APIC_ibs();
 }
 
 static int op_amd_check_ctrs(struct pt_regs * const regs,
@@ -493,16 +561,11 @@ static void op_amd_stop(struct op_msrs const * const msrs)
         op_amd_stop_ibs();
 }
 
-static int __init_ibs_nmi(void)
+static int setup_ibs_ctl(int ibs_eilvt_off)
 {
-#define IBSCTL_LVTOFFSETVAL             (1 << 8)
-#define IBSCTL                          0x1cc
         struct pci_dev *cpu_cfg;
         int nodes;
         u32 value = 0;
-        u8 ibs_eilvt_off;
-
-        ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 
         nodes = 0;
         cpu_cfg = NULL;
@@ -514,21 +577,60 @@ static int __init_ibs_nmi(void)
                         break;
                 ++nodes;
                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
-                                       | IBSCTL_LVTOFFSETVAL);
+                                       | IBSCTL_LVT_OFFSET_VALID);
                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
-                if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
+                if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
                         pci_dev_put(cpu_cfg);
                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
-                                "IBSCTL = 0x%08x", value);
-                        return 1;
+                               "IBSCTL = 0x%08x\n", value);
+                        return -EINVAL;
                 }
         } while (1);
 
         if (!nodes) {
-                printk(KERN_DEBUG "No CPU node configured for IBS");
-                return 1;
+                printk(KERN_DEBUG "No CPU node configured for IBS\n");
+                return -ENODEV;
+        }
+
+        return 0;
+}
+
+static int force_ibs_eilvt_setup(void)
+{
+        int i;
+        int ret;
+
+        /* find the next free available EILVT entry */
+        for (i = 1; i < 4; i++) {
+                if (!eilvt_is_available(i))
+                        continue;
+                ret = setup_ibs_ctl(i);
+                if (ret)
+                        return ret;
+                return 0;
         }
 
+        printk(KERN_DEBUG "No EILVT entry available\n");
+
+        return -EBUSY;
+}
+
+static int __init_ibs_nmi(void)
+{
+        int ret;
+
+        if (ibs_eilvt_valid())
+                return 0;
+
+        ret = force_ibs_eilvt_setup();
+        if (ret)
+                return ret;
+
+        if (!ibs_eilvt_valid())
+                return -EFAULT;
+
+        pr_err(FW_BUG "workaround enabled for IBS LVT offset\n");
+
         return 0;
 }
 
diff --git a/arch/x86/pci/olpc.c b/arch/x86/pci/olpc.c
index b34815408f58..13700ec8e2e4 100644
--- a/arch/x86/pci/olpc.c
+++ b/arch/x86/pci/olpc.c
@@ -304,7 +304,7 @@ static struct pci_raw_ops pci_olpc_conf = {
 
 int __init pci_olpc_init(void)
 {
-        printk(KERN_INFO "PCI: Using configuration type OLPC\n");
+        printk(KERN_INFO "PCI: Using configuration type OLPC XO-1\n");
         raw_pci_ops = &pci_olpc_conf;
         is_lx = is_geode_lx();
         return 0;
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 42086ac406af..b2363fcbcd0f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1969,7 +1969,6 @@ static const struct pv_mmu_ops xen_mmu_ops __initdata = {
         .alloc_pte = xen_alloc_pte_init,
         .release_pte = xen_release_pte_init,
         .alloc_pmd = xen_alloc_pmd_init,
-        .alloc_pmd_clone = paravirt_nop,
         .release_pmd = xen_release_pmd_init,
 
 #ifdef CONFIG_X86_64