Merge branch 'linus' into release

109 files changed, 6577 insertions, 5190 deletions

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index b62a7667828e..d364df03c1d6 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -12,6 +12,7 @@ CFLAGS_REMOVE_tsc.o = -pg
 CFLAGS_REMOVE_rtc.o = -pg
 CFLAGS_REMOVE_paravirt-spinlocks.o = -pg
 CFLAGS_REMOVE_ftrace.o = -pg
+CFLAGS_REMOVE_early_printk.o = -pg
 endif
 
 #
@@ -23,9 +24,9 @@ CFLAGS_vsyscall_64.o	:= $(PROFILING) -g0 $(nostackp)
 CFLAGS_hpet.o           := $(nostackp)
 CFLAGS_tsc.o            := $(nostackp)
 
-obj-y                   := process_$(BITS).o signal_$(BITS).o entry_$(BITS).o
+obj-y                   := process_$(BITS).o signal.o entry_$(BITS).o
 obj-y                   += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
-obj-y                   += time_$(BITS).o ioport.o ldt.o
+obj-y                   += time_$(BITS).o ioport.o ldt.o dumpstack.o
 obj-y                   += setup.o i8259.o irqinit_$(BITS).o setup_percpu.o
 obj-$(CONFIG_X86_VISWS) += visws_quirks.o
 obj-$(CONFIG_X86_32)    += probe_roms_32.o
@@ -65,6 +66,7 @@ obj-$(CONFIG_X86_LOCAL_APIC)	+= apic.o nmi.o
 obj-$(CONFIG_X86_IO_APIC)       += io_apic.o
 obj-$(CONFIG_X86_REBOOTFIXUPS)  += reboot_fixups_32.o
 obj-$(CONFIG_DYNAMIC_FTRACE)    += ftrace.o
+obj-$(CONFIG_FUNCTION_GRAPH_TRACER)     += ftrace.o
 obj-$(CONFIG_KEXEC)             += machine_kexec_$(BITS).o
 obj-$(CONFIG_KEXEC)             += relocate_kernel_$(BITS).o crash.o
 obj-$(CONFIG_CRASH_DUMP)        += crash_dump_$(BITS).o
@@ -105,6 +107,10 @@ microcode-$(CONFIG_MICROCODE_INTEL)	+= microcode_intel.o
 microcode-$(CONFIG_MICROCODE_AMD)       += microcode_amd.o
 obj-$(CONFIG_MICROCODE)                 += microcode.o
 
+obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o
+
+obj-$(CONFIG_SWIOTLB)                   += pci-swiotlb_64.o # NB rename without _64
+
 ###
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
@@ -118,7 +124,6 @@ ifeq ($(CONFIG_X86_64),y)
         obj-$(CONFIG_GART_IOMMU)        += pci-gart_64.o aperture_64.o
         obj-$(CONFIG_CALGARY_IOMMU)     += pci-calgary_64.o tce_64.o
         obj-$(CONFIG_AMD_IOMMU)         += amd_iommu_init.o amd_iommu.o
-        obj-$(CONFIG_SWIOTLB)           += pci-swiotlb_64.o
 
         obj-$(CONFIG_PCI_MMCONFIG)      += mmconf-fam10h_64.o
 endif
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 39ae3d0e3a4a..d37593c2f438 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -538,9 +538,10 @@ static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu)
         struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
         union acpi_object *obj;
         struct acpi_madt_local_apic *lapic;
-        cpumask_t tmp_map, new_map;
+        cpumask_var_t tmp_map, new_map;
         u8 physid;
         int cpu;
+        int retval = -ENOMEM;
 
         if (ACPI_FAILURE(acpi_evaluate_object(handle, "_MAT", NULL, &buffer)))
                 return -EINVAL;
@@ -569,23 +570,37 @@ static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu)
         buffer.length = ACPI_ALLOCATE_BUFFER;
         buffer.pointer = NULL;
 
-        tmp_map = cpu_present_map;
+        if (!alloc_cpumask_var(&tmp_map, GFP_KERNEL))
+                goto out;
+
+        if (!alloc_cpumask_var(&new_map, GFP_KERNEL))
+                goto free_tmp_map;
+
+        cpumask_copy(tmp_map, cpu_present_mask);
         acpi_register_lapic(physid, lapic->lapic_flags & ACPI_MADT_ENABLED);
 
         /*
          * If mp_register_lapic successfully generates a new logical cpu
          * number, then the following will get us exactly what was mapped
          */
-        cpus_andnot(new_map, cpu_present_map, tmp_map);
-        if (cpus_empty(new_map)) {
+        cpumask_andnot(new_map, cpu_present_mask, tmp_map);
+        if (cpumask_empty(new_map)) {
                 printk ("Unable to map lapic to logical cpu number\n");
-                return -EINVAL;
+                retval = -EINVAL;
+                goto free_new_map;
         }
 
-        cpu = first_cpu(new_map);
+        cpu = cpumask_first(new_map);
 
         *pcpu = cpu;
-        return 0;
+        retval = 0;
+
+free_new_map:
+        free_cpumask_var(new_map);
+free_tmp_map:
+        free_cpumask_var(tmp_map);
+out:
+        return retval;
 }
 
 /* wrapper to silence section mismatch warning */
@@ -598,7 +613,7 @@ EXPORT_SYMBOL(acpi_map_lsapic);
 int acpi_unmap_lsapic(int cpu)
 {
         per_cpu(x86_cpu_to_apicid, cpu) = -1;
-        cpu_clear(cpu, cpu_present_map);
+        set_cpu_present(cpu, false);
         num_processors--;
 
         return (0);
@@ -1371,6 +1386,17 @@ static void __init acpi_process_madt(void)
                         smp_found_config = 0;
                 }
         }
+
+        /*
+         * ACPI supports both logical (e.g. Hyper-Threading) and physical
+         * processors, where MPS only supports physical.
+         */
+        if (acpi_lapic && acpi_ioapic)
+                printk(KERN_INFO "Using ACPI (MADT) for SMP configuration "
+                       "information\n");
+        else if (acpi_lapic)
+                printk(KERN_INFO "Using ACPI for processor (LAPIC) "
+                       "configuration information\n");
 #endif
         return;
 }
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 0a60d60ed036..5113c080f0c4 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -20,10 +20,15 @@
 #include <linux/pci.h>
 #include <linux/gfp.h>
 #include <linux/bitops.h>
+#include <linux/debugfs.h>
 #include <linux/scatterlist.h>
 #include <linux/iommu-helper.h>
+#ifdef CONFIG_IOMMU_API
+#include <linux/iommu.h>
+#endif
 #include <asm/proto.h>
 #include <asm/iommu.h>
+#include <asm/gart.h>
 #include <asm/amd_iommu_types.h>
 #include <asm/amd_iommu.h>
 
@@ -37,6 +42,10 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock);
 static LIST_HEAD(iommu_pd_list);
 static DEFINE_SPINLOCK(iommu_pd_list_lock);
 
+#ifdef CONFIG_IOMMU_API
+static struct iommu_ops amd_iommu_ops;
+#endif
+
 /*
  * general struct to manage commands send to an IOMMU
  */
@@ -46,6 +55,68 @@ struct iommu_cmd {
 
 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
                              struct unity_map_entry *e);
+static struct dma_ops_domain *find_protection_domain(u16 devid);
+
+
+#ifdef CONFIG_AMD_IOMMU_STATS
+
+/*
+ * Initialization code for statistics collection
+ */
+
+DECLARE_STATS_COUNTER(compl_wait);
+DECLARE_STATS_COUNTER(cnt_map_single);
+DECLARE_STATS_COUNTER(cnt_unmap_single);
+DECLARE_STATS_COUNTER(cnt_map_sg);
+DECLARE_STATS_COUNTER(cnt_unmap_sg);
+DECLARE_STATS_COUNTER(cnt_alloc_coherent);
+DECLARE_STATS_COUNTER(cnt_free_coherent);
+DECLARE_STATS_COUNTER(cross_page);
+DECLARE_STATS_COUNTER(domain_flush_single);
+DECLARE_STATS_COUNTER(domain_flush_all);
+DECLARE_STATS_COUNTER(alloced_io_mem);
+DECLARE_STATS_COUNTER(total_map_requests);
+
+static struct dentry *stats_dir;
+static struct dentry *de_isolate;
+static struct dentry *de_fflush;
+
+static void amd_iommu_stats_add(struct __iommu_counter *cnt)
+{
+        if (stats_dir == NULL)
+                return;
+
+        cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
+                                       &cnt->value);
+}
+
+static void amd_iommu_stats_init(void)
+{
+        stats_dir = debugfs_create_dir("amd-iommu", NULL);
+        if (stats_dir == NULL)
+                return;
+
+        de_isolate = debugfs_create_bool("isolation", 0444, stats_dir,
+                                         (u32 *)&amd_iommu_isolate);
+
+        de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
+                                         (u32 *)&amd_iommu_unmap_flush);
+
+        amd_iommu_stats_add(&compl_wait);
+        amd_iommu_stats_add(&cnt_map_single);
+        amd_iommu_stats_add(&cnt_unmap_single);
+        amd_iommu_stats_add(&cnt_map_sg);
+        amd_iommu_stats_add(&cnt_unmap_sg);
+        amd_iommu_stats_add(&cnt_alloc_coherent);
+        amd_iommu_stats_add(&cnt_free_coherent);
+        amd_iommu_stats_add(&cross_page);
+        amd_iommu_stats_add(&domain_flush_single);
+        amd_iommu_stats_add(&domain_flush_all);
+        amd_iommu_stats_add(&alloced_io_mem);
+        amd_iommu_stats_add(&total_map_requests);
+}
+
+#endif
 
 /* returns !0 if the IOMMU is caching non-present entries in its TLB */
 static int iommu_has_npcache(struct amd_iommu *iommu)
@@ -188,13 +259,55 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
         spin_lock_irqsave(&iommu->lock, flags);
         ret = __iommu_queue_command(iommu, cmd);
         if (!ret)
-                iommu->need_sync = 1;
+                iommu->need_sync = true;
         spin_unlock_irqrestore(&iommu->lock, flags);
 
         return ret;
 }
 
 /*
+ * This function waits until an IOMMU has completed a completion
+ * wait command
+ */
+static void __iommu_wait_for_completion(struct amd_iommu *iommu)
+{
+        int ready = 0;
+        unsigned status = 0;
+        unsigned long i = 0;
+
+        INC_STATS_COUNTER(compl_wait);
+
+        while (!ready && (i < EXIT_LOOP_COUNT)) {
+                ++i;
+                /* wait for the bit to become one */
+                status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
+                ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
+        }
+
+        /* set bit back to zero */
+        status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
+        writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
+
+        if (unlikely(i == EXIT_LOOP_COUNT))
+                panic("AMD IOMMU: Completion wait loop failed\n");
+}
+
+/*
+ * This function queues a completion wait command into the command
+ * buffer of an IOMMU
+ */
+static int __iommu_completion_wait(struct amd_iommu *iommu)
+{
+        struct iommu_cmd cmd;
+
+         memset(&cmd, 0, sizeof(cmd));
+         cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
+         CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
+
+         return __iommu_queue_command(iommu, &cmd);
+}
+
+/*
  * This function is called whenever we need to ensure that the IOMMU has
  * completed execution of all commands we sent. It sends a
  * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
@@ -203,40 +316,22 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  */
 static int iommu_completion_wait(struct amd_iommu *iommu)
 {
-        int ret = 0, ready = 0;
-        unsigned status = 0;
-        struct iommu_cmd cmd;
-        unsigned long flags, i = 0;
-
-        memset(&cmd, 0, sizeof(cmd));
-        cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
-        CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
+        int ret = 0;
+        unsigned long flags;
 
         spin_lock_irqsave(&iommu->lock, flags);
 
         if (!iommu->need_sync)
                 goto out;
 
-        iommu->need_sync = 0;
+        ret = __iommu_completion_wait(iommu);
 
-        ret = __iommu_queue_command(iommu, &cmd);
+        iommu->need_sync = false;
 
         if (ret)
                 goto out;
 
-        while (!ready && (i < EXIT_LOOP_COUNT)) {
-                ++i;
-                /* wait for the bit to become one */
-                status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
-                ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
-        }
-
-        /* set bit back to zero */
-        status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
-        writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
-
-        if (unlikely(i == EXIT_LOOP_COUNT))
-                panic("AMD IOMMU: Completion wait loop failed\n");
+        __iommu_wait_for_completion(iommu);
 
 out:
         spin_unlock_irqrestore(&iommu->lock, flags);
@@ -263,6 +358,21 @@ static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
         return ret;
 }
 
+static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
+                                          u16 domid, int pde, int s)
+{
+        memset(cmd, 0, sizeof(*cmd));
+        address &= PAGE_MASK;
+        CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
+        cmd->data[1] |= domid;
+        cmd->data[2] = lower_32_bits(address);
+        cmd->data[3] = upper_32_bits(address);
+        if (s) /* size bit - we flush more than one 4kb page */
+                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
+        if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
+                cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
+}
+
 /*
  * Generic command send function for invalidaing TLB entries
  */
@@ -272,16 +382,7 @@ static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
         struct iommu_cmd cmd;
         int ret;
 
-        memset(&cmd, 0, sizeof(cmd));
-        address &= PAGE_MASK;
-        CMD_SET_TYPE(&cmd, CMD_INV_IOMMU_PAGES);
-        cmd.data[1] |= domid;
-        cmd.data[2] = lower_32_bits(address);
-        cmd.data[3] = upper_32_bits(address);
-        if (s) /* size bit - we flush more than one 4kb page */
-                cmd.data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
-        if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
-                cmd.data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
+        __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
 
         ret = iommu_queue_command(iommu, &cmd);
 
@@ -320,9 +421,35 @@ static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
 {
         u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
 
+        INC_STATS_COUNTER(domain_flush_single);
+
         iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
 }
 
+/*
+ * This function is used to flush the IO/TLB for a given protection domain
+ * on every IOMMU in the system
+ */
+static void iommu_flush_domain(u16 domid)
+{
+        unsigned long flags;
+        struct amd_iommu *iommu;
+        struct iommu_cmd cmd;
+
+        INC_STATS_COUNTER(domain_flush_all);
+
+        __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+                                      domid, 1, 1);
+
+        list_for_each_entry(iommu, &amd_iommu_list, list) {
+                spin_lock_irqsave(&iommu->lock, flags);
+                __iommu_queue_command(iommu, &cmd);
+                __iommu_completion_wait(iommu);
+                __iommu_wait_for_completion(iommu);
+                spin_unlock_irqrestore(&iommu->lock, flags);
+        }
+}
+
 /****************************************************************************
  *
  * The functions below are used the create the page table mappings for
@@ -337,10 +464,10 @@ static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
  * supporting all features of AMD IOMMU page tables like level skipping
  * and full 64 bit address spaces.
  */
-static int iommu_map(struct protection_domain *dom,
-                     unsigned long bus_addr,
-                     unsigned long phys_addr,
-                     int prot)
+static int iommu_map_page(struct protection_domain *dom,
+                          unsigned long bus_addr,
+                          unsigned long phys_addr,
+                          int prot)
 {
         u64 __pte, *pte, *page;
 
@@ -387,6 +514,28 @@ static int iommu_map(struct protection_domain *dom,
         return 0;
 }
 
+static void iommu_unmap_page(struct protection_domain *dom,
+                             unsigned long bus_addr)
+{
+        u64 *pte;
+
+        pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return;
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return;
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
+
+        *pte = 0;
+}
+
 /*
  * This function checks if a specific unity mapping entry is needed for
  * this specific IOMMU.
@@ -439,7 +588,7 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
 
         for (addr = e->address_start; addr < e->address_end;
              addr += PAGE_SIZE) {
-                ret = iommu_map(&dma_dom->domain, addr, addr, e->prot);
+                ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot);
                 if (ret)
                         return ret;
                 /*
@@ -570,6 +719,16 @@ static u16 domain_id_alloc(void)
         return id;
 }
 
+static void domain_id_free(int id)
+{
+        unsigned long flags;
+
+        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
+        if (id > 0 && id < MAX_DOMAIN_ID)
+                __clear_bit(id, amd_iommu_pd_alloc_bitmap);
+        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+}
+
 /*
  * Used to reserve address ranges in the aperture (e.g. for exclusion
  * ranges.
@@ -586,12 +745,12 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
         iommu_area_reserve(dom->bitmap, start_page, pages);
 }
 
-static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
+static void free_pagetable(struct protection_domain *domain)
 {
         int i, j;
         u64 *p1, *p2, *p3;
 
-        p1 = dma_dom->domain.pt_root;
+        p1 = domain->pt_root;
 
         if (!p1)
                 return;
@@ -612,6 +771,8 @@ static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
         }
 
         free_page((unsigned long)p1);
+
+        domain->pt_root = NULL;
 }
 
 /*
@@ -623,7 +784,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
         if (!dom)
                 return;
 
-        dma_ops_free_pagetable(dom);
+        free_pagetable(&dom->domain);
 
         kfree(dom->pte_pages);
 
@@ -662,6 +823,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
                 goto free_dma_dom;
         dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
         dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
+        dma_dom->domain.flags = PD_DMA_OPS_MASK;
         dma_dom->domain.priv = dma_dom;
         if (!dma_dom->domain.pt_root)
                 goto free_dma_dom;
@@ -724,6 +886,15 @@ free_dma_dom:
 }
 
 /*
+ * little helper function to check whether a given protection domain is a
+ * dma_ops domain
+ */
+static bool dma_ops_domain(struct protection_domain *domain)
+{
+        return domain->flags & PD_DMA_OPS_MASK;
+}
+
+/*
  * Find out the protection domain structure for a given PCI device. This
  * will give us the pointer to the page table root for example.
  */
@@ -743,14 +914,15 @@ static struct protection_domain *domain_for_device(u16 devid)
  * If a device is not yet associated with a domain, this function does
  * assigns it visible for the hardware
  */
-static void set_device_domain(struct amd_iommu *iommu,
-                              struct protection_domain *domain,
-                              u16 devid)
+static void attach_device(struct amd_iommu *iommu,
+                          struct protection_domain *domain,
+                          u16 devid)
 {
         unsigned long flags;
-
         u64 pte_root = virt_to_phys(domain->pt_root);
 
+        domain->dev_cnt += 1;
+
         pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
                     << DEV_ENTRY_MODE_SHIFT;
         pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
@@ -766,6 +938,116 @@ static void set_device_domain(struct amd_iommu *iommu,
         iommu_queue_inv_dev_entry(iommu, devid);
 }
 
+/*
+ * Removes a device from a protection domain (unlocked)
+ */
+static void __detach_device(struct protection_domain *domain, u16 devid)
+{
+
+        /* lock domain */
+        spin_lock(&domain->lock);
+
+        /* remove domain from the lookup table */
+        amd_iommu_pd_table[devid] = NULL;
+
+        /* remove entry from the device table seen by the hardware */
+        amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
+        amd_iommu_dev_table[devid].data[1] = 0;
+        amd_iommu_dev_table[devid].data[2] = 0;
+
+        /* decrease reference counter */
+        domain->dev_cnt -= 1;
+
+        /* ready */
+        spin_unlock(&domain->lock);
+}
+
+/*
+ * Removes a device from a protection domain (with devtable_lock held)
+ */
+static void detach_device(struct protection_domain *domain, u16 devid)
+{
+        unsigned long flags;
+
+        /* lock device table */
+        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
+        __detach_device(domain, devid);
+        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+}
+
+static int device_change_notifier(struct notifier_block *nb,
+                                  unsigned long action, void *data)
+{
+        struct device *dev = data;
+        struct pci_dev *pdev = to_pci_dev(dev);
+        u16 devid = calc_devid(pdev->bus->number, pdev->devfn);
+        struct protection_domain *domain;
+        struct dma_ops_domain *dma_domain;
+        struct amd_iommu *iommu;
+        int order = amd_iommu_aperture_order;
+        unsigned long flags;
+
+        if (devid > amd_iommu_last_bdf)
+                goto out;
+
+        devid = amd_iommu_alias_table[devid];
+
+        iommu = amd_iommu_rlookup_table[devid];
+        if (iommu == NULL)
+                goto out;
+
+        domain = domain_for_device(devid);
+
+        if (domain && !dma_ops_domain(domain))
+                WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
+                          "to a non-dma-ops domain\n", dev_name(dev));
+
+        switch (action) {
+        case BUS_NOTIFY_BOUND_DRIVER:
+                if (domain)
+                        goto out;
+                dma_domain = find_protection_domain(devid);
+                if (!dma_domain)
+                        dma_domain = iommu->default_dom;
+                attach_device(iommu, &dma_domain->domain, devid);
+                printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
+                       "device %s\n", dma_domain->domain.id, dev_name(dev));
+                break;
+        case BUS_NOTIFY_UNBIND_DRIVER:
+                if (!domain)
+                        goto out;
+                detach_device(domain, devid);
+                break;
+        case BUS_NOTIFY_ADD_DEVICE:
+                /* allocate a protection domain if a device is added */
+                dma_domain = find_protection_domain(devid);
+                if (dma_domain)
+                        goto out;
+                dma_domain = dma_ops_domain_alloc(iommu, order);
+                if (!dma_domain)
+                        goto out;
+                dma_domain->target_dev = devid;
+
+                spin_lock_irqsave(&iommu_pd_list_lock, flags);
+                list_add_tail(&dma_domain->list, &iommu_pd_list);
+                spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
+
+                break;
+        default:
+                goto out;
+        }
+
+        iommu_queue_inv_dev_entry(iommu, devid);
+        iommu_completion_wait(iommu);
+
+out:
+        return 0;
+}
+
+struct notifier_block device_nb = {
+        .notifier_call = device_change_notifier,
+};
+
 /*****************************************************************************
  *
  * The next functions belong to the dma_ops mapping/unmapping code.
@@ -801,7 +1083,6 @@ static struct dma_ops_domain *find_protection_domain(u16 devid)
         list_for_each_entry(entry, &iommu_pd_list, list) {
                 if (entry->target_dev == devid) {
                         ret = entry;
-                        list_del(&ret->list);
                         break;
                 }
         }
@@ -852,14 +1133,13 @@ static int get_device_resources(struct device *dev,
                 if (!dma_dom)
                         dma_dom = (*iommu)->default_dom;
                 *domain = &dma_dom->domain;
-                set_device_domain(*iommu, *domain, *bdf);
+                attach_device(*iommu, *domain, *bdf);
                 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
-                                "device ", (*domain)->id);
-                print_devid(_bdf, 1);
+                                "device %s\n", (*domain)->id, dev_name(dev));
         }
 
         if (domain_for_device(_bdf) == NULL)
-                set_device_domain(*iommu, *domain, _bdf);
+                attach_device(*iommu, *domain, _bdf);
 
         return 1;
 }
@@ -945,6 +1225,11 @@ static dma_addr_t __map_single(struct device *dev,
         pages = iommu_num_pages(paddr, size, PAGE_SIZE);
         paddr &= PAGE_MASK;
 
+        INC_STATS_COUNTER(total_map_requests);
+
+        if (pages > 1)
+                INC_STATS_COUNTER(cross_page);
+
         if (align)
                 align_mask = (1UL << get_order(size)) - 1;
 
@@ -961,6 +1246,8 @@ static dma_addr_t __map_single(struct device *dev,
         }
         address += offset;
 
+        ADD_STATS_COUNTER(alloced_io_mem, size);
+
         if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
                 iommu_flush_tlb(iommu, dma_dom->domain.id);
                 dma_dom->need_flush = false;
@@ -997,6 +1284,8 @@ static void __unmap_single(struct amd_iommu *iommu,
                 start += PAGE_SIZE;
         }
 
+        SUB_STATS_COUNTER(alloced_io_mem, size);
+
         dma_ops_free_addresses(dma_dom, dma_addr, pages);
 
         if (amd_iommu_unmap_flush || dma_dom->need_flush) {
@@ -1018,6 +1307,8 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
         dma_addr_t addr;
         u64 dma_mask;
 
+        INC_STATS_COUNTER(cnt_map_single);
+
         if (!check_device(dev))
                 return bad_dma_address;
 
@@ -1029,6 +1320,9 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
                 /* device not handled by any AMD IOMMU */
                 return (dma_addr_t)paddr;
 
+        if (!dma_ops_domain(domain))
+                return bad_dma_address;
+
         spin_lock_irqsave(&domain->lock, flags);
         addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
                             dma_mask);
@@ -1054,11 +1348,16 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,
         struct protection_domain *domain;
         u16 devid;
 
+        INC_STATS_COUNTER(cnt_unmap_single);
+
         if (!check_device(dev) ||
             !get_device_resources(dev, &iommu, &domain, &devid))
                 /* device not handled by any AMD IOMMU */
                 return;
 
+        if (!dma_ops_domain(domain))
+                return;
+
         spin_lock_irqsave(&domain->lock, flags);
 
         __unmap_single(iommu, domain->priv, dma_addr, size, dir);
@@ -1103,6 +1402,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
         int mapped_elems = 0;
         u64 dma_mask;
 
+        INC_STATS_COUNTER(cnt_map_sg);
+
         if (!check_device(dev))
                 return 0;
 
@@ -1113,6 +1414,9 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
         if (!iommu || !domain)
                 return map_sg_no_iommu(dev, sglist, nelems, dir);
 
+        if (!dma_ops_domain(domain))
+                return 0;
+
         spin_lock_irqsave(&domain->lock, flags);
 
         for_each_sg(sglist, s, nelems, i) {
@@ -1162,10 +1466,15 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
         u16 devid;
         int i;
 
+        INC_STATS_COUNTER(cnt_unmap_sg);
+
         if (!check_device(dev) ||
             !get_device_resources(dev, &iommu, &domain, &devid))
                 return;
 
+        if (!dma_ops_domain(domain))
+                return;
+
         spin_lock_irqsave(&domain->lock, flags);
 
         for_each_sg(sglist, s, nelems, i) {
@@ -1193,6 +1502,8 @@ static void *alloc_coherent(struct device *dev, size_t size,
         phys_addr_t paddr;
         u64 dma_mask = dev->coherent_dma_mask;
 
+        INC_STATS_COUNTER(cnt_alloc_coherent);
+
         if (!check_device(dev))
                 return NULL;
 
@@ -1211,6 +1522,9 @@ static void *alloc_coherent(struct device *dev, size_t size,
                 return virt_addr;
         }
 
+        if (!dma_ops_domain(domain))
+                goto out_free;
+
         if (!dma_mask)
                 dma_mask = *dev->dma_mask;
 
@@ -1219,18 +1533,20 @@ static void *alloc_coherent(struct device *dev, size_t size,
         *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
                                  size, DMA_BIDIRECTIONAL, true, dma_mask);
 
-        if (*dma_addr == bad_dma_address) {
-                free_pages((unsigned long)virt_addr, get_order(size));
-                virt_addr = NULL;
-                goto out;
-        }
+        if (*dma_addr == bad_dma_address)
+                goto out_free;
 
         iommu_completion_wait(iommu);
 
-out:
         spin_unlock_irqrestore(&domain->lock, flags);
 
         return virt_addr;
+
+out_free:
+
+        free_pages((unsigned long)virt_addr, get_order(size));
+
+        return NULL;
 }
 
 /*
@@ -1244,6 +1560,8 @@ static void free_coherent(struct device *dev, size_t size,
         struct protection_domain *domain;
         u16 devid;
 
+        INC_STATS_COUNTER(cnt_free_coherent);
+
         if (!check_device(dev))
                 return;
 
@@ -1252,6 +1570,9 @@ static void free_coherent(struct device *dev, size_t size,
         if (!iommu || !domain)
                 goto free_mem;
 
+        if (!dma_ops_domain(domain))
+                goto free_mem;
+
         spin_lock_irqsave(&domain->lock, flags);
 
         __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
@@ -1295,7 +1616,7 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
  * we don't need to preallocate the protection domains anymore.
  * For now we have to.
  */
-void prealloc_protection_domains(void)
+static void prealloc_protection_domains(void)
 {
         struct pci_dev *dev = NULL;
         struct dma_ops_domain *dma_dom;
@@ -1304,7 +1625,7 @@ void prealloc_protection_domains(void)
         u16 devid;
 
         while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
-                devid = (dev->bus->number << 8) | dev->devfn;
+                devid = calc_devid(dev->bus->number, dev->devfn);
                 if (devid > amd_iommu_last_bdf)
                         continue;
                 devid = amd_iommu_alias_table[devid];
@@ -1351,6 +1672,7 @@ int __init amd_iommu_init_dma_ops(void)
                 iommu->default_dom = dma_ops_domain_alloc(iommu, order);
                 if (iommu->default_dom == NULL)
                         return -ENOMEM;
+                iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
                 ret = iommu_init_unity_mappings(iommu);
                 if (ret)
                         goto free_domains;
@@ -1374,6 +1696,12 @@ int __init amd_iommu_init_dma_ops(void)
         /* Make the driver finally visible to the drivers */
         dma_ops = &amd_iommu_dma_ops;
 
+        register_iommu(&amd_iommu_ops);
+
+        bus_register_notifier(&pci_bus_type, &device_nb);
+
+        amd_iommu_stats_init();
+
         return 0;
 
 free_domains:
@@ -1385,3 +1713,224 @@ free_domains:
 
         return ret;
 }
+
+/*****************************************************************************
+ *
+ * The following functions belong to the exported interface of AMD IOMMU
+ *
+ * This interface allows access to lower level functions of the IOMMU
+ * like protection domain handling and assignement of devices to domains
+ * which is not possible with the dma_ops interface.
+ *
+ *****************************************************************************/
+
+static void cleanup_domain(struct protection_domain *domain)
+{
+        unsigned long flags;
+        u16 devid;
+
+        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
+
+        for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
+                if (amd_iommu_pd_table[devid] == domain)
+                        __detach_device(domain, devid);
+
+        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+}
+
+static int amd_iommu_domain_init(struct iommu_domain *dom)
+{
+        struct protection_domain *domain;
+
+        domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+        if (!domain)
+                return -ENOMEM;
+
+        spin_lock_init(&domain->lock);
+        domain->mode = PAGE_MODE_3_LEVEL;
+        domain->id = domain_id_alloc();
+        if (!domain->id)
+                goto out_free;
+        domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
+        if (!domain->pt_root)
+                goto out_free;
+
+        dom->priv = domain;
+
+        return 0;
+
+out_free:
+        kfree(domain);
+
+        return -ENOMEM;
+}
+
+static void amd_iommu_domain_destroy(struct iommu_domain *dom)
+{
+        struct protection_domain *domain = dom->priv;
+
+        if (!domain)
+                return;
+
+        if (domain->dev_cnt > 0)
+                cleanup_domain(domain);
+
+        BUG_ON(domain->dev_cnt != 0);
+
+        free_pagetable(domain);
+
+        domain_id_free(domain->id);
+
+        kfree(domain);
+
+        dom->priv = NULL;
+}
+
+static void amd_iommu_detach_device(struct iommu_domain *dom,
+                                    struct device *dev)
+{
+        struct protection_domain *domain = dom->priv;
+        struct amd_iommu *iommu;
+        struct pci_dev *pdev;
+        u16 devid;
+
+        if (dev->bus != &pci_bus_type)
+                return;
+
+        pdev = to_pci_dev(dev);
+
+        devid = calc_devid(pdev->bus->number, pdev->devfn);
+
+        if (devid > 0)
+                detach_device(domain, devid);
+
+        iommu = amd_iommu_rlookup_table[devid];
+        if (!iommu)
+                return;
+
+        iommu_queue_inv_dev_entry(iommu, devid);
+        iommu_completion_wait(iommu);
+}
+
+static int amd_iommu_attach_device(struct iommu_domain *dom,
+                                   struct device *dev)
+{
+        struct protection_domain *domain = dom->priv;
+        struct protection_domain *old_domain;
+        struct amd_iommu *iommu;
+        struct pci_dev *pdev;
+        u16 devid;
+
+        if (dev->bus != &pci_bus_type)
+                return -EINVAL;
+
+        pdev = to_pci_dev(dev);
+
+        devid = calc_devid(pdev->bus->number, pdev->devfn);
+
+        if (devid >= amd_iommu_last_bdf ||
+                        devid != amd_iommu_alias_table[devid])
+                return -EINVAL;
+
+        iommu = amd_iommu_rlookup_table[devid];
+        if (!iommu)
+                return -EINVAL;
+
+        old_domain = domain_for_device(devid);
+        if (old_domain)
+                return -EBUSY;
+
+        attach_device(iommu, domain, devid);
+
+        iommu_completion_wait(iommu);
+
+        return 0;
+}
+
+static int amd_iommu_map_range(struct iommu_domain *dom,
+                               unsigned long iova, phys_addr_t paddr,
+                               size_t size, int iommu_prot)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long i,  npages = iommu_num_pages(paddr, size, PAGE_SIZE);
+        int prot = 0;
+        int ret;
+
+        if (iommu_prot & IOMMU_READ)
+                prot |= IOMMU_PROT_IR;
+        if (iommu_prot & IOMMU_WRITE)
+                prot |= IOMMU_PROT_IW;
+
+        iova  &= PAGE_MASK;
+        paddr &= PAGE_MASK;
+
+        for (i = 0; i < npages; ++i) {
+                ret = iommu_map_page(domain, iova, paddr, prot);
+                if (ret)
+                        return ret;
+
+                iova  += PAGE_SIZE;
+                paddr += PAGE_SIZE;
+        }
+
+        return 0;
+}
+
+static void amd_iommu_unmap_range(struct iommu_domain *dom,
+                                  unsigned long iova, size_t size)
+{
+
+        struct protection_domain *domain = dom->priv;
+        unsigned long i,  npages = iommu_num_pages(iova, size, PAGE_SIZE);
+
+        iova  &= PAGE_MASK;
+
+        for (i = 0; i < npages; ++i) {
+                iommu_unmap_page(domain, iova);
+                iova  += PAGE_SIZE;
+        }
+
+        iommu_flush_domain(domain->id);
+}
+
+static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
+                                          unsigned long iova)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long offset = iova & ~PAGE_MASK;
+        phys_addr_t paddr;
+        u64 *pte;
+
+        pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(iova)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return 0;
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L1_INDEX(iova)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return 0;
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L0_INDEX(iova)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return 0;
+
+        paddr  = *pte & IOMMU_PAGE_MASK;
+        paddr |= offset;
+
+        return paddr;
+}
+
+static struct iommu_ops amd_iommu_ops = {
+        .domain_init = amd_iommu_domain_init,
+        .domain_destroy = amd_iommu_domain_destroy,
+        .attach_dev = amd_iommu_attach_device,
+        .detach_dev = amd_iommu_detach_device,
+        .map = amd_iommu_map_range,
+        .unmap = amd_iommu_unmap_range,
+        .iova_to_phys = amd_iommu_iova_to_phys,
+};
+
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index c6cc22815d35..42c33cebf00f 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -28,6 +28,7 @@
 #include <asm/amd_iommu_types.h>
 #include <asm/amd_iommu.h>
 #include <asm/iommu.h>
+#include <asm/gart.h>
 
 /*
  * definitions for the ACPI scanning code
@@ -121,7 +122,8 @@ u16 amd_iommu_last_bdf;			/* largest PCI device id we have
 LIST_HEAD(amd_iommu_unity_map);         /* a list of required unity mappings
                                            we find in ACPI */
 unsigned amd_iommu_aperture_order = 26; /* size of aperture in power of 2 */
-int amd_iommu_isolate = 1;              /* if 1, device isolation is enabled */
+bool amd_iommu_isolate = true;          /* if true, device isolation is
+                                           enabled */
 bool amd_iommu_unmap_flush;             /* if true, flush on every unmap */
 
 LIST_HEAD(amd_iommu_list);              /* list of all AMD IOMMUs in the
@@ -242,20 +244,16 @@ static void __init iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
 }
 
 /* Function to enable the hardware */
-void __init iommu_enable(struct amd_iommu *iommu)
+static void __init iommu_enable(struct amd_iommu *iommu)
 {
-        printk(KERN_INFO "AMD IOMMU: Enabling IOMMU "
-               "at %02x:%02x.%x cap 0x%hx\n",
-               iommu->dev->bus->number,
-               PCI_SLOT(iommu->dev->devfn),
-               PCI_FUNC(iommu->dev->devfn),
-               iommu->cap_ptr);
+        printk(KERN_INFO "AMD IOMMU: Enabling IOMMU at %s cap 0x%hx\n",
+               dev_name(&iommu->dev->dev), iommu->cap_ptr);
 
         iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
 }
 
 /* Function to enable IOMMU event logging and event interrupts */
-void __init iommu_enable_event_logging(struct amd_iommu *iommu)
+static void __init iommu_enable_event_logging(struct amd_iommu *iommu)
 {
         iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
         iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
@@ -1217,9 +1215,9 @@ static int __init parse_amd_iommu_options(char *str)
 {
         for (; *str; ++str) {
                 if (strncmp(str, "isolate", 7) == 0)
-                        amd_iommu_isolate = 1;
+                        amd_iommu_isolate = true;
                 if (strncmp(str, "share", 5) == 0)
-                        amd_iommu_isolate = 0;
+                        amd_iommu_isolate = false;
                 if (strncmp(str, "fullflush", 9) == 0)
                         amd_iommu_unmap_flush = true;
         }
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 9a32b37ee2ee..676debfc1702 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -1,8 +1,9 @@
 /*
  * Firmware replacement code.
  *
- * Work around broken BIOSes that don't set an aperture or only set the
- * aperture in the AGP bridge.
+ * Work around broken BIOSes that don't set an aperture, only set the
+ * aperture in the AGP bridge, or set too small aperture.
+ *
  * If all fails map the aperture over some low memory.  This is cheaper than
  * doing bounce buffering. The memory is lost. This is done at early boot
  * because only the bootmem allocator can allocate 32+MB.
diff --git a/arch/x86/kernel/apic.c b/arch/x86/kernel/apic.c
index 16f94879b525..b13d3c4dbd42 100644
--- a/arch/x86/kernel/apic.c
+++ b/arch/x86/kernel/apic.c
@@ -30,6 +30,7 @@
 #include <linux/module.h>
 #include <linux/dmi.h>
 #include <linux/dmar.h>
+#include <linux/ftrace.h>
 
 #include <asm/atomic.h>
 #include <asm/smp.h>
@@ -97,8 +98,8 @@ __setup("apicpmtimer", setup_apicpmtimer);
 #ifdef HAVE_X2APIC
 int x2apic;
 /* x2apic enabled before OS handover */
-int x2apic_preenabled;
-int disable_x2apic;
+static int x2apic_preenabled;
+static int disable_x2apic;
 static __init int setup_nox2apic(char *str)
 {
         disable_x2apic = 1;
@@ -118,8 +119,6 @@ EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
 
 int first_system_vector = 0xfe;
 
-char system_vectors[NR_VECTORS] = { [0 ... NR_VECTORS-1] = SYS_VECTOR_FREE};
-
 /*
  * Debug level, exported for io_apic.c
  */
@@ -141,7 +140,7 @@ static int lapic_next_event(unsigned long delta,
                             struct clock_event_device *evt);
 static void lapic_timer_setup(enum clock_event_mode mode,
                               struct clock_event_device *evt);
-static void lapic_timer_broadcast(cpumask_t mask);
+static void lapic_timer_broadcast(const struct cpumask *mask);
 static void apic_pm_activate(void);
 
 /*
@@ -227,7 +226,7 @@ void xapic_icr_write(u32 low, u32 id)
         apic_write(APIC_ICR, low);
 }
 
-u64 xapic_icr_read(void)
+static u64 xapic_icr_read(void)
 {
         u32 icr1, icr2;
 
@@ -267,7 +266,7 @@ void x2apic_icr_write(u32 low, u32 id)
         wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
 }
 
-u64 x2apic_icr_read(void)
+static u64 x2apic_icr_read(void)
 {
         unsigned long val;
 
@@ -441,6 +440,7 @@ static void lapic_timer_setup(enum clock_event_mode mode,
                 v = apic_read(APIC_LVTT);
                 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
                 apic_write(APIC_LVTT, v);
+                apic_write(APIC_TMICT, 0xffffffff);
                 break;
         case CLOCK_EVT_MODE_RESUME:
                 /* Nothing to do here */
@@ -453,7 +453,7 @@ static void lapic_timer_setup(enum clock_event_mode mode,
 /*
  * Local APIC timer broadcast function
  */
-static void lapic_timer_broadcast(cpumask_t mask)
+static void lapic_timer_broadcast(const struct cpumask *mask)
 {
 #ifdef CONFIG_SMP
         send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
@@ -469,7 +469,7 @@ static void __cpuinit setup_APIC_timer(void)
         struct clock_event_device *levt = &__get_cpu_var(lapic_events);
 
         memcpy(levt, &lapic_clockevent, sizeof(*levt));
-        levt->cpumask = cpumask_of_cpu(smp_processor_id());
+        levt->cpumask = cpumask_of(smp_processor_id());
 
         clockevents_register_device(levt);
 }
@@ -559,13 +559,13 @@ static int __init calibrate_by_pmtimer(long deltapm, long *delta)
         } else {
                 res = (((u64)deltapm) *  mult) >> 22;
                 do_div(res, 1000000);
-                printk(KERN_WARNING "APIC calibration not consistent "
+                pr_warning("APIC calibration not consistent "
                         "with PM Timer: %ldms instead of 100ms\n",
                         (long)res);
                 /* Correct the lapic counter value */
                 res = (((u64)(*delta)) * pm_100ms);
                 do_div(res, deltapm);
-                printk(KERN_INFO "APIC delta adjusted to PM-Timer: "
+                pr_info("APIC delta adjusted to PM-Timer: "
                         "%lu (%ld)\n", (unsigned long)res, *delta);
                 *delta = (long)res;
         }
@@ -645,8 +645,7 @@ static int __init calibrate_APIC_clock(void)
          */
         if (calibration_result < (1000000 / HZ)) {
                 local_irq_enable();
-                printk(KERN_WARNING
-                       "APIC frequency too slow, disabling apic timer\n");
+                pr_warning("APIC frequency too slow, disabling apic timer\n");
                 return -1;
         }
 
@@ -672,13 +671,9 @@ static int __init calibrate_APIC_clock(void)
                 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
                         cpu_relax();
 
-                local_irq_disable();
-
                 /* Stop the lapic timer */
                 lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);
 
-                local_irq_enable();
-
                 /* Jiffies delta */
                 deltaj = lapic_cal_j2 - lapic_cal_j1;
                 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
@@ -692,8 +687,7 @@ static int __init calibrate_APIC_clock(void)
                 local_irq_enable();
 
         if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
-                printk(KERN_WARNING
-                       "APIC timer disabled due to verification failure.\n");
+                pr_warning("APIC timer disabled due to verification failure.\n");
                         return -1;
         }
 
@@ -714,7 +708,7 @@ void __init setup_boot_APIC_clock(void)
          * broadcast mechanism is used. On UP systems simply ignore it.
          */
         if (disable_apic_timer) {
-                printk(KERN_INFO "Disabling APIC timer\n");
+                pr_info("Disabling APIC timer\n");
                 /* No broadcast on UP ! */
                 if (num_possible_cpus() > 1) {
                         lapic_clockevent.mult = 1;
@@ -741,7 +735,7 @@ void __init setup_boot_APIC_clock(void)
         if (nmi_watchdog != NMI_IO_APIC)
                 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
         else
-                printk(KERN_WARNING "APIC timer registered as dummy,"
+                pr_warning("APIC timer registered as dummy,"
                         " due to nmi_watchdog=%d!\n", nmi_watchdog);
 
         /* Setup the lapic or request the broadcast */
@@ -773,8 +767,7 @@ static void local_apic_timer_interrupt(void)
          * spurious.
          */
         if (!evt->event_handler) {
-                printk(KERN_WARNING
-                       "Spurious LAPIC timer interrupt on cpu %d\n", cpu);
+                pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
                 /* Switch it off */
                 lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
                 return;
@@ -783,11 +776,7 @@ static void local_apic_timer_interrupt(void)
         /*
          * the NMI deadlock-detector uses this.
          */
-#ifdef CONFIG_X86_64
-        add_pda(apic_timer_irqs, 1);
-#else
-        per_cpu(irq_stat, cpu).apic_timer_irqs++;
-#endif
+        inc_irq_stat(apic_timer_irqs);
 
         evt->event_handler(evt);
 }
@@ -800,7 +789,7 @@ static void local_apic_timer_interrupt(void)
  * [ if a single-CPU system runs an SMP kernel then we call the local
  *   interrupt as well. Thus we cannot inline the local irq ... ]
  */
-void smp_apic_timer_interrupt(struct pt_regs *regs)
+void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
 {
         struct pt_regs *old_regs = set_irq_regs(regs);
 
@@ -814,9 +803,7 @@ void smp_apic_timer_interrupt(struct pt_regs *regs)
          * Besides, if we don't timer interrupts ignore the global
          * interrupt lock, which is the WrongThing (tm) to do.
          */
-#ifdef CONFIG_X86_64
         exit_idle();
-#endif
         irq_enter();
         local_apic_timer_interrupt();
         irq_exit();
@@ -1093,7 +1080,7 @@ static void __cpuinit lapic_setup_esr(void)
         unsigned int oldvalue, value, maxlvt;
 
         if (!lapic_is_integrated()) {
-                printk(KERN_INFO "No ESR for 82489DX.\n");
+                pr_info("No ESR for 82489DX.\n");
                 return;
         }
 
@@ -1104,7 +1091,7 @@ static void __cpuinit lapic_setup_esr(void)
                  * ESR disabled - we can't do anything useful with the
                  * errors anyway - mbligh
                  */
-                printk(KERN_INFO "Leaving ESR disabled.\n");
+                pr_info("Leaving ESR disabled.\n");
                 return;
         }
 
@@ -1298,7 +1285,7 @@ void check_x2apic(void)
         rdmsr(MSR_IA32_APICBASE, msr, msr2);
 
         if (msr & X2APIC_ENABLE) {
-                printk("x2apic enabled by BIOS, switching to x2apic ops\n");
+                pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
                 x2apic_preenabled = x2apic = 1;
                 apic_ops = &x2apic_ops;
         }
@@ -1310,7 +1297,7 @@ void enable_x2apic(void)
 
         rdmsr(MSR_IA32_APICBASE, msr, msr2);
         if (!(msr & X2APIC_ENABLE)) {
-                printk("Enabling x2apic\n");
+                pr_info("Enabling x2apic\n");
                 wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, 0);
         }
 }
@@ -1325,9 +1312,8 @@ void __init enable_IR_x2apic(void)
                 return;
 
         if (!x2apic_preenabled && disable_x2apic) {
-                printk(KERN_INFO
-                       "Skipped enabling x2apic and Interrupt-remapping "
-                       "because of nox2apic\n");
+                pr_info("Skipped enabling x2apic and Interrupt-remapping "
+                        "because of nox2apic\n");
                 return;
         }
 
@@ -1335,22 +1321,19 @@ void __init enable_IR_x2apic(void)
                 panic("Bios already enabled x2apic, can't enforce nox2apic");
 
         if (!x2apic_preenabled && skip_ioapic_setup) {
-                printk(KERN_INFO
-                       "Skipped enabling x2apic and Interrupt-remapping "
-                       "because of skipping io-apic setup\n");
+                pr_info("Skipped enabling x2apic and Interrupt-remapping "
+                        "because of skipping io-apic setup\n");
                 return;
         }
 
         ret = dmar_table_init();
         if (ret) {
-                printk(KERN_INFO
-                       "dmar_table_init() failed with %d:\n", ret);
+                pr_info("dmar_table_init() failed with %d:\n", ret);
 
                 if (x2apic_preenabled)
                         panic("x2apic enabled by bios. But IR enabling failed");
                 else
-                        printk(KERN_INFO
-                               "Not enabling x2apic,Intr-remapping\n");
+                        pr_info("Not enabling x2apic,Intr-remapping\n");
                 return;
         }
 
@@ -1359,7 +1342,7 @@ void __init enable_IR_x2apic(void)
 
         ret = save_mask_IO_APIC_setup();
         if (ret) {
-                printk(KERN_INFO "Saving IO-APIC state failed: %d\n", ret);
+                pr_info("Saving IO-APIC state failed: %d\n", ret);
                 goto end;
         }
 
@@ -1394,14 +1377,11 @@ end:
 
         if (!ret) {
                 if (!x2apic_preenabled)
-                        printk(KERN_INFO
-                               "Enabled x2apic and interrupt-remapping\n");
+                        pr_info("Enabled x2apic and interrupt-remapping\n");
                 else
-                        printk(KERN_INFO
-                               "Enabled Interrupt-remapping\n");
+                        pr_info("Enabled Interrupt-remapping\n");
         } else
-                printk(KERN_ERR
-                       "Failed to enable Interrupt-remapping and x2apic\n");
+                pr_err("Failed to enable Interrupt-remapping and x2apic\n");
 #else
         if (!cpu_has_x2apic)
                 return;
@@ -1410,8 +1390,8 @@ end:
                 panic("x2apic enabled prior OS handover,"
                       " enable CONFIG_INTR_REMAP");
 
-        printk(KERN_INFO "Enable CONFIG_INTR_REMAP for enabling intr-remapping "
-               " and x2apic\n");
+        pr_info("Enable CONFIG_INTR_REMAP for enabling intr-remapping "
+                " and x2apic\n");
 #endif
 
         return;
@@ -1428,7 +1408,7 @@ end:
 static int __init detect_init_APIC(void)
 {
         if (!cpu_has_apic) {
-                printk(KERN_INFO "No local APIC present\n");
+                pr_info("No local APIC present\n");
                 return -1;
         }
 
@@ -1469,8 +1449,8 @@ static int __init detect_init_APIC(void)
                  * "lapic" specified.
                  */
                 if (!force_enable_local_apic) {
-                        printk(KERN_INFO "Local APIC disabled by BIOS -- "
-                               "you can enable it with \"lapic\"\n");
+                        pr_info("Local APIC disabled by BIOS -- "
+                                "you can enable it with \"lapic\"\n");
                         return -1;
                 }
                 /*
@@ -1480,8 +1460,7 @@ static int __init detect_init_APIC(void)
                  */
                 rdmsr(MSR_IA32_APICBASE, l, h);
                 if (!(l & MSR_IA32_APICBASE_ENABLE)) {
-                        printk(KERN_INFO
-                               "Local APIC disabled by BIOS -- reenabling.\n");
+                        pr_info("Local APIC disabled by BIOS -- reenabling.\n");
                         l &= ~MSR_IA32_APICBASE_BASE;
                         l |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE;
                         wrmsr(MSR_IA32_APICBASE, l, h);
@@ -1494,7 +1473,7 @@ static int __init detect_init_APIC(void)
          */
         features = cpuid_edx(1);
         if (!(features & (1 << X86_FEATURE_APIC))) {
-                printk(KERN_WARNING "Could not enable APIC!\n");
+                pr_warning("Could not enable APIC!\n");
                 return -1;
         }
         set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
@@ -1505,14 +1484,14 @@ static int __init detect_init_APIC(void)
         if (l & MSR_IA32_APICBASE_ENABLE)
                 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
 
-        printk(KERN_INFO "Found and enabled local APIC!\n");
+        pr_info("Found and enabled local APIC!\n");
 
         apic_pm_activate();
 
         return 0;
 
 no_apic:
-        printk(KERN_INFO "No local APIC present or hardware disabled\n");
+        pr_info("No local APIC present or hardware disabled\n");
         return -1;
 }
 #endif
@@ -1588,12 +1567,12 @@ int __init APIC_init_uniprocessor(void)
 {
 #ifdef CONFIG_X86_64
         if (disable_apic) {
-                printk(KERN_INFO "Apic disabled\n");
+                pr_info("Apic disabled\n");
                 return -1;
         }
         if (!cpu_has_apic) {
                 disable_apic = 1;
-                printk(KERN_INFO "Apic disabled by BIOS\n");
+                pr_info("Apic disabled by BIOS\n");
                 return -1;
         }
 #else
@@ -1605,8 +1584,8 @@ int __init APIC_init_uniprocessor(void)
          */
         if (!cpu_has_apic &&
             APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
-                printk(KERN_ERR "BIOS bug, local APIC 0x%x not detected!...\n",
-                       boot_cpu_physical_apicid);
+                pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
+                        boot_cpu_physical_apicid);
                 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
                 return -1;
         }
@@ -1682,9 +1661,7 @@ void smp_spurious_interrupt(struct pt_regs *regs)
 {
         u32 v;
 
-#ifdef CONFIG_X86_64
         exit_idle();
-#endif
         irq_enter();
         /*
          * Check if this really is a spurious interrupt and ACK it
@@ -1695,14 +1672,11 @@ void smp_spurious_interrupt(struct pt_regs *regs)
         if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
                 ack_APIC_irq();
 
-#ifdef CONFIG_X86_64
-        add_pda(irq_spurious_count, 1);
-#else
+        inc_irq_stat(irq_spurious_count);
+
         /* see sw-dev-man vol 3, chapter 7.4.13.5 */
-        printk(KERN_INFO "spurious APIC interrupt on CPU#%d, "
-               "should never happen.\n", smp_processor_id());
-        __get_cpu_var(irq_stat).irq_spurious_count++;
-#endif
+        pr_info("spurious APIC interrupt on CPU#%d, "
+                "should never happen.\n", smp_processor_id());
         irq_exit();
 }
 
@@ -1713,9 +1687,7 @@ void smp_error_interrupt(struct pt_regs *regs)
 {
         u32 v, v1;
 
-#ifdef CONFIG_X86_64
         exit_idle();
-#endif
         irq_enter();
         /* First tickle the hardware, only then report what went on. -- REW */
         v = apic_read(APIC_ESR);
@@ -1724,17 +1696,18 @@ void smp_error_interrupt(struct pt_regs *regs)
         ack_APIC_irq();
         atomic_inc(&irq_err_count);
 
-        /* Here is what the APIC error bits mean:
-           0: Send CS error
-           1: Receive CS error
-           2: Send accept error
-           3: Receive accept error
-           4: Reserved
-           5: Send illegal vector
-           6: Received illegal vector
-           7: Illegal register address
-        */
-        printk(KERN_DEBUG "APIC error on CPU%d: %02x(%02x)\n",
+        /*
+         * Here is what the APIC error bits mean:
+         * 0: Send CS error
+         * 1: Receive CS error
+         * 2: Send accept error
+         * 3: Receive accept error
+         * 4: Reserved
+         * 5: Send illegal vector
+         * 6: Received illegal vector
+         * 7: Illegal register address
+         */
+        pr_debug("APIC error on CPU%d: %02x(%02x)\n",
                 smp_processor_id(), v , v1);
         irq_exit();
 }
@@ -1832,28 +1805,32 @@ void disconnect_bsp_APIC(int virt_wire_setup)
 void __cpuinit generic_processor_info(int apicid, int version)
 {
         int cpu;
-        cpumask_t tmp_map;
 
         /*
          * Validate version
          */
         if (version == 0x0) {
-                printk(KERN_WARNING "BIOS bug, APIC version is 0 for CPU#%d! "
-                                "fixing up to 0x10. (tell your hw vendor)\n",
+                pr_warning("BIOS bug, APIC version is 0 for CPU#%d! "
+                           "fixing up to 0x10. (tell your hw vendor)\n",
                                 version);
                 version = 0x10;
         }
         apic_version[apicid] = version;
 
-        if (num_processors >= NR_CPUS) {
-                printk(KERN_WARNING "WARNING: NR_CPUS limit of %i reached."
-                        "  Processor ignored.\n", NR_CPUS);
+        if (num_processors >= nr_cpu_ids) {
+                int max = nr_cpu_ids;
+                int thiscpu = max + disabled_cpus;
+
+                pr_warning(
+                        "ACPI: NR_CPUS/possible_cpus limit of %i reached."
+                        "  Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
+
+                disabled_cpus++;
                 return;
         }
 
         num_processors++;
-        cpus_complement(tmp_map, cpu_present_map);
-        cpu = first_cpu(tmp_map);
+        cpu = cpumask_next_zero(-1, cpu_present_mask);
 
         physid_set(apicid, phys_cpu_present_map);
         if (apicid == boot_cpu_physical_apicid) {
@@ -1903,8 +1880,8 @@ void __cpuinit generic_processor_info(int apicid, int version)
         }
 #endif
 
-        cpu_set(cpu, cpu_possible_map);
-        cpu_set(cpu, cpu_present_map);
+        set_cpu_possible(cpu, true);
+        set_cpu_present(cpu, true);
 }
 
 #ifdef CONFIG_X86_64
@@ -2106,7 +2083,7 @@ __cpuinit int apic_is_clustered_box(void)
         bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
         bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
 
-        for (i = 0; i < NR_CPUS; i++) {
+        for (i = 0; i < nr_cpu_ids; i++) {
                 /* are we being called early in kernel startup? */
                 if (bios_cpu_apicid) {
                         id = bios_cpu_apicid[i];
@@ -2209,7 +2186,7 @@ static int __init apic_set_verbosity(char *arg)
         else if (strcmp("verbose", arg) == 0)
                 apic_verbosity = APIC_VERBOSE;
         else {
-                printk(KERN_WARNING "APIC Verbosity level %s not recognised"
+                pr_warning("APIC Verbosity level %s not recognised"
                         " use apic=verbose or apic=debug\n", arg);
                 return -EINVAL;
         }
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index 5145a6e72bbb..3a26525a3f31 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -391,11 +391,7 @@ static int power_off;
 #else
 static int power_off = 1;
 #endif
-#ifdef CONFIG_APM_REAL_MODE_POWER_OFF
-static int realmode_power_off = 1;
-#else
 static int realmode_power_off;
-#endif
 #ifdef CONFIG_APM_ALLOW_INTS
 static int allow_ints = 1;
 #else
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 6649d09ad88f..ee4df08feee6 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -11,7 +11,7 @@
 #include <linux/suspend.h>
 #include <linux/kbuild.h>
 #include <asm/ucontext.h>
-#include "sigframe.h"
+#include <asm/sigframe.h>
 #include <asm/pgtable.h>
 #include <asm/fixmap.h>
 #include <asm/processor.h>
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 7fcf63d22f8b..1d41d3f1edbc 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -20,6 +20,8 @@
 
 #include <xen/interface/xen.h>
 
+#include <asm/sigframe.h>
+
 #define __NO_STUBS 1
 #undef __SYSCALL
 #undef _ASM_X86_UNISTD_64_H
@@ -87,7 +89,7 @@ int main(void)
         BLANK();
 #undef ENTRY
         DEFINE(IA32_RT_SIGFRAME_sigcontext,
-               offsetof (struct rt_sigframe32, uc.uc_mcontext));
+               offsetof (struct rt_sigframe_ia32, uc.uc_mcontext));
         BLANK();
 #endif
         DEFINE(pbe_address, offsetof(struct pbe, address));
diff --git a/arch/x86/kernel/bios_uv.c b/arch/x86/kernel/bios_uv.c
index f0dfe6f17e7e..f63882728d91 100644
--- a/arch/x86/kernel/bios_uv.c
+++ b/arch/x86/kernel/bios_uv.c
@@ -25,7 +25,7 @@
 #include <asm/uv/bios.h>
 #include <asm/uv/uv_hub.h>
 
-struct uv_systab uv_systab;
+static struct uv_systab uv_systab;
 
 s64 uv_bios_call(enum uv_bios_cmd which, u64 a1, u64 a2, u64 a3, u64 a4, u64 a5)
 {
@@ -69,10 +69,10 @@ s64 uv_bios_call_reentrant(enum uv_bios_cmd which, u64 a1, u64 a2, u64 a3,
 
 long sn_partition_id;
 EXPORT_SYMBOL_GPL(sn_partition_id);
-long uv_coherency_id;
-EXPORT_SYMBOL_GPL(uv_coherency_id);
-long uv_region_size;
-EXPORT_SYMBOL_GPL(uv_region_size);
+long sn_coherency_id;
+EXPORT_SYMBOL_GPL(sn_coherency_id);
+long sn_region_size;
+EXPORT_SYMBOL_GPL(sn_region_size);
 int uv_type;
 
 
@@ -100,6 +100,56 @@ s64 uv_bios_get_sn_info(int fc, int *uvtype, long *partid, long *coher,
         return ret;
 }
 
+int
+uv_bios_mq_watchlist_alloc(int blade, unsigned long addr, unsigned int mq_size,
+                           unsigned long *intr_mmr_offset)
+{
+        union uv_watchlist_u size_blade;
+        u64 watchlist;
+        s64 ret;
+
+        size_blade.size = mq_size;
+        size_blade.blade = blade;
+
+        /*
+         * bios returns watchlist number or negative error number.
+         */
+        ret = (int)uv_bios_call_irqsave(UV_BIOS_WATCHLIST_ALLOC, addr,
+                        size_blade.val, (u64)intr_mmr_offset,
+                        (u64)&watchlist, 0);
+        if (ret < BIOS_STATUS_SUCCESS)
+                return ret;
+
+        return watchlist;
+}
+EXPORT_SYMBOL_GPL(uv_bios_mq_watchlist_alloc);
+
+int
+uv_bios_mq_watchlist_free(int blade, int watchlist_num)
+{
+        return (int)uv_bios_call_irqsave(UV_BIOS_WATCHLIST_FREE,
+                                blade, watchlist_num, 0, 0, 0);
+}
+EXPORT_SYMBOL_GPL(uv_bios_mq_watchlist_free);
+
+s64
+uv_bios_change_memprotect(u64 paddr, u64 len, enum uv_memprotect perms)
+{
+        return uv_bios_call_irqsave(UV_BIOS_MEMPROTECT, paddr, len,
+                                        perms, 0, 0);
+}
+EXPORT_SYMBOL_GPL(uv_bios_change_memprotect);
+
+s64
+uv_bios_reserved_page_pa(u64 buf, u64 *cookie, u64 *addr, u64 *len)
+{
+        s64 ret;
+
+        ret = uv_bios_call_irqsave(UV_BIOS_GET_PARTITION_ADDR, (u64)cookie,
+                                        (u64)addr, buf, (u64)len, 0);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(uv_bios_reserved_page_pa);
 
 s64 uv_bios_freq_base(u64 clock_type, u64 *ticks_per_second)
 {
diff --git a/arch/x86/kernel/check.c b/arch/x86/kernel/check.c
new file mode 100644
index 000000000000..2ac0ab71412a
--- /dev/null
+++ b/arch/x86/kernel/check.c
@@ -0,0 +1,161 @@
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/workqueue.h>
+#include <asm/e820.h>
+#include <asm/proto.h>
+
+/*
+ * Some BIOSes seem to corrupt the low 64k of memory during events
+ * like suspend/resume and unplugging an HDMI cable.  Reserve all
+ * remaining free memory in that area and fill it with a distinct
+ * pattern.
+ */
+#define MAX_SCAN_AREAS  8
+
+static int __read_mostly memory_corruption_check = -1;
+
+static unsigned __read_mostly corruption_check_size = 64*1024;
+static unsigned __read_mostly corruption_check_period = 60; /* seconds */
+
+static struct e820entry scan_areas[MAX_SCAN_AREAS];
+static int num_scan_areas;
+
+
+static __init int set_corruption_check(char *arg)
+{
+        char *end;
+
+        memory_corruption_check = simple_strtol(arg, &end, 10);
+
+        return (*end == 0) ? 0 : -EINVAL;
+}
+early_param("memory_corruption_check", set_corruption_check);
+
+static __init int set_corruption_check_period(char *arg)
+{
+        char *end;
+
+        corruption_check_period = simple_strtoul(arg, &end, 10);
+
+        return (*end == 0) ? 0 : -EINVAL;
+}
+early_param("memory_corruption_check_period", set_corruption_check_period);
+
+static __init int set_corruption_check_size(char *arg)
+{
+        char *end;
+        unsigned size;
+
+        size = memparse(arg, &end);
+
+        if (*end == '\0')
+                corruption_check_size = size;
+
+        return (size == corruption_check_size) ? 0 : -EINVAL;
+}
+early_param("memory_corruption_check_size", set_corruption_check_size);
+
+
+void __init setup_bios_corruption_check(void)
+{
+        u64 addr = PAGE_SIZE;   /* assume first page is reserved anyway */
+
+        if (memory_corruption_check == -1) {
+                memory_corruption_check =
+#ifdef CONFIG_X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK
+                        1
+#else
+                        0
+#endif
+                        ;
+        }
+
+        if (corruption_check_size == 0)
+                memory_corruption_check = 0;
+
+        if (!memory_corruption_check)
+                return;
+
+        corruption_check_size = round_up(corruption_check_size, PAGE_SIZE);
+
+        while (addr < corruption_check_size && num_scan_areas < MAX_SCAN_AREAS) {
+                u64 size;
+                addr = find_e820_area_size(addr, &size, PAGE_SIZE);
+
+                if (addr == 0)
+                        break;
+
+                if ((addr + size) > corruption_check_size)
+                        size = corruption_check_size - addr;
+
+                if (size == 0)
+                        break;
+
+                e820_update_range(addr, size, E820_RAM, E820_RESERVED);
+                scan_areas[num_scan_areas].addr = addr;
+                scan_areas[num_scan_areas].size = size;
+                num_scan_areas++;
+
+                /* Assume we've already mapped this early memory */
+                memset(__va(addr), 0, size);
+
+                addr += size;
+        }
+
+        printk(KERN_INFO "Scanning %d areas for low memory corruption\n",
+               num_scan_areas);
+        update_e820();
+}
+
+
+void check_for_bios_corruption(void)
+{
+        int i;
+        int corruption = 0;
+
+        if (!memory_corruption_check)
+                return;
+
+        for (i = 0; i < num_scan_areas; i++) {
+                unsigned long *addr = __va(scan_areas[i].addr);
+                unsigned long size = scan_areas[i].size;
+
+                for (; size; addr++, size -= sizeof(unsigned long)) {
+                        if (!*addr)
+                                continue;
+                        printk(KERN_ERR "Corrupted low memory at %p (%lx phys) = %08lx\n",
+                               addr, __pa(addr), *addr);
+                        corruption = 1;
+                        *addr = 0;
+                }
+        }
+
+        WARN_ONCE(corruption, KERN_ERR "Memory corruption detected in low memory\n");
+}
+
+static void check_corruption(struct work_struct *dummy);
+static DECLARE_DELAYED_WORK(bios_check_work, check_corruption);
+
+static void check_corruption(struct work_struct *dummy)
+{
+        check_for_bios_corruption();
+        schedule_delayed_work(&bios_check_work,
+                round_jiffies_relative(corruption_check_period*HZ)); 
+}
+
+static int start_periodic_check_for_corruption(void)
+{
+        if (!memory_corruption_check || corruption_check_period == 0)
+                return 0;
+
+        printk(KERN_INFO "Scanning for low memory corruption every %d seconds\n",
+               corruption_check_period);
+
+        /* First time we run the checks right away */
+        schedule_delayed_work(&bios_check_work, 0);
+        return 0;
+}
+
+module_init(start_periodic_check_for_corruption);
+
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 82ec6075c057..82db7f45e2de 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -2,8 +2,14 @@
 # Makefile for x86-compatible CPU details and quirks
 #
 
+# Don't trace early stages of a secondary CPU boot
+ifdef CONFIG_FUNCTION_TRACER
+CFLAGS_REMOVE_common.o = -pg
+endif
+
 obj-y                   := intel_cacheinfo.o addon_cpuid_features.o
 obj-y                   += proc.o capflags.o powerflags.o common.o
+obj-y                   += vmware.o hypervisor.o
 
 obj-$(CONFIG_X86_32)    += bugs.o cmpxchg.o
 obj-$(CONFIG_X86_64)    += bugs_64.o
diff --git a/arch/x86/kernel/cpu/addon_cpuid_features.c b/arch/x86/kernel/cpu/addon_cpuid_features.c
index ef8f831af823..2cf23634b6d9 100644
--- a/arch/x86/kernel/cpu/addon_cpuid_features.c
+++ b/arch/x86/kernel/cpu/addon_cpuid_features.c
@@ -120,9 +120,17 @@ void __cpuinit detect_extended_topology(struct cpuinfo_x86 *c)
         c->cpu_core_id = phys_pkg_id(c->initial_apicid, ht_mask_width)
                                                  & core_select_mask;
         c->phys_proc_id = phys_pkg_id(c->initial_apicid, core_plus_mask_width);
+        /*
+         * Reinit the apicid, now that we have extended initial_apicid.
+         */
+        c->apicid = phys_pkg_id(c->initial_apicid, 0);
 #else
         c->cpu_core_id = phys_pkg_id(ht_mask_width) & core_select_mask;
         c->phys_proc_id = phys_pkg_id(core_plus_mask_width);
+        /*
+         * Reinit the apicid, now that we have extended initial_apicid.
+         */
+        c->apicid = phys_pkg_id(0);
 #endif
         c->x86_max_cores = (core_level_siblings / smp_num_siblings);
 
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 8f1e31db2ad5..7c878f6aa919 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -283,9 +283,14 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
 {
         early_init_amd_mc(c);
 
-        /* c->x86_power is 8000_0007 edx. Bit 8 is constant TSC */
-        if (c->x86_power & (1<<8))
+        /*
+         * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
+         * with P/T states and does not stop in deep C-states
+         */
+        if (c->x86_power & (1 << 8)) {
                 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
+                set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
+        }
 
 #ifdef CONFIG_X86_64
         set_cpu_cap(c, X86_FEATURE_SYSCALL32);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index b9c9ea0217a9..3f95a40f718a 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -36,6 +36,7 @@
 #include <asm/proto.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
+#include <asm/hypervisor.h>
 
 #include "cpu.h"
 
@@ -354,7 +355,7 @@ void __cpuinit detect_ht(struct cpuinfo_x86 *c)
                 printk(KERN_INFO  "CPU: Hyper-Threading is disabled\n");
         } else if (smp_num_siblings > 1) {
 
-                if (smp_num_siblings > NR_CPUS) {
+                if (smp_num_siblings > nr_cpu_ids) {
                         printk(KERN_WARNING "CPU: Unsupported number of siblings %d",
                                         smp_num_siblings);
                         smp_num_siblings = 1;
@@ -703,6 +704,7 @@ static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
         detect_ht(c);
 #endif
 
+        init_hypervisor(c);
         /*
          * On SMP, boot_cpu_data holds the common feature set between
          * all CPUs; so make sure that we indicate which features are
@@ -862,7 +864,7 @@ EXPORT_SYMBOL(_cpu_pda);
 
 struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
 
-char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss;
+static char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss;
 
 void __cpuinit pda_init(int cpu)
 {
@@ -903,8 +905,8 @@ void __cpuinit pda_init(int cpu)
         }
 }
 
-char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ +
-                           DEBUG_STKSZ] __page_aligned_bss;
+static char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ +
+                                  DEBUG_STKSZ] __page_aligned_bss;
 
 extern asmlinkage void ignore_sysret(void);
 
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
index 8e48c5d4467d..28102ad1a363 100644
--- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
+++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
@@ -33,6 +33,7 @@
 #include <linux/cpufreq.h>
 #include <linux/compiler.h>
 #include <linux/dmi.h>
+#include <linux/ftrace.h>
 
 #include <linux/acpi.h>
 #include <acpi/processor.h>
@@ -391,6 +392,7 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
         unsigned int next_perf_state = 0; /* Index into perf table */
         unsigned int i;
         int result = 0;
+        struct power_trace it;
 
         dprintk("acpi_cpufreq_target %d (%d)\n", target_freq, policy->cpu);
 
@@ -427,6 +429,8 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
                 }
         }
 
+        trace_power_mark(&it, POWER_PSTATE, next_perf_state);
+
         switch (data->cpu_feature) {
         case SYSTEM_INTEL_MSR_CAPABLE:
                 cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
@@ -513,6 +517,17 @@ acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
         }
 }
 
+static void free_acpi_perf_data(void)
+{
+        unsigned int i;
+
+        /* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */
+        for_each_possible_cpu(i)
+                free_cpumask_var(per_cpu_ptr(acpi_perf_data, i)
+                                 ->shared_cpu_map);
+        free_percpu(acpi_perf_data);
+}
+
 /*
  * acpi_cpufreq_early_init - initialize ACPI P-States library
  *
@@ -523,6 +538,7 @@ acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
  */
 static int __init acpi_cpufreq_early_init(void)
 {
+        unsigned int i;
         dprintk("acpi_cpufreq_early_init\n");
 
         acpi_perf_data = alloc_percpu(struct acpi_processor_performance);
@@ -530,6 +546,16 @@ static int __init acpi_cpufreq_early_init(void)
                 dprintk("Memory allocation error for acpi_perf_data.\n");
                 return -ENOMEM;
         }
+        for_each_possible_cpu(i) {
+                if (!alloc_cpumask_var_node(
+                        &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map,
+                        GFP_KERNEL, cpu_to_node(i))) {
+
+                        /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */
+                        free_acpi_perf_data();
+                        return -ENOMEM;
+                }
+        }
 
         /* Do initialization in ACPI core */
         acpi_processor_preregister_performance(acpi_perf_data);
@@ -600,9 +626,9 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
          */
         if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
             policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
-                policy->cpus = perf->shared_cpu_map;
+                cpumask_copy(&policy->cpus, perf->shared_cpu_map);
         }
-        policy->related_cpus = perf->shared_cpu_map;
+        cpumask_copy(&policy->related_cpus, perf->shared_cpu_map);
 
 #ifdef CONFIG_SMP
         dmi_check_system(sw_any_bug_dmi_table);
@@ -791,7 +817,7 @@ static int __init acpi_cpufreq_init(void)
 
         ret = cpufreq_register_driver(&acpi_cpufreq_driver);
         if (ret)
-                free_percpu(acpi_perf_data);
+                free_acpi_perf_data();
 
         return ret;
 }
diff --git a/arch/x86/kernel/cpu/cpufreq/longhaul.c b/arch/x86/kernel/cpu/cpufreq/longhaul.c
index b0461856acfb..a4cff5d6e380 100644
--- a/arch/x86/kernel/cpu/cpufreq/longhaul.c
+++ b/arch/x86/kernel/cpu/cpufreq/longhaul.c
@@ -982,7 +982,7 @@ static int __init longhaul_init(void)
         case 10:
                 printk(KERN_ERR PFX "Use acpi-cpufreq driver for VIA C7\n");
         default:
-                ;;
+                ;
         }
 
         return -ENODEV;
diff --git a/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c b/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c
index b8e05ee4f736..beea4466b063 100644
--- a/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c
+++ b/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c
@@ -160,6 +160,7 @@ static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c)
                 switch (c->x86_model) {
                 case 0x0E: /* Core */
                 case 0x0F: /* Core Duo */
+                case 0x16: /* Celeron Core */
                         p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
                         return speedstep_get_processor_frequency(SPEEDSTEP_PROCESSOR_PCORE);
                 case 0x0D: /* Pentium M (Dothan) */
@@ -171,7 +172,9 @@ static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c)
         }
 
         if (c->x86 != 0xF) {
-                printk(KERN_WARNING PFX "Unknown p4-clockmod-capable CPU. Please send an e-mail to <cpufreq@vger.kernel.org>\n");
+                if (!cpu_has(c, X86_FEATURE_EST))
+                        printk(KERN_WARNING PFX "Unknown p4-clockmod-capable CPU. "
+                                "Please send an e-mail to <cpufreq@vger.kernel.org>\n");
                 return 0;
         }
 
@@ -274,6 +277,7 @@ static struct cpufreq_driver p4clockmod_driver = {
         .name           = "p4-clockmod",
         .owner          = THIS_MODULE,
         .attr           = p4clockmod_attr,
+        .hide_interface = 1,
 };
 
 
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k7.c b/arch/x86/kernel/cpu/cpufreq/powernow-k7.c
index 7c7d56b43136..1b446d79a8fd 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k7.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k7.c
@@ -310,6 +310,12 @@ static int powernow_acpi_init(void)
                 goto err0;
         }
 
+        if (!alloc_cpumask_var(&acpi_processor_perf->shared_cpu_map,
+                                                                GFP_KERNEL)) {
+                retval = -ENOMEM;
+                goto err05;
+        }
+
         if (acpi_processor_register_performance(acpi_processor_perf, 0)) {
                 retval = -EIO;
                 goto err1;
@@ -412,6 +418,8 @@ static int powernow_acpi_init(void)
 err2:
         acpi_processor_unregister_performance(acpi_processor_perf, 0);
 err1:
+        free_cpumask_var(acpi_processor_perf->shared_cpu_map);
+err05:
         kfree(acpi_processor_perf);
 err0:
         printk(KERN_WARNING PFX "ACPI perflib can not be used in this platform\n");
@@ -652,6 +660,7 @@ static int powernow_cpu_exit (struct cpufreq_policy *policy) {
 #ifdef CONFIG_X86_POWERNOW_K7_ACPI
         if (acpi_processor_perf) {
                 acpi_processor_unregister_performance(acpi_processor_perf, 0);
+                free_cpumask_var(acpi_processor_perf->shared_cpu_map);
                 kfree(acpi_processor_perf);
         }
 #endif
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
index 7f05f44b97e9..c3c9adbaa26f 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
@@ -766,7 +766,7 @@ static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned
 static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
 {
         struct cpufreq_frequency_table *powernow_table;
-        int ret_val;
+        int ret_val = -ENODEV;
 
         if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) {
                 dprintk("register performance failed: bad ACPI data\n");
@@ -815,6 +815,13 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
         /* notify BIOS that we exist */
         acpi_processor_notify_smm(THIS_MODULE);
 
+        if (!alloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) {
+                printk(KERN_ERR PFX
+                                "unable to alloc powernow_k8_data cpumask\n");
+                ret_val = -ENOMEM;
+                goto err_out_mem;
+        }
+
         return 0;
 
 err_out_mem:
@@ -826,7 +833,7 @@ err_out:
         /* data->acpi_data.state_count informs us at ->exit() whether ACPI was used */
         data->acpi_data.state_count = 0;
 
-        return -ENODEV;
+        return ret_val;
 }
 
 static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table)
@@ -929,6 +936,7 @@ static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data)
 {
         if (data->acpi_data.state_count)
                 acpi_processor_unregister_performance(&data->acpi_data, data->cpu);
+        free_cpumask_var(data->acpi_data.shared_cpu_map);
 }
 
 #else
@@ -1134,7 +1142,8 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
         data->cpu = pol->cpu;
         data->currpstate = HW_PSTATE_INVALID;
 
-        if (powernow_k8_cpu_init_acpi(data)) {
+        rc = powernow_k8_cpu_init_acpi(data);
+        if (rc) {
                 /*
                  * Use the PSB BIOS structure. This is only availabe on
                  * an UP version, and is deprecated by AMD.
@@ -1152,20 +1161,17 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
                                "ACPI maintainers and complain to your BIOS "
                                "vendor.\n");
 #endif
-                        kfree(data);
-                        return -ENODEV;
+                        goto err_out;
                 }
                 if (pol->cpu != 0) {
                         printk(KERN_ERR FW_BUG PFX "No ACPI _PSS objects for "
                                "CPU other than CPU0. Complain to your BIOS "
                                "vendor.\n");
-                        kfree(data);
-                        return -ENODEV;
+                        goto err_out;
                 }
                 rc = find_psb_table(data);
                 if (rc) {
-                        kfree(data);
-                        return -ENODEV;
+                        goto err_out;
                 }
         }
 
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
index 3b5f06423e77..f0ea6fa2f53c 100644
--- a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
+++ b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
@@ -459,9 +459,7 @@ static int centrino_verify (struct cpufreq_policy *policy)
  * Sets a new CPUFreq policy.
  */
 struct allmasks {
-        cpumask_t               online_policy_cpus;
         cpumask_t               saved_mask;
-        cpumask_t               set_mask;
         cpumask_t               covered_cpus;
 };
 
@@ -475,9 +473,7 @@ static int centrino_target (struct cpufreq_policy *policy,
         int                     retval = 0;
         unsigned int            j, k, first_cpu, tmp;
         CPUMASK_ALLOC(allmasks);
-        CPUMASK_PTR(online_policy_cpus, allmasks);
         CPUMASK_PTR(saved_mask, allmasks);
-        CPUMASK_PTR(set_mask, allmasks);
         CPUMASK_PTR(covered_cpus, allmasks);
 
         if (unlikely(allmasks == NULL))
@@ -497,30 +493,28 @@ static int centrino_target (struct cpufreq_policy *policy,
                 goto out;
         }
 
-#ifdef CONFIG_HOTPLUG_CPU
-        /* cpufreq holds the hotplug lock, so we are safe from here on */
-        cpus_and(*online_policy_cpus, cpu_online_map, policy->cpus);
-#else
-        *online_policy_cpus = policy->cpus;
-#endif
-
         *saved_mask = current->cpus_allowed;
         first_cpu = 1;
         cpus_clear(*covered_cpus);
-        for_each_cpu_mask_nr(j, *online_policy_cpus) {
+        for_each_cpu_mask_nr(j, policy->cpus) {
+                const cpumask_t *mask;
+
+                /* cpufreq holds the hotplug lock, so we are safe here */
+                if (!cpu_online(j))
+                        continue;
+
                 /*
                  * Support for SMP systems.
                  * Make sure we are running on CPU that wants to change freq
                  */
-                cpus_clear(*set_mask);
                 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY)
-                        cpus_or(*set_mask, *set_mask, *online_policy_cpus);
+                        mask = &policy->cpus;
                 else
-                        cpu_set(j, *set_mask);
+                        mask = &cpumask_of_cpu(j);
 
-                set_cpus_allowed_ptr(current, set_mask);
+                set_cpus_allowed_ptr(current, mask);
                 preempt_disable();
-                if (unlikely(!cpu_isset(smp_processor_id(), *set_mask))) {
+                if (unlikely(!cpu_isset(smp_processor_id(), *mask))) {
                         dprintk("couldn't limit to CPUs in this domain\n");
                         retval = -EAGAIN;
                         if (first_cpu) {
@@ -548,7 +542,9 @@ static int centrino_target (struct cpufreq_policy *policy,
                         dprintk("target=%dkHz old=%d new=%d msr=%04x\n",
                                 target_freq, freqs.old, freqs.new, msr);
 
-                        for_each_cpu_mask_nr(k, *online_policy_cpus) {
+                        for_each_cpu_mask_nr(k, policy->cpus) {
+                                if (!cpu_online(k))
+                                        continue;
                                 freqs.cpu = k;
                                 cpufreq_notify_transition(&freqs,
                                         CPUFREQ_PRECHANGE);
@@ -571,7 +567,9 @@ static int centrino_target (struct cpufreq_policy *policy,
                 preempt_enable();
         }
 
-        for_each_cpu_mask_nr(k, *online_policy_cpus) {
+        for_each_cpu_mask_nr(k, policy->cpus) {
+                if (!cpu_online(k))
+                        continue;
                 freqs.cpu = k;
                 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
         }
@@ -584,18 +582,17 @@ static int centrino_target (struct cpufreq_policy *policy,
                  * Best effort undo..
                  */
 
-                if (!cpus_empty(*covered_cpus))
-                        for_each_cpu_mask_nr(j, *covered_cpus) {
-                                set_cpus_allowed_ptr(current,
-                                                     &cpumask_of_cpu(j));
-                                wrmsr(MSR_IA32_PERF_CTL, oldmsr, h);
-                        }
+                for_each_cpu_mask_nr(j, *covered_cpus) {
+                        set_cpus_allowed_ptr(current, &cpumask_of_cpu(j));
+                        wrmsr(MSR_IA32_PERF_CTL, oldmsr, h);
+                }
 
                 tmp = freqs.new;
                 freqs.new = freqs.old;
                 freqs.old = tmp;
-                for_each_cpu_mask_nr(j, *online_policy_cpus) {
-                        freqs.cpu = j;
+                for_each_cpu_mask_nr(j, policy->cpus) {
+                        if (!cpu_online(j))
+                                continue;
                         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
                         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
                 }
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c
index 98d4fdb7dc04..cdac7d62369b 100644
--- a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c
+++ b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c
@@ -139,6 +139,15 @@ static unsigned int pentium_core_get_frequency(void)
         case 3:
                 fsb = 166667;
                 break;
+        case 2:
+                fsb = 200000;
+                break;
+        case 0:
+                fsb = 266667;
+                break;
+        case 4:
+                fsb = 333333;
+                break;
         default:
                 printk(KERN_ERR "PCORE - MSR_FSB_FREQ undefined value");
         }
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
new file mode 100644
index 000000000000..fb5b86af0b01
--- /dev/null
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -0,0 +1,58 @@
+/*
+ * Common hypervisor code
+ *
+ * Copyright (C) 2008, VMware, Inc.
+ * Author : Alok N Kataria <akataria@vmware.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+#include <asm/processor.h>
+#include <asm/vmware.h>
+#include <asm/hypervisor.h>
+
+static inline void __cpuinit
+detect_hypervisor_vendor(struct cpuinfo_x86 *c)
+{
+        if (vmware_platform()) {
+                c->x86_hyper_vendor = X86_HYPER_VENDOR_VMWARE;
+        } else {
+                c->x86_hyper_vendor = X86_HYPER_VENDOR_NONE;
+        }
+}
+
+unsigned long get_hypervisor_tsc_freq(void)
+{
+        if (boot_cpu_data.x86_hyper_vendor == X86_HYPER_VENDOR_VMWARE)
+                return vmware_get_tsc_khz();
+        return 0;
+}
+
+static inline void __cpuinit
+hypervisor_set_feature_bits(struct cpuinfo_x86 *c)
+{
+        if (boot_cpu_data.x86_hyper_vendor == X86_HYPER_VENDOR_VMWARE) {
+                vmware_set_feature_bits(c);
+                return;
+        }
+}
+
+void __cpuinit init_hypervisor(struct cpuinfo_x86 *c)
+{
+        detect_hypervisor_vendor(c);
+        hypervisor_set_feature_bits(c);
+}
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index cce0b6118d55..8ea6929e974c 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -11,7 +11,6 @@
 #include <asm/pgtable.h>
 #include <asm/msr.h>
 #include <asm/uaccess.h>
-#include <asm/ptrace.h>
 #include <asm/ds.h>
 #include <asm/bugs.h>
 
@@ -41,6 +40,16 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
         if (c->x86 == 15 && c->x86_cache_alignment == 64)
                 c->x86_cache_alignment = 128;
 #endif
+
+        /*
+         * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
+         * with P/T states and does not stop in deep C-states
+         */
+        if (c->x86_power & (1 << 8)) {
+                set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
+                set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
+        }
+
 }
 
 #ifdef CONFIG_X86_32
@@ -242,6 +251,13 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
 
         intel_workarounds(c);
 
+        /*
+         * Detect the extended topology information if available. This
+         * will reinitialise the initial_apicid which will be used
+         * in init_intel_cacheinfo()
+         */
+        detect_extended_topology(c);
+
         l2 = init_intel_cacheinfo(c);
         if (c->cpuid_level > 9) {
                 unsigned eax = cpuid_eax(10);
@@ -307,13 +323,8 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
                 set_cpu_cap(c, X86_FEATURE_P4);
         if (c->x86 == 6)
                 set_cpu_cap(c, X86_FEATURE_P3);
-
-        if (cpu_has_bts)
-                ptrace_bts_init_intel(c);
-
 #endif
 
-        detect_extended_topology(c);
         if (!cpu_has(c, X86_FEATURE_XTOPOLOGY)) {
                 /*
                  * let's use the legacy cpuid vector 0x1 and 0x4 for topology
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 3f46afbb1cf1..48533d77be78 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -534,31 +534,16 @@ static void __cpuinit free_cache_attributes(unsigned int cpu)
         per_cpu(cpuid4_info, cpu) = NULL;
 }
 
-static int __cpuinit detect_cache_attributes(unsigned int cpu)
+static void __cpuinit get_cpu_leaves(void *_retval)
 {
-        struct _cpuid4_info     *this_leaf;
-        unsigned long           j;
-        int                     retval;
-        cpumask_t               oldmask;
-
-        if (num_cache_leaves == 0)
-                return -ENOENT;
-
-        per_cpu(cpuid4_info, cpu) = kzalloc(
-            sizeof(struct _cpuid4_info) * num_cache_leaves, GFP_KERNEL);
-        if (per_cpu(cpuid4_info, cpu) == NULL)
-                return -ENOMEM;
-
-        oldmask = current->cpus_allowed;
-        retval = set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
-        if (retval)
-                goto out;
+        int j, *retval = _retval, cpu = smp_processor_id();
 
         /* Do cpuid and store the results */
         for (j = 0; j < num_cache_leaves; j++) {
+                struct _cpuid4_info *this_leaf;
                 this_leaf = CPUID4_INFO_IDX(cpu, j);
-                retval = cpuid4_cache_lookup(j, this_leaf);
-                if (unlikely(retval < 0)) {
+                *retval = cpuid4_cache_lookup(j, this_leaf);
+                if (unlikely(*retval < 0)) {
                         int i;
 
                         for (i = 0; i < j; i++)
@@ -567,9 +552,21 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu)
                 }
                 cache_shared_cpu_map_setup(cpu, j);
         }
-        set_cpus_allowed_ptr(current, &oldmask);
+}
+
+static int __cpuinit detect_cache_attributes(unsigned int cpu)
+{
+        int                     retval;
+
+        if (num_cache_leaves == 0)
+                return -ENOENT;
 
-out:
+        per_cpu(cpuid4_info, cpu) = kzalloc(
+            sizeof(struct _cpuid4_info) * num_cache_leaves, GFP_KERNEL);
+        if (per_cpu(cpuid4_info, cpu) == NULL)
+                return -ENOMEM;
+
+        smp_call_function_single(cpu, get_cpu_leaves, &retval, true);
         if (retval) {
                 kfree(per_cpu(cpuid4_info, cpu));
                 per_cpu(cpuid4_info, cpu) = NULL;
@@ -626,8 +623,8 @@ static ssize_t show_shared_cpu_map_func(struct _cpuid4_info *this_leaf,
                 cpumask_t *mask = &this_leaf->shared_cpu_map;
 
                 n = type?
-                        cpulist_scnprintf(buf, len-2, *mask):
-                        cpumask_scnprintf(buf, len-2, *mask);
+                        cpulist_scnprintf(buf, len-2, mask) :
+                        cpumask_scnprintf(buf, len-2, mask);
                 buf[n++] = '\n';
                 buf[n] = '\0';
         }
@@ -644,20 +641,17 @@ static inline ssize_t show_shared_cpu_list(struct _cpuid4_info *leaf, char *buf)
         return show_shared_cpu_map_func(leaf, 1, buf);
 }
 
-static ssize_t show_type(struct _cpuid4_info *this_leaf, char *buf) {
-        switch(this_leaf->eax.split.type) {
-            case CACHE_TYPE_DATA:
+static ssize_t show_type(struct _cpuid4_info *this_leaf, char *buf)
+{
+        switch (this_leaf->eax.split.type) {
+        case CACHE_TYPE_DATA:
                 return sprintf(buf, "Data\n");
-                break;
-            case CACHE_TYPE_INST:
+        case CACHE_TYPE_INST:
                 return sprintf(buf, "Instruction\n");
-                break;
-            case CACHE_TYPE_UNIFIED:
+        case CACHE_TYPE_UNIFIED:
                 return sprintf(buf, "Unified\n");
-                break;
-            default:
+        default:
                 return sprintf(buf, "Unknown\n");
-                break;
         }
 }
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
index 5eb390a4b2e9..a5a5e0530370 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
@@ -83,34 +83,41 @@ static DEFINE_PER_CPU(unsigned char, bank_map);	/* see which banks are on */
  * CPU Initialization
  */
 
+struct thresh_restart {
+        struct threshold_block *b;
+        int reset;
+        u16 old_limit;
+};
+
 /* must be called with correct cpu affinity */
-static void threshold_restart_bank(struct threshold_block *b,
-                                   int reset, u16 old_limit)
+static long threshold_restart_bank(void *_tr)
 {
+        struct thresh_restart *tr = _tr;
         u32 mci_misc_hi, mci_misc_lo;
 
-        rdmsr(b->address, mci_misc_lo, mci_misc_hi);
+        rdmsr(tr->b->address, mci_misc_lo, mci_misc_hi);
 
-        if (b->threshold_limit < (mci_misc_hi & THRESHOLD_MAX))
-                reset = 1;      /* limit cannot be lower than err count */
+        if (tr->b->threshold_limit < (mci_misc_hi & THRESHOLD_MAX))
+                tr->reset = 1;  /* limit cannot be lower than err count */
 
-        if (reset) {            /* reset err count and overflow bit */
+        if (tr->reset) {                /* reset err count and overflow bit */
                 mci_misc_hi =
                     (mci_misc_hi & ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI)) |
-                    (THRESHOLD_MAX - b->threshold_limit);
-        } else if (old_limit) { /* change limit w/o reset */
+                    (THRESHOLD_MAX - tr->b->threshold_limit);
+        } else if (tr->old_limit) {     /* change limit w/o reset */
                 int new_count = (mci_misc_hi & THRESHOLD_MAX) +
-                    (old_limit - b->threshold_limit);
+                    (tr->old_limit - tr->b->threshold_limit);
                 mci_misc_hi = (mci_misc_hi & ~MASK_ERR_COUNT_HI) |
                     (new_count & THRESHOLD_MAX);
         }
 
-        b->interrupt_enable ?
+        tr->b->interrupt_enable ?
             (mci_misc_hi = (mci_misc_hi & ~MASK_INT_TYPE_HI) | INT_TYPE_APIC) :
             (mci_misc_hi &= ~MASK_INT_TYPE_HI);
 
         mci_misc_hi |= MASK_COUNT_EN_HI;
-        wrmsr(b->address, mci_misc_lo, mci_misc_hi);
+        wrmsr(tr->b->address, mci_misc_lo, mci_misc_hi);
+        return 0;
 }
 
 /* cpu init entry point, called from mce.c with preempt off */
@@ -120,6 +127,7 @@ void __cpuinit mce_amd_feature_init(struct cpuinfo_x86 *c)
         unsigned int cpu = smp_processor_id();
         u8 lvt_off;
         u32 low = 0, high = 0, address = 0;
+        struct thresh_restart tr;
 
         for (bank = 0; bank < NR_BANKS; ++bank) {
                 for (block = 0; block < NR_BLOCKS; ++block) {
@@ -162,7 +170,10 @@ void __cpuinit mce_amd_feature_init(struct cpuinfo_x86 *c)
                         wrmsr(address, low, high);
 
                         threshold_defaults.address = address;
-                        threshold_restart_bank(&threshold_defaults, 0, 0);
+                        tr.b = &threshold_defaults;
+                        tr.reset = 0;
+                        tr.old_limit = 0;
+                        threshold_restart_bank(&tr);
                 }
         }
 }
@@ -237,7 +248,7 @@ asmlinkage void mce_threshold_interrupt(void)
                 }
         }
 out:
-        add_pda(irq_threshold_count, 1);
+        inc_irq_stat(irq_threshold_count);
         irq_exit();
 }
 
@@ -251,20 +262,6 @@ struct threshold_attr {
         ssize_t(*store) (struct threshold_block *, const char *, size_t count);
 };
 
-static void affinity_set(unsigned int cpu, cpumask_t *oldmask,
-                                           cpumask_t *newmask)
-{
-        *oldmask = current->cpus_allowed;
-        cpus_clear(*newmask);
-        cpu_set(cpu, *newmask);
-        set_cpus_allowed_ptr(current, newmask);
-}
-
-static void affinity_restore(const cpumask_t *oldmask)
-{
-        set_cpus_allowed_ptr(current, oldmask);
-}
-
 #define SHOW_FIELDS(name)                                           \
 static ssize_t show_ ## name(struct threshold_block * b, char *buf) \
 {                                                                   \
@@ -277,15 +274,16 @@ static ssize_t store_interrupt_enable(struct threshold_block *b,
                                       const char *buf, size_t count)
 {
         char *end;
-        cpumask_t oldmask, newmask;
+        struct thresh_restart tr;
         unsigned long new = simple_strtoul(buf, &end, 0);
         if (end == buf)
                 return -EINVAL;
         b->interrupt_enable = !!new;
 
-        affinity_set(b->cpu, &oldmask, &newmask);
-        threshold_restart_bank(b, 0, 0);
-        affinity_restore(&oldmask);
+        tr.b = b;
+        tr.reset = 0;
+        tr.old_limit = 0;
+        work_on_cpu(b->cpu, threshold_restart_bank, &tr);
 
         return end - buf;
 }
@@ -294,8 +292,7 @@ static ssize_t store_threshold_limit(struct threshold_block *b,
                                      const char *buf, size_t count)
 {
         char *end;
-        cpumask_t oldmask, newmask;
-        u16 old;
+        struct thresh_restart tr;
         unsigned long new = simple_strtoul(buf, &end, 0);
         if (end == buf)
                 return -EINVAL;
@@ -303,34 +300,36 @@ static ssize_t store_threshold_limit(struct threshold_block *b,
                 new = THRESHOLD_MAX;
         if (new < 1)
                 new = 1;
-        old = b->threshold_limit;
+        tr.old_limit = b->threshold_limit;
         b->threshold_limit = new;
+        tr.b = b;
+        tr.reset = 0;
 
-        affinity_set(b->cpu, &oldmask, &newmask);
-        threshold_restart_bank(b, 0, old);
-        affinity_restore(&oldmask);
+        work_on_cpu(b->cpu, threshold_restart_bank, &tr);
 
         return end - buf;
 }
 
-static ssize_t show_error_count(struct threshold_block *b, char *buf)
+static long local_error_count(void *_b)
 {
-        u32 high, low;
-        cpumask_t oldmask, newmask;
-        affinity_set(b->cpu, &oldmask, &newmask);
+        struct threshold_block *b = _b;
+        u32 low, high;
+
         rdmsr(b->address, low, high);
-        affinity_restore(&oldmask);
-        return sprintf(buf, "%x\n",
-                       (high & 0xFFF) - (THRESHOLD_MAX - b->threshold_limit));
+        return (high & 0xFFF) - (THRESHOLD_MAX - b->threshold_limit);
+}
+
+static ssize_t show_error_count(struct threshold_block *b, char *buf)
+{
+        return sprintf(buf, "%lx\n", work_on_cpu(b->cpu, local_error_count, b));
 }
 
 static ssize_t store_error_count(struct threshold_block *b,
                                  const char *buf, size_t count)
 {
-        cpumask_t oldmask, newmask;
-        affinity_set(b->cpu, &oldmask, &newmask);
-        threshold_restart_bank(b, 1, 0);
-        affinity_restore(&oldmask);
+        struct thresh_restart tr = { .b = b, .reset = 1, .old_limit = 0 };
+
+        work_on_cpu(b->cpu, threshold_restart_bank, &tr);
         return 1;
 }
 
@@ -463,12 +462,19 @@ out_free:
         return err;
 }
 
+static long local_allocate_threshold_blocks(void *_bank)
+{
+        unsigned int *bank = _bank;
+
+        return allocate_threshold_blocks(smp_processor_id(), *bank, 0,
+                                         MSR_IA32_MC0_MISC + *bank * 4);
+}
+
 /* symlinks sibling shared banks to first core.  first core owns dir/files. */
 static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 {
         int i, err = 0;
         struct threshold_bank *b = NULL;
-        cpumask_t oldmask, newmask;
         char name[32];
 
         sprintf(name, "threshold_bank%i", bank);
@@ -519,11 +525,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 
         per_cpu(threshold_banks, cpu)[bank] = b;
 
-        affinity_set(cpu, &oldmask, &newmask);
-        err = allocate_threshold_blocks(cpu, bank, 0,
-                                        MSR_IA32_MC0_MISC + bank * 4);
-        affinity_restore(&oldmask);
-
+        err = work_on_cpu(cpu, local_allocate_threshold_blocks, &bank);
         if (err)
                 goto out_free;
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
index c17eaf5dd6dd..4b48f251fd39 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
@@ -26,7 +26,7 @@ asmlinkage void smp_thermal_interrupt(void)
         if (therm_throt_process(msr_val & 1))
                 mce_log_therm_throt_event(smp_processor_id(), msr_val);
 
-        add_pda(irq_thermal_count, 1);
+        inc_irq_stat(irq_thermal_count);
         irq_exit();
 }
 
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 4e8d77f01eeb..b59ddcc88cd8 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -14,14 +14,6 @@
 #include <asm/pat.h>
 #include "mtrr.h"
 
-struct mtrr_state {
-        struct mtrr_var_range var_ranges[MAX_VAR_RANGES];
-        mtrr_type fixed_ranges[NUM_FIXED_RANGES];
-        unsigned char enabled;
-        unsigned char have_fixed;
-        mtrr_type def_type;
-};
-
 struct fixed_range_block {
         int base_msr; /* start address of an MTRR block */
         int ranges;   /* number of MTRRs in this block  */
@@ -35,10 +27,12 @@ static struct fixed_range_block fixed_range_blocks[] = {
 };
 
 static unsigned long smp_changes_mask;
-static struct mtrr_state mtrr_state = {};
 static int mtrr_state_set;
 u64 mtrr_tom2;
 
+struct mtrr_state_type mtrr_state = {};
+EXPORT_SYMBOL_GPL(mtrr_state);
+
 #undef MODULE_PARAM_PREFIX
 #define MODULE_PARAM_PREFIX "mtrr."
 
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index c78c04821ea1..d259e5d2e054 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -49,7 +49,7 @@
 
 u32 num_var_ranges = 0;
 
-unsigned int mtrr_usage_table[MAX_VAR_RANGES];
+unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
 static DEFINE_MUTEX(mtrr_mutex);
 
 u64 size_or_mask, size_and_mask;
@@ -574,7 +574,7 @@ struct mtrr_value {
         unsigned long   lsize;
 };
 
-static struct mtrr_value mtrr_state[MAX_VAR_RANGES];
+static struct mtrr_value mtrr_state[MTRR_MAX_VAR_RANGES];
 
 static int mtrr_save(struct sys_device * sysdev, pm_message_t state)
 {
@@ -803,6 +803,7 @@ x86_get_mtrr_mem_range(struct res_range *range, int nr_range,
 }
 
 static struct res_range __initdata range[RANGE_NUM];
+static int __initdata nr_range;
 
 #ifdef CONFIG_MTRR_SANITIZER
 
@@ -823,16 +824,14 @@ static int enable_mtrr_cleanup __initdata =
 
 static int __init disable_mtrr_cleanup_setup(char *str)
 {
-        if (enable_mtrr_cleanup != -1)
-                enable_mtrr_cleanup = 0;
+        enable_mtrr_cleanup = 0;
         return 0;
 }
 early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup);
 
 static int __init enable_mtrr_cleanup_setup(char *str)
 {
-        if (enable_mtrr_cleanup != -1)
-                enable_mtrr_cleanup = 1;
+        enable_mtrr_cleanup = 1;
         return 0;
 }
 early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup);
@@ -1206,39 +1205,43 @@ struct mtrr_cleanup_result {
 #define PSHIFT          (PAGE_SHIFT - 10)
 
 static struct mtrr_cleanup_result __initdata result[NUM_RESULT];
-static struct res_range __initdata range_new[RANGE_NUM];
 static unsigned long __initdata min_loss_pfn[RANGE_NUM];
 
-static int __init mtrr_cleanup(unsigned address_bits)
+static void __init print_out_mtrr_range_state(void)
 {
-        unsigned long extra_remove_base, extra_remove_size;
-        unsigned long base, size, def, dummy;
-        mtrr_type type;
-        int nr_range, nr_range_new;
-        u64 chunk_size, gran_size;
-        unsigned long range_sums, range_sums_new;
-        int index_good;
-        int num_reg_good;
         int i;
+        char start_factor = 'K', size_factor = 'K';
+        unsigned long start_base, size_base;
+        mtrr_type type;
 
-        /* extra one for all 0 */
-        int num[MTRR_NUM_TYPES + 1];
+        for (i = 0; i < num_var_ranges; i++) {
 
-        if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
-                return 0;
-        rdmsr(MTRRdefType_MSR, def, dummy);
-        def &= 0xff;
-        if (def != MTRR_TYPE_UNCACHABLE)
-                return 0;
+                size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10);
+                if (!size_base)
+                        continue;
 
-        /* get it and store it aside */
-        memset(range_state, 0, sizeof(range_state));
-        for (i = 0; i < num_var_ranges; i++) {
-                mtrr_if->get(i, &base, &size, &type);
-                range_state[i].base_pfn = base;
-                range_state[i].size_pfn = size;
-                range_state[i].type = type;
+                size_base = to_size_factor(size_base, &size_factor),
+                start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10);
+                start_base = to_size_factor(start_base, &start_factor),
+                type = range_state[i].type;
+
+                printk(KERN_DEBUG "reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
+                        i, start_base, start_factor,
+                        size_base, size_factor,
+                        (type == MTRR_TYPE_UNCACHABLE) ? "UC" :
+                            ((type == MTRR_TYPE_WRPROT) ? "WP" :
+                             ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other"))
+                        );
         }
+}
+
+static int __init mtrr_need_cleanup(void)
+{
+        int i;
+        mtrr_type type;
+        unsigned long size;
+        /* extra one for all 0 */
+        int num[MTRR_NUM_TYPES + 1];
 
         /* check entries number */
         memset(num, 0, sizeof(num));
@@ -1263,29 +1266,133 @@ static int __init mtrr_cleanup(unsigned address_bits)
                 num_var_ranges - num[MTRR_NUM_TYPES])
                 return 0;
 
-        /* print original var MTRRs at first, for debugging: */
-        printk(KERN_DEBUG "original variable MTRRs\n");
-        for (i = 0; i < num_var_ranges; i++) {
-                char start_factor = 'K', size_factor = 'K';
-                unsigned long start_base, size_base;
+        return 1;
+}
 
-                size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10);
-                if (!size_base)
-                        continue;
+static unsigned long __initdata range_sums;
+static void __init mtrr_calc_range_state(u64 chunk_size, u64 gran_size,
+                                         unsigned long extra_remove_base,
+                                         unsigned long extra_remove_size,
+                                         int i)
+{
+        int num_reg;
+        static struct res_range range_new[RANGE_NUM];
+        static int nr_range_new;
+        unsigned long range_sums_new;
+
+        /* convert ranges to var ranges state */
+        num_reg = x86_setup_var_mtrrs(range, nr_range,
+                                                chunk_size, gran_size);
+
+        /* we got new setting in range_state, check it */
+        memset(range_new, 0, sizeof(range_new));
+        nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
+                                extra_remove_base, extra_remove_size);
+        range_sums_new = sum_ranges(range_new, nr_range_new);
+
+        result[i].chunk_sizek = chunk_size >> 10;
+        result[i].gran_sizek = gran_size >> 10;
+        result[i].num_reg = num_reg;
+        if (range_sums < range_sums_new) {
+                result[i].lose_cover_sizek =
+                        (range_sums_new - range_sums) << PSHIFT;
+                result[i].bad = 1;
+        } else
+                result[i].lose_cover_sizek =
+                        (range_sums - range_sums_new) << PSHIFT;
 
-                size_base = to_size_factor(size_base, &size_factor),
-                start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10);
-                start_base = to_size_factor(start_base, &start_factor),
-                type = range_state[i].type;
+        /* double check it */
+        if (!result[i].bad && !result[i].lose_cover_sizek) {
+                if (nr_range_new != nr_range ||
+                        memcmp(range, range_new, sizeof(range)))
+                                result[i].bad = 1;
+        }
 
-                printk(KERN_DEBUG "reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
-                        i, start_base, start_factor,
-                        size_base, size_factor,
-                        (type == MTRR_TYPE_UNCACHABLE) ? "UC" :
-                            ((type == MTRR_TYPE_WRPROT) ? "WP" :
-                             ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other"))
-                        );
+        if (!result[i].bad && (range_sums - range_sums_new <
+                                min_loss_pfn[num_reg])) {
+                min_loss_pfn[num_reg] =
+                        range_sums - range_sums_new;
         }
+}
+
+static void __init mtrr_print_out_one_result(int i)
+{
+        char gran_factor, chunk_factor, lose_factor;
+        unsigned long gran_base, chunk_base, lose_base;
+
+        gran_base = to_size_factor(result[i].gran_sizek, &gran_factor),
+        chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor),
+        lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor),
+        printk(KERN_INFO "%sgran_size: %ld%c \tchunk_size: %ld%c \t",
+                        result[i].bad ? "*BAD*" : " ",
+                        gran_base, gran_factor, chunk_base, chunk_factor);
+        printk(KERN_CONT "num_reg: %d  \tlose cover RAM: %s%ld%c\n",
+                        result[i].num_reg, result[i].bad ? "-" : "",
+                        lose_base, lose_factor);
+}
+
+static int __init mtrr_search_optimal_index(void)
+{
+        int i;
+        int num_reg_good;
+        int index_good;
+
+        if (nr_mtrr_spare_reg >= num_var_ranges)
+                nr_mtrr_spare_reg = num_var_ranges - 1;
+        num_reg_good = -1;
+        for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
+                if (!min_loss_pfn[i])
+                        num_reg_good = i;
+        }
+
+        index_good = -1;
+        if (num_reg_good != -1) {
+                for (i = 0; i < NUM_RESULT; i++) {
+                        if (!result[i].bad &&
+                            result[i].num_reg == num_reg_good &&
+                            !result[i].lose_cover_sizek) {
+                                index_good = i;
+                                break;
+                        }
+                }
+        }
+
+        return index_good;
+}
+
+
+static int __init mtrr_cleanup(unsigned address_bits)
+{
+        unsigned long extra_remove_base, extra_remove_size;
+        unsigned long base, size, def, dummy;
+        mtrr_type type;
+        u64 chunk_size, gran_size;
+        int index_good;
+        int i;
+
+        if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
+                return 0;
+        rdmsr(MTRRdefType_MSR, def, dummy);
+        def &= 0xff;
+        if (def != MTRR_TYPE_UNCACHABLE)
+                return 0;
+
+        /* get it and store it aside */
+        memset(range_state, 0, sizeof(range_state));
+        for (i = 0; i < num_var_ranges; i++) {
+                mtrr_if->get(i, &base, &size, &type);
+                range_state[i].base_pfn = base;
+                range_state[i].size_pfn = size;
+                range_state[i].type = type;
+        }
+
+        /* check if we need handle it and can handle it */
+        if (!mtrr_need_cleanup())
+                return 0;
+
+        /* print original var MTRRs at first, for debugging: */
+        printk(KERN_DEBUG "original variable MTRRs\n");
+        print_out_mtrr_range_state();
 
         memset(range, 0, sizeof(range));
         extra_remove_size = 0;
@@ -1309,176 +1416,64 @@ static int __init mtrr_cleanup(unsigned address_bits)
                range_sums >> (20 - PAGE_SHIFT));
 
         if (mtrr_chunk_size && mtrr_gran_size) {
-                int num_reg;
-                char gran_factor, chunk_factor, lose_factor;
-                unsigned long gran_base, chunk_base, lose_base;
-
-                debug_print++;
-                /* convert ranges to var ranges state */
-                num_reg = x86_setup_var_mtrrs(range, nr_range, mtrr_chunk_size,
-                                              mtrr_gran_size);
+                i = 0;
+                mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size,
+                                      extra_remove_base, extra_remove_size, i);
 
-                /* we got new setting in range_state, check it */
-                memset(range_new, 0, sizeof(range_new));
-                nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
-                                                      extra_remove_base,
-                                                      extra_remove_size);
-                range_sums_new = sum_ranges(range_new, nr_range_new);
+                mtrr_print_out_one_result(i);
 
-                i = 0;
-                result[i].chunk_sizek = mtrr_chunk_size >> 10;
-                result[i].gran_sizek = mtrr_gran_size >> 10;
-                result[i].num_reg = num_reg;
-                if (range_sums < range_sums_new) {
-                        result[i].lose_cover_sizek =
-                                (range_sums_new - range_sums) << PSHIFT;
-                        result[i].bad = 1;
-                } else
-                        result[i].lose_cover_sizek =
-                                (range_sums - range_sums_new) << PSHIFT;
-
-                gran_base = to_size_factor(result[i].gran_sizek, &gran_factor),
-                chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor),
-                lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor),
-                printk(KERN_INFO "%sgran_size: %ld%c \tchunk_size: %ld%c \t",
-                         result[i].bad?"*BAD*":" ",
-                         gran_base, gran_factor, chunk_base, chunk_factor);
-                printk(KERN_CONT "num_reg: %d  \tlose cover RAM: %s%ld%c\n",
-                         result[i].num_reg, result[i].bad?"-":"",
-                         lose_base, lose_factor);
                 if (!result[i].bad) {
                         set_var_mtrr_all(address_bits);
                         return 1;
                 }
                 printk(KERN_INFO "invalid mtrr_gran_size or mtrr_chunk_size, "
                        "will find optimal one\n");
-                debug_print--;
-                memset(result, 0, sizeof(result[0]));
         }
 
         i = 0;
         memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn));
         memset(result, 0, sizeof(result));
         for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) {
-                char gran_factor;
-                unsigned long gran_base;
-
-                if (debug_print)
-                        gran_base = to_size_factor(gran_size >> 10, &gran_factor);
 
                 for (chunk_size = gran_size; chunk_size < (1ULL<<32);
                      chunk_size <<= 1) {
-                        int num_reg;
 
-                        if (debug_print) {
-                                char chunk_factor;
-                                unsigned long chunk_base;
-
-                                chunk_base = to_size_factor(chunk_size>>10, &chunk_factor),
-                                printk(KERN_INFO "\n");
-                                printk(KERN_INFO "gran_size: %ld%c   chunk_size: %ld%c \n",
-                                       gran_base, gran_factor, chunk_base, chunk_factor);
-                        }
                         if (i >= NUM_RESULT)
                                 continue;
 
-                        /* convert ranges to var ranges state */
-                        num_reg = x86_setup_var_mtrrs(range, nr_range,
-                                                         chunk_size, gran_size);
-
-                        /* we got new setting in range_state, check it */
-                        memset(range_new, 0, sizeof(range_new));
-                        nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
-                                         extra_remove_base, extra_remove_size);
-                        range_sums_new = sum_ranges(range_new, nr_range_new);
-
-                        result[i].chunk_sizek = chunk_size >> 10;
-                        result[i].gran_sizek = gran_size >> 10;
-                        result[i].num_reg = num_reg;
-                        if (range_sums < range_sums_new) {
-                                result[i].lose_cover_sizek =
-                                        (range_sums_new - range_sums) << PSHIFT;
-                                result[i].bad = 1;
-                        } else
-                                result[i].lose_cover_sizek =
-                                        (range_sums - range_sums_new) << PSHIFT;
-
-                        /* double check it */
-                        if (!result[i].bad && !result[i].lose_cover_sizek) {
-                                if (nr_range_new != nr_range ||
-                                        memcmp(range, range_new, sizeof(range)))
-                                                result[i].bad = 1;
+                        mtrr_calc_range_state(chunk_size, gran_size,
+                                      extra_remove_base, extra_remove_size, i);
+                        if (debug_print) {
+                                mtrr_print_out_one_result(i);
+                                printk(KERN_INFO "\n");
                         }
 
-                        if (!result[i].bad && (range_sums - range_sums_new <
-                                               min_loss_pfn[num_reg])) {
-                                min_loss_pfn[num_reg] =
-                                        range_sums - range_sums_new;
-                        }
                         i++;
                 }
         }
 
-        /* print out all */
-        for (i = 0; i < NUM_RESULT; i++) {
-                char gran_factor, chunk_factor, lose_factor;
-                unsigned long gran_base, chunk_base, lose_base;
-
-                gran_base = to_size_factor(result[i].gran_sizek, &gran_factor),
-                chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor),
-                lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor),
-                printk(KERN_INFO "%sgran_size: %ld%c \tchunk_size: %ld%c \t",
-                         result[i].bad?"*BAD*":" ",
-                         gran_base, gran_factor, chunk_base, chunk_factor);
-                printk(KERN_CONT "num_reg: %d  \tlose cover RAM: %s%ld%c\n",
-                         result[i].num_reg, result[i].bad?"-":"",
-                         lose_base, lose_factor);
-        }
-
         /* try to find the optimal index */
-        if (nr_mtrr_spare_reg >= num_var_ranges)
-                nr_mtrr_spare_reg = num_var_ranges - 1;
-        num_reg_good = -1;
-        for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
-                if (!min_loss_pfn[i])
-                        num_reg_good = i;
-        }
-
-        index_good = -1;
-        if (num_reg_good != -1) {
-                for (i = 0; i < NUM_RESULT; i++) {
-                        if (!result[i].bad &&
-                            result[i].num_reg == num_reg_good &&
-                            !result[i].lose_cover_sizek) {
-                                index_good = i;
-                                break;
-                        }
-                }
-        }
+        index_good = mtrr_search_optimal_index();
 
         if (index_good != -1) {
-                char gran_factor, chunk_factor, lose_factor;
-                unsigned long gran_base, chunk_base, lose_base;
-
                 printk(KERN_INFO "Found optimal setting for mtrr clean up\n");
                 i = index_good;
-                gran_base = to_size_factor(result[i].gran_sizek, &gran_factor),
-                chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor),
-                lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor),
-                printk(KERN_INFO "gran_size: %ld%c \tchunk_size: %ld%c \t",
-                         gran_base, gran_factor, chunk_base, chunk_factor);
-                printk(KERN_CONT "num_reg: %d  \tlose RAM: %ld%c\n",
-                         result[i].num_reg, lose_base, lose_factor);
+                mtrr_print_out_one_result(i);
+
                 /* convert ranges to var ranges state */
                 chunk_size = result[i].chunk_sizek;
                 chunk_size <<= 10;
                 gran_size = result[i].gran_sizek;
                 gran_size <<= 10;
-                debug_print++;
                 x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
-                debug_print--;
                 set_var_mtrr_all(address_bits);
+                printk(KERN_DEBUG "New variable MTRRs\n");
+                print_out_mtrr_range_state();
                 return 1;
+        } else {
+                /* print out all */
+                for (i = 0; i < NUM_RESULT; i++)
+                        mtrr_print_out_one_result(i);
         }
 
         printk(KERN_INFO "mtrr_cleanup: can not find optimal value\n");
@@ -1562,7 +1557,6 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
 {
         unsigned long i, base, size, highest_pfn = 0, def, dummy;
         mtrr_type type;
-        int nr_range;
         u64 total_trim_size;
 
         /* extra one for all 0 */
diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.h b/arch/x86/kernel/cpu/mtrr/mtrr.h
index 2dc4ec656b23..ffd60409cc6d 100644
--- a/arch/x86/kernel/cpu/mtrr/mtrr.h
+++ b/arch/x86/kernel/cpu/mtrr/mtrr.h
@@ -8,11 +8,6 @@
 #define MTRRcap_MSR     0x0fe
 #define MTRRdefType_MSR 0x2ff
 
-#define MTRRphysBase_MSR(reg) (0x200 + 2 * (reg))
-#define MTRRphysMask_MSR(reg) (0x200 + 2 * (reg) + 1)
-
-#define NUM_FIXED_RANGES 88
-#define MAX_VAR_RANGES 256
 #define MTRRfix64K_00000_MSR 0x250
 #define MTRRfix16K_80000_MSR 0x258
 #define MTRRfix16K_A0000_MSR 0x259
@@ -29,11 +24,7 @@
 #define MTRR_CHANGE_MASK_VARIABLE  0x02
 #define MTRR_CHANGE_MASK_DEFTYPE   0x04
 
-/* In the Intel processor's MTRR interface, the MTRR type is always held in
-   an 8 bit field: */
-typedef u8 mtrr_type;
-
-extern unsigned int mtrr_usage_table[MAX_VAR_RANGES];
+extern unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
 
 struct mtrr_ops {
         u32     vendor;
@@ -70,13 +61,6 @@ struct set_mtrr_context {
         u32 ccr3;
 };
 
-struct mtrr_var_range {
-        u32 base_lo;
-        u32 base_hi;
-        u32 mask_lo;
-        u32 mask_hi;
-};
-
 void set_mtrr_done(struct set_mtrr_context *ctxt);
 void set_mtrr_cache_disable(struct set_mtrr_context *ctxt);
 void set_mtrr_prepare_save(struct set_mtrr_context *ctxt);
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
new file mode 100644
index 000000000000..284c399e3234
--- /dev/null
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -0,0 +1,112 @@
+/*
+ * VMware Detection code.
+ *
+ * Copyright (C) 2008, VMware, Inc.
+ * Author : Alok N Kataria <akataria@vmware.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+#include <linux/dmi.h>
+#include <asm/div64.h>
+#include <asm/vmware.h>
+
+#define CPUID_VMWARE_INFO_LEAF  0x40000000
+#define VMWARE_HYPERVISOR_MAGIC 0x564D5868
+#define VMWARE_HYPERVISOR_PORT  0x5658
+
+#define VMWARE_PORT_CMD_GETVERSION      10
+#define VMWARE_PORT_CMD_GETHZ           45
+
+#define VMWARE_PORT(cmd, eax, ebx, ecx, edx)                            \
+        __asm__("inl (%%dx)" :                                          \
+                        "=a"(eax), "=c"(ecx), "=d"(edx), "=b"(ebx) :    \
+                        "0"(VMWARE_HYPERVISOR_MAGIC),                   \
+                        "1"(VMWARE_PORT_CMD_##cmd),                     \
+                        "2"(VMWARE_HYPERVISOR_PORT), "3"(UINT_MAX) :    \
+                        "memory");
+
+static inline int __vmware_platform(void)
+{
+        uint32_t eax, ebx, ecx, edx;
+        VMWARE_PORT(GETVERSION, eax, ebx, ecx, edx);
+        return eax != (uint32_t)-1 && ebx == VMWARE_HYPERVISOR_MAGIC;
+}
+
+static unsigned long __vmware_get_tsc_khz(void)
+{
+        uint64_t tsc_hz;
+        uint32_t eax, ebx, ecx, edx;
+
+        VMWARE_PORT(GETHZ, eax, ebx, ecx, edx);
+
+        if (ebx == UINT_MAX)
+                return 0;
+        tsc_hz = eax | (((uint64_t)ebx) << 32);
+        do_div(tsc_hz, 1000);
+        BUG_ON(tsc_hz >> 32);
+        return tsc_hz;
+}
+
+/*
+ * While checking the dmi string infomation, just checking the product
+ * serial key should be enough, as this will always have a VMware
+ * specific string when running under VMware hypervisor.
+ */
+int vmware_platform(void)
+{
+        if (cpu_has_hypervisor) {
+                unsigned int eax, ebx, ecx, edx;
+                char hyper_vendor_id[13];
+
+                cpuid(CPUID_VMWARE_INFO_LEAF, &eax, &ebx, &ecx, &edx);
+                memcpy(hyper_vendor_id + 0, &ebx, 4);
+                memcpy(hyper_vendor_id + 4, &ecx, 4);
+                memcpy(hyper_vendor_id + 8, &edx, 4);
+                hyper_vendor_id[12] = '\0';
+                if (!strcmp(hyper_vendor_id, "VMwareVMware"))
+                        return 1;
+        } else if (dmi_available && dmi_name_in_serial("VMware") &&
+                   __vmware_platform())
+                return 1;
+
+        return 0;
+}
+
+unsigned long vmware_get_tsc_khz(void)
+{
+        BUG_ON(!vmware_platform());
+        return __vmware_get_tsc_khz();
+}
+
+/*
+ * VMware hypervisor takes care of exporting a reliable TSC to the guest.
+ * Still, due to timing difference when running on virtual cpus, the TSC can
+ * be marked as unstable in some cases. For example, the TSC sync check at
+ * bootup can fail due to a marginal offset between vcpus' TSCs (though the
+ * TSCs do not drift from each other).  Also, the ACPI PM timer clocksource
+ * is not suitable as a watchdog when running on a hypervisor because the
+ * kernel may miss a wrap of the counter if the vcpu is descheduled for a
+ * long time. To skip these checks at runtime we set these capability bits,
+ * so that the kernel could just trust the hypervisor with providing a
+ * reliable virtual TSC that is suitable for timekeeping.
+ */
+void __cpuinit vmware_set_feature_bits(struct cpuinfo_x86 *c)
+{
+        set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
+        set_cpu_cap(c, X86_FEATURE_TSC_RELIABLE);
+}
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 72cefd1e649b..2ac1f0c2beb3 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -39,10 +39,10 @@
 #include <linux/device.h>
 #include <linux/cpu.h>
 #include <linux/notifier.h>
+#include <linux/uaccess.h>
 
 #include <asm/processor.h>
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/system.h>
 
 static struct class *cpuid_class;
@@ -82,7 +82,7 @@ static loff_t cpuid_seek(struct file *file, loff_t offset, int orig)
 }
 
 static ssize_t cpuid_read(struct file *file, char __user *buf,
-                          size_t count, loff_t * ppos)
+                          size_t count, loff_t *ppos)
 {
         char __user *tmp = buf;
         struct cpuid_regs cmd;
@@ -117,11 +117,11 @@ static int cpuid_open(struct inode *inode, struct file *file)
         unsigned int cpu;
         struct cpuinfo_x86 *c;
         int ret = 0;
-        
+
         lock_kernel();
 
         cpu = iminor(file->f_path.dentry->d_inode);
-        if (cpu >= NR_CPUS || !cpu_online(cpu)) {
+        if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
                 ret = -ENXIO;   /* No such CPU */
                 goto out;
         }
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index 268553817909..c689d19e35ab 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -26,37 +26,21 @@
 #include <linux/kdebug.h>
 #include <asm/smp.h>
 #include <asm/reboot.h>
+#include <asm/virtext.h>
 
 #include <mach_ipi.h>
 
-/* This keeps a track of which one is crashing cpu. */
-static int crashing_cpu;
 
 #if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC)
-static atomic_t waiting_for_crash_ipi;
 
-static int crash_nmi_callback(struct notifier_block *self,
-                        unsigned long val, void *data)
+static void kdump_nmi_callback(int cpu, struct die_args *args)
 {
         struct pt_regs *regs;
 #ifdef CONFIG_X86_32
         struct pt_regs fixed_regs;
 #endif
-        int cpu;
 
-        if (val != DIE_NMI_IPI)
-                return NOTIFY_OK;
-
-        regs = ((struct die_args *)data)->regs;
-        cpu = raw_smp_processor_id();
-
-        /* Don't do anything if this handler is invoked on crashing cpu.
-         * Otherwise, system will completely hang. Crashing cpu can get
-         * an NMI if system was initially booted with nmi_watchdog parameter.
-         */
-        if (cpu == crashing_cpu)
-                return NOTIFY_STOP;
-        local_irq_disable();
+        regs = args->regs;
 
 #ifdef CONFIG_X86_32
         if (!user_mode_vm(regs)) {
@@ -65,54 +49,28 @@ static int crash_nmi_callback(struct notifier_block *self,
         }
 #endif
         crash_save_cpu(regs, cpu);
-        disable_local_APIC();
-        atomic_dec(&waiting_for_crash_ipi);
-        /* Assume hlt works */
-        halt();
-        for (;;)
-                cpu_relax();
 
-        return 1;
-}
+        /* Disable VMX or SVM if needed.
+         *
+         * We need to disable virtualization on all CPUs.
+         * Having VMX or SVM enabled on any CPU may break rebooting
+         * after the kdump kernel has finished its task.
+         */
+        cpu_emergency_vmxoff();
+        cpu_emergency_svm_disable();
 
-static void smp_send_nmi_allbutself(void)
-{
-        cpumask_t mask = cpu_online_map;
-        cpu_clear(safe_smp_processor_id(), mask);
-        if (!cpus_empty(mask))
-                send_IPI_mask(mask, NMI_VECTOR);
+        disable_local_APIC();
 }
 
-static struct notifier_block crash_nmi_nb = {
-        .notifier_call = crash_nmi_callback,
-};
-
-static void nmi_shootdown_cpus(void)
+static void kdump_nmi_shootdown_cpus(void)
 {
-        unsigned long msecs;
-
-        atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1);
-        /* Would it be better to replace the trap vector here? */
-        if (register_die_notifier(&crash_nmi_nb))
-                return;         /* return what? */
-        /* Ensure the new callback function is set before sending
-         * out the NMI
-         */
-        wmb();
-
-        smp_send_nmi_allbutself();
-
-        msecs = 1000; /* Wait at most a second for the other cpus to stop */
-        while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) {
-                mdelay(1);
-                msecs--;
-        }
+        nmi_shootdown_cpus(kdump_nmi_callback);
 
-        /* Leave the nmi callback set */
         disable_local_APIC();
 }
+
 #else
-static void nmi_shootdown_cpus(void)
+static void kdump_nmi_shootdown_cpus(void)
 {
         /* There are no cpus to shootdown */
 }
@@ -131,9 +89,15 @@ void native_machine_crash_shutdown(struct pt_regs *regs)
         /* The kernel is broken so disable interrupts */
         local_irq_disable();
 
-        /* Make a note of crashing cpu. Will be used in NMI callback.*/
-        crashing_cpu = safe_smp_processor_id();
-        nmi_shootdown_cpus();
+        kdump_nmi_shootdown_cpus();
+
+        /* Booting kdump kernel with VMX or SVM enabled won't work,
+         * because (among other limitations) we can't disable paging
+         * with the virt flags.
+         */
+        cpu_emergency_vmxoff();
+        cpu_emergency_svm_disable();
+
         lapic_shutdown();
 #if defined(CONFIG_X86_IO_APIC)
         disable_IO_APIC();
diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c
index a2d1176c38ee..da91701a2348 100644
--- a/arch/x86/kernel/ds.c
+++ b/arch/x86/kernel/ds.c
@@ -6,14 +6,13 @@
  * precise-event based sampling (PEBS).
  *
  * It manages:
- * - per-thread and per-cpu allocation of BTS and PEBS
- * - buffer memory allocation (optional)
- * - buffer overflow handling
+ * - DS and BTS hardware configuration
+ * - buffer overflow handling (to be done)
  * - buffer access
  *
- * It assumes:
- * - get_task_struct on all parameter tasks
- * - current is allowed to trace parameter tasks
+ * It does not do:
+ * - security checking (is the caller allowed to trace the task)
+ * - buffer allocation (memory accounting)
  *
  *
  * Copyright (C) 2007-2008 Intel Corporation.
@@ -28,22 +27,69 @@
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
+#include <linux/kernel.h>
 
 
 /*
  * The configuration for a particular DS hardware implementation.
  */
 struct ds_configuration {
-        /* the size of the DS structure in bytes */
-        unsigned char  sizeof_ds;
-        /* the size of one pointer-typed field in the DS structure in bytes;
-           this covers the first 8 fields related to buffer management. */
+        /* the name of the configuration */
+        const char *name;
+        /* the size of one pointer-typed field in the DS structure and
+           in the BTS and PEBS buffers in bytes;
+           this covers the first 8 DS fields related to buffer management. */
         unsigned char  sizeof_field;
         /* the size of a BTS/PEBS record in bytes */
         unsigned char  sizeof_rec[2];
+        /* a series of bit-masks to control various features indexed
+         * by enum ds_feature */
+        unsigned long ctl[dsf_ctl_max];
 };
-static struct ds_configuration ds_cfg;
+static DEFINE_PER_CPU(struct ds_configuration, ds_cfg_array);
 
+#define ds_cfg per_cpu(ds_cfg_array, smp_processor_id())
+
+#define MAX_SIZEOF_DS (12 * 8)  /* maximal size of a DS configuration */
+#define MAX_SIZEOF_BTS (3 * 8)  /* maximal size of a BTS record */
+#define DS_ALIGNMENT (1 << 3)   /* BTS and PEBS buffer alignment */
+
+#define BTS_CONTROL \
+ (ds_cfg.ctl[dsf_bts] | ds_cfg.ctl[dsf_bts_kernel] | ds_cfg.ctl[dsf_bts_user] |\
+  ds_cfg.ctl[dsf_bts_overflow])
+
+
+/*
+ * A BTS or PEBS tracer.
+ *
+ * This holds the configuration of the tracer and serves as a handle
+ * to identify tracers.
+ */
+struct ds_tracer {
+        /* the DS context (partially) owned by this tracer */
+        struct ds_context *context;
+        /* the buffer provided on ds_request() and its size in bytes */
+        void *buffer;
+        size_t size;
+};
+
+struct bts_tracer {
+        /* the common DS part */
+        struct ds_tracer ds;
+        /* the trace including the DS configuration */
+        struct bts_trace trace;
+        /* buffer overflow notification function */
+        bts_ovfl_callback_t ovfl;
+};
+
+struct pebs_tracer {
+        /* the common DS part */
+        struct ds_tracer ds;
+        /* the trace including the DS configuration */
+        struct pebs_trace trace;
+        /* buffer overflow notification function */
+        pebs_ovfl_callback_t ovfl;
+};
 
 /*
  * Debug Store (DS) save area configuration (see Intel64 and IA32
@@ -109,32 +155,9 @@ static inline void ds_set(unsigned char *base, enum ds_qualifier qual,
 
 
 /*
- * Locking is done only for allocating BTS or PEBS resources and for
- * guarding context and buffer memory allocation.
- *
- * Most functions require the current task to own the ds context part
- * they are going to access. All the locking is done when validating
- * access to the context.
+ * Locking is done only for allocating BTS or PEBS resources.
  */
-static spinlock_t ds_lock = __SPIN_LOCK_UNLOCKED(ds_lock);
-
-/*
- * Validate that the current task is allowed to access the BTS/PEBS
- * buffer of the parameter task.
- *
- * Returns 0, if access is granted; -Eerrno, otherwise.
- */
-static inline int ds_validate_access(struct ds_context *context,
-                                     enum ds_qualifier qual)
-{
-        if (!context)
-                return -EPERM;
-
-        if (context->owner[qual] == current)
-                return 0;
-
-        return -EPERM;
-}
+static DEFINE_SPINLOCK(ds_lock);
 
 
 /*
@@ -150,27 +173,32 @@ static inline int ds_validate_access(struct ds_context *context,
  *   >0  number of per-thread tracers
  *   <0  number of per-cpu tracers
  *
- * The below functions to get and put tracers and to check the
- * allocation type require the ds_lock to be held by the caller.
- *
  * Tracers essentially gives the number of ds contexts for a certain
  * type of allocation.
  */
-static long tracers;
+static atomic_t tracers = ATOMIC_INIT(0);
 
 static inline void get_tracer(struct task_struct *task)
 {
-        tracers += (task ? 1 : -1);
+        if (task)
+                atomic_inc(&tracers);
+        else
+                atomic_dec(&tracers);
 }
 
 static inline void put_tracer(struct task_struct *task)
 {
-        tracers -= (task ? 1 : -1);
+        if (task)
+                atomic_dec(&tracers);
+        else
+                atomic_inc(&tracers);
 }
 
 static inline int check_tracer(struct task_struct *task)
 {
-        return (task ? (tracers >= 0) : (tracers <= 0));
+        return task ?
+                (atomic_read(&tracers) >= 0) :
+                (atomic_read(&tracers) <= 0);
 }
 
 
@@ -183,99 +211,70 @@ static inline int check_tracer(struct task_struct *task)
  *
  * Contexts are use-counted. They are allocated on first access and
  * deallocated when the last user puts the context.
- *
- * We distinguish between an allocating and a non-allocating get of a
- * context:
- * - the allocating get is used for requesting BTS/PEBS resources. It
- *   requires the caller to hold the global ds_lock.
- * - the non-allocating get is used for all other cases. A
- *   non-existing context indicates an error. It acquires and releases
- *   the ds_lock itself for obtaining the context.
- *
- * A context and its DS configuration are allocated and deallocated
- * together. A context always has a DS configuration of the
- * appropriate size.
- */
-static DEFINE_PER_CPU(struct ds_context *, system_context);
-
-#define this_system_context per_cpu(system_context, smp_processor_id())
-
-/*
- * Returns the pointer to the parameter task's context or to the
- * system-wide context, if task is NULL.
- *
- * Increases the use count of the returned context, if not NULL.
  */
-static inline struct ds_context *ds_get_context(struct task_struct *task)
-{
-        struct ds_context *context;
-        unsigned long irq;
+struct ds_context {
+        /* pointer to the DS configuration; goes into MSR_IA32_DS_AREA */
+        unsigned char ds[MAX_SIZEOF_DS];
+        /* the owner of the BTS and PEBS configuration, respectively */
+        struct bts_tracer *bts_master;
+        struct pebs_tracer *pebs_master;
+        /* use count */
+        unsigned long count;
+        /* a pointer to the context location inside the thread_struct
+         * or the per_cpu context array */
+        struct ds_context **this;
+        /* a pointer to the task owning this context, or NULL, if the
+         * context is owned by a cpu */
+        struct task_struct *task;
+};
 
-        spin_lock_irqsave(&ds_lock, irq);
+static DEFINE_PER_CPU(struct ds_context *, system_context_array);
 
-        context = (task ? task->thread.ds_ctx : this_system_context);
-        if (context)
-                context->count++;
+#define system_context per_cpu(system_context_array, smp_processor_id())
 
-        spin_unlock_irqrestore(&ds_lock, irq);
-
-        return context;
-}
 
-/*
- * Same as ds_get_context, but allocates the context and it's DS
- * structure, if necessary; returns NULL; if out of memory.
- */
-static inline struct ds_context *ds_alloc_context(struct task_struct *task)
+static inline struct ds_context *ds_get_context(struct task_struct *task)
 {
         struct ds_context **p_context =
-                (task ? &task->thread.ds_ctx : &this_system_context);
-        struct ds_context *context = *p_context;
+                (task ? &task->thread.ds_ctx : &system_context);
+        struct ds_context *context = NULL;
+        struct ds_context *new_context = NULL;
         unsigned long irq;
 
-        if (!context) {
-                context = kzalloc(sizeof(*context), GFP_KERNEL);
-                if (!context)
-                        return NULL;
-
-                context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL);
-                if (!context->ds) {
-                        kfree(context);
-                        return NULL;
-                }
+        /* Chances are small that we already have a context. */
+        new_context = kzalloc(sizeof(*new_context), GFP_KERNEL);
+        if (!new_context)
+                return NULL;
 
-                spin_lock_irqsave(&ds_lock, irq);
+        spin_lock_irqsave(&ds_lock, irq);
 
-                if (*p_context) {
-                        kfree(context->ds);
-                        kfree(context);
+        context = *p_context;
+        if (!context) {
+                context = new_context;
 
-                        context = *p_context;
-                } else {
-                        *p_context = context;
+                context->this = p_context;
+                context->task = task;
+                context->count = 0;
 
-                        context->this = p_context;
-                        context->task = task;
+                if (task)
+                        set_tsk_thread_flag(task, TIF_DS_AREA_MSR);
 
-                        if (task)
-                                set_tsk_thread_flag(task, TIF_DS_AREA_MSR);
+                if (!task || (task == current))
+                        wrmsrl(MSR_IA32_DS_AREA, (unsigned long)context->ds);
 
-                        if (!task || (task == current))
-                                wrmsrl(MSR_IA32_DS_AREA,
-                                       (unsigned long)context->ds);
-                }
-                spin_unlock_irqrestore(&ds_lock, irq);
+                *p_context = context;
         }
 
         context->count++;
 
+        spin_unlock_irqrestore(&ds_lock, irq);
+
+        if (context != new_context)
+                kfree(new_context);
+
         return context;
 }
 
-/*
- * Decreases the use count of the parameter context, if not NULL.
- * Deallocates the context, if the use count reaches zero.
- */
 static inline void ds_put_context(struct ds_context *context)
 {
         unsigned long irq;
@@ -285,8 +284,10 @@ static inline void ds_put_context(struct ds_context *context)
 
         spin_lock_irqsave(&ds_lock, irq);
 
-        if (--context->count)
-                goto out;
+        if (--context->count) {
+                spin_unlock_irqrestore(&ds_lock, irq);
+                return;
+        }
 
         *(context->this) = NULL;
 
@@ -296,135 +297,263 @@ static inline void ds_put_context(struct ds_context *context)
         if (!context->task || (context->task == current))
                 wrmsrl(MSR_IA32_DS_AREA, 0);
 
-        put_tracer(context->task);
+        spin_unlock_irqrestore(&ds_lock, irq);
 
-        /* free any leftover buffers from tracers that did not
-         * deallocate them properly. */
-        kfree(context->buffer[ds_bts]);
-        kfree(context->buffer[ds_pebs]);
-        kfree(context->ds);
         kfree(context);
- out:
-        spin_unlock_irqrestore(&ds_lock, irq);
 }
 
 
 /*
- * Handle a buffer overflow
+ * Call the tracer's callback on a buffer overflow.
  *
- * task: the task whose buffers are overflowing;
- *       NULL for a buffer overflow on the current cpu
  * context: the ds context
  * qual: the buffer type
  */
-static void ds_overflow(struct task_struct *task, struct ds_context *context,
-                        enum ds_qualifier qual)
+static void ds_overflow(struct ds_context *context, enum ds_qualifier qual)
 {
-        if (!context)
-                return;
-
-        if (context->callback[qual])
-                (*context->callback[qual])(task);
-
-        /* todo: do some more overflow handling */
+        switch (qual) {
+        case ds_bts:
+                if (context->bts_master &&
+                    context->bts_master->ovfl)
+                        context->bts_master->ovfl(context->bts_master);
+                break;
+        case ds_pebs:
+                if (context->pebs_master &&
+                    context->pebs_master->ovfl)
+                        context->pebs_master->ovfl(context->pebs_master);
+                break;
+        }
 }
 
 
 /*
- * Allocate a non-pageable buffer of the parameter size.
- * Checks the memory and the locked memory rlimit.
+ * Write raw data into the BTS or PEBS buffer.
  *
- * Returns the buffer, if successful;
- *         NULL, if out of memory or rlimit exceeded.
+ * The remainder of any partially written record is zeroed out.
  *
- * size: the requested buffer size in bytes
- * pages (out): if not NULL, contains the number of pages reserved
+ * context: the DS context
+ * qual: the buffer type
+ * record: the data to write
+ * size: the size of the data
  */
-static inline void *ds_allocate_buffer(size_t size, unsigned int *pages)
+static int ds_write(struct ds_context *context, enum ds_qualifier qual,
+                    const void *record, size_t size)
 {
-        unsigned long rlim, vm, pgsz;
-        void *buffer;
+        int bytes_written = 0;
 
-        pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
+        if (!record)
+                return -EINVAL;
 
-        rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
-        vm   = current->mm->total_vm  + pgsz;
-        if (rlim < vm)
-                return NULL;
+        while (size) {
+                unsigned long base, index, end, write_end, int_th;
+                unsigned long write_size, adj_write_size;
 
-        rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
-        vm   = current->mm->locked_vm  + pgsz;
-        if (rlim < vm)
-                return NULL;
+                /*
+                 * write as much as possible without producing an
+                 * overflow interrupt.
+                 *
+                 * interrupt_threshold must either be
+                 * - bigger than absolute_maximum or
+                 * - point to a record between buffer_base and absolute_maximum
+                 *
+                 * index points to a valid record.
+                 */
+                base   = ds_get(context->ds, qual, ds_buffer_base);
+                index  = ds_get(context->ds, qual, ds_index);
+                end    = ds_get(context->ds, qual, ds_absolute_maximum);
+                int_th = ds_get(context->ds, qual, ds_interrupt_threshold);
 
-        buffer = kzalloc(size, GFP_KERNEL);
-        if (!buffer)
-                return NULL;
+                write_end = min(end, int_th);
 
-        current->mm->total_vm  += pgsz;
-        current->mm->locked_vm += pgsz;
+                /* if we are already beyond the interrupt threshold,
+                 * we fill the entire buffer */
+                if (write_end <= index)
+                        write_end = end;
 
-        if (pages)
-                *pages = pgsz;
+                if (write_end <= index)
+                        break;
+
+                write_size = min((unsigned long) size, write_end - index);
+                memcpy((void *)index, record, write_size);
 
-        return buffer;
+                record = (const char *)record + write_size;
+                size -= write_size;
+                bytes_written += write_size;
+
+                adj_write_size = write_size / ds_cfg.sizeof_rec[qual];
+                adj_write_size *= ds_cfg.sizeof_rec[qual];
+
+                /* zero out trailing bytes */
+                memset((char *)index + write_size, 0,
+                       adj_write_size - write_size);
+                index += adj_write_size;
+
+                if (index >= end)
+                        index = base;
+                ds_set(context->ds, qual, ds_index, index);
+
+                if (index >= int_th)
+                        ds_overflow(context, qual);
+        }
+
+        return bytes_written;
 }
 
-static int ds_request(struct task_struct *task, void *base, size_t size,
-                      ds_ovfl_callback_t ovfl, enum ds_qualifier qual)
+
+/*
+ * Branch Trace Store (BTS) uses the following format. Different
+ * architectures vary in the size of those fields.
+ * - source linear address
+ * - destination linear address
+ * - flags
+ *
+ * Later architectures use 64bit pointers throughout, whereas earlier
+ * architectures use 32bit pointers in 32bit mode.
+ *
+ * We compute the base address for the first 8 fields based on:
+ * - the field size stored in the DS configuration
+ * - the relative field position
+ *
+ * In order to store additional information in the BTS buffer, we use
+ * a special source address to indicate that the record requires
+ * special interpretation.
+ *
+ * Netburst indicated via a bit in the flags field whether the branch
+ * was predicted; this is ignored.
+ *
+ * We use two levels of abstraction:
+ * - the raw data level defined here
+ * - an arch-independent level defined in ds.h
+ */
+
+enum bts_field {
+        bts_from,
+        bts_to,
+        bts_flags,
+
+        bts_qual = bts_from,
+        bts_jiffies = bts_to,
+        bts_pid = bts_flags,
+
+        bts_qual_mask = (bts_qual_max - 1),
+        bts_escape = ((unsigned long)-1 & ~bts_qual_mask)
+};
+
+static inline unsigned long bts_get(const char *base, enum bts_field field)
 {
-        struct ds_context *context;
-        unsigned long buffer, adj;
-        const unsigned long alignment = (1 << 3);
-        unsigned long irq;
-        int error = 0;
+        base += (ds_cfg.sizeof_field * field);
+        return *(unsigned long *)base;
+}
+
+static inline void bts_set(char *base, enum bts_field field, unsigned long val)
+{
+        base += (ds_cfg.sizeof_field * field);;
+        (*(unsigned long *)base) = val;
+}
 
-        if (!ds_cfg.sizeof_ds)
-                return -EOPNOTSUPP;
 
-        /* we require some space to do alignment adjustments below */
-        if (size < (alignment + ds_cfg.sizeof_rec[qual]))
+/*
+ * The raw BTS data is architecture dependent.
+ *
+ * For higher-level users, we give an arch-independent view.
+ * - ds.h defines struct bts_struct
+ * - bts_read translates one raw bts record into a bts_struct
+ * - bts_write translates one bts_struct into the raw format and
+ *   writes it into the top of the parameter tracer's buffer.
+ *
+ * return: bytes read/written on success; -Eerrno, otherwise
+ */
+static int bts_read(struct bts_tracer *tracer, const void *at,
+                    struct bts_struct *out)
+{
+        if (!tracer)
                 return -EINVAL;
 
-        /* buffer overflow notification is not yet implemented */
-        if (ovfl)
-                return -EOPNOTSUPP;
+        if (at < tracer->trace.ds.begin)
+                return -EINVAL;
 
+        if (tracer->trace.ds.end < (at + tracer->trace.ds.size))
+                return -EINVAL;
 
-        context = ds_alloc_context(task);
-        if (!context)
-                return -ENOMEM;
+        memset(out, 0, sizeof(*out));
+        if ((bts_get(at, bts_qual) & ~bts_qual_mask) == bts_escape) {
+                out->qualifier = (bts_get(at, bts_qual) & bts_qual_mask);
+                out->variant.timestamp.jiffies = bts_get(at, bts_jiffies);
+                out->variant.timestamp.pid = bts_get(at, bts_pid);
+        } else {
+                out->qualifier = bts_branch;
+                out->variant.lbr.from = bts_get(at, bts_from);
+                out->variant.lbr.to   = bts_get(at, bts_to);
+
+                if (!out->variant.lbr.from && !out->variant.lbr.to)
+                        out->qualifier = bts_invalid;
+        }
 
-        spin_lock_irqsave(&ds_lock, irq);
+        return ds_cfg.sizeof_rec[ds_bts];
+}
 
-        error = -EPERM;
-        if (!check_tracer(task))
-                goto out_unlock;
+static int bts_write(struct bts_tracer *tracer, const struct bts_struct *in)
+{
+        unsigned char raw[MAX_SIZEOF_BTS];
 
-        get_tracer(task);
+        if (!tracer)
+                return -EINVAL;
 
-        error = -EALREADY;
-        if (context->owner[qual] == current)
-                goto out_put_tracer;
-        error = -EPERM;
-        if (context->owner[qual] != NULL)
-                goto out_put_tracer;
-        context->owner[qual] = current;
+        if (MAX_SIZEOF_BTS < ds_cfg.sizeof_rec[ds_bts])
+                return -EOVERFLOW;
 
-        spin_unlock_irqrestore(&ds_lock, irq);
+        switch (in->qualifier) {
+        case bts_invalid:
+                bts_set(raw, bts_from, 0);
+                bts_set(raw, bts_to, 0);
+                bts_set(raw, bts_flags, 0);
+                break;
+        case bts_branch:
+                bts_set(raw, bts_from, in->variant.lbr.from);
+                bts_set(raw, bts_to,   in->variant.lbr.to);
+                bts_set(raw, bts_flags, 0);
+                break;
+        case bts_task_arrives:
+        case bts_task_departs:
+                bts_set(raw, bts_qual, (bts_escape | in->qualifier));
+                bts_set(raw, bts_jiffies, in->variant.timestamp.jiffies);
+                bts_set(raw, bts_pid, in->variant.timestamp.pid);
+                break;
+        default:
+                return -EINVAL;
+        }
 
+        return ds_write(tracer->ds.context, ds_bts, raw,
+                        ds_cfg.sizeof_rec[ds_bts]);
+}
 
-        error = -ENOMEM;
-        if (!base) {
-                base = ds_allocate_buffer(size, &context->pages[qual]);
-                if (!base)
-                        goto out_release;
 
-                context->buffer[qual]   = base;
-        }
-        error = 0;
+static void ds_write_config(struct ds_context *context,
+                            struct ds_trace *cfg, enum ds_qualifier qual)
+{
+        unsigned char *ds = context->ds;
+
+        ds_set(ds, qual, ds_buffer_base, (unsigned long)cfg->begin);
+        ds_set(ds, qual, ds_index, (unsigned long)cfg->top);
+        ds_set(ds, qual, ds_absolute_maximum, (unsigned long)cfg->end);
+        ds_set(ds, qual, ds_interrupt_threshold, (unsigned long)cfg->ith);
+}
+
+static void ds_read_config(struct ds_context *context,
+                           struct ds_trace *cfg, enum ds_qualifier qual)
+{
+        unsigned char *ds = context->ds;
 
-        context->callback[qual] = ovfl;
+        cfg->begin = (void *)ds_get(ds, qual, ds_buffer_base);
+        cfg->top = (void *)ds_get(ds, qual, ds_index);
+        cfg->end = (void *)ds_get(ds, qual, ds_absolute_maximum);
+        cfg->ith = (void *)ds_get(ds, qual, ds_interrupt_threshold);
+}
+
+static void ds_init_ds_trace(struct ds_trace *trace, enum ds_qualifier qual,
+                             void *base, size_t size, size_t ith,
+                             unsigned int flags) {
+        unsigned long buffer, adj;
 
         /* adjust the buffer address and size to meet alignment
          * constraints:
@@ -436,410 +565,383 @@ static int ds_request(struct task_struct *task, void *base, size_t size,
          */
         buffer = (unsigned long)base;
 
-        adj = ALIGN(buffer, alignment) - buffer;
+        adj = ALIGN(buffer, DS_ALIGNMENT) - buffer;
         buffer += adj;
         size   -= adj;
 
-        size /= ds_cfg.sizeof_rec[qual];
-        size *= ds_cfg.sizeof_rec[qual];
-
-        ds_set(context->ds, qual, ds_buffer_base, buffer);
-        ds_set(context->ds, qual, ds_index, buffer);
-        ds_set(context->ds, qual, ds_absolute_maximum, buffer + size);
+        trace->n = size / ds_cfg.sizeof_rec[qual];
+        trace->size = ds_cfg.sizeof_rec[qual];
 
-        if (ovfl) {
-                /* todo: select a suitable interrupt threshold */
-        } else
-                ds_set(context->ds, qual,
-                       ds_interrupt_threshold, buffer + size + 1);
+        size = (trace->n * trace->size);
 
-        /* we keep the context until ds_release */
-        return error;
-
- out_release:
-        context->owner[qual] = NULL;
-        ds_put_context(context);
-        put_tracer(task);
-        return error;
-
- out_put_tracer:
-        spin_unlock_irqrestore(&ds_lock, irq);
-        ds_put_context(context);
-        put_tracer(task);
-        return error;
+        trace->begin = (void *)buffer;
+        trace->top = trace->begin;
+        trace->end = (void *)(buffer + size);
+        /* The value for 'no threshold' is -1, which will set the
+         * threshold outside of the buffer, just like we want it.
+         */
+        trace->ith = (void *)(buffer + size - ith);
 
- out_unlock:
-        spin_unlock_irqrestore(&ds_lock, irq);
-        ds_put_context(context);
-        return error;
+        trace->flags = flags;
 }
 
-int ds_request_bts(struct task_struct *task, void *base, size_t size,
-                   ds_ovfl_callback_t ovfl)
-{
-        return ds_request(task, base, size, ovfl, ds_bts);
-}
 
-int ds_request_pebs(struct task_struct *task, void *base, size_t size,
-                    ds_ovfl_callback_t ovfl)
-{
-        return ds_request(task, base, size, ovfl, ds_pebs);
-}
-
-static int ds_release(struct task_struct *task, enum ds_qualifier qual)
+static int ds_request(struct ds_tracer *tracer, struct ds_trace *trace,
+                      enum ds_qualifier qual, struct task_struct *task,
+                      void *base, size_t size, size_t th, unsigned int flags)
 {
         struct ds_context *context;
         int error;
 
-        context = ds_get_context(task);
-        error = ds_validate_access(context, qual);
-        if (error < 0)
+        error = -EINVAL;
+        if (!base)
                 goto out;
 
-        kfree(context->buffer[qual]);
-        context->buffer[qual] = NULL;
-
-        current->mm->total_vm  -= context->pages[qual];
-        current->mm->locked_vm -= context->pages[qual];
-        context->pages[qual] = 0;
-        context->owner[qual] = NULL;
-
-        /*
-         * we put the context twice:
-         *   once for the ds_get_context
-         *   once for the corresponding ds_request
-         */
-        ds_put_context(context);
- out:
-        ds_put_context(context);
-        return error;
-}
+        /* we require some space to do alignment adjustments below */
+        error = -EINVAL;
+        if (size < (DS_ALIGNMENT + ds_cfg.sizeof_rec[qual]))
+                goto out;
 
-int ds_release_bts(struct task_struct *task)
-{
-        return ds_release(task, ds_bts);
-}
+        if (th != (size_t)-1) {
+                th *= ds_cfg.sizeof_rec[qual];
 
-int ds_release_pebs(struct task_struct *task)
-{
-        return ds_release(task, ds_pebs);
-}
+                error = -EINVAL;
+                if (size <= th)
+                        goto out;
+        }
 
-static int ds_get_index(struct task_struct *task, size_t *pos,
-                        enum ds_qualifier qual)
-{
-        struct ds_context *context;
-        unsigned long base, index;
-        int error;
+        tracer->buffer = base;
+        tracer->size = size;
 
+        error = -ENOMEM;
         context = ds_get_context(task);
-        error = ds_validate_access(context, qual);
-        if (error < 0)
+        if (!context)
                 goto out;
+        tracer->context = context;
 
-        base  = ds_get(context->ds, qual, ds_buffer_base);
-        index = ds_get(context->ds, qual, ds_index);
+        ds_init_ds_trace(trace, qual, base, size, th, flags);
 
-        error = ((index - base) / ds_cfg.sizeof_rec[qual]);
-        if (pos)
-                *pos = error;
+        error = 0;
  out:
-        ds_put_context(context);
         return error;
 }
 
-int ds_get_bts_index(struct task_struct *task, size_t *pos)
-{
-        return ds_get_index(task, pos, ds_bts);
-}
-
-int ds_get_pebs_index(struct task_struct *task, size_t *pos)
+struct bts_tracer *ds_request_bts(struct task_struct *task,
+                                  void *base, size_t size,
+                                  bts_ovfl_callback_t ovfl, size_t th,
+                                  unsigned int flags)
 {
-        return ds_get_index(task, pos, ds_pebs);
-}
-
-static int ds_get_end(struct task_struct *task, size_t *pos,
-                      enum ds_qualifier qual)
-{
-        struct ds_context *context;
-        unsigned long base, end;
+        struct bts_tracer *tracer;
+        unsigned long irq;
         int error;
 
-        context = ds_get_context(task);
-        error = ds_validate_access(context, qual);
-        if (error < 0)
+        error = -EOPNOTSUPP;
+        if (!ds_cfg.ctl[dsf_bts])
                 goto out;
 
-        base = ds_get(context->ds, qual, ds_buffer_base);
-        end  = ds_get(context->ds, qual, ds_absolute_maximum);
+        /* buffer overflow notification is not yet implemented */
+        error = -EOPNOTSUPP;
+        if (ovfl)
+                goto out;
 
-        error = ((end - base) / ds_cfg.sizeof_rec[qual]);
-        if (pos)
-                *pos = error;
- out:
-        ds_put_context(context);
-        return error;
-}
+        error = -ENOMEM;
+        tracer = kzalloc(sizeof(*tracer), GFP_KERNEL);
+        if (!tracer)
+                goto out;
+        tracer->ovfl = ovfl;
 
-int ds_get_bts_end(struct task_struct *task, size_t *pos)
-{
-        return ds_get_end(task, pos, ds_bts);
-}
+        error = ds_request(&tracer->ds, &tracer->trace.ds,
+                           ds_bts, task, base, size, th, flags);
+        if (error < 0)
+                goto out_tracer;
 
-int ds_get_pebs_end(struct task_struct *task, size_t *pos)
-{
-        return ds_get_end(task, pos, ds_pebs);
-}
 
-static int ds_access(struct task_struct *task, size_t index,
-                     const void **record, enum ds_qualifier qual)
-{
-        struct ds_context *context;
-        unsigned long base, idx;
-        int error;
+        spin_lock_irqsave(&ds_lock, irq);
 
-        if (!record)
-                return -EINVAL;
+        error = -EPERM;
+        if (!check_tracer(task))
+                goto out_unlock;
+        get_tracer(task);
 
-        context = ds_get_context(task);
-        error = ds_validate_access(context, qual);
-        if (error < 0)
-                goto out;
+        error = -EPERM;
+        if (tracer->ds.context->bts_master)
+                goto out_put_tracer;
+        tracer->ds.context->bts_master = tracer;
 
-        base = ds_get(context->ds, qual, ds_buffer_base);
-        idx = base + (index * ds_cfg.sizeof_rec[qual]);
+        spin_unlock_irqrestore(&ds_lock, irq);
 
-        error = -EINVAL;
-        if (idx > ds_get(context->ds, qual, ds_absolute_maximum))
-                goto out;
 
-        *record = (const void *)idx;
-        error = ds_cfg.sizeof_rec[qual];
- out:
-        ds_put_context(context);
-        return error;
-}
+        tracer->trace.read  = bts_read;
+        tracer->trace.write = bts_write;
 
-int ds_access_bts(struct task_struct *task, size_t index, const void **record)
-{
-        return ds_access(task, index, record, ds_bts);
-}
+        ds_write_config(tracer->ds.context, &tracer->trace.ds, ds_bts);
+        ds_resume_bts(tracer);
 
-int ds_access_pebs(struct task_struct *task, size_t index, const void **record)
-{
-        return ds_access(task, index, record, ds_pebs);
+        return tracer;
+
+ out_put_tracer:
+        put_tracer(task);
+ out_unlock:
+        spin_unlock_irqrestore(&ds_lock, irq);
+        ds_put_context(tracer->ds.context);
+ out_tracer:
+        kfree(tracer);
+ out:
+        return ERR_PTR(error);
 }
 
-static int ds_write(struct task_struct *task, const void *record, size_t size,
-                    enum ds_qualifier qual, int force)
+struct pebs_tracer *ds_request_pebs(struct task_struct *task,
+                                    void *base, size_t size,
+                                    pebs_ovfl_callback_t ovfl, size_t th,
+                                    unsigned int flags)
 {
-        struct ds_context *context;
+        struct pebs_tracer *tracer;
+        unsigned long irq;
         int error;
 
-        if (!record)
-                return -EINVAL;
+        /* buffer overflow notification is not yet implemented */
+        error = -EOPNOTSUPP;
+        if (ovfl)
+                goto out;
 
-        error = -EPERM;
-        context = ds_get_context(task);
-        if (!context)
+        error = -ENOMEM;
+        tracer = kzalloc(sizeof(*tracer), GFP_KERNEL);
+        if (!tracer)
                 goto out;
+        tracer->ovfl = ovfl;
 
-        if (!force) {
-                error = ds_validate_access(context, qual);
-                if (error < 0)
-                        goto out;
-        }
+        error = ds_request(&tracer->ds, &tracer->trace.ds,
+                           ds_pebs, task, base, size, th, flags);
+        if (error < 0)
+                goto out_tracer;
 
-        error = 0;
-        while (size) {
-                unsigned long base, index, end, write_end, int_th;
-                unsigned long write_size, adj_write_size;
+        spin_lock_irqsave(&ds_lock, irq);
 
-                /*
-                 * write as much as possible without producing an
-                 * overflow interrupt.
-                 *
-                 * interrupt_threshold must either be
-                 * - bigger than absolute_maximum or
-                 * - point to a record between buffer_base and absolute_maximum
-                 *
-                 * index points to a valid record.
-                 */
-                base   = ds_get(context->ds, qual, ds_buffer_base);
-                index  = ds_get(context->ds, qual, ds_index);
-                end    = ds_get(context->ds, qual, ds_absolute_maximum);
-                int_th = ds_get(context->ds, qual, ds_interrupt_threshold);
+        error = -EPERM;
+        if (!check_tracer(task))
+                goto out_unlock;
+        get_tracer(task);
 
-                write_end = min(end, int_th);
+        error = -EPERM;
+        if (tracer->ds.context->pebs_master)
+                goto out_put_tracer;
+        tracer->ds.context->pebs_master = tracer;
 
-                /* if we are already beyond the interrupt threshold,
-                 * we fill the entire buffer */
-                if (write_end <= index)
-                        write_end = end;
+        spin_unlock_irqrestore(&ds_lock, irq);
 
-                if (write_end <= index)
-                        goto out;
+        ds_write_config(tracer->ds.context, &tracer->trace.ds, ds_bts);
+        ds_resume_pebs(tracer);
 
-                write_size = min((unsigned long) size, write_end - index);
-                memcpy((void *)index, record, write_size);
+        return tracer;
 
-                record = (const char *)record + write_size;
-                size  -= write_size;
-                error += write_size;
+ out_put_tracer:
+        put_tracer(task);
+ out_unlock:
+        spin_unlock_irqrestore(&ds_lock, irq);
+        ds_put_context(tracer->ds.context);
+ out_tracer:
+        kfree(tracer);
+ out:
+        return ERR_PTR(error);
+}
 
-                adj_write_size = write_size / ds_cfg.sizeof_rec[qual];
-                adj_write_size *= ds_cfg.sizeof_rec[qual];
+void ds_release_bts(struct bts_tracer *tracer)
+{
+        if (!tracer)
+                return;
 
-                /* zero out trailing bytes */
-                memset((char *)index + write_size, 0,
-                       adj_write_size - write_size);
-                index += adj_write_size;
+        ds_suspend_bts(tracer);
 
-                if (index >= end)
-                        index = base;
-                ds_set(context->ds, qual, ds_index, index);
+        WARN_ON_ONCE(tracer->ds.context->bts_master != tracer);
+        tracer->ds.context->bts_master = NULL;
 
-                if (index >= int_th)
-                        ds_overflow(task, context, qual);
-        }
+        put_tracer(tracer->ds.context->task);
+        ds_put_context(tracer->ds.context);
 
- out:
-        ds_put_context(context);
-        return error;
+        kfree(tracer);
 }
 
-int ds_write_bts(struct task_struct *task, const void *record, size_t size)
+void ds_suspend_bts(struct bts_tracer *tracer)
 {
-        return ds_write(task, record, size, ds_bts, /* force = */ 0);
-}
+        struct task_struct *task;
 
-int ds_write_pebs(struct task_struct *task, const void *record, size_t size)
-{
-        return ds_write(task, record, size, ds_pebs, /* force = */ 0);
-}
+        if (!tracer)
+                return;
 
-int ds_unchecked_write_bts(struct task_struct *task,
-                           const void *record, size_t size)
-{
-        return ds_write(task, record, size, ds_bts, /* force = */ 1);
-}
+        task = tracer->ds.context->task;
 
-int ds_unchecked_write_pebs(struct task_struct *task,
-                            const void *record, size_t size)
-{
-        return ds_write(task, record, size, ds_pebs, /* force = */ 1);
+        if (!task || (task == current))
+                update_debugctlmsr(get_debugctlmsr() & ~BTS_CONTROL);
+
+        if (task) {
+                task->thread.debugctlmsr &= ~BTS_CONTROL;
+
+                if (!task->thread.debugctlmsr)
+                        clear_tsk_thread_flag(task, TIF_DEBUGCTLMSR);
+        }
 }
 
-static int ds_reset_or_clear(struct task_struct *task,
-                             enum ds_qualifier qual, int clear)
+void ds_resume_bts(struct bts_tracer *tracer)
 {
-        struct ds_context *context;
-        unsigned long base, end;
-        int error;
+        struct task_struct *task;
+        unsigned long control;
 
-        context = ds_get_context(task);
-        error = ds_validate_access(context, qual);
-        if (error < 0)
-                goto out;
+        if (!tracer)
+                return;
 
-        base = ds_get(context->ds, qual, ds_buffer_base);
-        end  = ds_get(context->ds, qual, ds_absolute_maximum);
+        task = tracer->ds.context->task;
 
-        if (clear)
-                memset((void *)base, 0, end - base);
+        control = ds_cfg.ctl[dsf_bts];
+        if (!(tracer->trace.ds.flags & BTS_KERNEL))
+                control |= ds_cfg.ctl[dsf_bts_kernel];
+        if (!(tracer->trace.ds.flags & BTS_USER))
+                control |= ds_cfg.ctl[dsf_bts_user];
 
-        ds_set(context->ds, qual, ds_index, base);
+        if (task) {
+                task->thread.debugctlmsr |= control;
+                set_tsk_thread_flag(task, TIF_DEBUGCTLMSR);
+        }
 
-        error = 0;
- out:
-        ds_put_context(context);
-        return error;
+        if (!task || (task == current))
+                update_debugctlmsr(get_debugctlmsr() | control);
 }
 
-int ds_reset_bts(struct task_struct *task)
+void ds_release_pebs(struct pebs_tracer *tracer)
 {
-        return ds_reset_or_clear(task, ds_bts, /* clear = */ 0);
+        if (!tracer)
+                return;
+
+        ds_suspend_pebs(tracer);
+
+        WARN_ON_ONCE(tracer->ds.context->pebs_master != tracer);
+        tracer->ds.context->pebs_master = NULL;
+
+        put_tracer(tracer->ds.context->task);
+        ds_put_context(tracer->ds.context);
+
+        kfree(tracer);
 }
 
-int ds_reset_pebs(struct task_struct *task)
+void ds_suspend_pebs(struct pebs_tracer *tracer)
 {
-        return ds_reset_or_clear(task, ds_pebs, /* clear = */ 0);
+
 }
 
-int ds_clear_bts(struct task_struct *task)
+void ds_resume_pebs(struct pebs_tracer *tracer)
 {
-        return ds_reset_or_clear(task, ds_bts, /* clear = */ 1);
+
 }
 
-int ds_clear_pebs(struct task_struct *task)
+const struct bts_trace *ds_read_bts(struct bts_tracer *tracer)
 {
-        return ds_reset_or_clear(task, ds_pebs, /* clear = */ 1);
+        if (!tracer)
+                return NULL;
+
+        ds_read_config(tracer->ds.context, &tracer->trace.ds, ds_bts);
+        return &tracer->trace;
 }
 
-int ds_get_pebs_reset(struct task_struct *task, u64 *value)
+const struct pebs_trace *ds_read_pebs(struct pebs_tracer *tracer)
 {
-        struct ds_context *context;
-        int error;
+        if (!tracer)
+                return NULL;
+
+        ds_read_config(tracer->ds.context, &tracer->trace.ds, ds_pebs);
+        tracer->trace.reset_value =
+                *(u64 *)(tracer->ds.context->ds + (ds_cfg.sizeof_field * 8));
 
-        if (!value)
+        return &tracer->trace;
+}
+
+int ds_reset_bts(struct bts_tracer *tracer)
+{
+        if (!tracer)
                 return -EINVAL;
 
-        context = ds_get_context(task);
-        error = ds_validate_access(context, ds_pebs);
-        if (error < 0)
-                goto out;
+        tracer->trace.ds.top = tracer->trace.ds.begin;
 
-        *value = *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8));
+        ds_set(tracer->ds.context->ds, ds_bts, ds_index,
+               (unsigned long)tracer->trace.ds.top);
 
-        error = 0;
- out:
-        ds_put_context(context);
-        return error;
+        return 0;
 }
 
-int ds_set_pebs_reset(struct task_struct *task, u64 value)
+int ds_reset_pebs(struct pebs_tracer *tracer)
 {
-        struct ds_context *context;
-        int error;
+        if (!tracer)
+                return -EINVAL;
 
-        context = ds_get_context(task);
-        error = ds_validate_access(context, ds_pebs);
-        if (error < 0)
-                goto out;
+        tracer->trace.ds.top = tracer->trace.ds.begin;
 
-        *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)) = value;
+        ds_set(tracer->ds.context->ds, ds_bts, ds_index,
+               (unsigned long)tracer->trace.ds.top);
 
-        error = 0;
- out:
-        ds_put_context(context);
-        return error;
+        return 0;
+}
+
+int ds_set_pebs_reset(struct pebs_tracer *tracer, u64 value)
+{
+        if (!tracer)
+                return -EINVAL;
+
+        *(u64 *)(tracer->ds.context->ds + (ds_cfg.sizeof_field * 8)) = value;
+
+        return 0;
 }
 
-static const struct ds_configuration ds_cfg_var = {
-        .sizeof_ds    = sizeof(long) * 12,
-        .sizeof_field = sizeof(long),
-        .sizeof_rec[ds_bts]   = sizeof(long) * 3,
+static const struct ds_configuration ds_cfg_netburst = {
+        .name = "netburst",
+        .ctl[dsf_bts]           = (1 << 2) | (1 << 3),
+        .ctl[dsf_bts_kernel]    = (1 << 5),
+        .ctl[dsf_bts_user]      = (1 << 6),
+
+        .sizeof_field           = sizeof(long),
+        .sizeof_rec[ds_bts]     = sizeof(long) * 3,
 #ifdef __i386__
-        .sizeof_rec[ds_pebs]  = sizeof(long) * 10
+        .sizeof_rec[ds_pebs]    = sizeof(long) * 10,
 #else
-        .sizeof_rec[ds_pebs]  = sizeof(long) * 18
+        .sizeof_rec[ds_pebs]    = sizeof(long) * 18,
 #endif
 };
-static const struct ds_configuration ds_cfg_64 = {
-        .sizeof_ds    = 8 * 12,
-        .sizeof_field = 8,
-        .sizeof_rec[ds_bts]   = 8 * 3,
+static const struct ds_configuration ds_cfg_pentium_m = {
+        .name = "pentium m",
+        .ctl[dsf_bts]           = (1 << 6) | (1 << 7),
+
+        .sizeof_field           = sizeof(long),
+        .sizeof_rec[ds_bts]     = sizeof(long) * 3,
 #ifdef __i386__
-        .sizeof_rec[ds_pebs]  = 8 * 10
+        .sizeof_rec[ds_pebs]    = sizeof(long) * 10,
 #else
-        .sizeof_rec[ds_pebs]  = 8 * 18
+        .sizeof_rec[ds_pebs]    = sizeof(long) * 18,
 #endif
 };
+static const struct ds_configuration ds_cfg_core2 = {
+        .name = "core 2",
+        .ctl[dsf_bts]           = (1 << 6) | (1 << 7),
+        .ctl[dsf_bts_kernel]    = (1 << 9),
+        .ctl[dsf_bts_user]      = (1 << 10),
+
+        .sizeof_field           = 8,
+        .sizeof_rec[ds_bts]     = 8 * 3,
+        .sizeof_rec[ds_pebs]    = 8 * 18,
+};
 
-static inline void
+static void
 ds_configure(const struct ds_configuration *cfg)
 {
+        memset(&ds_cfg, 0, sizeof(ds_cfg));
         ds_cfg = *cfg;
+
+        printk(KERN_INFO "[ds] using %s configuration\n", ds_cfg.name);
+
+        if (!cpu_has_bts) {
+                ds_cfg.ctl[dsf_bts] = 0;
+                printk(KERN_INFO "[ds] bts not available\n");
+        }
+        if (!cpu_has_pebs)
+                printk(KERN_INFO "[ds] pebs not available\n");
+
+        WARN_ON_ONCE(MAX_SIZEOF_DS < (12 * ds_cfg.sizeof_field));
 }
 
 void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
@@ -847,16 +949,15 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
         switch (c->x86) {
         case 0x6:
                 switch (c->x86_model) {
+                case 0 ... 0xC:
+                        /* sorry, don't know about them */
+                        break;
                 case 0xD:
                 case 0xE: /* Pentium M */
-                        ds_configure(&ds_cfg_var);
+                        ds_configure(&ds_cfg_pentium_m);
                         break;
-                case 0xF: /* Core2 */
-                case 0x1C: /* Atom */
-                        ds_configure(&ds_cfg_64);
-                        break;
-                default:
-                        /* sorry, don't know about them */
+                default: /* Core2, Atom, ... */
+                        ds_configure(&ds_cfg_core2);
                         break;
                 }
                 break;
@@ -865,7 +966,7 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
                 case 0x0:
                 case 0x1:
                 case 0x2: /* Netburst */
-                        ds_configure(&ds_cfg_var);
+                        ds_configure(&ds_cfg_netburst);
                         break;
                 default:
                         /* sorry, don't know about them */
@@ -878,12 +979,52 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
         }
 }
 
-void ds_free(struct ds_context *context)
+/*
+ * Change the DS configuration from tracing prev to tracing next.
+ */
+void ds_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+        struct ds_context *prev_ctx = prev->thread.ds_ctx;
+        struct ds_context *next_ctx = next->thread.ds_ctx;
+
+        if (prev_ctx) {
+                update_debugctlmsr(0);
+
+                if (prev_ctx->bts_master &&
+                    (prev_ctx->bts_master->trace.ds.flags & BTS_TIMESTAMPS)) {
+                        struct bts_struct ts = {
+                                .qualifier = bts_task_departs,
+                                .variant.timestamp.jiffies = jiffies_64,
+                                .variant.timestamp.pid = prev->pid
+                        };
+                        bts_write(prev_ctx->bts_master, &ts);
+                }
+        }
+
+        if (next_ctx) {
+                if (next_ctx->bts_master &&
+                    (next_ctx->bts_master->trace.ds.flags & BTS_TIMESTAMPS)) {
+                        struct bts_struct ts = {
+                                .qualifier = bts_task_arrives,
+                                .variant.timestamp.jiffies = jiffies_64,
+                                .variant.timestamp.pid = next->pid
+                        };
+                        bts_write(next_ctx->bts_master, &ts);
+                }
+
+                wrmsrl(MSR_IA32_DS_AREA, (unsigned long)next_ctx->ds);
+        }
+
+        update_debugctlmsr(next->thread.debugctlmsr);
+}
+
+void ds_copy_thread(struct task_struct *tsk, struct task_struct *father)
+{
+        clear_tsk_thread_flag(tsk, TIF_DS_AREA_MSR);
+        tsk->thread.ds_ctx = NULL;
+}
+
+void ds_exit_thread(struct task_struct *tsk)
 {
-        /* This is called when the task owning the parameter context
-         * is dying. There should not be any user of that context left
-         * to disturb us, anymore. */
-        unsigned long leftovers = context->count;
-        while (leftovers--)
-                ds_put_context(context);
+        WARN_ON(tsk->thread.ds_ctx);
 }
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
new file mode 100644
index 000000000000..6b1f6f6f8661
--- /dev/null
+++ b/arch/x86/kernel/dumpstack.c
@@ -0,0 +1,351 @@
+/*
+ *  Copyright (C) 1991, 1992  Linus Torvalds
+ *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
+ */
+#include <linux/kallsyms.h>
+#include <linux/kprobes.h>
+#include <linux/uaccess.h>
+#include <linux/utsname.h>
+#include <linux/hardirq.h>
+#include <linux/kdebug.h>
+#include <linux/module.h>
+#include <linux/ptrace.h>
+#include <linux/kexec.h>
+#include <linux/bug.h>
+#include <linux/nmi.h>
+#include <linux/sysfs.h>
+
+#include <asm/stacktrace.h>
+
+#include "dumpstack.h"
+
+int panic_on_unrecovered_nmi;
+unsigned int code_bytes = 64;
+int kstack_depth_to_print = 3 * STACKSLOTS_PER_LINE;
+static int die_counter;
+
+void printk_address(unsigned long address, int reliable)
+{
+        printk(" [<%p>] %s%pS\n", (void *) address,
+                        reliable ? "" : "? ", (void *) address);
+}
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+static void
+print_ftrace_graph_addr(unsigned long addr, void *data,
+                        const struct stacktrace_ops *ops,
+                        struct thread_info *tinfo, int *graph)
+{
+        struct task_struct *task = tinfo->task;
+        unsigned long ret_addr;
+        int index = task->curr_ret_stack;
+
+        if (addr != (unsigned long)return_to_handler)
+                return;
+
+        if (!task->ret_stack || index < *graph)
+                return;
+
+        index -= *graph;
+        ret_addr = task->ret_stack[index].ret;
+
+        ops->address(data, ret_addr, 1);
+
+        (*graph)++;
+}
+#else
+static inline void
+print_ftrace_graph_addr(unsigned long addr, void *data,
+                        const struct stacktrace_ops *ops,
+                        struct thread_info *tinfo, int *graph)
+{ }
+#endif
+
+/*
+ * x86-64 can have up to three kernel stacks:
+ * process stack
+ * interrupt stack
+ * severe exception (double fault, nmi, stack fault, debug, mce) hardware stack
+ */
+
+static inline int valid_stack_ptr(struct thread_info *tinfo,
+                        void *p, unsigned int size, void *end)
+{
+        void *t = tinfo;
+        if (end) {
+                if (p < end && p >= (end-THREAD_SIZE))
+                        return 1;
+                else
+                        return 0;
+        }
+        return p > t && p < t + THREAD_SIZE - size;
+}
+
+unsigned long
+print_context_stack(struct thread_info *tinfo,
+                unsigned long *stack, unsigned long bp,
+                const struct stacktrace_ops *ops, void *data,
+                unsigned long *end, int *graph)
+{
+        struct stack_frame *frame = (struct stack_frame *)bp;
+
+        while (valid_stack_ptr(tinfo, stack, sizeof(*stack), end)) {
+                unsigned long addr;
+
+                addr = *stack;
+                if (__kernel_text_address(addr)) {
+                        if ((unsigned long) stack == bp + sizeof(long)) {
+                                ops->address(data, addr, 1);
+                                frame = frame->next_frame;
+                                bp = (unsigned long) frame;
+                        } else {
+                                ops->address(data, addr, bp == 0);
+                        }
+                        print_ftrace_graph_addr(addr, data, ops, tinfo, graph);
+                }
+                stack++;
+        }
+        return bp;
+}
+
+
+static void
+print_trace_warning_symbol(void *data, char *msg, unsigned long symbol)
+{
+        printk(data);
+        print_symbol(msg, symbol);
+        printk("\n");
+}
+
+static void print_trace_warning(void *data, char *msg)
+{
+        printk("%s%s\n", (char *)data, msg);
+}
+
+static int print_trace_stack(void *data, char *name)
+{
+        printk("%s <%s> ", (char *)data, name);
+        return 0;
+}
+
+/*
+ * Print one address/symbol entries per line.
+ */
+static void print_trace_address(void *data, unsigned long addr, int reliable)
+{
+        touch_nmi_watchdog();
+        printk(data);
+        printk_address(addr, reliable);
+}
+
+static const struct stacktrace_ops print_trace_ops = {
+        .warning = print_trace_warning,
+        .warning_symbol = print_trace_warning_symbol,
+        .stack = print_trace_stack,
+        .address = print_trace_address,
+};
+
+void
+show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
+                unsigned long *stack, unsigned long bp, char *log_lvl)
+{
+        printk("%sCall Trace:\n", log_lvl);
+        dump_trace(task, regs, stack, bp, &print_trace_ops, log_lvl);
+}
+
+void show_trace(struct task_struct *task, struct pt_regs *regs,
+                unsigned long *stack, unsigned long bp)
+{
+        show_trace_log_lvl(task, regs, stack, bp, "");
+}
+
+void show_stack(struct task_struct *task, unsigned long *sp)
+{
+        show_stack_log_lvl(task, NULL, sp, 0, "");
+}
+
+/*
+ * The architecture-independent dump_stack generator
+ */
+void dump_stack(void)
+{
+        unsigned long bp = 0;
+        unsigned long stack;
+
+#ifdef CONFIG_FRAME_POINTER
+        if (!bp)
+                get_bp(bp);
+#endif
+
+        printk("Pid: %d, comm: %.20s %s %s %.*s\n",
+                current->pid, current->comm, print_tainted(),
+                init_utsname()->release,
+                (int)strcspn(init_utsname()->version, " "),
+                init_utsname()->version);
+        show_trace(NULL, NULL, &stack, bp);
+}
+EXPORT_SYMBOL(dump_stack);
+
+static raw_spinlock_t die_lock = __RAW_SPIN_LOCK_UNLOCKED;
+static int die_owner = -1;
+static unsigned int die_nest_count;
+
+unsigned __kprobes long oops_begin(void)
+{
+        int cpu;
+        unsigned long flags;
+
+        oops_enter();
+
+        /* racy, but better than risking deadlock. */
+        raw_local_irq_save(flags);
+        cpu = smp_processor_id();
+        if (!__raw_spin_trylock(&die_lock)) {
+                if (cpu == die_owner)
+                        /* nested oops. should stop eventually */;
+                else
+                        __raw_spin_lock(&die_lock);
+        }
+        die_nest_count++;
+        die_owner = cpu;
+        console_verbose();
+        bust_spinlocks(1);
+        return flags;
+}
+
+void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
+{
+        if (regs && kexec_should_crash(current))
+                crash_kexec(regs);
+
+        bust_spinlocks(0);
+        die_owner = -1;
+        add_taint(TAINT_DIE);
+        die_nest_count--;
+        if (!die_nest_count)
+                /* Nest count reaches zero, release the lock. */
+                __raw_spin_unlock(&die_lock);
+        raw_local_irq_restore(flags);
+        oops_exit();
+
+        if (!signr)
+                return;
+        if (in_interrupt())
+                panic("Fatal exception in interrupt");
+        if (panic_on_oops)
+                panic("Fatal exception");
+        do_exit(signr);
+}
+
+int __kprobes __die(const char *str, struct pt_regs *regs, long err)
+{
+#ifdef CONFIG_X86_32
+        unsigned short ss;
+        unsigned long sp;
+#endif
+        printk(KERN_EMERG "%s: %04lx [#%d] ", str, err & 0xffff, ++die_counter);
+#ifdef CONFIG_PREEMPT
+        printk("PREEMPT ");
+#endif
+#ifdef CONFIG_SMP
+        printk("SMP ");
+#endif
+#ifdef CONFIG_DEBUG_PAGEALLOC
+        printk("DEBUG_PAGEALLOC");
+#endif
+        printk("\n");
+        sysfs_printk_last_file();
+        if (notify_die(DIE_OOPS, str, regs, err,
+                        current->thread.trap_no, SIGSEGV) == NOTIFY_STOP)
+                return 1;
+
+        show_registers(regs);
+#ifdef CONFIG_X86_32
+        sp = (unsigned long) (&regs->sp);
+        savesegment(ss, ss);
+        if (user_mode(regs)) {
+                sp = regs->sp;
+                ss = regs->ss & 0xffff;
+        }
+        printk(KERN_EMERG "EIP: [<%08lx>] ", regs->ip);
+        print_symbol("%s", regs->ip);
+        printk(" SS:ESP %04x:%08lx\n", ss, sp);
+#else
+        /* Executive summary in case the oops scrolled away */
+        printk(KERN_ALERT "RIP ");
+        printk_address(regs->ip, 1);
+        printk(" RSP <%016lx>\n", regs->sp);
+#endif
+        return 0;
+}
+
+/*
+ * This is gone through when something in the kernel has done something bad
+ * and is about to be terminated:
+ */
+void die(const char *str, struct pt_regs *regs, long err)
+{
+        unsigned long flags = oops_begin();
+        int sig = SIGSEGV;
+
+        if (!user_mode_vm(regs))
+                report_bug(regs->ip, regs);
+
+        if (__die(str, regs, err))
+                sig = 0;
+        oops_end(flags, regs, sig);
+}
+
+void notrace __kprobes
+die_nmi(char *str, struct pt_regs *regs, int do_panic)
+{
+        unsigned long flags;
+
+        if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
+                return;
+
+        /*
+         * We are in trouble anyway, lets at least try
+         * to get a message out.
+         */
+        flags = oops_begin();
+        printk(KERN_EMERG "%s", str);
+        printk(" on CPU%d, ip %08lx, registers:\n",
+                smp_processor_id(), regs->ip);
+        show_registers(regs);
+        oops_end(flags, regs, 0);
+        if (do_panic || panic_on_oops)
+                panic("Non maskable interrupt");
+        nmi_exit();
+        local_irq_enable();
+        do_exit(SIGBUS);
+}
+
+static int __init oops_setup(char *s)
+{
+        if (!s)
+                return -EINVAL;
+        if (!strcmp(s, "panic"))
+                panic_on_oops = 1;
+        return 0;
+}
+early_param("oops", oops_setup);
+
+static int __init kstack_setup(char *s)
+{
+        if (!s)
+                return -EINVAL;
+        kstack_depth_to_print = simple_strtoul(s, NULL, 0);
+        return 0;
+}
+early_param("kstack", kstack_setup);
+
+static int __init code_bytes_setup(char *s)
+{
+        code_bytes = simple_strtoul(s, NULL, 0);
+        if (code_bytes > 8192)
+                code_bytes = 8192;
+
+        return 1;
+}
+__setup("code_bytes=", code_bytes_setup);
diff --git a/arch/x86/kernel/dumpstack.h b/arch/x86/kernel/dumpstack.h
new file mode 100644
index 000000000000..da87590b8698
--- /dev/null
+++ b/arch/x86/kernel/dumpstack.h
@@ -0,0 +1,39 @@
+/*
+ *  Copyright (C) 1991, 1992  Linus Torvalds
+ *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
+ */
+
+#ifndef DUMPSTACK_H
+#define DUMPSTACK_H
+
+#ifdef CONFIG_X86_32
+#define STACKSLOTS_PER_LINE 8
+#define get_bp(bp) asm("movl %%ebp, %0" : "=r" (bp) :)
+#else
+#define STACKSLOTS_PER_LINE 4
+#define get_bp(bp) asm("movq %%rbp, %0" : "=r" (bp) :)
+#endif
+
+extern unsigned long
+print_context_stack(struct thread_info *tinfo,
+                unsigned long *stack, unsigned long bp,
+                const struct stacktrace_ops *ops, void *data,
+                unsigned long *end, int *graph);
+
+extern void
+show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
+                unsigned long *stack, unsigned long bp, char *log_lvl);
+
+extern void
+show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
+                unsigned long *sp, unsigned long bp, char *log_lvl);
+
+extern unsigned int code_bytes;
+extern int kstack_depth_to_print;
+
+/* The form of the top of the frame on the stack */
+struct stack_frame {
+        struct stack_frame *next_frame;
+        unsigned long return_address;
+};
+#endif
diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index b3614752197b..d593cd1f58dc 100644
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -17,69 +17,14 @@
 
 #include <asm/stacktrace.h>
 
-#define STACKSLOTS_PER_LINE 8
-#define get_bp(bp) asm("movl %%ebp, %0" : "=r" (bp) :)
-
-int panic_on_unrecovered_nmi;
-int kstack_depth_to_print = 3 * STACKSLOTS_PER_LINE;
-static unsigned int code_bytes = 64;
-static int die_counter;
-
-void printk_address(unsigned long address, int reliable)
-{
-        printk(" [<%p>] %s%pS\n", (void *) address,
-                        reliable ? "" : "? ", (void *) address);
-}
-
-static inline int valid_stack_ptr(struct thread_info *tinfo,
-                        void *p, unsigned int size, void *end)
-{
-        void *t = tinfo;
-        if (end) {
-                if (p < end && p >= (end-THREAD_SIZE))
-                        return 1;
-                else
-                        return 0;
-        }
-        return p > t && p < t + THREAD_SIZE - size;
-}
-
-/* The form of the top of the frame on the stack */
-struct stack_frame {
-        struct stack_frame *next_frame;
-        unsigned long return_address;
-};
-
-static inline unsigned long
-print_context_stack(struct thread_info *tinfo,
-                unsigned long *stack, unsigned long bp,
-                const struct stacktrace_ops *ops, void *data,
-                unsigned long *end)
-{
-        struct stack_frame *frame = (struct stack_frame *)bp;
-
-        while (valid_stack_ptr(tinfo, stack, sizeof(*stack), end)) {
-                unsigned long addr;
-
-                addr = *stack;
-                if (__kernel_text_address(addr)) {
-                        if ((unsigned long) stack == bp + sizeof(long)) {
-                                ops->address(data, addr, 1);
-                                frame = frame->next_frame;
-                                bp = (unsigned long) frame;
-                        } else {
-                                ops->address(data, addr, bp == 0);
-                        }
-                }
-                stack++;
-        }
-        return bp;
-}
+#include "dumpstack.h"
 
 void dump_trace(struct task_struct *task, struct pt_regs *regs,
                 unsigned long *stack, unsigned long bp,
                 const struct stacktrace_ops *ops, void *data)
 {
+        int graph = 0;
+
         if (!task)
                 task = current;
 
@@ -107,7 +52,8 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 
                 context = (struct thread_info *)
                         ((unsigned long)stack & (~(THREAD_SIZE - 1)));
-                bp = print_context_stack(context, stack, bp, ops, data, NULL);
+                bp = print_context_stack(context, stack, bp, ops,
+                                         data, NULL, &graph);
 
                 stack = (unsigned long *)context->previous_esp;
                 if (!stack)
@@ -119,57 +65,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 }
 EXPORT_SYMBOL(dump_trace);
 
-static void
-print_trace_warning_symbol(void *data, char *msg, unsigned long symbol)
-{
-        printk(data);
-        print_symbol(msg, symbol);
-        printk("\n");
-}
-
-static void print_trace_warning(void *data, char *msg)
-{
-        printk("%s%s\n", (char *)data, msg);
-}
-
-static int print_trace_stack(void *data, char *name)
-{
-        printk("%s <%s> ", (char *)data, name);
-        return 0;
-}
-
-/*
- * Print one address/symbol entries per line.
- */
-static void print_trace_address(void *data, unsigned long addr, int reliable)
-{
-        touch_nmi_watchdog();
-        printk(data);
-        printk_address(addr, reliable);
-}
-
-static const struct stacktrace_ops print_trace_ops = {
-        .warning = print_trace_warning,
-        .warning_symbol = print_trace_warning_symbol,
-        .stack = print_trace_stack,
-        .address = print_trace_address,
-};
-
-static void
-show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
-                unsigned long *stack, unsigned long bp, char *log_lvl)
-{
-        printk("%sCall Trace:\n", log_lvl);
-        dump_trace(task, regs, stack, bp, &print_trace_ops, log_lvl);
-}
-
-void show_trace(struct task_struct *task, struct pt_regs *regs,
-                unsigned long *stack, unsigned long bp)
-{
-        show_trace_log_lvl(task, regs, stack, bp, "");
-}
-
-static void
+void
 show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
                 unsigned long *sp, unsigned long bp, char *log_lvl)
 {
@@ -196,33 +92,6 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
         show_trace_log_lvl(task, regs, sp, bp, log_lvl);
 }
 
-void show_stack(struct task_struct *task, unsigned long *sp)
-{
-        show_stack_log_lvl(task, NULL, sp, 0, "");
-}
-
-/*
- * The architecture-independent dump_stack generator
- */
-void dump_stack(void)
-{
-        unsigned long bp = 0;
-        unsigned long stack;
-
-#ifdef CONFIG_FRAME_POINTER
-        if (!bp)
-                get_bp(bp);
-#endif
-
-        printk("Pid: %d, comm: %.20s %s %s %.*s\n",
-                current->pid, current->comm, print_tainted(),
-                init_utsname()->release,
-                (int)strcspn(init_utsname()->version, " "),
-                init_utsname()->version);
-        show_trace(NULL, NULL, &stack, bp);
-}
-
-EXPORT_SYMBOL(dump_stack);
 
 void show_registers(struct pt_regs *regs)
 {
@@ -283,167 +152,3 @@ int is_valid_bugaddr(unsigned long ip)
         return ud2 == 0x0b0f;
 }
 
-static raw_spinlock_t die_lock = __RAW_SPIN_LOCK_UNLOCKED;
-static int die_owner = -1;
-static unsigned int die_nest_count;
-
-unsigned __kprobes long oops_begin(void)
-{
-        unsigned long flags;
-
-        oops_enter();
-
-        if (die_owner != raw_smp_processor_id()) {
-                console_verbose();
-                raw_local_irq_save(flags);
-                __raw_spin_lock(&die_lock);
-                die_owner = smp_processor_id();
-                die_nest_count = 0;
-                bust_spinlocks(1);
-        } else {
-                raw_local_irq_save(flags);
-        }
-        die_nest_count++;
-        return flags;
-}
-
-void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
-{
-        bust_spinlocks(0);
-        die_owner = -1;
-        add_taint(TAINT_DIE);
-        __raw_spin_unlock(&die_lock);
-        raw_local_irq_restore(flags);
-
-        if (!regs)
-                return;
-
-        if (kexec_should_crash(current))
-                crash_kexec(regs);
-        if (in_interrupt())
-                panic("Fatal exception in interrupt");
-        if (panic_on_oops)
-                panic("Fatal exception");
-        oops_exit();
-        do_exit(signr);
-}
-
-int __kprobes __die(const char *str, struct pt_regs *regs, long err)
-{
-        unsigned short ss;
-        unsigned long sp;
-
-        printk(KERN_EMERG "%s: %04lx [#%d] ", str, err & 0xffff, ++die_counter);
-#ifdef CONFIG_PREEMPT
-        printk("PREEMPT ");
-#endif
-#ifdef CONFIG_SMP
-        printk("SMP ");
-#endif
-#ifdef CONFIG_DEBUG_PAGEALLOC
-        printk("DEBUG_PAGEALLOC");
-#endif
-        printk("\n");
-        sysfs_printk_last_file();
-        if (notify_die(DIE_OOPS, str, regs, err,
-                        current->thread.trap_no, SIGSEGV) == NOTIFY_STOP)
-                return 1;
-
-        show_registers(regs);
-        /* Executive summary in case the oops scrolled away */
-        sp = (unsigned long) (&regs->sp);
-        savesegment(ss, ss);
-        if (user_mode(regs)) {
-                sp = regs->sp;
-                ss = regs->ss & 0xffff;
-        }
-        printk(KERN_EMERG "EIP: [<%08lx>] ", regs->ip);
-        print_symbol("%s", regs->ip);
-        printk(" SS:ESP %04x:%08lx\n", ss, sp);
-        return 0;
-}
-
-/*
- * This is gone through when something in the kernel has done something bad
- * and is about to be terminated:
- */
-void die(const char *str, struct pt_regs *regs, long err)
-{
-        unsigned long flags = oops_begin();
-
-        if (die_nest_count < 3) {
-                report_bug(regs->ip, regs);
-
-                if (__die(str, regs, err))
-                        regs = NULL;
-        } else {
-                printk(KERN_EMERG "Recursive die() failure, output suppressed\n");
-        }
-
-        oops_end(flags, regs, SIGSEGV);
-}
-
-static DEFINE_SPINLOCK(nmi_print_lock);
-
-void notrace __kprobes
-die_nmi(char *str, struct pt_regs *regs, int do_panic)
-{
-        if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
-                return;
-
-        spin_lock(&nmi_print_lock);
-        /*
-        * We are in trouble anyway, lets at least try
-        * to get a message out:
-        */
-        bust_spinlocks(1);
-        printk(KERN_EMERG "%s", str);
-        printk(" on CPU%d, ip %08lx, registers:\n",
-                smp_processor_id(), regs->ip);
-        show_registers(regs);
-        if (do_panic)
-                panic("Non maskable interrupt");
-        console_silent();
-        spin_unlock(&nmi_print_lock);
-
-        /*
-         * If we are in kernel we are probably nested up pretty bad
-         * and might aswell get out now while we still can:
-         */
-        if (!user_mode_vm(regs)) {
-                current->thread.trap_no = 2;
-                crash_kexec(regs);
-        }
-
-        bust_spinlocks(0);
-        do_exit(SIGSEGV);
-}
-
-static int __init oops_setup(char *s)
-{
-        if (!s)
-                return -EINVAL;
-        if (!strcmp(s, "panic"))
-                panic_on_oops = 1;
-        return 0;
-}
-early_param("oops", oops_setup);
-
-static int __init kstack_setup(char *s)
-{
-        if (!s)
-                return -EINVAL;
-        kstack_depth_to_print = simple_strtoul(s, NULL, 0);
-        return 0;
-}
-early_param("kstack", kstack_setup);
-
-static int __init code_bytes_setup(char *s)
-{
-        code_bytes = simple_strtoul(s, NULL, 0);
-        if (code_bytes > 8192)
-                code_bytes = 8192;
-
-        return 1;
-}
-__setup("code_bytes=", code_bytes_setup);
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 96a5db7da8a7..c302d0707048 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -17,19 +17,7 @@
 
 #include <asm/stacktrace.h>
 
-#define STACKSLOTS_PER_LINE 4
-#define get_bp(bp) asm("movq %%rbp, %0" : "=r" (bp) :)
-
-int panic_on_unrecovered_nmi;
-int kstack_depth_to_print = 3 * STACKSLOTS_PER_LINE;
-static unsigned int code_bytes = 64;
-static int die_counter;
-
-void printk_address(unsigned long address, int reliable)
-{
-        printk(" [<%p>] %s%pS\n", (void *) address,
-                        reliable ? "" : "? ", (void *) address);
-}
+#include "dumpstack.h"
 
 static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
                                         unsigned *usedp, char **idp)
@@ -113,51 +101,6 @@ static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
  * severe exception (double fault, nmi, stack fault, debug, mce) hardware stack
  */
 
-static inline int valid_stack_ptr(struct thread_info *tinfo,
-                        void *p, unsigned int size, void *end)
-{
-        void *t = tinfo;
-        if (end) {
-                if (p < end && p >= (end-THREAD_SIZE))
-                        return 1;
-                else
-                        return 0;
-        }
-        return p > t && p < t + THREAD_SIZE - size;
-}
-
-/* The form of the top of the frame on the stack */
-struct stack_frame {
-        struct stack_frame *next_frame;
-        unsigned long return_address;
-};
-
-static inline unsigned long
-print_context_stack(struct thread_info *tinfo,
-                unsigned long *stack, unsigned long bp,
-                const struct stacktrace_ops *ops, void *data,
-                unsigned long *end)
-{
-        struct stack_frame *frame = (struct stack_frame *)bp;
-
-        while (valid_stack_ptr(tinfo, stack, sizeof(*stack), end)) {
-                unsigned long addr;
-
-                addr = *stack;
-                if (__kernel_text_address(addr)) {
-                        if ((unsigned long) stack == bp + sizeof(long)) {
-                                ops->address(data, addr, 1);
-                                frame = frame->next_frame;
-                                bp = (unsigned long) frame;
-                        } else {
-                                ops->address(data, addr, bp == 0);
-                        }
-                }
-                stack++;
-        }
-        return bp;
-}
-
 void dump_trace(struct task_struct *task, struct pt_regs *regs,
                 unsigned long *stack, unsigned long bp,
                 const struct stacktrace_ops *ops, void *data)
@@ -166,6 +109,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
         unsigned long *irqstack_end = (unsigned long *)cpu_pda(cpu)->irqstackptr;
         unsigned used = 0;
         struct thread_info *tinfo;
+        int graph = 0;
 
         if (!task)
                 task = current;
@@ -206,7 +150,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                                 break;
 
                         bp = print_context_stack(tinfo, stack, bp, ops,
-                                                        data, estack_end);
+                                                 data, estack_end, &graph);
                         ops->stack(data, "<EOE>");
                         /*
                          * We link to the next stack via the
@@ -225,7 +169,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                                 if (ops->stack(data, "IRQ") < 0)
                                         break;
                                 bp = print_context_stack(tinfo, stack, bp,
-                                                ops, data, irqstack_end);
+                                        ops, data, irqstack_end, &graph);
                                 /*
                                  * We link to the next stack (which would be
                                  * the process stack normally) the last
@@ -243,62 +187,12 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
         /*
          * This handles the process stack:
          */
-        bp = print_context_stack(tinfo, stack, bp, ops, data, NULL);
+        bp = print_context_stack(tinfo, stack, bp, ops, data, NULL, &graph);
         put_cpu();
 }
 EXPORT_SYMBOL(dump_trace);
 
-static void
-print_trace_warning_symbol(void *data, char *msg, unsigned long symbol)
-{
-        printk(data);
-        print_symbol(msg, symbol);
-        printk("\n");
-}
-
-static void print_trace_warning(void *data, char *msg)
-{
-        printk("%s%s\n", (char *)data, msg);
-}
-
-static int print_trace_stack(void *data, char *name)
-{
-        printk("%s <%s> ", (char *)data, name);
-        return 0;
-}
-
-/*
- * Print one address/symbol entries per line.
- */
-static void print_trace_address(void *data, unsigned long addr, int reliable)
-{
-        touch_nmi_watchdog();
-        printk(data);
-        printk_address(addr, reliable);
-}
-
-static const struct stacktrace_ops print_trace_ops = {
-        .warning = print_trace_warning,
-        .warning_symbol = print_trace_warning_symbol,
-        .stack = print_trace_stack,
-        .address = print_trace_address,
-};
-
-static void
-show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
-                unsigned long *stack, unsigned long bp, char *log_lvl)
-{
-        printk("%sCall Trace:\n", log_lvl);
-        dump_trace(task, regs, stack, bp, &print_trace_ops, log_lvl);
-}
-
-void show_trace(struct task_struct *task, struct pt_regs *regs,
-                unsigned long *stack, unsigned long bp)
-{
-        show_trace_log_lvl(task, regs, stack, bp, "");
-}
-
-static void
+void
 show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
                 unsigned long *sp, unsigned long bp, char *log_lvl)
 {
@@ -342,33 +236,6 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
         show_trace_log_lvl(task, regs, sp, bp, log_lvl);
 }
 
-void show_stack(struct task_struct *task, unsigned long *sp)
-{
-        show_stack_log_lvl(task, NULL, sp, 0, "");
-}
-
-/*
- * The architecture-independent dump_stack generator
- */
-void dump_stack(void)
-{
-        unsigned long bp = 0;
-        unsigned long stack;
-
-#ifdef CONFIG_FRAME_POINTER
-        if (!bp)
-                get_bp(bp);
-#endif
-
-        printk("Pid: %d, comm: %.20s %s %s %.*s\n",
-                current->pid, current->comm, print_tainted(),
-                init_utsname()->release,
-                (int)strcspn(init_utsname()->version, " "),
-                init_utsname()->version);
-        show_trace(NULL, NULL, &stack, bp);
-}
-EXPORT_SYMBOL(dump_stack);
-
 void show_registers(struct pt_regs *regs)
 {
         int i;
@@ -429,147 +296,3 @@ int is_valid_bugaddr(unsigned long ip)
         return ud2 == 0x0b0f;
 }
 
-static raw_spinlock_t die_lock = __RAW_SPIN_LOCK_UNLOCKED;
-static int die_owner = -1;
-static unsigned int die_nest_count;
-
-unsigned __kprobes long oops_begin(void)
-{
-        int cpu;
-        unsigned long flags;
-
-        oops_enter();
-
-        /* racy, but better than risking deadlock. */
-        raw_local_irq_save(flags);
-        cpu = smp_processor_id();
-        if (!__raw_spin_trylock(&die_lock)) {
-                if (cpu == die_owner)
-                        /* nested oops. should stop eventually */;
-                else
-                        __raw_spin_lock(&die_lock);
-        }
-        die_nest_count++;
-        die_owner = cpu;
-        console_verbose();
-        bust_spinlocks(1);
-        return flags;
-}
-
-void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
-{
-        die_owner = -1;
-        bust_spinlocks(0);
-        die_nest_count--;
-        if (!die_nest_count)
-                /* Nest count reaches zero, release the lock. */
-                __raw_spin_unlock(&die_lock);
-        raw_local_irq_restore(flags);
-        if (!regs) {
-                oops_exit();
-                return;
-        }
-        if (in_interrupt())
-                panic("Fatal exception in interrupt");
-        if (panic_on_oops)
-                panic("Fatal exception");
-        oops_exit();
-        do_exit(signr);
-}
-
-int __kprobes __die(const char *str, struct pt_regs *regs, long err)
-{
-        printk(KERN_EMERG "%s: %04lx [#%d] ", str, err & 0xffff, ++die_counter);
-#ifdef CONFIG_PREEMPT
-        printk("PREEMPT ");
-#endif
-#ifdef CONFIG_SMP
-        printk("SMP ");
-#endif
-#ifdef CONFIG_DEBUG_PAGEALLOC
-        printk("DEBUG_PAGEALLOC");
-#endif
-        printk("\n");
-        sysfs_printk_last_file();
-        if (notify_die(DIE_OOPS, str, regs, err,
-                        current->thread.trap_no, SIGSEGV) == NOTIFY_STOP)
-                return 1;
-
-        show_registers(regs);
-        add_taint(TAINT_DIE);
-        /* Executive summary in case the oops scrolled away */
-        printk(KERN_ALERT "RIP ");
-        printk_address(regs->ip, 1);
-        printk(" RSP <%016lx>\n", regs->sp);
-        if (kexec_should_crash(current))
-                crash_kexec(regs);
-        return 0;
-}
-
-void die(const char *str, struct pt_regs *regs, long err)
-{
-        unsigned long flags = oops_begin();
-
-        if (!user_mode(regs))
-                report_bug(regs->ip, regs);
-
-        if (__die(str, regs, err))
-                regs = NULL;
-        oops_end(flags, regs, SIGSEGV);
-}
-
-notrace __kprobes void
-die_nmi(char *str, struct pt_regs *regs, int do_panic)
-{
-        unsigned long flags;
-
-        if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
-                return;
-
-        flags = oops_begin();
-        /*
-         * We are in trouble anyway, lets at least try
-         * to get a message out.
-         */
-        printk(KERN_EMERG "%s", str);
-        printk(" on CPU%d, ip %08lx, registers:\n",
-                smp_processor_id(), regs->ip);
-        show_registers(regs);
-        if (kexec_should_crash(current))
-                crash_kexec(regs);
-        if (do_panic || panic_on_oops)
-                panic("Non maskable interrupt");
-        oops_end(flags, NULL, SIGBUS);
-        nmi_exit();
-        local_irq_enable();
-        do_exit(SIGBUS);
-}
-
-static int __init oops_setup(char *s)
-{
-        if (!s)
-                return -EINVAL;
-        if (!strcmp(s, "panic"))
-                panic_on_oops = 1;
-        return 0;
-}
-early_param("oops", oops_setup);
-
-static int __init kstack_setup(char *s)
-{
-        if (!s)
-                return -EINVAL;
-        kstack_depth_to_print = simple_strtoul(s, NULL, 0);
-        return 0;
-}
-early_param("kstack", kstack_setup);
-
-static int __init code_bytes_setup(char *s)
-{
-        code_bytes = simple_strtoul(s, NULL, 0);
-        if (code_bytes > 8192)
-                code_bytes = 8192;
-
-        return 1;
-}
-__setup("code_bytes=", code_bytes_setup);
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 74c6a21fdc8c..e85826829cf2 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -698,22 +698,6 @@ struct early_res {
 };
 static struct early_res early_res[MAX_EARLY_RES] __initdata = {
         { 0, PAGE_SIZE, "BIOS data page" },     /* BIOS data page */
-#if defined(CONFIG_X86_64) && defined(CONFIG_X86_TRAMPOLINE)
-        { TRAMPOLINE_BASE, TRAMPOLINE_BASE + 2 * PAGE_SIZE, "TRAMPOLINE" },
-#endif
-#if defined(CONFIG_X86_32) && defined(CONFIG_SMP)
-        /*
-         * But first pinch a few for the stack/trampoline stuff
-         * FIXME: Don't need the extra page at 4K, but need to fix
-         * trampoline before removing it. (see the GDT stuff)
-         */
-        { PAGE_SIZE, PAGE_SIZE + PAGE_SIZE, "EX TRAMPOLINE" },
-        /*
-         * Has to be in very low memory so we can execute
-         * real-mode AP code.
-         */
-        { TRAMPOLINE_BASE, TRAMPOLINE_BASE + PAGE_SIZE, "TRAMPOLINE" },
-#endif
         {}
 };
 
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 1b894b72c0f5..744aa7fc49d5 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -17,6 +17,7 @@
 #include <asm/io_apic.h>
 #include <asm/apic.h>
 #include <asm/iommu.h>
+#include <asm/gart.h>
 
 static void __init fix_hypertransport_config(int num, int slot, int func)
 {
diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c
index 34ad997d3834..504ad198e4ad 100644
--- a/arch/x86/kernel/early_printk.c
+++ b/arch/x86/kernel/early_printk.c
@@ -875,49 +875,6 @@ static struct console early_dbgp_console = {
 };
 #endif
 
-/* Console interface to a host file on AMD's SimNow! */
-
-static int simnow_fd;
-
-enum {
-        MAGIC1 = 0xBACCD00A,
-        MAGIC2 = 0xCA110000,
-        XOPEN = 5,
-        XWRITE = 4,
-};
-
-static noinline long simnow(long cmd, long a, long b, long c)
-{
-        long ret;
-
-        asm volatile("cpuid" :
-                     "=a" (ret) :
-                     "b" (a), "c" (b), "d" (c), "0" (MAGIC1), "D" (cmd + MAGIC2));
-        return ret;
-}
-
-static void __init simnow_init(char *str)
-{
-        char *fn = "klog";
-
-        if (*str == '=')
-                fn = ++str;
-        /* error ignored */
-        simnow_fd = simnow(XOPEN, (unsigned long)fn, O_WRONLY|O_APPEND|O_CREAT, 0644);
-}
-
-static void simnow_write(struct console *con, const char *s, unsigned n)
-{
-        simnow(XWRITE, simnow_fd, (unsigned long)s, n);
-}
-
-static struct console simnow_console = {
-        .name =         "simnow",
-        .write =        simnow_write,
-        .flags =        CON_PRINTBUFFER,
-        .index =        -1,
-};
-
 /* Direct interface for emergencies */
 static struct console *early_console = &early_vga_console;
 static int __initdata early_console_initialized;
@@ -929,7 +886,7 @@ asmlinkage void early_printk(const char *fmt, ...)
         va_list ap;
 
         va_start(ap, fmt);
-        n = vscnprintf(buf, 512, fmt, ap);
+        n = vscnprintf(buf, sizeof(buf), fmt, ap);
         early_console->write(early_console, buf, n);
         va_end(ap);
 }
@@ -960,10 +917,6 @@ static int __init setup_early_printk(char *buf)
                 max_ypos = boot_params.screen_info.orig_video_lines;
                 current_ypos = boot_params.screen_info.orig_y;
                 early_console = &early_vga_console;
-        } else if (!strncmp(buf, "simnow", 6)) {
-                simnow_init(buf + 6);
-                early_console = &simnow_console;
-                keep_early = 1;
 #ifdef CONFIG_EARLY_PRINTK_DBGP
         } else if (!strncmp(buf, "dbgp", 4)) {
                 if (early_dbgp_init(buf+4) < 0)
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 28b597ef9ca1..d6f0490a7391 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -619,28 +619,37 @@ END(syscall_badsys)
 27:;
 
 /*
- * Build the entry stubs and pointer table with
- * some assembler magic.
+ * Build the entry stubs and pointer table with some assembler magic.
+ * We pack 7 stubs into a single 32-byte chunk, which will fit in a
+ * single cache line on all modern x86 implementations.
  */
-.section .rodata,"a"
+.section .init.rodata,"a"
 ENTRY(interrupt)
 .text
-
+        .p2align 5
+        .p2align CONFIG_X86_L1_CACHE_SHIFT
 ENTRY(irq_entries_start)
         RING0_INT_FRAME
-vector=0
-.rept NR_VECTORS
-        ALIGN
- .if vector
+vector=FIRST_EXTERNAL_VECTOR
+.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
+        .balign 32
+  .rept 7
+    .if vector < NR_VECTORS
+      .if vector <> FIRST_EXTERNAL_VECTOR
         CFI_ADJUST_CFA_OFFSET -4
- .endif
-1:      pushl $~(vector)
+      .endif
+1:      pushl $(~vector+0x80)   /* Note: always in signed byte range */
         CFI_ADJUST_CFA_OFFSET 4
-        jmp common_interrupt
- .previous
+      .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
+        jmp 2f
+      .endif
+      .previous
         .long 1b
- .text
+      .text
 vector=vector+1
+    .endif
+  .endr
+2:      jmp common_interrupt
 .endr
 END(irq_entries_start)
 
@@ -652,8 +661,9 @@ END(interrupt)
  * the CPU automatically disables interrupts when executing an IRQ vector,
  * so IRQ-flags tracing has to follow that:
  */
-        ALIGN
+        .p2align CONFIG_X86_L1_CACHE_SHIFT
 common_interrupt:
+        addl $-0x80,(%esp)      /* Adjust vector into the [-256,-1] range */
         SAVE_ALL
         TRACE_IRQS_OFF
         movl %esp,%eax
@@ -678,65 +688,6 @@ ENDPROC(name)
 /* The include is where all of the SMP etc. interrupts come from */
 #include "entry_arch.h"
 
-KPROBE_ENTRY(page_fault)
-        RING0_EC_FRAME
-        pushl $do_page_fault
-        CFI_ADJUST_CFA_OFFSET 4
-        ALIGN
-error_code:
-        /* the function address is in %fs's slot on the stack */
-        pushl %es
-        CFI_ADJUST_CFA_OFFSET 4
-        /*CFI_REL_OFFSET es, 0*/
-        pushl %ds
-        CFI_ADJUST_CFA_OFFSET 4
-        /*CFI_REL_OFFSET ds, 0*/
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET eax, 0
-        pushl %ebp
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET ebp, 0
-        pushl %edi
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET edi, 0
-        pushl %esi
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET esi, 0
-        pushl %edx
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET edx, 0
-        pushl %ecx
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET ecx, 0
-        pushl %ebx
-        CFI_ADJUST_CFA_OFFSET 4
-        CFI_REL_OFFSET ebx, 0
-        cld
-        pushl %fs
-        CFI_ADJUST_CFA_OFFSET 4
-        /*CFI_REL_OFFSET fs, 0*/
-        movl $(__KERNEL_PERCPU), %ecx
-        movl %ecx, %fs
-        UNWIND_ESPFIX_STACK
-        popl %ecx
-        CFI_ADJUST_CFA_OFFSET -4
-        /*CFI_REGISTER es, ecx*/
-        movl PT_FS(%esp), %edi          # get the function address
-        movl PT_ORIG_EAX(%esp), %edx    # get the error code
-        movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
-        mov  %ecx, PT_FS(%esp)
-        /*CFI_REL_OFFSET fs, ES*/
-        movl $(__USER_DS), %ecx
-        movl %ecx, %ds
-        movl %ecx, %es
-        TRACE_IRQS_OFF
-        movl %esp,%eax                  # pt_regs pointer
-        call *%edi
-        jmp ret_from_exception
-        CFI_ENDPROC
-KPROBE_END(page_fault)
-
 ENTRY(coprocessor_error)
         RING0_INT_FRAME
         pushl $0
@@ -767,140 +718,6 @@ ENTRY(device_not_available)
         CFI_ENDPROC
 END(device_not_available)
 
-/*
- * Debug traps and NMI can happen at the one SYSENTER instruction
- * that sets up the real kernel stack. Check here, since we can't
- * allow the wrong stack to be used.
- *
- * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
- * already pushed 3 words if it hits on the sysenter instruction:
- * eflags, cs and eip.
- *
- * We just load the right stack, and push the three (known) values
- * by hand onto the new stack - while updating the return eip past
- * the instruction that would have done it for sysenter.
- */
-#define FIX_STACK(offset, ok, label)            \
-        cmpw $__KERNEL_CS,4(%esp);              \
-        jne ok;                                 \
-label:                                          \
-        movl TSS_sysenter_sp0+offset(%esp),%esp;        \
-        CFI_DEF_CFA esp, 0;                     \
-        CFI_UNDEFINED eip;                      \
-        pushfl;                                 \
-        CFI_ADJUST_CFA_OFFSET 4;                \
-        pushl $__KERNEL_CS;                     \
-        CFI_ADJUST_CFA_OFFSET 4;                \
-        pushl $sysenter_past_esp;               \
-        CFI_ADJUST_CFA_OFFSET 4;                \
-        CFI_REL_OFFSET eip, 0
-
-KPROBE_ENTRY(debug)
-        RING0_INT_FRAME
-        cmpl $ia32_sysenter_target,(%esp)
-        jne debug_stack_correct
-        FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
-debug_stack_correct:
-        pushl $-1                       # mark this as an int
-        CFI_ADJUST_CFA_OFFSET 4
-        SAVE_ALL
-        TRACE_IRQS_OFF
-        xorl %edx,%edx                  # error code 0
-        movl %esp,%eax                  # pt_regs pointer
-        call do_debug
-        jmp ret_from_exception
-        CFI_ENDPROC
-KPROBE_END(debug)
-
-/*
- * NMI is doubly nasty. It can happen _while_ we're handling
- * a debug fault, and the debug fault hasn't yet been able to
- * clear up the stack. So we first check whether we got  an
- * NMI on the sysenter entry path, but after that we need to
- * check whether we got an NMI on the debug path where the debug
- * fault happened on the sysenter path.
- */
-KPROBE_ENTRY(nmi)
-        RING0_INT_FRAME
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
-        movl %ss, %eax
-        cmpw $__ESPFIX_SS, %ax
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
-        je nmi_espfix_stack
-        cmpl $ia32_sysenter_target,(%esp)
-        je nmi_stack_fixup
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
-        movl %esp,%eax
-        /* Do not access memory above the end of our stack page,
-         * it might not exist.
-         */
-        andl $(THREAD_SIZE-1),%eax
-        cmpl $(THREAD_SIZE-20),%eax
-        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4
-        jae nmi_stack_correct
-        cmpl $ia32_sysenter_target,12(%esp)
-        je nmi_debug_stack_check
-nmi_stack_correct:
-        /* We have a RING0_INT_FRAME here */
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
-        SAVE_ALL
-        TRACE_IRQS_OFF
-        xorl %edx,%edx          # zero error code
-        movl %esp,%eax          # pt_regs pointer
-        call do_nmi
-        jmp restore_nocheck_notrace
-        CFI_ENDPROC
-
-nmi_stack_fixup:
-        RING0_INT_FRAME
-        FIX_STACK(12,nmi_stack_correct, 1)
-        jmp nmi_stack_correct
-
-nmi_debug_stack_check:
-        /* We have a RING0_INT_FRAME here */
-        cmpw $__KERNEL_CS,16(%esp)
-        jne nmi_stack_correct
-        cmpl $debug,(%esp)
-        jb nmi_stack_correct
-        cmpl $debug_esp_fix_insn,(%esp)
-        ja nmi_stack_correct
-        FIX_STACK(24,nmi_stack_correct, 1)
-        jmp nmi_stack_correct
-
-nmi_espfix_stack:
-        /* We have a RING0_INT_FRAME here.
-         *
-         * create the pointer to lss back
-         */
-        pushl %ss
-        CFI_ADJUST_CFA_OFFSET 4
-        pushl %esp
-        CFI_ADJUST_CFA_OFFSET 4
-        addw $4, (%esp)
-        /* copy the iret frame of 12 bytes */
-        .rept 3
-        pushl 16(%esp)
-        CFI_ADJUST_CFA_OFFSET 4
-        .endr
-        pushl %eax
-        CFI_ADJUST_CFA_OFFSET 4
-        SAVE_ALL
-        TRACE_IRQS_OFF
-        FIXUP_ESPFIX_STACK              # %eax == %esp
-        xorl %edx,%edx                  # zero error code
-        call do_nmi
-        RESTORE_REGS
-        lss 12+4(%esp), %esp            # back to espfix stack
-        CFI_ADJUST_CFA_OFFSET -24
-        jmp irq_return
-        CFI_ENDPROC
-KPROBE_END(nmi)
-
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
         iret
@@ -916,19 +733,6 @@ ENTRY(native_irq_enable_sysexit)
 END(native_irq_enable_sysexit)
 #endif
 
-KPROBE_ENTRY(int3)
-        RING0_INT_FRAME
-        pushl $-1                       # mark this as an int
-        CFI_ADJUST_CFA_OFFSET 4
-        SAVE_ALL
-        TRACE_IRQS_OFF
-        xorl %edx,%edx          # zero error code
-        movl %esp,%eax          # pt_regs pointer
-        call do_int3
-        jmp ret_from_exception
-        CFI_ENDPROC
-KPROBE_END(int3)
-
 ENTRY(overflow)
         RING0_INT_FRAME
         pushl $0
@@ -993,14 +797,6 @@ ENTRY(stack_segment)
         CFI_ENDPROC
 END(stack_segment)
 
-KPROBE_ENTRY(general_protection)
-        RING0_EC_FRAME
-        pushl $do_general_protection
-        CFI_ADJUST_CFA_OFFSET 4
-        jmp error_code
-        CFI_ENDPROC
-KPROBE_END(general_protection)
-
 ENTRY(alignment_check)
         RING0_EC_FRAME
         pushl $do_alignment_check
@@ -1051,6 +847,7 @@ ENTRY(kernel_thread_helper)
         push %eax
         CFI_ADJUST_CFA_OFFSET 4
         call do_exit
+        ud2                     # padding for call trace
         CFI_ENDPROC
 ENDPROC(kernel_thread_helper)
 
@@ -1157,6 +954,9 @@ ENTRY(mcount)
 END(mcount)
 
 ENTRY(ftrace_caller)
+        cmpl $0, function_trace_stop
+        jne  ftrace_stub
+
         pushl %eax
         pushl %ecx
         pushl %edx
@@ -1171,6 +971,11 @@ ftrace_call:
         popl %edx
         popl %ecx
         popl %eax
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+.globl ftrace_graph_call
+ftrace_graph_call:
+        jmp ftrace_stub
+#endif
 
 .globl ftrace_stub
 ftrace_stub:
@@ -1180,8 +985,18 @@ END(ftrace_caller)
 #else /* ! CONFIG_DYNAMIC_FTRACE */
 
 ENTRY(mcount)
+        cmpl $0, function_trace_stop
+        jne  ftrace_stub
+
         cmpl $ftrace_stub, ftrace_trace_function
         jnz trace
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+        cmpl $ftrace_stub, ftrace_graph_return
+        jnz ftrace_graph_caller
+
+        cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
+        jnz ftrace_graph_caller
+#endif
 .globl ftrace_stub
 ftrace_stub:
         ret
@@ -1200,13 +1015,268 @@ trace:
         popl %edx
         popl %ecx
         popl %eax
-
         jmp ftrace_stub
 END(mcount)
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #endif /* CONFIG_FUNCTION_TRACER */
 
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(ftrace_graph_caller)
+        cmpl $0, function_trace_stop
+        jne ftrace_stub
+
+        pushl %eax
+        pushl %ecx
+        pushl %edx
+        movl 0xc(%esp), %edx
+        lea 0x4(%ebp), %eax
+        subl $MCOUNT_INSN_SIZE, %edx
+        call prepare_ftrace_return
+        popl %edx
+        popl %ecx
+        popl %eax
+        ret
+END(ftrace_graph_caller)
+
+.globl return_to_handler
+return_to_handler:
+        pushl $0
+        pushl %eax
+        pushl %ecx
+        pushl %edx
+        call ftrace_return_to_handler
+        movl %eax, 0xc(%esp)
+        popl %edx
+        popl %ecx
+        popl %eax
+        ret
+#endif
+
 .section .rodata,"a"
 #include "syscall_table_32.S"
 
 syscall_table_size=(.-sys_call_table)
+
+/*
+ * Some functions should be protected against kprobes
+ */
+        .pushsection .kprobes.text, "ax"
+
+ENTRY(page_fault)
+        RING0_EC_FRAME
+        pushl $do_page_fault
+        CFI_ADJUST_CFA_OFFSET 4
+        ALIGN
+error_code:
+        /* the function address is in %fs's slot on the stack */
+        pushl %es
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET es, 0*/
+        pushl %ds
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET ds, 0*/
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET eax, 0
+        pushl %ebp
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ebp, 0
+        pushl %edi
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET edi, 0
+        pushl %esi
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET esi, 0
+        pushl %edx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET edx, 0
+        pushl %ecx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ecx, 0
+        pushl %ebx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ebx, 0
+        cld
+        pushl %fs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET fs, 0*/
+        movl $(__KERNEL_PERCPU), %ecx
+        movl %ecx, %fs
+        UNWIND_ESPFIX_STACK
+        popl %ecx
+        CFI_ADJUST_CFA_OFFSET -4
+        /*CFI_REGISTER es, ecx*/
+        movl PT_FS(%esp), %edi          # get the function address
+        movl PT_ORIG_EAX(%esp), %edx    # get the error code
+        movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
+        mov  %ecx, PT_FS(%esp)
+        /*CFI_REL_OFFSET fs, ES*/
+        movl $(__USER_DS), %ecx
+        movl %ecx, %ds
+        movl %ecx, %es
+        TRACE_IRQS_OFF
+        movl %esp,%eax                  # pt_regs pointer
+        call *%edi
+        jmp ret_from_exception
+        CFI_ENDPROC
+END(page_fault)
+
+/*
+ * Debug traps and NMI can happen at the one SYSENTER instruction
+ * that sets up the real kernel stack. Check here, since we can't
+ * allow the wrong stack to be used.
+ *
+ * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
+ * already pushed 3 words if it hits on the sysenter instruction:
+ * eflags, cs and eip.
+ *
+ * We just load the right stack, and push the three (known) values
+ * by hand onto the new stack - while updating the return eip past
+ * the instruction that would have done it for sysenter.
+ */
+#define FIX_STACK(offset, ok, label)            \
+        cmpw $__KERNEL_CS,4(%esp);              \
+        jne ok;                                 \
+label:                                          \
+        movl TSS_sysenter_sp0+offset(%esp),%esp;        \
+        CFI_DEF_CFA esp, 0;                     \
+        CFI_UNDEFINED eip;                      \
+        pushfl;                                 \
+        CFI_ADJUST_CFA_OFFSET 4;                \
+        pushl $__KERNEL_CS;                     \
+        CFI_ADJUST_CFA_OFFSET 4;                \
+        pushl $sysenter_past_esp;               \
+        CFI_ADJUST_CFA_OFFSET 4;                \
+        CFI_REL_OFFSET eip, 0
+
+ENTRY(debug)
+        RING0_INT_FRAME
+        cmpl $ia32_sysenter_target,(%esp)
+        jne debug_stack_correct
+        FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
+debug_stack_correct:
+        pushl $-1                       # mark this as an int
+        CFI_ADJUST_CFA_OFFSET 4
+        SAVE_ALL
+        TRACE_IRQS_OFF
+        xorl %edx,%edx                  # error code 0
+        movl %esp,%eax                  # pt_regs pointer
+        call do_debug
+        jmp ret_from_exception
+        CFI_ENDPROC
+END(debug)
+
+/*
+ * NMI is doubly nasty. It can happen _while_ we're handling
+ * a debug fault, and the debug fault hasn't yet been able to
+ * clear up the stack. So we first check whether we got  an
+ * NMI on the sysenter entry path, but after that we need to
+ * check whether we got an NMI on the debug path where the debug
+ * fault happened on the sysenter path.
+ */
+ENTRY(nmi)
+        RING0_INT_FRAME
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        movl %ss, %eax
+        cmpw $__ESPFIX_SS, %ax
+        popl %eax
+        CFI_ADJUST_CFA_OFFSET -4
+        je nmi_espfix_stack
+        cmpl $ia32_sysenter_target,(%esp)
+        je nmi_stack_fixup
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        movl %esp,%eax
+        /* Do not access memory above the end of our stack page,
+         * it might not exist.
+         */
+        andl $(THREAD_SIZE-1),%eax
+        cmpl $(THREAD_SIZE-20),%eax
+        popl %eax
+        CFI_ADJUST_CFA_OFFSET -4
+        jae nmi_stack_correct
+        cmpl $ia32_sysenter_target,12(%esp)
+        je nmi_debug_stack_check
+nmi_stack_correct:
+        /* We have a RING0_INT_FRAME here */
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        SAVE_ALL
+        TRACE_IRQS_OFF
+        xorl %edx,%edx          # zero error code
+        movl %esp,%eax          # pt_regs pointer
+        call do_nmi
+        jmp restore_nocheck_notrace
+        CFI_ENDPROC
+
+nmi_stack_fixup:
+        RING0_INT_FRAME
+        FIX_STACK(12,nmi_stack_correct, 1)
+        jmp nmi_stack_correct
+
+nmi_debug_stack_check:
+        /* We have a RING0_INT_FRAME here */
+        cmpw $__KERNEL_CS,16(%esp)
+        jne nmi_stack_correct
+        cmpl $debug,(%esp)
+        jb nmi_stack_correct
+        cmpl $debug_esp_fix_insn,(%esp)
+        ja nmi_stack_correct
+        FIX_STACK(24,nmi_stack_correct, 1)
+        jmp nmi_stack_correct
+
+nmi_espfix_stack:
+        /* We have a RING0_INT_FRAME here.
+         *
+         * create the pointer to lss back
+         */
+        pushl %ss
+        CFI_ADJUST_CFA_OFFSET 4
+        pushl %esp
+        CFI_ADJUST_CFA_OFFSET 4
+        addw $4, (%esp)
+        /* copy the iret frame of 12 bytes */
+        .rept 3
+        pushl 16(%esp)
+        CFI_ADJUST_CFA_OFFSET 4
+        .endr
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        SAVE_ALL
+        TRACE_IRQS_OFF
+        FIXUP_ESPFIX_STACK              # %eax == %esp
+        xorl %edx,%edx                  # zero error code
+        call do_nmi
+        RESTORE_REGS
+        lss 12+4(%esp), %esp            # back to espfix stack
+        CFI_ADJUST_CFA_OFFSET -24
+        jmp irq_return
+        CFI_ENDPROC
+END(nmi)
+
+ENTRY(int3)
+        RING0_INT_FRAME
+        pushl $-1                       # mark this as an int
+        CFI_ADJUST_CFA_OFFSET 4
+        SAVE_ALL
+        TRACE_IRQS_OFF
+        xorl %edx,%edx          # zero error code
+        movl %esp,%eax          # pt_regs pointer
+        call do_int3
+        jmp ret_from_exception
+        CFI_ENDPROC
+END(int3)
+
+ENTRY(general_protection)
+        RING0_EC_FRAME
+        pushl $do_general_protection
+        CFI_ADJUST_CFA_OFFSET 4
+        jmp error_code
+        CFI_ENDPROC
+END(general_protection)
+
+/*
+ * End of kprobes section
+ */
+        .popsection
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index b86f332c96a6..e28c7a987793 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -11,15 +11,15 @@
  *
  * NOTE: This code handles signal-recognition, which happens every time
  * after an interrupt and after each system call.
- * 
- * Normal syscalls and interrupts don't save a full stack frame, this is 
+ *
+ * Normal syscalls and interrupts don't save a full stack frame, this is
  * only done for syscall tracing, signals or fork/exec et.al.
- * 
- * A note on terminology:        
- * - top of stack: Architecture defined interrupt frame from SS to RIP 
- * at the top of the kernel process stack.      
+ *
+ * A note on terminology:
+ * - top of stack: Architecture defined interrupt frame from SS to RIP
+ * at the top of the kernel process stack.
  * - partial stack frame: partially saved registers upto R11.
- * - full stack frame: Like partial stack frame, but all register saved. 
+ * - full stack frame: Like partial stack frame, but all register saved.
  *
  * Some macro usage:
  * - CFI macros are used to generate dwarf2 unwind information for better
@@ -60,7 +60,6 @@
 #define __AUDIT_ARCH_LE    0x40000000
 
         .code64
-
 #ifdef CONFIG_FUNCTION_TRACER
 #ifdef CONFIG_DYNAMIC_FTRACE
 ENTRY(mcount)
@@ -68,16 +67,10 @@ ENTRY(mcount)
 END(mcount)
 
 ENTRY(ftrace_caller)
+        cmpl $0, function_trace_stop
+        jne  ftrace_stub
 
-        /* taken from glibc */
-        subq $0x38, %rsp
-        movq %rax, (%rsp)
-        movq %rcx, 8(%rsp)
-        movq %rdx, 16(%rsp)
-        movq %rsi, 24(%rsp)
-        movq %rdi, 32(%rsp)
-        movq %r8, 40(%rsp)
-        movq %r9, 48(%rsp)
+        MCOUNT_SAVE_FRAME
 
         movq 0x38(%rsp), %rdi
         movq 8(%rbp), %rsi
@@ -87,14 +80,13 @@ ENTRY(ftrace_caller)
 ftrace_call:
         call ftrace_stub
 
-        movq 48(%rsp), %r9
-        movq 40(%rsp), %r8
-        movq 32(%rsp), %rdi
-        movq 24(%rsp), %rsi
-        movq 16(%rsp), %rdx
-        movq 8(%rsp), %rcx
-        movq (%rsp), %rax
-        addq $0x38, %rsp
+        MCOUNT_RESTORE_FRAME
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+.globl ftrace_graph_call
+ftrace_graph_call:
+        jmp ftrace_stub
+#endif
 
 .globl ftrace_stub
 ftrace_stub:
@@ -103,15 +95,63 @@ END(ftrace_caller)
 
 #else /* ! CONFIG_DYNAMIC_FTRACE */
 ENTRY(mcount)
+        cmpl $0, function_trace_stop
+        jne  ftrace_stub
+
         cmpq $ftrace_stub, ftrace_trace_function
         jnz trace
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+        cmpq $ftrace_stub, ftrace_graph_return
+        jnz ftrace_graph_caller
+
+        cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
+        jnz ftrace_graph_caller
+#endif
+
 .globl ftrace_stub
 ftrace_stub:
         retq
 
 trace:
-        /* taken from glibc */
-        subq $0x38, %rsp
+        MCOUNT_SAVE_FRAME
+
+        movq 0x38(%rsp), %rdi
+        movq 8(%rbp), %rsi
+        subq $MCOUNT_INSN_SIZE, %rdi
+
+        call   *ftrace_trace_function
+
+        MCOUNT_RESTORE_FRAME
+
+        jmp ftrace_stub
+END(mcount)
+#endif /* CONFIG_DYNAMIC_FTRACE */
+#endif /* CONFIG_FUNCTION_TRACER */
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(ftrace_graph_caller)
+        cmpl $0, function_trace_stop
+        jne ftrace_stub
+
+        MCOUNT_SAVE_FRAME
+
+        leaq 8(%rbp), %rdi
+        movq 0x38(%rsp), %rsi
+        subq $MCOUNT_INSN_SIZE, %rsi
+
+        call    prepare_ftrace_return
+
+        MCOUNT_RESTORE_FRAME
+
+        retq
+END(ftrace_graph_caller)
+
+
+.globl return_to_handler
+return_to_handler:
+        subq  $80, %rsp
+
         movq %rax, (%rsp)
         movq %rcx, 8(%rsp)
         movq %rdx, 16(%rsp)
@@ -119,13 +159,14 @@ trace:
         movq %rdi, 32(%rsp)
         movq %r8, 40(%rsp)
         movq %r9, 48(%rsp)
+        movq %r10, 56(%rsp)
+        movq %r11, 64(%rsp)
 
-        movq 0x38(%rsp), %rdi
-        movq 8(%rbp), %rsi
-        subq $MCOUNT_INSN_SIZE, %rdi
-
-        call   *ftrace_trace_function
+        call ftrace_return_to_handler
 
+        movq %rax, 72(%rsp)
+        movq 64(%rsp), %r11
+        movq 56(%rsp), %r10
         movq 48(%rsp), %r9
         movq 40(%rsp), %r8
         movq 32(%rsp), %rdi
@@ -133,16 +174,14 @@ trace:
         movq 16(%rsp), %rdx
         movq 8(%rsp), %rcx
         movq (%rsp), %rax
-        addq $0x38, %rsp
+        addq $72, %rsp
+        retq
+#endif
 
-        jmp ftrace_stub
-END(mcount)
-#endif /* CONFIG_DYNAMIC_FTRACE */
-#endif /* CONFIG_FUNCTION_TRACER */
 
 #ifndef CONFIG_PREEMPT
 #define retint_kernel retint_restore_args
-#endif  
+#endif
 
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_usergs_sysret64)
@@ -161,29 +200,29 @@ ENTRY(native_usergs_sysret64)
 .endm
 
 /*
- * C code is not supposed to know about undefined top of stack. Every time 
- * a C function with an pt_regs argument is called from the SYSCALL based 
+ * C code is not supposed to know about undefined top of stack. Every time
+ * a C function with an pt_regs argument is called from the SYSCALL based
  * fast path FIXUP_TOP_OF_STACK is needed.
  * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
  * manipulation.
- */             
-                
-        /* %rsp:at FRAMEEND */ 
-        .macro FIXUP_TOP_OF_STACK tmp
-        movq    %gs:pda_oldrsp,\tmp
-        movq    \tmp,RSP(%rsp)
-        movq    $__USER_DS,SS(%rsp)
-        movq    $__USER_CS,CS(%rsp)
-        movq    $-1,RCX(%rsp)
-        movq    R11(%rsp),\tmp  /* get eflags */
-        movq    \tmp,EFLAGS(%rsp)
+ */
+
+        /* %rsp:at FRAMEEND */
+        .macro FIXUP_TOP_OF_STACK tmp offset=0
+        movq %gs:pda_oldrsp,\tmp
+        movq \tmp,RSP+\offset(%rsp)
+        movq $__USER_DS,SS+\offset(%rsp)
+        movq $__USER_CS,CS+\offset(%rsp)
+        movq $-1,RCX+\offset(%rsp)
+        movq R11+\offset(%rsp),\tmp  /* get eflags */
+        movq \tmp,EFLAGS+\offset(%rsp)
         .endm
 
-        .macro RESTORE_TOP_OF_STACK tmp,offset=0
-        movq   RSP-\offset(%rsp),\tmp
-        movq   \tmp,%gs:pda_oldrsp
-        movq   EFLAGS-\offset(%rsp),\tmp
-        movq   \tmp,R11-\offset(%rsp)
+        .macro RESTORE_TOP_OF_STACK tmp offset=0
+        movq RSP+\offset(%rsp),\tmp
+        movq \tmp,%gs:pda_oldrsp
+        movq EFLAGS+\offset(%rsp),\tmp
+        movq \tmp,R11+\offset(%rsp)
         .endm
 
         .macro FAKE_STACK_FRAME child_rip
@@ -195,7 +234,7 @@ ENTRY(native_usergs_sysret64)
         pushq %rax /* rsp */
         CFI_ADJUST_CFA_OFFSET   8
         CFI_REL_OFFSET  rsp,0
-        pushq $(1<<9) /* eflags - interrupts on */
+        pushq $X86_EFLAGS_IF /* eflags - interrupts on */
         CFI_ADJUST_CFA_OFFSET   8
         /*CFI_REL_OFFSET        rflags,0*/
         pushq $__KERNEL_CS /* cs */
@@ -213,62 +252,184 @@ ENTRY(native_usergs_sysret64)
         CFI_ADJUST_CFA_OFFSET   -(6*8)
         .endm
 
-        .macro  CFI_DEFAULT_STACK start=1
+/*
+ * initial frame state for interrupts (and exceptions without error code)
+ */
+        .macro EMPTY_FRAME start=1 offset=0
         .if \start
-        CFI_STARTPROC   simple
+        CFI_STARTPROC simple
         CFI_SIGNAL_FRAME
-        CFI_DEF_CFA     rsp,SS+8
+        CFI_DEF_CFA rsp,8+\offset
         .else
-        CFI_DEF_CFA_OFFSET SS+8
+        CFI_DEF_CFA_OFFSET 8+\offset
         .endif
-        CFI_REL_OFFSET  r15,R15
-        CFI_REL_OFFSET  r14,R14
-        CFI_REL_OFFSET  r13,R13
-        CFI_REL_OFFSET  r12,R12
-        CFI_REL_OFFSET  rbp,RBP
-        CFI_REL_OFFSET  rbx,RBX
-        CFI_REL_OFFSET  r11,R11
-        CFI_REL_OFFSET  r10,R10
-        CFI_REL_OFFSET  r9,R9
-        CFI_REL_OFFSET  r8,R8
-        CFI_REL_OFFSET  rax,RAX
-        CFI_REL_OFFSET  rcx,RCX
-        CFI_REL_OFFSET  rdx,RDX
-        CFI_REL_OFFSET  rsi,RSI
-        CFI_REL_OFFSET  rdi,RDI
-        CFI_REL_OFFSET  rip,RIP
-        /*CFI_REL_OFFSET        cs,CS*/
-        /*CFI_REL_OFFSET        rflags,EFLAGS*/
-        CFI_REL_OFFSET  rsp,RSP
-        /*CFI_REL_OFFSET        ss,SS*/
         .endm
+
+/*
+ * initial frame state for interrupts (and exceptions without error code)
+ */
+        .macro INTR_FRAME start=1 offset=0
+        EMPTY_FRAME \start, SS+8+\offset-RIP
+        /*CFI_REL_OFFSET ss, SS+\offset-RIP*/
+        CFI_REL_OFFSET rsp, RSP+\offset-RIP
+        /*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/
+        /*CFI_REL_OFFSET cs, CS+\offset-RIP*/
+        CFI_REL_OFFSET rip, RIP+\offset-RIP
+        .endm
+
+/*
+ * initial frame state for exceptions with error code (and interrupts
+ * with vector already pushed)
+ */
+        .macro XCPT_FRAME start=1 offset=0
+        INTR_FRAME \start, RIP+\offset-ORIG_RAX
+        /*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/
+        .endm
+
 /*
- * A newly forked process directly context switches into this.
- */     
-/* rdi: prev */ 
+ * frame that enables calling into C.
+ */
+        .macro PARTIAL_FRAME start=1 offset=0
+        XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET
+        CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET
+        CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET
+        CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET
+        CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET
+        CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET
+        CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET
+        CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET
+        CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET
+        CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET
+        .endm
+
+/*
+ * frame that enables passing a complete pt_regs to a C function.
+ */
+        .macro DEFAULT_FRAME start=1 offset=0
+        PARTIAL_FRAME \start, R11+\offset-R15
+        CFI_REL_OFFSET rbx, RBX+\offset
+        CFI_REL_OFFSET rbp, RBP+\offset
+        CFI_REL_OFFSET r12, R12+\offset
+        CFI_REL_OFFSET r13, R13+\offset
+        CFI_REL_OFFSET r14, R14+\offset
+        CFI_REL_OFFSET r15, R15+\offset
+        .endm
+
+/* save partial stack frame */
+ENTRY(save_args)
+        XCPT_FRAME
+        cld
+        movq_cfi rdi, RDI+16-ARGOFFSET
+        movq_cfi rsi, RSI+16-ARGOFFSET
+        movq_cfi rdx, RDX+16-ARGOFFSET
+        movq_cfi rcx, RCX+16-ARGOFFSET
+        movq_cfi rax, RAX+16-ARGOFFSET
+        movq_cfi  r8,  R8+16-ARGOFFSET
+        movq_cfi  r9,  R9+16-ARGOFFSET
+        movq_cfi r10, R10+16-ARGOFFSET
+        movq_cfi r11, R11+16-ARGOFFSET
+
+        leaq -ARGOFFSET+16(%rsp),%rdi   /* arg1 for handler */
+        movq_cfi rbp, 8         /* push %rbp */
+        leaq 8(%rsp), %rbp              /* mov %rsp, %ebp */
+        testl $3, CS(%rdi)
+        je 1f
+        SWAPGS
+        /*
+         * irqcount is used to check if a CPU is already on an interrupt stack
+         * or not. While this is essentially redundant with preempt_count it is
+         * a little cheaper to use a separate counter in the PDA (short of
+         * moving irq_enter into assembly, which would be too much work)
+         */
+1:      incl %gs:pda_irqcount
+        jne 2f
+        popq_cfi %rax                   /* move return address... */
+        mov %gs:pda_irqstackptr,%rsp
+        EMPTY_FRAME 0
+        pushq_cfi %rax                  /* ... to the new stack */
+        /*
+         * We entered an interrupt context - irqs are off:
+         */
+2:      TRACE_IRQS_OFF
+        ret
+        CFI_ENDPROC
+END(save_args)
+
+ENTRY(save_rest)
+        PARTIAL_FRAME 1 REST_SKIP+8
+        movq 5*8+16(%rsp), %r11 /* save return address */
+        movq_cfi rbx, RBX+16
+        movq_cfi rbp, RBP+16
+        movq_cfi r12, R12+16
+        movq_cfi r13, R13+16
+        movq_cfi r14, R14+16
+        movq_cfi r15, R15+16
+        movq %r11, 8(%rsp)      /* return address */
+        FIXUP_TOP_OF_STACK %r11, 16
+        ret
+        CFI_ENDPROC
+END(save_rest)
+
+/* save complete stack frame */
+ENTRY(save_paranoid)
+        XCPT_FRAME 1 RDI+8
+        cld
+        movq_cfi rdi, RDI+8
+        movq_cfi rsi, RSI+8
+        movq_cfi rdx, RDX+8
+        movq_cfi rcx, RCX+8
+        movq_cfi rax, RAX+8
+        movq_cfi r8, R8+8
+        movq_cfi r9, R9+8
+        movq_cfi r10, R10+8
+        movq_cfi r11, R11+8
+        movq_cfi rbx, RBX+8
+        movq_cfi rbp, RBP+8
+        movq_cfi r12, R12+8
+        movq_cfi r13, R13+8
+        movq_cfi r14, R14+8
+        movq_cfi r15, R15+8
+        movl $1,%ebx
+        movl $MSR_GS_BASE,%ecx
+        rdmsr
+        testl %edx,%edx
+        js 1f   /* negative -> in kernel */
+        SWAPGS
+        xorl %ebx,%ebx
+1:      ret
+        CFI_ENDPROC
+END(save_paranoid)
+
+/*
+ * A newly forked process directly context switches into this address.
+ *
+ * rdi: prev task we switched from
+ */
 ENTRY(ret_from_fork)
-        CFI_DEFAULT_STACK
+        DEFAULT_FRAME
+
         push kernel_eflags(%rip)
         CFI_ADJUST_CFA_OFFSET 8
-        popf                            # reset kernel eflags
+        popf                                    # reset kernel eflags
         CFI_ADJUST_CFA_OFFSET -8
-        call schedule_tail
+
+        call schedule_tail                      # rdi: 'prev' task parameter
+
         GET_THREAD_INFO(%rcx)
-        testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
-        jnz rff_trace
-rff_action:     
+
+        CFI_REMEMBER_STATE
         RESTORE_REST
-        testl $3,CS-ARGOFFSET(%rsp)     # from kernel_thread?
+
+        testl $3, CS-ARGOFFSET(%rsp)            # from kernel_thread?
         je   int_ret_from_sys_call
-        testl $_TIF_IA32,TI_flags(%rcx)
+
+        testl $_TIF_IA32, TI_flags(%rcx)        # 32-bit compat task needs IRET
         jnz  int_ret_from_sys_call
-        RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
-        jmp ret_from_sys_call
-rff_trace:
-        movq %rsp,%rdi
-        call syscall_trace_leave
-        GET_THREAD_INFO(%rcx)   
-        jmp rff_action
+
+        RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET
+        jmp ret_from_sys_call                   # go to the SYSRET fastpath
+
+        CFI_RESTORE_STATE
         CFI_ENDPROC
 END(ret_from_fork)
 
@@ -278,20 +439,20 @@ END(ret_from_fork)
  * SYSCALL does not save anything on the stack and does not change the
  * stack pointer.
  */
-                
+
 /*
- * Register setup:      
+ * Register setup:
  * rax  system call number
  * rdi  arg0
- * rcx  return address for syscall/sysret, C arg3 
+ * rcx  return address for syscall/sysret, C arg3
  * rsi  arg1
- * rdx  arg2    
+ * rdx  arg2
  * r10  arg3    (--> moved to rcx for C)
  * r8   arg4
  * r9   arg5
  * r11  eflags for syscall/sysret, temporary for C
- * r12-r15,rbp,rbx saved by C code, not touched.                
- * 
+ * r12-r15,rbp,rbx saved by C code, not touched.
+ *
  * Interrupts are off on entry.
  * Only called from user space.
  *
@@ -301,7 +462,7 @@ END(ret_from_fork)
  * When user can change the frames always force IRET. That is because
  * it deals with uncanonical addresses better. SYSRET has trouble
  * with them due to bugs in both AMD and Intel CPUs.
- */                                     
+ */
 
 ENTRY(system_call)
         CFI_STARTPROC   simple
@@ -317,7 +478,7 @@ ENTRY(system_call)
          */
 ENTRY(system_call_after_swapgs)
 
-        movq    %rsp,%gs:pda_oldrsp 
+        movq    %rsp,%gs:pda_oldrsp
         movq    %gs:pda_kernelstack,%rsp
         /*
          * No need to follow this irqs off/on section - it's straight
@@ -325,7 +486,7 @@ ENTRY(system_call_after_swapgs)
          */
         ENABLE_INTERRUPTS(CLBR_NONE)
         SAVE_ARGS 8,1
-        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
+        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
         movq  %rcx,RIP-ARGOFFSET(%rsp)
         CFI_REL_OFFSET rip,RIP-ARGOFFSET
         GET_THREAD_INFO(%rcx)
@@ -339,19 +500,19 @@ system_call_fastpath:
         movq %rax,RAX-ARGOFFSET(%rsp)
 /*
  * Syscall return path ending with SYSRET (fast path)
- * Has incomplete stack frame and undefined top of stack. 
- */             
+ * Has incomplete stack frame and undefined top of stack.
+ */
 ret_from_sys_call:
         movl $_TIF_ALLWORK_MASK,%edi
         /* edi: flagmask */
-sysret_check:           
+sysret_check:
         LOCKDEP_SYS_EXIT
         GET_THREAD_INFO(%rcx)
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         movl TI_flags(%rcx),%edx
         andl %edi,%edx
-        jnz  sysret_careful 
+        jnz  sysret_careful
         CFI_REMEMBER_STATE
         /*
          * sysretq will re-enable interrupts:
@@ -366,7 +527,7 @@ sysret_check:
 
         CFI_RESTORE_STATE
         /* Handle reschedules */
-        /* edx: work, edi: workmask */  
+        /* edx: work, edi: workmask */
 sysret_careful:
         bt $TIF_NEED_RESCHED,%edx
         jnc sysret_signal
@@ -379,7 +540,7 @@ sysret_careful:
         CFI_ADJUST_CFA_OFFSET -8
         jmp sysret_check
 
-        /* Handle a signal */ 
+        /* Handle a signal */
 sysret_signal:
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
@@ -388,17 +549,20 @@ sysret_signal:
         jc sysret_audit
 #endif
         /* edx: work flags (arg3) */
-        leaq do_notify_resume(%rip),%rax
         leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
         xorl %esi,%esi # oldset -> arg2
-        call ptregscall_common
+        SAVE_REST
+        FIXUP_TOP_OF_STACK %r11
+        call do_notify_resume
+        RESTORE_TOP_OF_STACK %r11
+        RESTORE_REST
         movl $_TIF_WORK_MASK,%edi
         /* Use IRET because user could have changed frame. This
            works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp int_with_check
-        
+
 badsys:
         movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
         jmp ret_from_sys_call
@@ -437,7 +601,7 @@ sysret_audit:
 #endif  /* CONFIG_AUDITSYSCALL */
 
         /* Do syscall tracing */
-tracesys:                        
+tracesys:
 #ifdef CONFIG_AUDITSYSCALL
         testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
         jz auditsys
@@ -460,8 +624,8 @@ tracesys:
         call *sys_call_table(,%rax,8)
         movq %rax,RAX-ARGOFFSET(%rsp)
         /* Use IRET because user could have changed frame */
-                
-/* 
+
+/*
  * Syscall return path ending with IRET.
  * Has correct top of stack, but partial stack frame.
  */
@@ -505,18 +669,18 @@ int_very_careful:
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
         SAVE_REST
-        /* Check for syscall exit trace */      
+        /* Check for syscall exit trace */
         testl $_TIF_WORK_SYSCALL_EXIT,%edx
         jz int_signal
         pushq %rdi
         CFI_ADJUST_CFA_OFFSET 8
-        leaq 8(%rsp),%rdi       # &ptregs -> arg1       
+        leaq 8(%rsp),%rdi       # &ptregs -> arg1
         call syscall_trace_leave
         popq %rdi
         CFI_ADJUST_CFA_OFFSET -8
         andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
         jmp int_restore_rest
-        
+
 int_signal:
         testl $_TIF_DO_NOTIFY_MASK,%edx
         jz 1f
@@ -531,22 +695,24 @@ int_restore_rest:
         jmp int_with_check
         CFI_ENDPROC
 END(system_call)
-                
-/* 
+
+/*
  * Certain special system calls that need to save a complete full stack frame.
- */                                                             
-        
+ */
         .macro PTREGSCALL label,func,arg
-        .globl \label
-\label:
-        leaq    \func(%rip),%rax
-        leaq    -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
-        jmp     ptregscall_common
+ENTRY(\label)
+        PARTIAL_FRAME 1 8               /* offset 8: return address */
+        subq $REST_SKIP, %rsp
+        CFI_ADJUST_CFA_OFFSET REST_SKIP
+        call save_rest
+        DEFAULT_FRAME 0 8               /* offset 8: return address */
+        leaq 8(%rsp), \arg      /* pt_regs pointer */
+        call \func
+        jmp ptregscall_common
+        CFI_ENDPROC
 END(\label)
         .endm
 
-        CFI_STARTPROC
-
         PTREGSCALL stub_clone, sys_clone, %r8
         PTREGSCALL stub_fork, sys_fork, %rdi
         PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -554,25 +720,18 @@ END(\label)
         PTREGSCALL stub_iopl, sys_iopl, %rsi
 
 ENTRY(ptregscall_common)
-        popq %r11
-        CFI_ADJUST_CFA_OFFSET -8
-        CFI_REGISTER rip, r11
-        SAVE_REST
-        movq %r11, %r15
-        CFI_REGISTER rip, r15
-        FIXUP_TOP_OF_STACK %r11
-        call *%rax
-        RESTORE_TOP_OF_STACK %r11
-        movq %r15, %r11
-        CFI_REGISTER rip, r11
-        RESTORE_REST
-        pushq %r11
-        CFI_ADJUST_CFA_OFFSET 8
-        CFI_REL_OFFSET rip, 0
-        ret
+        DEFAULT_FRAME 1 8       /* offset 8: return address */
+        RESTORE_TOP_OF_STACK %r11, 8
+        movq_cfi_restore R15+8, r15
+        movq_cfi_restore R14+8, r14
+        movq_cfi_restore R13+8, r13
+        movq_cfi_restore R12+8, r12
+        movq_cfi_restore RBP+8, rbp
+        movq_cfi_restore RBX+8, rbx
+        ret $REST_SKIP          /* pop extended registers */
         CFI_ENDPROC
 END(ptregscall_common)
-        
+
 ENTRY(stub_execve)
         CFI_STARTPROC
         popq %r11
@@ -588,11 +747,11 @@ ENTRY(stub_execve)
         jmp int_ret_from_sys_call
         CFI_ENDPROC
 END(stub_execve)
-        
+
 /*
  * sigreturn is special because it needs to restore all registers on return.
  * This cannot be done with SYSRET, so use the IRET return path instead.
- */                
+ */
 ENTRY(stub_rt_sigreturn)
         CFI_STARTPROC
         addq $8, %rsp
@@ -608,70 +767,70 @@ ENTRY(stub_rt_sigreturn)
 END(stub_rt_sigreturn)
 
 /*
- * initial frame state for interrupts and exceptions
+ * Build the entry stubs and pointer table with some assembler magic.
+ * We pack 7 stubs into a single 32-byte chunk, which will fit in a
+ * single cache line on all modern x86 implementations.
  */
-        .macro _frame ref
-        CFI_STARTPROC simple
-        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA rsp,SS+8-\ref
-        /*CFI_REL_OFFSET ss,SS-\ref*/
-        CFI_REL_OFFSET rsp,RSP-\ref
-        /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
-        /*CFI_REL_OFFSET cs,CS-\ref*/
-        CFI_REL_OFFSET rip,RIP-\ref
-        .endm
+        .section .init.rodata,"a"
+ENTRY(interrupt)
+        .text
+        .p2align 5
+        .p2align CONFIG_X86_L1_CACHE_SHIFT
+ENTRY(irq_entries_start)
+        INTR_FRAME
+vector=FIRST_EXTERNAL_VECTOR
+.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
+        .balign 32
+  .rept 7
+    .if vector < NR_VECTORS
+      .if vector <> FIRST_EXTERNAL_VECTOR
+        CFI_ADJUST_CFA_OFFSET -8
+      .endif
+1:      pushq $(~vector+0x80)   /* Note: always in signed byte range */
+        CFI_ADJUST_CFA_OFFSET 8
+      .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
+        jmp 2f
+      .endif
+      .previous
+        .quad 1b
+      .text
+vector=vector+1
+    .endif
+  .endr
+2:      jmp common_interrupt
+.endr
+        CFI_ENDPROC
+END(irq_entries_start)
 
-/* initial frame state for interrupts (and exceptions without error code) */
-#define INTR_FRAME _frame RIP
-/* initial frame state for exceptions with error code (and interrupts with
-   vector already pushed) */
-#define XCPT_FRAME _frame ORIG_RAX
+.previous
+END(interrupt)
+.previous
 
-/* 
+/*
  * Interrupt entry/exit.
  *
  * Interrupt entry points save only callee clobbered registers in fast path.
- *      
- * Entry runs with interrupts off.      
- */ 
+ *
+ * Entry runs with interrupts off.
+ */
 
-/* 0(%rsp): interrupt number */ 
+/* 0(%rsp): ~(interrupt number) */
         .macro interrupt func
-        cld
-        SAVE_ARGS
-        leaq -ARGOFFSET(%rsp),%rdi      # arg1 for handler
-        pushq %rbp
-        /*
-         * Save rbp twice: One is for marking the stack frame, as usual, and the
-         * other, to fill pt_regs properly. This is because bx comes right
-         * before the last saved register in that structure, and not bp. If the
-         * base pointer were in the place bx is today, this would not be needed.
-         */
-        movq %rbp, -8(%rsp)
-        CFI_ADJUST_CFA_OFFSET   8
-        CFI_REL_OFFSET          rbp, 0
-        movq %rsp,%rbp
-        CFI_DEF_CFA_REGISTER    rbp
-        testl $3,CS(%rdi)
-        je 1f
-        SWAPGS
-        /* irqcount is used to check if a CPU is already on an interrupt
-           stack or not. While this is essentially redundant with preempt_count
-           it is a little cheaper to use a separate counter in the PDA
-           (short of moving irq_enter into assembly, which would be too
-            much work) */
-1:      incl    %gs:pda_irqcount
-        cmoveq %gs:pda_irqstackptr,%rsp
-        push    %rbp                    # backlink for old unwinder
-        /*
-         * We entered an interrupt context - irqs are off:
-         */
-        TRACE_IRQS_OFF
+        subq $10*8, %rsp
+        CFI_ADJUST_CFA_OFFSET 10*8
+        call save_args
+        PARTIAL_FRAME 0
         call \func
         .endm
 
-ENTRY(common_interrupt)
+        /*
+         * The interrupt stubs push (~vector+0x80) onto the stack and
+         * then jump to common_interrupt.
+         */
+        .p2align CONFIG_X86_L1_CACHE_SHIFT
+common_interrupt:
         XCPT_FRAME
+        addq $-0x80,(%rsp)              /* Adjust vector to [-256,-1] range */
         interrupt do_IRQ
         /* 0(%rsp): oldrsp-ARGOFFSET */
 ret_from_intr:
@@ -685,12 +844,12 @@ exit_intr:
         GET_THREAD_INFO(%rcx)
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_kernel
-        
+
         /* Interrupt came from user space */
         /*
          * Has a correct top of stack, but a partial stack frame
          * %rcx: thread info. Interrupts off.
-         */             
+         */
 retint_with_reschedule:
         movl $_TIF_WORK_MASK,%edi
 retint_check:
@@ -763,20 +922,20 @@ retint_careful:
         pushq %rdi
         CFI_ADJUST_CFA_OFFSET   8
         call  schedule
-        popq %rdi               
+        popq %rdi
         CFI_ADJUST_CFA_OFFSET   -8
         GET_THREAD_INFO(%rcx)
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp retint_check
-        
+
 retint_signal:
         testl $_TIF_DO_NOTIFY_MASK,%edx
         jz    retint_swapgs
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
         SAVE_REST
-        movq $-1,ORIG_RAX(%rsp)                         
+        movq $-1,ORIG_RAX(%rsp)
         xorl %esi,%esi          # oldset
         movq %rsp,%rdi          # &pt_regs
         call do_notify_resume
@@ -798,324 +957,211 @@ ENTRY(retint_kernel)
         jnc  retint_restore_args
         call preempt_schedule_irq
         jmp exit_intr
-#endif  
+#endif
 
         CFI_ENDPROC
 END(common_interrupt)
-        
+
 /*
  * APIC interrupts.
- */             
-        .macro apicinterrupt num,func
+ */
+.macro apicinterrupt num sym do_sym
+ENTRY(\sym)
         INTR_FRAME
         pushq $~(\num)
         CFI_ADJUST_CFA_OFFSET 8
-        interrupt \func
+        interrupt \do_sym
         jmp ret_from_intr
         CFI_ENDPROC
-        .endm
-
-ENTRY(thermal_interrupt)
-        apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
-END(thermal_interrupt)
-
-ENTRY(threshold_interrupt)
-        apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
-END(threshold_interrupt)
-
-#ifdef CONFIG_SMP       
-ENTRY(reschedule_interrupt)
-        apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
-END(reschedule_interrupt)
-
-        .macro INVALIDATE_ENTRY num
-ENTRY(invalidate_interrupt\num)
-        apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt 
-END(invalidate_interrupt\num)
-        .endm
+END(\sym)
+.endm
 
-        INVALIDATE_ENTRY 0
-        INVALIDATE_ENTRY 1
-        INVALIDATE_ENTRY 2
-        INVALIDATE_ENTRY 3
-        INVALIDATE_ENTRY 4
-        INVALIDATE_ENTRY 5
-        INVALIDATE_ENTRY 6
-        INVALIDATE_ENTRY 7
-
-ENTRY(call_function_interrupt)
-        apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
-END(call_function_interrupt)
-ENTRY(call_function_single_interrupt)
-        apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
-END(call_function_single_interrupt)
-ENTRY(irq_move_cleanup_interrupt)
-        apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
-END(irq_move_cleanup_interrupt)
+#ifdef CONFIG_SMP
+apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \
+        irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
 #endif
 
-ENTRY(apic_timer_interrupt)
-        apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
-END(apic_timer_interrupt)
+apicinterrupt UV_BAU_MESSAGE \
+        uv_bau_message_intr1 uv_bau_message_interrupt
+apicinterrupt LOCAL_TIMER_VECTOR \
+        apic_timer_interrupt smp_apic_timer_interrupt
+
+#ifdef CONFIG_SMP
+apicinterrupt INVALIDATE_TLB_VECTOR_START+0 \
+        invalidate_interrupt0 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+1 \
+        invalidate_interrupt1 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+2 \
+        invalidate_interrupt2 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+3 \
+        invalidate_interrupt3 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+4 \
+        invalidate_interrupt4 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+5 \
+        invalidate_interrupt5 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+6 \
+        invalidate_interrupt6 smp_invalidate_interrupt
+apicinterrupt INVALIDATE_TLB_VECTOR_START+7 \
+        invalidate_interrupt7 smp_invalidate_interrupt
+#endif
 
-ENTRY(uv_bau_message_intr1)
-        apicinterrupt 220,uv_bau_message_interrupt
-END(uv_bau_message_intr1)
+apicinterrupt THRESHOLD_APIC_VECTOR \
+        threshold_interrupt mce_threshold_interrupt
+apicinterrupt THERMAL_APIC_VECTOR \
+        thermal_interrupt smp_thermal_interrupt
+
+#ifdef CONFIG_SMP
+apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \
+        call_function_single_interrupt smp_call_function_single_interrupt
+apicinterrupt CALL_FUNCTION_VECTOR \
+        call_function_interrupt smp_call_function_interrupt
+apicinterrupt RESCHEDULE_VECTOR \
+        reschedule_interrupt smp_reschedule_interrupt
+#endif
 
-ENTRY(error_interrupt)
-        apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
-END(error_interrupt)
+apicinterrupt ERROR_APIC_VECTOR \
+        error_interrupt smp_error_interrupt
+apicinterrupt SPURIOUS_APIC_VECTOR \
+        spurious_interrupt smp_spurious_interrupt
 
-ENTRY(spurious_interrupt)
-        apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
-END(spurious_interrupt)
-                                
 /*
  * Exception entry points.
- */             
-        .macro zeroentry sym
+ */
+.macro zeroentry sym do_sym
+ENTRY(\sym)
         INTR_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $0        /* push error code/oldrax */ 
-        CFI_ADJUST_CFA_OFFSET 8
-        pushq %rax      /* push real oldrax to the rdi slot */ 
-        CFI_ADJUST_CFA_OFFSET 8
-        CFI_REL_OFFSET rax,0
-        leaq  \sym(%rip),%rax
-        jmp error_entry
+        pushq_cfi $-1           /* ORIG_RAX: no syscall to restart */
+        subq $15*8,%rsp
+        CFI_ADJUST_CFA_OFFSET 15*8
+        call error_entry
+        DEFAULT_FRAME 0
+        movq %rsp,%rdi          /* pt_regs pointer */
+        xorl %esi,%esi          /* no error code */
+        call \do_sym
+        jmp error_exit          /* %ebx: no swapgs flag */
         CFI_ENDPROC
-        .endm   
+END(\sym)
+.endm
 
-        .macro errorentry sym
-        XCPT_FRAME
+.macro paranoidzeroentry sym do_sym
+ENTRY(\sym)
+        INTR_FRAME
         PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq %rax
+        pushq $-1               /* ORIG_RAX: no syscall to restart */
         CFI_ADJUST_CFA_OFFSET 8
-        CFI_REL_OFFSET rax,0
-        leaq  \sym(%rip),%rax
-        jmp error_entry
+        subq $15*8, %rsp
+        call save_paranoid
+        TRACE_IRQS_OFF
+        movq %rsp,%rdi          /* pt_regs pointer */
+        xorl %esi,%esi          /* no error code */
+        call \do_sym
+        jmp paranoid_exit       /* %ebx: no swapgs flag */
         CFI_ENDPROC
-        .endm
+END(\sym)
+.endm
 
-        /* error code is on the stack already */
-        /* handle NMI like exceptions that can happen everywhere */
-        .macro paranoidentry sym, ist=0, irqtrace=1
-        SAVE_ALL
-        cld
-        movl $1,%ebx
-        movl  $MSR_GS_BASE,%ecx
-        rdmsr
-        testl %edx,%edx
-        js    1f
-        SWAPGS
-        xorl  %ebx,%ebx
-1:
-        .if \ist
-        movq    %gs:pda_data_offset, %rbp
-        .endif
-        .if \irqtrace
-        TRACE_IRQS_OFF
-        .endif
-        movq %rsp,%rdi
-        movq ORIG_RAX(%rsp),%rsi
-        movq $-1,ORIG_RAX(%rsp)
-        .if \ist
-        subq    $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
-        .endif
-        call \sym
-        .if \ist
-        addq    $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
-        .endif
-        DISABLE_INTERRUPTS(CLBR_NONE)
-        .if \irqtrace
+.macro paranoidzeroentry_ist sym do_sym ist
+ENTRY(\sym)
+        INTR_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
+        pushq $-1               /* ORIG_RAX: no syscall to restart */
+        CFI_ADJUST_CFA_OFFSET 8
+        subq $15*8, %rsp
+        call save_paranoid
         TRACE_IRQS_OFF
-        .endif
-        .endm
+        movq %rsp,%rdi          /* pt_regs pointer */
+        xorl %esi,%esi          /* no error code */
+        movq %gs:pda_data_offset, %rbp
+        subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        call \do_sym
+        addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        jmp paranoid_exit       /* %ebx: no swapgs flag */
+        CFI_ENDPROC
+END(\sym)
+.endm
 
-        /*
-         * "Paranoid" exit path from exception stack.
-         * Paranoid because this is used by NMIs and cannot take
-         * any kernel state for granted.
-         * We don't do kernel preemption checks here, because only
-         * NMI should be common and it does not enable IRQs and
-         * cannot get reschedule ticks.
-         *
-         * "trace" is 0 for the NMI handler only, because irq-tracing
-         * is fundamentally NMI-unsafe. (we cannot change the soft and
-         * hard flags at once, atomically)
-         */
-        .macro paranoidexit trace=1
-        /* ebx: no swapgs flag */
-paranoid_exit\trace:
-        testl %ebx,%ebx                         /* swapgs needed? */
-        jnz paranoid_restore\trace
-        testl $3,CS(%rsp)
-        jnz   paranoid_userspace\trace
-paranoid_swapgs\trace:
-        .if \trace
-        TRACE_IRQS_IRETQ 0
-        .endif
-        SWAPGS_UNSAFE_STACK
-paranoid_restore\trace:
-        RESTORE_ALL 8
-        jmp irq_return
-paranoid_userspace\trace:
-        GET_THREAD_INFO(%rcx)
-        movl TI_flags(%rcx),%ebx
-        andl $_TIF_WORK_MASK,%ebx
-        jz paranoid_swapgs\trace
-        movq %rsp,%rdi                  /* &pt_regs */
-        call sync_regs
-        movq %rax,%rsp                  /* switch stack for scheduling */
-        testl $_TIF_NEED_RESCHED,%ebx
-        jnz paranoid_schedule\trace
-        movl %ebx,%edx                  /* arg3: thread flags */
-        .if \trace
-        TRACE_IRQS_ON
-        .endif
-        ENABLE_INTERRUPTS(CLBR_NONE)
-        xorl %esi,%esi                  /* arg2: oldset */
-        movq %rsp,%rdi                  /* arg1: &pt_regs */
-        call do_notify_resume
-        DISABLE_INTERRUPTS(CLBR_NONE)
-        .if \trace
-        TRACE_IRQS_OFF
-        .endif
-        jmp paranoid_userspace\trace
-paranoid_schedule\trace:
-        .if \trace
-        TRACE_IRQS_ON
-        .endif
-        ENABLE_INTERRUPTS(CLBR_ANY)
-        call schedule
-        DISABLE_INTERRUPTS(CLBR_ANY)
-        .if \trace
-        TRACE_IRQS_OFF
-        .endif
-        jmp paranoid_userspace\trace
+.macro errorentry sym do_sym
+ENTRY(\sym)
+        XCPT_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
+        subq $15*8,%rsp
+        CFI_ADJUST_CFA_OFFSET 15*8
+        call error_entry
+        DEFAULT_FRAME 0
+        movq %rsp,%rdi                  /* pt_regs pointer */
+        movq ORIG_RAX(%rsp),%rsi        /* get error code */
+        movq $-1,ORIG_RAX(%rsp)         /* no syscall to restart */
+        call \do_sym
+        jmp error_exit                  /* %ebx: no swapgs flag */
         CFI_ENDPROC
-        .endm
+END(\sym)
+.endm
 
-/*
- * Exception entry point. This expects an error code/orig_rax on the stack
- * and the exception handler in %rax.   
- */                                             
-KPROBE_ENTRY(error_entry)
-        _frame RDI
-        CFI_REL_OFFSET rax,0
-        /* rdi slot contains rax, oldrax contains error code */
-        cld     
-        subq  $14*8,%rsp
-        CFI_ADJUST_CFA_OFFSET   (14*8)
-        movq %rsi,13*8(%rsp)
-        CFI_REL_OFFSET  rsi,RSI
-        movq 14*8(%rsp),%rsi    /* load rax from rdi slot */
-        CFI_REGISTER    rax,rsi
-        movq %rdx,12*8(%rsp)
-        CFI_REL_OFFSET  rdx,RDX
-        movq %rcx,11*8(%rsp)
-        CFI_REL_OFFSET  rcx,RCX
-        movq %rsi,10*8(%rsp)    /* store rax */ 
-        CFI_REL_OFFSET  rax,RAX
-        movq %r8, 9*8(%rsp)
-        CFI_REL_OFFSET  r8,R8
-        movq %r9, 8*8(%rsp)
-        CFI_REL_OFFSET  r9,R9
-        movq %r10,7*8(%rsp)
-        CFI_REL_OFFSET  r10,R10
-        movq %r11,6*8(%rsp)
-        CFI_REL_OFFSET  r11,R11
-        movq %rbx,5*8(%rsp) 
-        CFI_REL_OFFSET  rbx,RBX
-        movq %rbp,4*8(%rsp) 
-        CFI_REL_OFFSET  rbp,RBP
-        movq %r12,3*8(%rsp) 
-        CFI_REL_OFFSET  r12,R12
-        movq %r13,2*8(%rsp) 
-        CFI_REL_OFFSET  r13,R13
-        movq %r14,1*8(%rsp) 
-        CFI_REL_OFFSET  r14,R14
-        movq %r15,(%rsp) 
-        CFI_REL_OFFSET  r15,R15
-        xorl %ebx,%ebx  
-        testl $3,CS(%rsp)
-        je  error_kernelspace
-error_swapgs:   
-        SWAPGS
-error_sti:
-        TRACE_IRQS_OFF
-        movq %rdi,RDI(%rsp)     
-        CFI_REL_OFFSET  rdi,RDI
-        movq %rsp,%rdi
-        movq ORIG_RAX(%rsp),%rsi        /* get error code */ 
-        movq $-1,ORIG_RAX(%rsp)
-        call *%rax
-        /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
-error_exit:
-        movl %ebx,%eax
-        RESTORE_REST
-        DISABLE_INTERRUPTS(CLBR_NONE)
+        /* error code is on the stack already */
+.macro paranoiderrorentry sym do_sym
+ENTRY(\sym)
+        XCPT_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
+        subq $15*8,%rsp
+        CFI_ADJUST_CFA_OFFSET 15*8
+        call save_paranoid
+        DEFAULT_FRAME 0
         TRACE_IRQS_OFF
-        GET_THREAD_INFO(%rcx)   
-        testl %eax,%eax
-        jne  retint_kernel
-        LOCKDEP_SYS_EXIT_IRQ
-        movl  TI_flags(%rcx),%edx
-        movl  $_TIF_WORK_MASK,%edi
-        andl  %edi,%edx
-        jnz  retint_careful
-        jmp retint_swapgs
+        movq %rsp,%rdi                  /* pt_regs pointer */
+        movq ORIG_RAX(%rsp),%rsi        /* get error code */
+        movq $-1,ORIG_RAX(%rsp)         /* no syscall to restart */
+        call \do_sym
+        jmp paranoid_exit               /* %ebx: no swapgs flag */
         CFI_ENDPROC
+END(\sym)
+.endm
 
-error_kernelspace:
-        incl %ebx
-       /* There are two places in the kernel that can potentially fault with
-          usergs. Handle them here. The exception handlers after
-           iret run with kernel gs again, so don't set the user space flag.
-           B stepping K8s sometimes report an truncated RIP for IRET 
-           exceptions returning to compat mode. Check for these here too. */
-        leaq irq_return(%rip),%rcx
-        cmpq %rcx,RIP(%rsp)
-        je   error_swapgs
-        movl %ecx,%ecx  /* zero extend */
-        cmpq %rcx,RIP(%rsp)
-        je   error_swapgs
-        cmpq $gs_change,RIP(%rsp)
-        je   error_swapgs
-        jmp  error_sti
-KPROBE_END(error_entry)
-        
-       /* Reload gs selector with exception handling */
-       /* edi:  new selector */ 
+zeroentry divide_error do_divide_error
+zeroentry overflow do_overflow
+zeroentry bounds do_bounds
+zeroentry invalid_op do_invalid_op
+zeroentry device_not_available do_device_not_available
+paranoiderrorentry double_fault do_double_fault
+zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
+errorentry invalid_TSS do_invalid_TSS
+errorentry segment_not_present do_segment_not_present
+zeroentry spurious_interrupt_bug do_spurious_interrupt_bug
+zeroentry coprocessor_error do_coprocessor_error
+errorentry alignment_check do_alignment_check
+zeroentry simd_coprocessor_error do_simd_coprocessor_error
+
+        /* Reload gs selector with exception handling */
+        /* edi:  new selector */
 ENTRY(native_load_gs_index)
         CFI_STARTPROC
         pushf
         CFI_ADJUST_CFA_OFFSET 8
         DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
-        SWAPGS
-gs_change:     
-        movl %edi,%gs   
+        SWAPGS
+gs_change:
+        movl %edi,%gs
 2:      mfence          /* workaround */
         SWAPGS
-        popf
+        popf
         CFI_ADJUST_CFA_OFFSET -8
-        ret
+        ret
         CFI_ENDPROC
-ENDPROC(native_load_gs_index)
-       
-        .section __ex_table,"a"
-        .align 8
-        .quad gs_change,bad_gs
-        .previous
-        .section .fixup,"ax"
+END(native_load_gs_index)
+
+        .section __ex_table,"a"
+        .align 8
+        .quad gs_change,bad_gs
+        .previous
+        .section .fixup,"ax"
         /* running with kernelgs */
-bad_gs: 
+bad_gs:
         SWAPGS                  /* switch back to user gs */
         xorl %eax,%eax
-        movl %eax,%gs
-        jmp  2b
-        .previous       
-        
+        movl %eax,%gs
+        jmp  2b
+        .previous
+
 /*
  * Create a kernel thread.
  *
@@ -1138,7 +1184,7 @@ ENTRY(kernel_thread)
 
         xorl %r8d,%r8d
         xorl %r9d,%r9d
-        
+
         # clone now
         call do_fork
         movq %rax,RAX(%rsp)
@@ -1149,15 +1195,15 @@ ENTRY(kernel_thread)
          * so internally to the x86_64 port you can rely on kernel_thread()
          * not to reschedule the child before returning, this avoids the need
          * of hacks for example to fork off the per-CPU idle tasks.
-         * [Hopefully no generic code relies on the reschedule -AK]     
+         * [Hopefully no generic code relies on the reschedule -AK]
          */
         RESTORE_ALL
         UNFAKE_STACK_FRAME
         ret
         CFI_ENDPROC
-ENDPROC(kernel_thread)
-        
-child_rip:
+END(kernel_thread)
+
+ENTRY(child_rip)
         pushq $0                # fake return address
         CFI_STARTPROC
         /*
@@ -1170,8 +1216,9 @@ child_rip:
         # exit
         mov %eax, %edi
         call do_exit
+        ud2                     # padding for call trace
         CFI_ENDPROC
-ENDPROC(child_rip)
+END(child_rip)
 
 /*
  * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
@@ -1191,10 +1238,10 @@ ENDPROC(child_rip)
 ENTRY(kernel_execve)
         CFI_STARTPROC
         FAKE_STACK_FRAME $0
-        SAVE_ALL        
+        SAVE_ALL
         movq %rsp,%rcx
         call sys_execve
-        movq %rax, RAX(%rsp)    
+        movq %rax, RAX(%rsp)
         RESTORE_REST
         testq %rax,%rax
         je int_ret_from_sys_call
@@ -1202,129 +1249,7 @@ ENTRY(kernel_execve)
         UNFAKE_STACK_FRAME
         ret
         CFI_ENDPROC
-ENDPROC(kernel_execve)
-
-KPROBE_ENTRY(page_fault)
-        errorentry do_page_fault
-KPROBE_END(page_fault)
-
-ENTRY(coprocessor_error)
-        zeroentry do_coprocessor_error
-END(coprocessor_error)
-
-ENTRY(simd_coprocessor_error)
-        zeroentry do_simd_coprocessor_error     
-END(simd_coprocessor_error)
-
-ENTRY(device_not_available)
-        zeroentry do_device_not_available
-END(device_not_available)
-
-        /* runs on exception stack */
-KPROBE_ENTRY(debug)
-        INTR_FRAME
-        PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $0
-        CFI_ADJUST_CFA_OFFSET 8         
-        paranoidentry do_debug, DEBUG_STACK
-        paranoidexit
-KPROBE_END(debug)
-
-        /* runs on exception stack */   
-KPROBE_ENTRY(nmi)
-        INTR_FRAME
-        PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $-1
-        CFI_ADJUST_CFA_OFFSET 8
-        paranoidentry do_nmi, 0, 0
-#ifdef CONFIG_TRACE_IRQFLAGS
-        paranoidexit 0
-#else
-        jmp paranoid_exit1
-        CFI_ENDPROC
-#endif
-KPROBE_END(nmi)
-
-KPROBE_ENTRY(int3)
-        INTR_FRAME
-        PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $0
-        CFI_ADJUST_CFA_OFFSET 8
-        paranoidentry do_int3, DEBUG_STACK
-        jmp paranoid_exit1
-        CFI_ENDPROC
-KPROBE_END(int3)
-
-ENTRY(overflow)
-        zeroentry do_overflow
-END(overflow)
-
-ENTRY(bounds)
-        zeroentry do_bounds
-END(bounds)
-
-ENTRY(invalid_op)
-        zeroentry do_invalid_op 
-END(invalid_op)
-
-ENTRY(coprocessor_segment_overrun)
-        zeroentry do_coprocessor_segment_overrun
-END(coprocessor_segment_overrun)
-
-        /* runs on exception stack */
-ENTRY(double_fault)
-        XCPT_FRAME
-        PARAVIRT_ADJUST_EXCEPTION_FRAME
-        paranoidentry do_double_fault
-        jmp paranoid_exit1
-        CFI_ENDPROC
-END(double_fault)
-
-ENTRY(invalid_TSS)
-        errorentry do_invalid_TSS
-END(invalid_TSS)
-
-ENTRY(segment_not_present)
-        errorentry do_segment_not_present
-END(segment_not_present)
-
-        /* runs on exception stack */
-ENTRY(stack_segment)
-        XCPT_FRAME
-        PARAVIRT_ADJUST_EXCEPTION_FRAME
-        paranoidentry do_stack_segment
-        jmp paranoid_exit1
-        CFI_ENDPROC
-END(stack_segment)
-
-KPROBE_ENTRY(general_protection)
-        errorentry do_general_protection
-KPROBE_END(general_protection)
-
-ENTRY(alignment_check)
-        errorentry do_alignment_check
-END(alignment_check)
-
-ENTRY(divide_error)
-        zeroentry do_divide_error
-END(divide_error)
-
-ENTRY(spurious_interrupt_bug)
-        zeroentry do_spurious_interrupt_bug
-END(spurious_interrupt_bug)
-
-#ifdef CONFIG_X86_MCE
-        /* runs on exception stack */
-ENTRY(machine_check)
-        INTR_FRAME
-        PARAVIRT_ADJUST_EXCEPTION_FRAME
-        pushq $0
-        CFI_ADJUST_CFA_OFFSET 8 
-        paranoidentry do_machine_check
-        jmp paranoid_exit1
-        CFI_ENDPROC
-END(machine_check)
-#endif
+END(kernel_execve)
 
 /* Call softirq on interrupt stack. Interrupts are off. */
 ENTRY(call_softirq)
@@ -1344,40 +1269,33 @@ ENTRY(call_softirq)
         decl %gs:pda_irqcount
         ret
         CFI_ENDPROC
-ENDPROC(call_softirq)
-
-KPROBE_ENTRY(ignore_sysret)
-        CFI_STARTPROC
-        mov $-ENOSYS,%eax
-        sysret
-        CFI_ENDPROC
-ENDPROC(ignore_sysret)
+END(call_softirq)
 
 #ifdef CONFIG_XEN
-ENTRY(xen_hypervisor_callback)
-        zeroentry xen_do_hypervisor_callback
-END(xen_hypervisor_callback)
+zeroentry xen_hypervisor_callback xen_do_hypervisor_callback
 
 /*
-# A note on the "critical region" in our callback handler.
-# We want to avoid stacking callback handlers due to events occurring
-# during handling of the last event. To do this, we keep events disabled
-# until we've done all processing. HOWEVER, we must enable events before
-# popping the stack frame (can't be done atomically) and so it would still
-# be possible to get enough handler activations to overflow the stack.
-# Although unlikely, bugs of that kind are hard to track down, so we'd
-# like to avoid the possibility.
-# So, on entry to the handler we detect whether we interrupted an
-# existing activation in its critical region -- if so, we pop the current
-# activation and restart the handler using the previous one.
-*/
+ * A note on the "critical region" in our callback handler.
+ * We want to avoid stacking callback handlers due to events occurring
+ * during handling of the last event. To do this, we keep events disabled
+ * until we've done all processing. HOWEVER, we must enable events before
+ * popping the stack frame (can't be done atomically) and so it would still
+ * be possible to get enough handler activations to overflow the stack.
+ * Although unlikely, bugs of that kind are hard to track down, so we'd
+ * like to avoid the possibility.
+ * So, on entry to the handler we detect whether we interrupted an
+ * existing activation in its critical region -- if so, we pop the current
+ * activation and restart the handler using the previous one.
+ */
 ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
         CFI_STARTPROC
-/* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
-   see the correct pointer to the pt_regs */
+/*
+ * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
+ * see the correct pointer to the pt_regs
+ */
         movq %rdi, %rsp            # we don't return, adjust the stack frame
         CFI_ENDPROC
-        CFI_DEFAULT_STACK
+        DEFAULT_FRAME
 11:     incl %gs:pda_irqcount
         movq %rsp,%rbp
         CFI_DEF_CFA_REGISTER rbp
@@ -1392,23 +1310,26 @@ ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
 END(do_hypervisor_callback)
 
 /*
-# Hypervisor uses this for application faults while it executes.
-# We get here for two reasons:
-#  1. Fault while reloading DS, ES, FS or GS
-#  2. Fault while executing IRET
-# Category 1 we do not need to fix up as Xen has already reloaded all segment
-# registers that could be reloaded and zeroed the others.
-# Category 2 we fix up by killing the current process. We cannot use the
-# normal Linux return path in this case because if we use the IRET hypercall
-# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
-# We distinguish between categories by comparing each saved segment register
-# with its current contents: any discrepancy means we in category 1.
-*/
+ * Hypervisor uses this for application faults while it executes.
+ * We get here for two reasons:
+ *  1. Fault while reloading DS, ES, FS or GS
+ *  2. Fault while executing IRET
+ * Category 1 we do not need to fix up as Xen has already reloaded all segment
+ * registers that could be reloaded and zeroed the others.
+ * Category 2 we fix up by killing the current process. We cannot use the
+ * normal Linux return path in this case because if we use the IRET hypercall
+ * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
+ * We distinguish between categories by comparing each saved segment register
+ * with its current contents: any discrepancy means we in category 1.
+ */
 ENTRY(xen_failsafe_callback)
-        framesz = (RIP-0x30)    /* workaround buggy gas */
-        _frame framesz
-        CFI_REL_OFFSET rcx, 0
-        CFI_REL_OFFSET r11, 8
+        INTR_FRAME 1 (6*8)
+        /*CFI_REL_OFFSET gs,GS*/
+        /*CFI_REL_OFFSET fs,FS*/
+        /*CFI_REL_OFFSET es,ES*/
+        /*CFI_REL_OFFSET ds,DS*/
+        CFI_REL_OFFSET r11,8
+        CFI_REL_OFFSET rcx,0
         movw %ds,%cx
         cmpw %cx,0x10(%rsp)
         CFI_REMEMBER_STATE
@@ -1429,12 +1350,9 @@ ENTRY(xen_failsafe_callback)
         CFI_RESTORE r11
         addq $0x30,%rsp
         CFI_ADJUST_CFA_OFFSET -0x30
-        pushq $0
-        CFI_ADJUST_CFA_OFFSET 8
-        pushq %r11
-        CFI_ADJUST_CFA_OFFSET 8
-        pushq %rcx
-        CFI_ADJUST_CFA_OFFSET 8
+        pushq_cfi $0    /* RIP */
+        pushq_cfi %r11
+        pushq_cfi %rcx
         jmp general_protection
         CFI_RESTORE_STATE
 1:      /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
@@ -1444,11 +1362,223 @@ ENTRY(xen_failsafe_callback)
         CFI_RESTORE r11
         addq $0x30,%rsp
         CFI_ADJUST_CFA_OFFSET -0x30
-        pushq $0
-        CFI_ADJUST_CFA_OFFSET 8
+        pushq_cfi $0
         SAVE_ALL
         jmp error_exit
         CFI_ENDPROC
 END(xen_failsafe_callback)
 
 #endif /* CONFIG_XEN */
+
+/*
+ * Some functions should be protected against kprobes
+ */
+        .pushsection .kprobes.text, "ax"
+
+paranoidzeroentry_ist debug do_debug DEBUG_STACK
+paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
+paranoiderrorentry stack_segment do_stack_segment
+errorentry general_protection do_general_protection
+errorentry page_fault do_page_fault
+#ifdef CONFIG_X86_MCE
+paranoidzeroentry machine_check do_machine_check
+#endif
+
+        /*
+         * "Paranoid" exit path from exception stack.
+         * Paranoid because this is used by NMIs and cannot take
+         * any kernel state for granted.
+         * We don't do kernel preemption checks here, because only
+         * NMI should be common and it does not enable IRQs and
+         * cannot get reschedule ticks.
+         *
+         * "trace" is 0 for the NMI handler only, because irq-tracing
+         * is fundamentally NMI-unsafe. (we cannot change the soft and
+         * hard flags at once, atomically)
+         */
+
+        /* ebx: no swapgs flag */
+ENTRY(paranoid_exit)
+        INTR_FRAME
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        TRACE_IRQS_OFF
+        testl %ebx,%ebx                         /* swapgs needed? */
+        jnz paranoid_restore
+        testl $3,CS(%rsp)
+        jnz   paranoid_userspace
+paranoid_swapgs:
+        TRACE_IRQS_IRETQ 0
+        SWAPGS_UNSAFE_STACK
+paranoid_restore:
+        RESTORE_ALL 8
+        jmp irq_return
+paranoid_userspace:
+        GET_THREAD_INFO(%rcx)
+        movl TI_flags(%rcx),%ebx
+        andl $_TIF_WORK_MASK,%ebx
+        jz paranoid_swapgs
+        movq %rsp,%rdi                  /* &pt_regs */
+        call sync_regs
+        movq %rax,%rsp                  /* switch stack for scheduling */
+        testl $_TIF_NEED_RESCHED,%ebx
+        jnz paranoid_schedule
+        movl %ebx,%edx                  /* arg3: thread flags */
+        TRACE_IRQS_ON
+        ENABLE_INTERRUPTS(CLBR_NONE)
+        xorl %esi,%esi                  /* arg2: oldset */
+        movq %rsp,%rdi                  /* arg1: &pt_regs */
+        call do_notify_resume
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        TRACE_IRQS_OFF
+        jmp paranoid_userspace
+paranoid_schedule:
+        TRACE_IRQS_ON
+        ENABLE_INTERRUPTS(CLBR_ANY)
+        call schedule
+        DISABLE_INTERRUPTS(CLBR_ANY)
+        TRACE_IRQS_OFF
+        jmp paranoid_userspace
+        CFI_ENDPROC
+END(paranoid_exit)
+
+/*
+ * Exception entry point. This expects an error code/orig_rax on the stack.
+ * returns in "no swapgs flag" in %ebx.
+ */
+ENTRY(error_entry)
+        XCPT_FRAME
+        CFI_ADJUST_CFA_OFFSET 15*8
+        /* oldrax contains error code */
+        cld
+        movq_cfi rdi, RDI+8
+        movq_cfi rsi, RSI+8
+        movq_cfi rdx, RDX+8
+        movq_cfi rcx, RCX+8
+        movq_cfi rax, RAX+8
+        movq_cfi  r8,  R8+8
+        movq_cfi  r9,  R9+8
+        movq_cfi r10, R10+8
+        movq_cfi r11, R11+8
+        movq_cfi rbx, RBX+8
+        movq_cfi rbp, RBP+8
+        movq_cfi r12, R12+8
+        movq_cfi r13, R13+8
+        movq_cfi r14, R14+8
+        movq_cfi r15, R15+8
+        xorl %ebx,%ebx
+        testl $3,CS+8(%rsp)
+        je error_kernelspace
+error_swapgs:
+        SWAPGS
+error_sti:
+        TRACE_IRQS_OFF
+        ret
+        CFI_ENDPROC
+
+/*
+ * There are two places in the kernel that can potentially fault with
+ * usergs. Handle them here. The exception handlers after iret run with
+ * kernel gs again, so don't set the user space flag. B stepping K8s
+ * sometimes report an truncated RIP for IRET exceptions returning to
+ * compat mode. Check for these here too.
+ */
+error_kernelspace:
+        incl %ebx
+        leaq irq_return(%rip),%rcx
+        cmpq %rcx,RIP+8(%rsp)
+        je error_swapgs
+        movl %ecx,%ecx  /* zero extend */
+        cmpq %rcx,RIP+8(%rsp)
+        je error_swapgs
+        cmpq $gs_change,RIP+8(%rsp)
+        je error_swapgs
+        jmp error_sti
+END(error_entry)
+
+
+/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
+ENTRY(error_exit)
+        DEFAULT_FRAME
+        movl %ebx,%eax
+        RESTORE_REST
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        TRACE_IRQS_OFF
+        GET_THREAD_INFO(%rcx)
+        testl %eax,%eax
+        jne retint_kernel
+        LOCKDEP_SYS_EXIT_IRQ
+        movl TI_flags(%rcx),%edx
+        movl $_TIF_WORK_MASK,%edi
+        andl %edi,%edx
+        jnz retint_careful
+        jmp retint_swapgs
+        CFI_ENDPROC
+END(error_exit)
+
+
+        /* runs on exception stack */
+ENTRY(nmi)
+        INTR_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
+        pushq_cfi $-1
+        subq $15*8, %rsp
+        CFI_ADJUST_CFA_OFFSET 15*8
+        call save_paranoid
+        DEFAULT_FRAME 0
+        /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
+        movq %rsp,%rdi
+        movq $-1,%rsi
+        call do_nmi
+#ifdef CONFIG_TRACE_IRQFLAGS
+        /* paranoidexit; without TRACE_IRQS_OFF */
+        /* ebx: no swapgs flag */
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        testl %ebx,%ebx                         /* swapgs needed? */
+        jnz nmi_restore
+        testl $3,CS(%rsp)
+        jnz nmi_userspace
+nmi_swapgs:
+        SWAPGS_UNSAFE_STACK
+nmi_restore:
+        RESTORE_ALL 8
+        jmp irq_return
+nmi_userspace:
+        GET_THREAD_INFO(%rcx)
+        movl TI_flags(%rcx),%ebx
+        andl $_TIF_WORK_MASK,%ebx
+        jz nmi_swapgs
+        movq %rsp,%rdi                  /* &pt_regs */
+        call sync_regs
+        movq %rax,%rsp                  /* switch stack for scheduling */
+        testl $_TIF_NEED_RESCHED,%ebx
+        jnz nmi_schedule
+        movl %ebx,%edx                  /* arg3: thread flags */
+        ENABLE_INTERRUPTS(CLBR_NONE)
+        xorl %esi,%esi                  /* arg2: oldset */
+        movq %rsp,%rdi                  /* arg1: &pt_regs */
+        call do_notify_resume
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        jmp nmi_userspace
+nmi_schedule:
+        ENABLE_INTERRUPTS(CLBR_ANY)
+        call schedule
+        DISABLE_INTERRUPTS(CLBR_ANY)
+        jmp nmi_userspace
+        CFI_ENDPROC
+#else
+        jmp paranoid_exit
+        CFI_ENDPROC
+#endif
+END(nmi)
+
+ENTRY(ignore_sysret)
+        CFI_STARTPROC
+        mov $-ENOSYS,%eax
+        sysret
+        CFI_ENDPROC
+END(ignore_sysret)
+
+/*
+ * End of kprobes section
+ */
+        .popsection
diff --git a/arch/x86/kernel/es7000_32.c b/arch/x86/kernel/es7000_32.c
index 0aa2c443d600..53699c931ad4 100644
--- a/arch/x86/kernel/es7000_32.c
+++ b/arch/x86/kernel/es7000_32.c
@@ -38,8 +38,11 @@
 #include <asm/io.h>
 #include <asm/nmi.h>
 #include <asm/smp.h>
+#include <asm/atomic.h>
 #include <asm/apicdef.h>
 #include <mach_mpparse.h>
+#include <asm/genapic.h>
+#include <asm/setup.h>
 
 /*
  * ES7000 chipsets
@@ -161,6 +164,43 @@ es7000_rename_gsi(int ioapic, int gsi)
         return gsi;
 }
 
+static int wakeup_secondary_cpu_via_mip(int cpu, unsigned long eip)
+{
+        unsigned long vect = 0, psaival = 0;
+
+        if (psai == NULL)
+                return -1;
+
+        vect = ((unsigned long)__pa(eip)/0x1000) << 16;
+        psaival = (0x1000000 | vect | cpu);
+
+        while (*psai & 0x1000000)
+                ;
+
+        *psai = psaival;
+
+        return 0;
+}
+
+static void noop_wait_for_deassert(atomic_t *deassert_not_used)
+{
+}
+
+static int __init es7000_update_genapic(void)
+{
+        genapic->wakeup_cpu = wakeup_secondary_cpu_via_mip;
+
+        /* MPENTIUMIII */
+        if (boot_cpu_data.x86 == 6 &&
+            (boot_cpu_data.x86_model >= 7 || boot_cpu_data.x86_model <= 11)) {
+                es7000_update_genapic_to_cluster();
+                genapic->wait_for_init_deassert = noop_wait_for_deassert;
+                genapic->wakeup_cpu = wakeup_secondary_cpu_via_mip;
+        }
+
+        return 0;
+}
+
 void __init
 setup_unisys(void)
 {
@@ -176,6 +216,8 @@ setup_unisys(void)
         else
                 es7000_plat = ES7000_CLASSIC;
         ioapic_renumber_irq = es7000_rename_gsi;
+
+        x86_quirks->update_genapic = es7000_update_genapic;
 }
 
 /*
@@ -317,26 +359,6 @@ es7000_mip_write(struct mip_reg *mip_reg)
         return status;
 }
 
-int
-es7000_start_cpu(int cpu, unsigned long eip)
-{
-        unsigned long vect = 0, psaival = 0;
-
-        if (psai == NULL)
-                return -1;
-
-        vect = ((unsigned long)__pa(eip)/0x1000) << 16;
-        psaival = (0x1000000 | vect | cpu);
-
-        while (*psai & 0x1000000)
-                ;
-
-        *psai = psaival;
-
-        return 0;
-
-}
-
 void __init
 es7000_sw_apic(void)
 {
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 50ea0ac8c9bf..1b43086b097a 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -14,14 +14,17 @@
 #include <linux/uaccess.h>
 #include <linux/ftrace.h>
 #include <linux/percpu.h>
+#include <linux/sched.h>
 #include <linux/init.h>
 #include <linux/list.h>
 
 #include <asm/ftrace.h>
+#include <linux/ftrace.h>
 #include <asm/nops.h>
+#include <asm/nmi.h>
 
 
-static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
+#ifdef CONFIG_DYNAMIC_FTRACE
 
 union ftrace_code_union {
         char code[MCOUNT_INSN_SIZE];
@@ -31,18 +34,12 @@ union ftrace_code_union {
         } __attribute__((packed));
 };
 
-
 static int ftrace_calc_offset(long ip, long addr)
 {
         return (int)(addr - ip);
 }
 
-unsigned char *ftrace_nop_replace(void)
-{
-        return ftrace_nop;
-}
-
-unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
+static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
 {
         static union ftrace_code_union calc;
 
@@ -56,7 +53,142 @@ unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
         return calc.code;
 }
 
-int
+/*
+ * Modifying code must take extra care. On an SMP machine, if
+ * the code being modified is also being executed on another CPU
+ * that CPU will have undefined results and possibly take a GPF.
+ * We use kstop_machine to stop other CPUS from exectuing code.
+ * But this does not stop NMIs from happening. We still need
+ * to protect against that. We separate out the modification of
+ * the code to take care of this.
+ *
+ * Two buffers are added: An IP buffer and a "code" buffer.
+ *
+ * 1) Put the instruction pointer into the IP buffer
+ *    and the new code into the "code" buffer.
+ * 2) Set a flag that says we are modifying code
+ * 3) Wait for any running NMIs to finish.
+ * 4) Write the code
+ * 5) clear the flag.
+ * 6) Wait for any running NMIs to finish.
+ *
+ * If an NMI is executed, the first thing it does is to call
+ * "ftrace_nmi_enter". This will check if the flag is set to write
+ * and if it is, it will write what is in the IP and "code" buffers.
+ *
+ * The trick is, it does not matter if everyone is writing the same
+ * content to the code location. Also, if a CPU is executing code
+ * it is OK to write to that code location if the contents being written
+ * are the same as what exists.
+ */
+
+static atomic_t in_nmi = ATOMIC_INIT(0);
+static int mod_code_status;             /* holds return value of text write */
+static int mod_code_write;              /* set when NMI should do the write */
+static void *mod_code_ip;               /* holds the IP to write to */
+static void *mod_code_newcode;          /* holds the text to write to the IP */
+
+static unsigned nmi_wait_count;
+static atomic_t nmi_update_count = ATOMIC_INIT(0);
+
+int ftrace_arch_read_dyn_info(char *buf, int size)
+{
+        int r;
+
+        r = snprintf(buf, size, "%u %u",
+                     nmi_wait_count,
+                     atomic_read(&nmi_update_count));
+        return r;
+}
+
+static void ftrace_mod_code(void)
+{
+        /*
+         * Yes, more than one CPU process can be writing to mod_code_status.
+         *    (and the code itself)
+         * But if one were to fail, then they all should, and if one were
+         * to succeed, then they all should.
+         */
+        mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
+                                             MCOUNT_INSN_SIZE);
+}
+
+void ftrace_nmi_enter(void)
+{
+        atomic_inc(&in_nmi);
+        /* Must have in_nmi seen before reading write flag */
+        smp_mb();
+        if (mod_code_write) {
+                ftrace_mod_code();
+                atomic_inc(&nmi_update_count);
+        }
+}
+
+void ftrace_nmi_exit(void)
+{
+        /* Finish all executions before clearing in_nmi */
+        smp_wmb();
+        atomic_dec(&in_nmi);
+}
+
+static void wait_for_nmi(void)
+{
+        int waited = 0;
+
+        while (atomic_read(&in_nmi)) {
+                waited = 1;
+                cpu_relax();
+        }
+
+        if (waited)
+                nmi_wait_count++;
+}
+
+static int
+do_ftrace_mod_code(unsigned long ip, void *new_code)
+{
+        mod_code_ip = (void *)ip;
+        mod_code_newcode = new_code;
+
+        /* The buffers need to be visible before we let NMIs write them */
+        smp_wmb();
+
+        mod_code_write = 1;
+
+        /* Make sure write bit is visible before we wait on NMIs */
+        smp_mb();
+
+        wait_for_nmi();
+
+        /* Make sure all running NMIs have finished before we write the code */
+        smp_mb();
+
+        ftrace_mod_code();
+
+        /* Make sure the write happens before clearing the bit */
+        smp_wmb();
+
+        mod_code_write = 0;
+
+        /* make sure NMIs see the cleared bit */
+        smp_mb();
+
+        wait_for_nmi();
+
+        return mod_code_status;
+}
+
+
+
+
+static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
+
+static unsigned char *ftrace_nop_replace(void)
+{
+        return ftrace_nop;
+}
+
+static int
 ftrace_modify_code(unsigned long ip, unsigned char *old_code,
                    unsigned char *new_code)
 {
@@ -81,7 +213,7 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
                 return -EINVAL;
 
         /* replace the text with the new text */
-        if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
+        if (do_ftrace_mod_code(ip, new_code))
                 return -EPERM;
 
         sync_core();
@@ -89,6 +221,29 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
         return 0;
 }
 
+int ftrace_make_nop(struct module *mod,
+                    struct dyn_ftrace *rec, unsigned long addr)
+{
+        unsigned char *new, *old;
+        unsigned long ip = rec->ip;
+
+        old = ftrace_call_replace(ip, addr);
+        new = ftrace_nop_replace();
+
+        return ftrace_modify_code(rec->ip, old, new);
+}
+
+int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
+{
+        unsigned char *new, *old;
+        unsigned long ip = rec->ip;
+
+        old = ftrace_nop_replace();
+        new = ftrace_call_replace(ip, addr);
+
+        return ftrace_modify_code(rec->ip, old, new);
+}
+
 int ftrace_update_ftrace_func(ftrace_func_t func)
 {
         unsigned long ip = (unsigned long)(&ftrace_call);
@@ -165,3 +320,218 @@ int __init ftrace_dyn_arch_init(void *data)
 
         return 0;
 }
+#endif
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+extern void ftrace_graph_call(void);
+
+static int ftrace_mod_jmp(unsigned long ip,
+                          int old_offset, int new_offset)
+{
+        unsigned char code[MCOUNT_INSN_SIZE];
+
+        if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
+                return -EFAULT;
+
+        if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
+                return -EINVAL;
+
+        *(int *)(&code[1]) = new_offset;
+
+        if (do_ftrace_mod_code(ip, &code))
+                return -EPERM;
+
+        return 0;
+}
+
+int ftrace_enable_ftrace_graph_caller(void)
+{
+        unsigned long ip = (unsigned long)(&ftrace_graph_call);
+        int old_offset, new_offset;
+
+        old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
+        new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
+
+        return ftrace_mod_jmp(ip, old_offset, new_offset);
+}
+
+int ftrace_disable_ftrace_graph_caller(void)
+{
+        unsigned long ip = (unsigned long)(&ftrace_graph_call);
+        int old_offset, new_offset;
+
+        old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
+        new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
+
+        return ftrace_mod_jmp(ip, old_offset, new_offset);
+}
+
+#else /* CONFIG_DYNAMIC_FTRACE */
+
+/*
+ * These functions are picked from those used on
+ * this page for dynamic ftrace. They have been
+ * simplified to ignore all traces in NMI context.
+ */
+static atomic_t in_nmi;
+
+void ftrace_nmi_enter(void)
+{
+        atomic_inc(&in_nmi);
+}
+
+void ftrace_nmi_exit(void)
+{
+        atomic_dec(&in_nmi);
+}
+
+#endif /* !CONFIG_DYNAMIC_FTRACE */
+
+/* Add a function return address to the trace stack on thread info.*/
+static int push_return_trace(unsigned long ret, unsigned long long time,
+                                unsigned long func, int *depth)
+{
+        int index;
+
+        if (!current->ret_stack)
+                return -EBUSY;
+
+        /* The return trace stack is full */
+        if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
+                atomic_inc(&current->trace_overrun);
+                return -EBUSY;
+        }
+
+        index = ++current->curr_ret_stack;
+        barrier();
+        current->ret_stack[index].ret = ret;
+        current->ret_stack[index].func = func;
+        current->ret_stack[index].calltime = time;
+        *depth = index;
+
+        return 0;
+}
+
+/* Retrieve a function return address to the trace stack on thread info.*/
+static void pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
+{
+        int index;
+
+        index = current->curr_ret_stack;
+
+        if (unlikely(index < 0)) {
+                ftrace_graph_stop();
+                WARN_ON(1);
+                /* Might as well panic, otherwise we have no where to go */
+                *ret = (unsigned long)panic;
+                return;
+        }
+
+        *ret = current->ret_stack[index].ret;
+        trace->func = current->ret_stack[index].func;
+        trace->calltime = current->ret_stack[index].calltime;
+        trace->overrun = atomic_read(&current->trace_overrun);
+        trace->depth = index;
+        barrier();
+        current->curr_ret_stack--;
+
+}
+
+/*
+ * Send the trace to the ring-buffer.
+ * @return the original return address.
+ */
+unsigned long ftrace_return_to_handler(void)
+{
+        struct ftrace_graph_ret trace;
+        unsigned long ret;
+
+        pop_return_trace(&trace, &ret);
+        trace.rettime = cpu_clock(raw_smp_processor_id());
+        ftrace_graph_return(&trace);
+
+        if (unlikely(!ret)) {
+                ftrace_graph_stop();
+                WARN_ON(1);
+                /* Might as well panic. What else to do? */
+                ret = (unsigned long)panic;
+        }
+
+        return ret;
+}
+
+/*
+ * Hook the return address and push it in the stack of return addrs
+ * in current thread info.
+ */
+void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
+{
+        unsigned long old;
+        unsigned long long calltime;
+        int faulted;
+        struct ftrace_graph_ent trace;
+        unsigned long return_hooker = (unsigned long)
+                                &return_to_handler;
+
+        /* Nmi's are currently unsupported */
+        if (unlikely(atomic_read(&in_nmi)))
+                return;
+
+        if (unlikely(atomic_read(&current->tracing_graph_pause)))
+                return;
+
+        /*
+         * Protect against fault, even if it shouldn't
+         * happen. This tool is too much intrusive to
+         * ignore such a protection.
+         */
+        asm volatile(
+                "1: " _ASM_MOV " (%[parent_old]), %[old]\n"
+                "2: " _ASM_MOV " %[return_hooker], (%[parent_replaced])\n"
+                "   movl $0, %[faulted]\n"
+
+                ".section .fixup, \"ax\"\n"
+                "3: movl $1, %[faulted]\n"
+                ".previous\n"
+
+                _ASM_EXTABLE(1b, 3b)
+                _ASM_EXTABLE(2b, 3b)
+
+                : [parent_replaced] "=r" (parent), [old] "=r" (old),
+                  [faulted] "=r" (faulted)
+                : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker)
+                : "memory"
+        );
+
+        if (unlikely(faulted)) {
+                ftrace_graph_stop();
+                WARN_ON(1);
+                return;
+        }
+
+        if (unlikely(!__kernel_text_address(old))) {
+                ftrace_graph_stop();
+                *parent = old;
+                WARN_ON(1);
+                return;
+        }
+
+        calltime = cpu_clock(raw_smp_processor_id());
+
+        if (push_return_trace(old, calltime,
+                                self_addr, &trace.depth) == -EBUSY) {
+                *parent = old;
+                return;
+        }
+
+        trace.func = self_addr;
+
+        /* Only trace if the calling function expects to */
+        if (!ftrace_graph_entry(&trace)) {
+                current->curr_ret_stack--;
+                *parent = old;
+        }
+}
+#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/x86/kernel/genapic_64.c b/arch/x86/kernel/genapic_64.c
index 6c9bfc9e1e95..2bced78b0b8e 100644
--- a/arch/x86/kernel/genapic_64.c
+++ b/arch/x86/kernel/genapic_64.c
@@ -21,6 +21,7 @@
 #include <asm/smp.h>
 #include <asm/ipi.h>
 #include <asm/genapic.h>
+#include <asm/setup.h>
 
 extern struct genapic apic_flat;
 extern struct genapic apic_physflat;
@@ -53,6 +54,9 @@ void __init setup_apic_routing(void)
                         genapic = &apic_physflat;
                 printk(KERN_INFO "Setting APIC routing to %s\n", genapic->name);
         }
+
+        if (x86_quirks->update_genapic)
+                x86_quirks->update_genapic();
 }
 
 /* Same for both flat and physical. */
diff --git a/arch/x86/kernel/genapic_flat_64.c b/arch/x86/kernel/genapic_flat_64.c
index c0262791bda4..34185488e4fb 100644
--- a/arch/x86/kernel/genapic_flat_64.c
+++ b/arch/x86/kernel/genapic_flat_64.c
@@ -30,12 +30,12 @@ static int flat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
         return 1;
 }
 
-static cpumask_t flat_target_cpus(void)
+static const struct cpumask *flat_target_cpus(void)
 {
-        return cpu_online_map;
+        return cpu_online_mask;
 }
 
-static cpumask_t flat_vector_allocation_domain(int cpu)
+static void flat_vector_allocation_domain(int cpu, struct cpumask *retmask)
 {
         /* Careful. Some cpus do not strictly honor the set of cpus
          * specified in the interrupt destination when using lowest
@@ -45,8 +45,8 @@ static cpumask_t flat_vector_allocation_domain(int cpu)
          * deliver interrupts to the wrong hyperthread when only one
          * hyperthread was specified in the interrupt desitination.
          */
-        cpumask_t domain = { { [0] = APIC_ALL_CPUS, } };
-        return domain;
+        cpumask_clear(retmask);
+        cpumask_bits(retmask)[0] = APIC_ALL_CPUS;
 }
 
 /*
@@ -69,9 +69,8 @@ static void flat_init_apic_ldr(void)
         apic_write(APIC_LDR, val);
 }
 
-static void flat_send_IPI_mask(cpumask_t cpumask, int vector)
+static inline void _flat_send_IPI_mask(unsigned long mask, int vector)
 {
-        unsigned long mask = cpus_addr(cpumask)[0];
         unsigned long flags;
 
         local_irq_save(flags);
@@ -79,20 +78,41 @@ static void flat_send_IPI_mask(cpumask_t cpumask, int vector)
         local_irq_restore(flags);
 }
 
+static void flat_send_IPI_mask(const struct cpumask *cpumask, int vector)
+{
+        unsigned long mask = cpumask_bits(cpumask)[0];
+
+        _flat_send_IPI_mask(mask, vector);
+}
+
+static void flat_send_IPI_mask_allbutself(const struct cpumask *cpumask,
+                                          int vector)
+{
+        unsigned long mask = cpumask_bits(cpumask)[0];
+        int cpu = smp_processor_id();
+
+        if (cpu < BITS_PER_LONG)
+                clear_bit(cpu, &mask);
+        _flat_send_IPI_mask(mask, vector);
+}
+
 static void flat_send_IPI_allbutself(int vector)
 {
+        int cpu = smp_processor_id();
 #ifdef  CONFIG_HOTPLUG_CPU
         int hotplug = 1;
 #else
         int hotplug = 0;
 #endif
         if (hotplug || vector == NMI_VECTOR) {
-                cpumask_t allbutme = cpu_online_map;
+                if (!cpumask_equal(cpu_online_mask, cpumask_of(cpu))) {
+                        unsigned long mask = cpumask_bits(cpu_online_mask)[0];
 
-                cpu_clear(smp_processor_id(), allbutme);
+                        if (cpu < BITS_PER_LONG)
+                                clear_bit(cpu, &mask);
 
-                if (!cpus_empty(allbutme))
-                        flat_send_IPI_mask(allbutme, vector);
+                        _flat_send_IPI_mask(mask, vector);
+                }
         } else if (num_online_cpus() > 1) {
                 __send_IPI_shortcut(APIC_DEST_ALLBUT, vector,APIC_DEST_LOGICAL);
         }
@@ -101,7 +121,7 @@ static void flat_send_IPI_allbutself(int vector)
 static void flat_send_IPI_all(int vector)
 {
         if (vector == NMI_VECTOR)
-                flat_send_IPI_mask(cpu_online_map, vector);
+                flat_send_IPI_mask(cpu_online_mask, vector);
         else
                 __send_IPI_shortcut(APIC_DEST_ALLINC, vector, APIC_DEST_LOGICAL);
 }
@@ -135,9 +155,18 @@ static int flat_apic_id_registered(void)
         return physid_isset(read_xapic_id(), phys_cpu_present_map);
 }
 
-static unsigned int flat_cpu_mask_to_apicid(cpumask_t cpumask)
+static unsigned int flat_cpu_mask_to_apicid(const struct cpumask *cpumask)
+{
+        return cpumask_bits(cpumask)[0] & APIC_ALL_CPUS;
+}
+
+static unsigned int flat_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+                                                const struct cpumask *andmask)
 {
-        return cpus_addr(cpumask)[0] & APIC_ALL_CPUS;
+        unsigned long mask1 = cpumask_bits(cpumask)[0] & APIC_ALL_CPUS;
+        unsigned long mask2 = cpumask_bits(andmask)[0] & APIC_ALL_CPUS;
+
+        return mask1 & mask2;
 }
 
 static unsigned int phys_pkg_id(int index_msb)
@@ -157,8 +186,10 @@ struct genapic apic_flat =  {
         .send_IPI_all = flat_send_IPI_all,
         .send_IPI_allbutself = flat_send_IPI_allbutself,
         .send_IPI_mask = flat_send_IPI_mask,
+        .send_IPI_mask_allbutself = flat_send_IPI_mask_allbutself,
         .send_IPI_self = apic_send_IPI_self,
         .cpu_mask_to_apicid = flat_cpu_mask_to_apicid,
+        .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and,
         .phys_pkg_id = phys_pkg_id,
         .get_apic_id = get_apic_id,
         .set_apic_id = set_apic_id,
@@ -188,35 +219,39 @@ static int physflat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
         return 0;
 }
 
-static cpumask_t physflat_target_cpus(void)
+static const struct cpumask *physflat_target_cpus(void)
 {
-        return cpu_online_map;
+        return cpu_online_mask;
 }
 
-static cpumask_t physflat_vector_allocation_domain(int cpu)
+static void physflat_vector_allocation_domain(int cpu, struct cpumask *retmask)
 {
-        return cpumask_of_cpu(cpu);
+        cpumask_clear(retmask);
+        cpumask_set_cpu(cpu, retmask);
 }
 
-static void physflat_send_IPI_mask(cpumask_t cpumask, int vector)
+static void physflat_send_IPI_mask(const struct cpumask *cpumask, int vector)
 {
         send_IPI_mask_sequence(cpumask, vector);
 }
 
-static void physflat_send_IPI_allbutself(int vector)
+static void physflat_send_IPI_mask_allbutself(const struct cpumask *cpumask,
+                                              int vector)
 {
-        cpumask_t allbutme = cpu_online_map;
+        send_IPI_mask_allbutself(cpumask, vector);
+}
 
-        cpu_clear(smp_processor_id(), allbutme);
-        physflat_send_IPI_mask(allbutme, vector);
+static void physflat_send_IPI_allbutself(int vector)
+{
+        send_IPI_mask_allbutself(cpu_online_mask, vector);
 }
 
 static void physflat_send_IPI_all(int vector)
 {
-        physflat_send_IPI_mask(cpu_online_map, vector);
+        physflat_send_IPI_mask(cpu_online_mask, vector);
 }
 
-static unsigned int physflat_cpu_mask_to_apicid(cpumask_t cpumask)
+static unsigned int physflat_cpu_mask_to_apicid(const struct cpumask *cpumask)
 {
         int cpu;
 
@@ -224,13 +259,31 @@ static unsigned int physflat_cpu_mask_to_apicid(cpumask_t cpumask)
          * We're using fixed IRQ delivery, can only return one phys APIC ID.
          * May as well be the first.
          */
-        cpu = first_cpu(cpumask);
+        cpu = cpumask_first(cpumask);
         if ((unsigned)cpu < nr_cpu_ids)
                 return per_cpu(x86_cpu_to_apicid, cpu);
         else
                 return BAD_APICID;
 }
 
+static unsigned int
+physflat_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+                                const struct cpumask *andmask)
+{
+        int cpu;
+
+        /*
+         * We're using fixed IRQ delivery, can only return one phys APIC ID.
+         * May as well be the first.
+         */
+        for_each_cpu_and(cpu, cpumask, andmask)
+                if (cpumask_test_cpu(cpu, cpu_online_mask))
+                        break;
+        if (cpu < nr_cpu_ids)
+                return per_cpu(x86_cpu_to_apicid, cpu);
+        return BAD_APICID;
+}
+
 struct genapic apic_physflat =  {
         .name = "physical flat",
         .acpi_madt_oem_check = physflat_acpi_madt_oem_check,
@@ -243,8 +296,10 @@ struct genapic apic_physflat =  {
         .send_IPI_all = physflat_send_IPI_all,
         .send_IPI_allbutself = physflat_send_IPI_allbutself,
         .send_IPI_mask = physflat_send_IPI_mask,
+        .send_IPI_mask_allbutself = physflat_send_IPI_mask_allbutself,
         .send_IPI_self = apic_send_IPI_self,
         .cpu_mask_to_apicid = physflat_cpu_mask_to_apicid,
+        .cpu_mask_to_apicid_and = physflat_cpu_mask_to_apicid_and,
         .phys_pkg_id = phys_pkg_id,
         .get_apic_id = get_apic_id,
         .set_apic_id = set_apic_id,
diff --git a/arch/x86/kernel/genx2apic_cluster.c b/arch/x86/kernel/genx2apic_cluster.c
index f6a2c8eb48a6..6ce497cc372d 100644
--- a/arch/x86/kernel/genx2apic_cluster.c
+++ b/arch/x86/kernel/genx2apic_cluster.c
@@ -22,19 +22,18 @@ static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 
 /* Start with all IRQs pointing to boot CPU.  IRQ balancing will shift them. */
 
-static cpumask_t x2apic_target_cpus(void)
+static const struct cpumask *x2apic_target_cpus(void)
 {
-        return cpumask_of_cpu(0);
+        return cpumask_of(0);
 }
 
 /*
  * for now each logical cpu is in its own vector allocation domain.
  */
-static cpumask_t x2apic_vector_allocation_domain(int cpu)
+static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
 {
-        cpumask_t domain = CPU_MASK_NONE;
-        cpu_set(cpu, domain);
-        return domain;
+        cpumask_clear(retmask);
+        cpumask_set_cpu(cpu, retmask);
 }
 
 static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
@@ -56,32 +55,53 @@ static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
  * at once. We have 16 cpu's in a cluster. This will minimize IPI register
  * writes.
  */
-static void x2apic_send_IPI_mask(cpumask_t mask, int vector)
+static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
 {
         unsigned long flags;
         unsigned long query_cpu;
 
         local_irq_save(flags);
-        for_each_cpu_mask(query_cpu, mask) {
-                __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_logical_apicid, query_cpu),
-                                       vector, APIC_DEST_LOGICAL);
-        }
+        for_each_cpu(query_cpu, mask)
+                __x2apic_send_IPI_dest(
+                        per_cpu(x86_cpu_to_logical_apicid, query_cpu),
+                        vector, APIC_DEST_LOGICAL);
         local_irq_restore(flags);
 }
 
-static void x2apic_send_IPI_allbutself(int vector)
+static void x2apic_send_IPI_mask_allbutself(const struct cpumask *mask,
+                                            int vector)
 {
-        cpumask_t mask = cpu_online_map;
+        unsigned long flags;
+        unsigned long query_cpu;
+        unsigned long this_cpu = smp_processor_id();
 
-        cpu_clear(smp_processor_id(), mask);
+        local_irq_save(flags);
+        for_each_cpu(query_cpu, mask)
+                if (query_cpu != this_cpu)
+                        __x2apic_send_IPI_dest(
+                                per_cpu(x86_cpu_to_logical_apicid, query_cpu),
+                                vector, APIC_DEST_LOGICAL);
+        local_irq_restore(flags);
+}
+
+static void x2apic_send_IPI_allbutself(int vector)
+{
+        unsigned long flags;
+        unsigned long query_cpu;
+        unsigned long this_cpu = smp_processor_id();
 
-        if (!cpus_empty(mask))
-                x2apic_send_IPI_mask(mask, vector);
+        local_irq_save(flags);
+        for_each_online_cpu(query_cpu)
+                if (query_cpu != this_cpu)
+                        __x2apic_send_IPI_dest(
+                                per_cpu(x86_cpu_to_logical_apicid, query_cpu),
+                                vector, APIC_DEST_LOGICAL);
+        local_irq_restore(flags);
 }
 
 static void x2apic_send_IPI_all(int vector)
 {
-        x2apic_send_IPI_mask(cpu_online_map, vector);
+        x2apic_send_IPI_mask(cpu_online_mask, vector);
 }
 
 static int x2apic_apic_id_registered(void)
@@ -89,21 +109,38 @@ static int x2apic_apic_id_registered(void)
         return 1;
 }
 
-static unsigned int x2apic_cpu_mask_to_apicid(cpumask_t cpumask)
+static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
 {
         int cpu;
 
         /*
-         * We're using fixed IRQ delivery, can only return one phys APIC ID.
+         * We're using fixed IRQ delivery, can only return one logical APIC ID.
          * May as well be the first.
          */
-        cpu = first_cpu(cpumask);
-        if ((unsigned)cpu < NR_CPUS)
+        cpu = cpumask_first(cpumask);
+        if ((unsigned)cpu < nr_cpu_ids)
                 return per_cpu(x86_cpu_to_logical_apicid, cpu);
         else
                 return BAD_APICID;
 }
 
+static unsigned int x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+                                                  const struct cpumask *andmask)
+{
+        int cpu;
+
+        /*
+         * We're using fixed IRQ delivery, can only return one logical APIC ID.
+         * May as well be the first.
+         */
+        for_each_cpu_and(cpu, cpumask, andmask)
+                if (cpumask_test_cpu(cpu, cpu_online_mask))
+                        break;
+        if (cpu < nr_cpu_ids)
+                return per_cpu(x86_cpu_to_logical_apicid, cpu);
+        return BAD_APICID;
+}
+
 static unsigned int get_apic_id(unsigned long x)
 {
         unsigned int id;
@@ -150,8 +187,10 @@ struct genapic apic_x2apic_cluster = {
         .send_IPI_all = x2apic_send_IPI_all,
         .send_IPI_allbutself = x2apic_send_IPI_allbutself,
         .send_IPI_mask = x2apic_send_IPI_mask,
+        .send_IPI_mask_allbutself = x2apic_send_IPI_mask_allbutself,
         .send_IPI_self = x2apic_send_IPI_self,
         .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
+        .cpu_mask_to_apicid_and = x2apic_cpu_mask_to_apicid_and,
         .phys_pkg_id = phys_pkg_id,
         .get_apic_id = get_apic_id,
         .set_apic_id = set_apic_id,
diff --git a/arch/x86/kernel/genx2apic_phys.c b/arch/x86/kernel/genx2apic_phys.c
index d042211768b7..21bcc0e098ba 100644
--- a/arch/x86/kernel/genx2apic_phys.c
+++ b/arch/x86/kernel/genx2apic_phys.c
@@ -29,16 +29,15 @@ static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 
 /* Start with all IRQs pointing to boot CPU.  IRQ balancing will shift them. */
 
-static cpumask_t x2apic_target_cpus(void)
+static const struct cpumask *x2apic_target_cpus(void)
 {
-        return cpumask_of_cpu(0);
+        return cpumask_of(0);
 }
 
-static cpumask_t x2apic_vector_allocation_domain(int cpu)
+static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
 {
-        cpumask_t domain = CPU_MASK_NONE;
-        cpu_set(cpu, domain);
-        return domain;
+        cpumask_clear(retmask);
+        cpumask_set_cpu(cpu, retmask);
 }
 
 static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
@@ -54,32 +53,54 @@ static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
         x2apic_icr_write(cfg, apicid);
 }
 
-static void x2apic_send_IPI_mask(cpumask_t mask, int vector)
+static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
 {
         unsigned long flags;
         unsigned long query_cpu;
 
         local_irq_save(flags);
-        for_each_cpu_mask(query_cpu, mask) {
+        for_each_cpu(query_cpu, mask) {
                 __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
                                        vector, APIC_DEST_PHYSICAL);
         }
         local_irq_restore(flags);
 }
 
-static void x2apic_send_IPI_allbutself(int vector)
+static void x2apic_send_IPI_mask_allbutself(const struct cpumask *mask,
+                                            int vector)
 {
-        cpumask_t mask = cpu_online_map;
+        unsigned long flags;
+        unsigned long query_cpu;
+        unsigned long this_cpu = smp_processor_id();
+
+        local_irq_save(flags);
+        for_each_cpu(query_cpu, mask) {
+                if (query_cpu != this_cpu)
+                        __x2apic_send_IPI_dest(
+                                per_cpu(x86_cpu_to_apicid, query_cpu),
+                                vector, APIC_DEST_PHYSICAL);
+        }
+        local_irq_restore(flags);
+}
 
-        cpu_clear(smp_processor_id(), mask);
+static void x2apic_send_IPI_allbutself(int vector)
+{
+        unsigned long flags;
+        unsigned long query_cpu;
+        unsigned long this_cpu = smp_processor_id();
 
-        if (!cpus_empty(mask))
-                x2apic_send_IPI_mask(mask, vector);
+        local_irq_save(flags);
+        for_each_online_cpu(query_cpu)
+                if (query_cpu != this_cpu)
+                        __x2apic_send_IPI_dest(
+                                per_cpu(x86_cpu_to_apicid, query_cpu),
+                                vector, APIC_DEST_PHYSICAL);
+        local_irq_restore(flags);
 }
 
 static void x2apic_send_IPI_all(int vector)
 {
-        x2apic_send_IPI_mask(cpu_online_map, vector);
+        x2apic_send_IPI_mask(cpu_online_mask, vector);
 }
 
 static int x2apic_apic_id_registered(void)
@@ -87,7 +108,7 @@ static int x2apic_apic_id_registered(void)
         return 1;
 }
 
-static unsigned int x2apic_cpu_mask_to_apicid(cpumask_t cpumask)
+static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
 {
         int cpu;
 
@@ -95,13 +116,30 @@ static unsigned int x2apic_cpu_mask_to_apicid(cpumask_t cpumask)
          * We're using fixed IRQ delivery, can only return one phys APIC ID.
          * May as well be the first.
          */
-        cpu = first_cpu(cpumask);
-        if ((unsigned)cpu < NR_CPUS)
+        cpu = cpumask_first(cpumask);
+        if ((unsigned)cpu < nr_cpu_ids)
                 return per_cpu(x86_cpu_to_apicid, cpu);
         else
                 return BAD_APICID;
 }
 
+static unsigned int x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+                                                  const struct cpumask *andmask)
+{
+        int cpu;
+
+        /*
+         * We're using fixed IRQ delivery, can only return one phys APIC ID.
+         * May as well be the first.
+         */
+        for_each_cpu_and(cpu, cpumask, andmask)
+                if (cpumask_test_cpu(cpu, cpu_online_mask))
+                        break;
+        if (cpu < nr_cpu_ids)
+                return per_cpu(x86_cpu_to_apicid, cpu);
+        return BAD_APICID;
+}
+
 static unsigned int get_apic_id(unsigned long x)
 {
         unsigned int id;
@@ -123,12 +161,12 @@ static unsigned int phys_pkg_id(int index_msb)
         return current_cpu_data.initial_apicid >> index_msb;
 }
 
-void x2apic_send_IPI_self(int vector)
+static void x2apic_send_IPI_self(int vector)
 {
         apic_write(APIC_SELF_IPI, vector);
 }
 
-void init_x2apic_ldr(void)
+static void init_x2apic_ldr(void)
 {
         return;
 }
@@ -145,8 +183,10 @@ struct genapic apic_x2apic_phys = {
         .send_IPI_all = x2apic_send_IPI_all,
         .send_IPI_allbutself = x2apic_send_IPI_allbutself,
         .send_IPI_mask = x2apic_send_IPI_mask,
+        .send_IPI_mask_allbutself = x2apic_send_IPI_mask_allbutself,
         .send_IPI_self = x2apic_send_IPI_self,
         .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
+        .cpu_mask_to_apicid_and = x2apic_cpu_mask_to_apicid_and,
         .phys_pkg_id = phys_pkg_id,
         .get_apic_id = get_apic_id,
         .set_apic_id = set_apic_id,
diff --git a/arch/x86/kernel/genx2apic_uv_x.c b/arch/x86/kernel/genx2apic_uv_x.c
index 2c7dbdb98278..b193e082f6ce 100644
--- a/arch/x86/kernel/genx2apic_uv_x.c
+++ b/arch/x86/kernel/genx2apic_uv_x.c
@@ -10,6 +10,7 @@
 
 #include <linux/kernel.h>
 #include <linux/threads.h>
+#include <linux/cpu.h>
 #include <linux/cpumask.h>
 #include <linux/string.h>
 #include <linux/ctype.h>
@@ -17,6 +18,9 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 #include <linux/hardirq.h>
+#include <linux/timer.h>
+#include <linux/proc_fs.h>
+#include <asm/current.h>
 #include <asm/smp.h>
 #include <asm/ipi.h>
 #include <asm/genapic.h>
@@ -75,16 +79,15 @@ EXPORT_SYMBOL(sn_rtc_cycles_per_second);
 
 /* Start with all IRQs pointing to boot CPU.  IRQ balancing will shift them. */
 
-static cpumask_t uv_target_cpus(void)
+static const struct cpumask *uv_target_cpus(void)
 {
-        return cpumask_of_cpu(0);
+        return cpumask_of(0);
 }
 
-static cpumask_t uv_vector_allocation_domain(int cpu)
+static void uv_vector_allocation_domain(int cpu, struct cpumask *retmask)
 {
-        cpumask_t domain = CPU_MASK_NONE;
-        cpu_set(cpu, domain);
-        return domain;
+        cpumask_clear(retmask);
+        cpumask_set_cpu(cpu, retmask);
 }
 
 int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip)
@@ -123,28 +126,37 @@ static void uv_send_IPI_one(int cpu, int vector)
         uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
 }
 
-static void uv_send_IPI_mask(cpumask_t mask, int vector)
+static void uv_send_IPI_mask(const struct cpumask *mask, int vector)
 {
         unsigned int cpu;
 
-        for_each_possible_cpu(cpu)
-                if (cpu_isset(cpu, mask))
+        for_each_cpu(cpu, mask)
+                uv_send_IPI_one(cpu, vector);
+}
+
+static void uv_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
+{
+        unsigned int cpu;
+        unsigned int this_cpu = smp_processor_id();
+
+        for_each_cpu(cpu, mask)
+                if (cpu != this_cpu)
                         uv_send_IPI_one(cpu, vector);
 }
 
 static void uv_send_IPI_allbutself(int vector)
 {
-        cpumask_t mask = cpu_online_map;
-
-        cpu_clear(smp_processor_id(), mask);
+        unsigned int cpu;
+        unsigned int this_cpu = smp_processor_id();
 
-        if (!cpus_empty(mask))
-                uv_send_IPI_mask(mask, vector);
+        for_each_online_cpu(cpu)
+                if (cpu != this_cpu)
+                        uv_send_IPI_one(cpu, vector);
 }
 
 static void uv_send_IPI_all(int vector)
 {
-        uv_send_IPI_mask(cpu_online_map, vector);
+        uv_send_IPI_mask(cpu_online_mask, vector);
 }
 
 static int uv_apic_id_registered(void)
@@ -156,7 +168,7 @@ static void uv_init_apic_ldr(void)
 {
 }
 
-static unsigned int uv_cpu_mask_to_apicid(cpumask_t cpumask)
+static unsigned int uv_cpu_mask_to_apicid(const struct cpumask *cpumask)
 {
         int cpu;
 
@@ -164,13 +176,30 @@ static unsigned int uv_cpu_mask_to_apicid(cpumask_t cpumask)
          * We're using fixed IRQ delivery, can only return one phys APIC ID.
          * May as well be the first.
          */
-        cpu = first_cpu(cpumask);
+        cpu = cpumask_first(cpumask);
         if ((unsigned)cpu < nr_cpu_ids)
                 return per_cpu(x86_cpu_to_apicid, cpu);
         else
                 return BAD_APICID;
 }
 
+static unsigned int uv_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+                                              const struct cpumask *andmask)
+{
+        int cpu;
+
+        /*
+         * We're using fixed IRQ delivery, can only return one phys APIC ID.
+         * May as well be the first.
+         */
+        for_each_cpu_and(cpu, cpumask, andmask)
+                if (cpumask_test_cpu(cpu, cpu_online_mask))
+                        break;
+        if (cpu < nr_cpu_ids)
+                return per_cpu(x86_cpu_to_apicid, cpu);
+        return BAD_APICID;
+}
+
 static unsigned int get_apic_id(unsigned long x)
 {
         unsigned int id;
@@ -218,8 +247,10 @@ struct genapic apic_x2apic_uv_x = {
         .send_IPI_all = uv_send_IPI_all,
         .send_IPI_allbutself = uv_send_IPI_allbutself,
         .send_IPI_mask = uv_send_IPI_mask,
+        .send_IPI_mask_allbutself = uv_send_IPI_mask_allbutself,
         .send_IPI_self = uv_send_IPI_self,
         .cpu_mask_to_apicid = uv_cpu_mask_to_apicid,
+        .cpu_mask_to_apicid_and = uv_cpu_mask_to_apicid_and,
         .phys_pkg_id = phys_pkg_id,
         .get_apic_id = get_apic_id,
         .set_apic_id = set_apic_id,
@@ -356,6 +387,103 @@ static __init void uv_rtc_init(void)
 }
 
 /*
+ * percpu heartbeat timer
+ */
+static void uv_heartbeat(unsigned long ignored)
+{
+        struct timer_list *timer = &uv_hub_info->scir.timer;
+        unsigned char bits = uv_hub_info->scir.state;
+
+        /* flip heartbeat bit */
+        bits ^= SCIR_CPU_HEARTBEAT;
+
+        /* is this cpu idle? */
+        if (idle_cpu(raw_smp_processor_id()))
+                bits &= ~SCIR_CPU_ACTIVITY;
+        else
+                bits |= SCIR_CPU_ACTIVITY;
+
+        /* update system controller interface reg */
+        uv_set_scir_bits(bits);
+
+        /* enable next timer period */
+        mod_timer(timer, jiffies + SCIR_CPU_HB_INTERVAL);
+}
+
+static void __cpuinit uv_heartbeat_enable(int cpu)
+{
+        if (!uv_cpu_hub_info(cpu)->scir.enabled) {
+                struct timer_list *timer = &uv_cpu_hub_info(cpu)->scir.timer;
+
+                uv_set_cpu_scir_bits(cpu, SCIR_CPU_HEARTBEAT|SCIR_CPU_ACTIVITY);
+                setup_timer(timer, uv_heartbeat, cpu);
+                timer->expires = jiffies + SCIR_CPU_HB_INTERVAL;
+                add_timer_on(timer, cpu);
+                uv_cpu_hub_info(cpu)->scir.enabled = 1;
+        }
+
+        /* check boot cpu */
+        if (!uv_cpu_hub_info(0)->scir.enabled)
+                uv_heartbeat_enable(0);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void __cpuinit uv_heartbeat_disable(int cpu)
+{
+        if (uv_cpu_hub_info(cpu)->scir.enabled) {
+                uv_cpu_hub_info(cpu)->scir.enabled = 0;
+                del_timer(&uv_cpu_hub_info(cpu)->scir.timer);
+        }
+        uv_set_cpu_scir_bits(cpu, 0xff);
+}
+
+/*
+ * cpu hotplug notifier
+ */
+static __cpuinit int uv_scir_cpu_notify(struct notifier_block *self,
+                                       unsigned long action, void *hcpu)
+{
+        long cpu = (long)hcpu;
+
+        switch (action) {
+        case CPU_ONLINE:
+                uv_heartbeat_enable(cpu);
+                break;
+        case CPU_DOWN_PREPARE:
+                uv_heartbeat_disable(cpu);
+                break;
+        default:
+                break;
+        }
+        return NOTIFY_OK;
+}
+
+static __init void uv_scir_register_cpu_notifier(void)
+{
+        hotcpu_notifier(uv_scir_cpu_notify, 0);
+}
+
+#else /* !CONFIG_HOTPLUG_CPU */
+
+static __init void uv_scir_register_cpu_notifier(void)
+{
+}
+
+static __init int uv_init_heartbeat(void)
+{
+        int cpu;
+
+        if (is_uv_system())
+                for_each_online_cpu(cpu)
+                        uv_heartbeat_enable(cpu);
+        return 0;
+}
+
+late_initcall(uv_init_heartbeat);
+
+#endif /* !CONFIG_HOTPLUG_CPU */
+
+/*
  * Called on each cpu to initialize the per_cpu UV data area.
  *      ZZZ hotplug not supported yet
  */
@@ -428,7 +556,7 @@ void __init uv_system_init(void)
 
         uv_bios_init();
         uv_bios_get_sn_info(0, &uv_type, &sn_partition_id,
-                            &uv_coherency_id, &uv_region_size);
+                            &sn_coherency_id, &sn_region_size);
         uv_rtc_init();
 
         for_each_present_cpu(cpu) {
@@ -439,8 +567,7 @@ void __init uv_system_init(void)
                 uv_blade_info[blade].nr_possible_cpus++;
 
                 uv_cpu_hub_info(cpu)->lowmem_remap_base = lowmem_redir_base;
-                uv_cpu_hub_info(cpu)->lowmem_remap_top =
-                                        lowmem_redir_base + lowmem_redir_size;
+                uv_cpu_hub_info(cpu)->lowmem_remap_top = lowmem_redir_size;
                 uv_cpu_hub_info(cpu)->m_val = m_val;
                 uv_cpu_hub_info(cpu)->n_val = m_val;
                 uv_cpu_hub_info(cpu)->numa_blade_id = blade;
@@ -450,7 +577,8 @@ void __init uv_system_init(void)
                 uv_cpu_hub_info(cpu)->gpa_mask = (1 << (m_val + n_val)) - 1;
                 uv_cpu_hub_info(cpu)->gnode_upper = gnode_upper;
                 uv_cpu_hub_info(cpu)->global_mmr_base = mmr_base;
-                uv_cpu_hub_info(cpu)->coherency_domain_number = uv_coherency_id;
+                uv_cpu_hub_info(cpu)->coherency_domain_number = sn_coherency_id;
+                uv_cpu_hub_info(cpu)->scir.offset = SCIR_LOCAL_MMR_BASE + lcpu;
                 uv_node_to_blade[nid] = blade;
                 uv_cpu_to_blade[cpu] = blade;
                 max_pnode = max(pnode, max_pnode);
@@ -467,4 +595,6 @@ void __init uv_system_init(void)
         map_mmioh_high(max_pnode);
 
         uv_cpu_init();
+        uv_scir_register_cpu_notifier();
+        proc_mkdir("sgi_uv", NULL);
 }
diff --git a/arch/x86/kernel/head.c b/arch/x86/kernel/head.c
index 1dcb0f13897e..3e66bd364a9d 100644
--- a/arch/x86/kernel/head.c
+++ b/arch/x86/kernel/head.c
@@ -35,7 +35,6 @@ void __init reserve_ebda_region(void)
 
         /* start of EBDA area */
         ebda_addr = get_bios_ebda();
-        printk(KERN_INFO "BIOS EBDA/lowmem at: %08x/%08x\n", ebda_addr, lowmem);
 
         /* Fixup: bios puts an EBDA in the top 64K segment */
         /* of conventional memory, but does not adjust lowmem. */
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index fa1d25dd83e3..ac108d1fe182 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -12,9 +12,12 @@
 #include <asm/sections.h>
 #include <asm/e820.h>
 #include <asm/bios_ebda.h>
+#include <asm/trampoline.h>
 
 void __init i386_start_kernel(void)
 {
+        reserve_trampoline_memory();
+
         reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
 
 #ifdef CONFIG_BLK_DEV_INITRD
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index d16084f90649..b9a4d8c4b935 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -24,9 +24,10 @@
 #include <asm/kdebug.h>
 #include <asm/e820.h>
 #include <asm/bios_ebda.h>
+#include <asm/trampoline.h>
 
 /* boot cpu pda */
-static struct x8664_pda _boot_cpu_pda __read_mostly;
+static struct x8664_pda _boot_cpu_pda;
 
 #ifdef CONFIG_SMP
 /*
@@ -120,6 +121,8 @@ void __init x86_64_start_reservations(char *real_mode_data)
 {
         copy_bootdata(__va(real_mode_data));
 
+        reserve_trampoline_memory();
+
         reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
 
 #ifdef CONFIG_BLK_DEV_INITRD
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index 067d8de913f6..cd759ad90690 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -33,7 +33,9 @@
  * HPET address is set in acpi/boot.c, when an ACPI entry exists
  */
 unsigned long                           hpet_address;
-unsigned long                           hpet_num_timers;
+#ifdef CONFIG_PCI_MSI
+static unsigned long                    hpet_num_timers;
+#endif
 static void __iomem                     *hpet_virt_address;
 
 struct hpet_dev {
@@ -246,7 +248,7 @@ static void hpet_legacy_clockevent_register(void)
          * Start hpet with the boot cpu mask and make it
          * global after the IO_APIC has been initialized.
          */
-        hpet_clockevent.cpumask = cpumask_of_cpu(smp_processor_id());
+        hpet_clockevent.cpumask = cpumask_of(smp_processor_id());
         clockevents_register_device(&hpet_clockevent);
         global_clock_event = &hpet_clockevent;
         printk(KERN_DEBUG "hpet clockevent registered\n");
@@ -301,7 +303,7 @@ static void hpet_set_mode(enum clock_event_mode mode,
                         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
                         hpet_setup_msi_irq(hdev->irq);
                         disable_irq(hdev->irq);
-                        irq_set_affinity(hdev->irq, cpumask_of_cpu(hdev->cpu));
+                        irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu));
                         enable_irq(hdev->irq);
                 }
                 break;
@@ -449,7 +451,7 @@ static int hpet_setup_irq(struct hpet_dev *dev)
                 return -1;
 
         disable_irq(dev->irq);
-        irq_set_affinity(dev->irq, cpumask_of_cpu(dev->cpu));
+        irq_set_affinity(dev->irq, cpumask_of(dev->cpu));
         enable_irq(dev->irq);
 
         printk(KERN_DEBUG "hpet: %s irq %d for MSI\n",
@@ -500,7 +502,7 @@ static void init_one_hpet_msi_clockevent(struct hpet_dev *hdev, int cpu)
         /* 5 usec minimum reprogramming delta. */
         evt->min_delta_ns = 5000;
 
-        evt->cpumask = cpumask_of_cpu(hdev->cpu);
+        evt->cpumask = cpumask_of(hdev->cpu);
         clockevents_register_device(evt);
 }
 
@@ -811,7 +813,7 @@ int __init hpet_enable(void)
 
 out_nohpet:
         hpet_clear_mapping();
-        boot_hpet_disable = 1;
+        hpet_address = 0;
         return 0;
 }
 
@@ -834,10 +836,11 @@ static __init int hpet_late_init(void)
 
                 hpet_address = force_hpet_address;
                 hpet_enable();
-                if (!hpet_virt_address)
-                        return -ENODEV;
         }
 
+        if (!hpet_virt_address)
+                return -ENODEV;
+
         hpet_reserve_platform_timers(hpet_readl(HPET_ID));
 
         for_each_online_cpu(cpu) {
diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c
index c1b5e3ece1f2..10f92fb532f3 100644
--- a/arch/x86/kernel/i8253.c
+++ b/arch/x86/kernel/i8253.c
@@ -114,7 +114,7 @@ void __init setup_pit_timer(void)
          * Start pit with the boot cpu mask and make it global after the
          * IO_APIC has been initialized.
          */
-        pit_clockevent.cpumask = cpumask_of_cpu(smp_processor_id());
+        pit_clockevent.cpumask = cpumask_of(smp_processor_id());
         pit_clockevent.mult = div_sc(CLOCK_TICK_RATE, NSEC_PER_SEC,
                                      pit_clockevent.shift);
         pit_clockevent.max_delta_ns =
diff --git a/arch/x86/kernel/init_task.c b/arch/x86/kernel/init_task.c
index a4f93b4120c1..df3bf269beab 100644
--- a/arch/x86/kernel/init_task.c
+++ b/arch/x86/kernel/init_task.c
@@ -10,11 +10,9 @@
 #include <asm/pgtable.h>
 #include <asm/desc.h>
 
-static struct fs_struct init_fs = INIT_FS;
 static struct signal_struct init_signals = INIT_SIGNALS(init_signals);
 static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand);
 struct mm_struct init_mm = INIT_MM(init_mm);
-EXPORT_UNUSED_SYMBOL(init_mm); /* will be removed in 2.6.26 */
 
 /*
  * Initial thread structure.
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c
index 9043251210fb..3639442aa7a4 100644
--- a/arch/x86/kernel/io_apic.c
+++ b/arch/x86/kernel/io_apic.c
@@ -108,93 +108,276 @@ static int __init parse_noapic(char *str)
 early_param("noapic", parse_noapic);
 
 struct irq_pin_list;
+
+/*
+ * This is performance-critical, we want to do it O(1)
+ *
+ * the indexing order of this array favors 1:1 mappings
+ * between pins and IRQs.
+ */
+
+struct irq_pin_list {
+        int apic, pin;
+        struct irq_pin_list *next;
+};
+
+static struct irq_pin_list *get_one_free_irq_2_pin(int cpu)
+{
+        struct irq_pin_list *pin;
+        int node;
+
+        node = cpu_to_node(cpu);
+
+        pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
+        printk(KERN_DEBUG "  alloc irq_2_pin on cpu %d node %d\n", cpu, node);
+
+        return pin;
+}
+
 struct irq_cfg {
-        unsigned int irq;
         struct irq_pin_list *irq_2_pin;
-        cpumask_t domain;
-        cpumask_t old_domain;
+        cpumask_var_t domain;
+        cpumask_var_t old_domain;
         unsigned move_cleanup_count;
         u8 vector;
         u8 move_in_progress : 1;
+#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
+        u8 move_desc_pending : 1;
+#endif
 };
 
 /* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
+#ifdef CONFIG_SPARSE_IRQ
+static struct irq_cfg irq_cfgx[] = {
+#else
 static struct irq_cfg irq_cfgx[NR_IRQS] = {
-        [0]  = { .irq =  0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR,  },
-        [1]  = { .irq =  1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR,  },
-        [2]  = { .irq =  2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR,  },
-        [3]  = { .irq =  3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR,  },
-        [4]  = { .irq =  4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR,  },
-        [5]  = { .irq =  5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR,  },
-        [6]  = { .irq =  6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR,  },
-        [7]  = { .irq =  7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR,  },
-        [8]  = { .irq =  8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR,  },
-        [9]  = { .irq =  9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR,  },
-        [10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
-        [11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
-        [12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
-        [13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
-        [14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
-        [15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
+#endif
+        [0]  = { .vector = IRQ0_VECTOR,  },
+        [1]  = { .vector = IRQ1_VECTOR,  },
+        [2]  = { .vector = IRQ2_VECTOR,  },
+        [3]  = { .vector = IRQ3_VECTOR,  },
+        [4]  = { .vector = IRQ4_VECTOR,  },
+        [5]  = { .vector = IRQ5_VECTOR,  },
+        [6]  = { .vector = IRQ6_VECTOR,  },
+        [7]  = { .vector = IRQ7_VECTOR,  },
+        [8]  = { .vector = IRQ8_VECTOR,  },
+        [9]  = { .vector = IRQ9_VECTOR,  },
+        [10] = { .vector = IRQ10_VECTOR, },
+        [11] = { .vector = IRQ11_VECTOR, },
+        [12] = { .vector = IRQ12_VECTOR, },
+        [13] = { .vector = IRQ13_VECTOR, },
+        [14] = { .vector = IRQ14_VECTOR, },
+        [15] = { .vector = IRQ15_VECTOR, },
 };
 
-#define for_each_irq_cfg(irq, cfg)              \
-        for (irq = 0, cfg = irq_cfgx; irq < nr_irqs; irq++, cfg++)
+int __init arch_early_irq_init(void)
+{
+        struct irq_cfg *cfg;
+        struct irq_desc *desc;
+        int count;
+        int i;
+
+        cfg = irq_cfgx;
+        count = ARRAY_SIZE(irq_cfgx);
 
+        for (i = 0; i < count; i++) {
+                desc = irq_to_desc(i);
+                desc->chip_data = &cfg[i];
+                alloc_bootmem_cpumask_var(&cfg[i].domain);
+                alloc_bootmem_cpumask_var(&cfg[i].old_domain);
+                if (i < NR_IRQS_LEGACY)
+                        cpumask_setall(cfg[i].domain);
+        }
+
+        return 0;
+}
+
+#ifdef CONFIG_SPARSE_IRQ
 static struct irq_cfg *irq_cfg(unsigned int irq)
 {
-        return irq < nr_irqs ? irq_cfgx + irq : NULL;
+        struct irq_cfg *cfg = NULL;
+        struct irq_desc *desc;
+
+        desc = irq_to_desc(irq);
+        if (desc)
+                cfg = desc->chip_data;
+
+        return cfg;
 }
 
-static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
+static struct irq_cfg *get_one_free_irq_cfg(int cpu)
 {
-        return irq_cfg(irq);
+        struct irq_cfg *cfg;
+        int node;
+
+        node = cpu_to_node(cpu);
+
+        cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
+        if (cfg) {
+                if (!alloc_cpumask_var_node(&cfg->domain, GFP_ATOMIC, node)) {
+                        kfree(cfg);
+                        cfg = NULL;
+                } else if (!alloc_cpumask_var_node(&cfg->old_domain,
+                                                          GFP_ATOMIC, node)) {
+                        free_cpumask_var(cfg->domain);
+                        kfree(cfg);
+                        cfg = NULL;
+                } else {
+                        cpumask_clear(cfg->domain);
+                        cpumask_clear(cfg->old_domain);
+                }
+        }
+        printk(KERN_DEBUG "  alloc irq_cfg on cpu %d node %d\n", cpu, node);
+
+        return cfg;
 }
 
-/*
- * Rough estimation of how many shared IRQs there are, can be changed
- * anytime.
- */
-#define MAX_PLUS_SHARED_IRQS NR_IRQS
-#define PIN_MAP_SIZE (MAX_PLUS_SHARED_IRQS + NR_IRQS)
+int arch_init_chip_data(struct irq_desc *desc, int cpu)
+{
+        struct irq_cfg *cfg;
 
-/*
- * This is performance-critical, we want to do it O(1)
- *
- * the indexing order of this array favors 1:1 mappings
- * between pins and IRQs.
- */
+        cfg = desc->chip_data;
+        if (!cfg) {
+                desc->chip_data = get_one_free_irq_cfg(cpu);
+                if (!desc->chip_data) {
+                        printk(KERN_ERR "can not alloc irq_cfg\n");
+                        BUG_ON(1);
+                }
+        }
 
-struct irq_pin_list {
-        int apic, pin;
-        struct irq_pin_list *next;
-};
+        return 0;
+}
 
-static struct irq_pin_list irq_2_pin_head[PIN_MAP_SIZE];
-static struct irq_pin_list *irq_2_pin_ptr;
+#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
 
-static void __init irq_2_pin_init(void)
+static void
+init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
 {
-        struct irq_pin_list *pin = irq_2_pin_head;
-        int i;
+        struct irq_pin_list *old_entry, *head, *tail, *entry;
+
+        cfg->irq_2_pin = NULL;
+        old_entry = old_cfg->irq_2_pin;
+        if (!old_entry)
+                return;
+
+        entry = get_one_free_irq_2_pin(cpu);
+        if (!entry)
+                return;
+
+        entry->apic     = old_entry->apic;
+        entry->pin      = old_entry->pin;
+        head            = entry;
+        tail            = entry;
+        old_entry       = old_entry->next;
+        while (old_entry) {
+                entry = get_one_free_irq_2_pin(cpu);
+                if (!entry) {
+                        entry = head;
+                        while (entry) {
+                                head = entry->next;
+                                kfree(entry);
+                                entry = head;
+                        }
+                        /* still use the old one */
+                        return;
+                }
+                entry->apic     = old_entry->apic;
+                entry->pin      = old_entry->pin;
+                tail->next      = entry;
+                tail            = entry;
+                old_entry       = old_entry->next;
+        }
 
-        for (i = 1; i < PIN_MAP_SIZE; i++)
-                pin[i-1].next = &pin[i];
+        tail->next = NULL;
+        cfg->irq_2_pin = head;
+}
+
+static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
+{
+        struct irq_pin_list *entry, *next;
+
+        if (old_cfg->irq_2_pin == cfg->irq_2_pin)
+                return;
 
-        irq_2_pin_ptr = &pin[0];
+        entry = old_cfg->irq_2_pin;
+
+        while (entry) {
+                next = entry->next;
+                kfree(entry);
+                entry = next;
+        }
+        old_cfg->irq_2_pin = NULL;
 }
 
-static struct irq_pin_list *get_one_free_irq_2_pin(void)
+void arch_init_copy_chip_data(struct irq_desc *old_desc,
+                                 struct irq_desc *desc, int cpu)
 {
-        struct irq_pin_list *pin = irq_2_pin_ptr;
+        struct irq_cfg *cfg;
+        struct irq_cfg *old_cfg;
 
-        if (!pin)
-                panic("can not get more irq_2_pin\n");
+        cfg = get_one_free_irq_cfg(cpu);
 
-        irq_2_pin_ptr = pin->next;
-        pin->next = NULL;
-        return pin;
+        if (!cfg)
+                return;
+
+        desc->chip_data = cfg;
+
+        old_cfg = old_desc->chip_data;
+
+        memcpy(cfg, old_cfg, sizeof(struct irq_cfg));
+
+        init_copy_irq_2_pin(old_cfg, cfg, cpu);
+}
+
+static void free_irq_cfg(struct irq_cfg *old_cfg)
+{
+        kfree(old_cfg);
+}
+
+void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
+{
+        struct irq_cfg *old_cfg, *cfg;
+
+        old_cfg = old_desc->chip_data;
+        cfg = desc->chip_data;
+
+        if (old_cfg == cfg)
+                return;
+
+        if (old_cfg) {
+                free_irq_2_pin(old_cfg, cfg);
+                free_irq_cfg(old_cfg);
+                old_desc->chip_data = NULL;
+        }
+}
+
+static void
+set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
+{
+        struct irq_cfg *cfg = desc->chip_data;
+
+        if (!cfg->move_in_progress) {
+                /* it means that domain is not changed */
+                if (!cpumask_intersects(&desc->affinity, mask))
+                        cfg->move_desc_pending = 1;
+        }
 }
+#endif
+
+#else
+static struct irq_cfg *irq_cfg(unsigned int irq)
+{
+        return irq < nr_irqs ? irq_cfgx + irq : NULL;
+}
+
+#endif
+
+#ifndef CONFIG_NUMA_MIGRATE_IRQ_DESC
+static inline void
+set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
+{
+}
+#endif
 
 struct io_apic {
         unsigned int index;
@@ -237,11 +420,10 @@ static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned
         writel(value, &io_apic->data);
 }
 
-static bool io_apic_level_ack_pending(unsigned int irq)
+static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
 {
         struct irq_pin_list *entry;
         unsigned long flags;
-        struct irq_cfg *cfg = irq_cfg(irq);
 
         spin_lock_irqsave(&ioapic_lock, flags);
         entry = cfg->irq_2_pin;
@@ -323,13 +505,32 @@ static void ioapic_mask_entry(int apic, int pin)
 }
 
 #ifdef CONFIG_SMP
-static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
+static void send_cleanup_vector(struct irq_cfg *cfg)
+{
+        cpumask_var_t cleanup_mask;
+
+        if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
+                unsigned int i;
+                cfg->move_cleanup_count = 0;
+                for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
+                        cfg->move_cleanup_count++;
+                for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
+                        send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
+        } else {
+                cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
+                cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
+                send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
+                free_cpumask_var(cleanup_mask);
+        }
+        cfg->move_in_progress = 0;
+}
+
+static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
 {
         int apic, pin;
-        struct irq_cfg *cfg;
         struct irq_pin_list *entry;
+        u8 vector = cfg->vector;
 
-        cfg = irq_cfg(irq);
         entry = cfg->irq_2_pin;
         for (;;) {
                 unsigned int reg;
@@ -359,36 +560,61 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
         }
 }
 
-static int assign_irq_vector(int irq, cpumask_t mask);
+static int
+assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
 
-static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
+/*
+ * Either sets desc->affinity to a valid value, and returns cpu_mask_to_apicid
+ * of that, or returns BAD_APICID and leaves desc->affinity untouched.
+ */
+static unsigned int
+set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
+{
+        struct irq_cfg *cfg;
+        unsigned int irq;
+
+        if (!cpumask_intersects(mask, cpu_online_mask))
+                return BAD_APICID;
+
+        irq = desc->irq;
+        cfg = desc->chip_data;
+        if (assign_irq_vector(irq, cfg, mask))
+                return BAD_APICID;
+
+        cpumask_and(&desc->affinity, cfg->domain, mask);
+        set_extra_move_desc(desc, mask);
+        return cpu_mask_to_apicid_and(&desc->affinity, cpu_online_mask);
+}
+
+static void
+set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
 {
         struct irq_cfg *cfg;
         unsigned long flags;
         unsigned int dest;
-        cpumask_t tmp;
-        struct irq_desc *desc;
+        unsigned int irq;
 
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
-                return;
+        irq = desc->irq;
+        cfg = desc->chip_data;
 
-        cfg = irq_cfg(irq);
-        if (assign_irq_vector(irq, mask))
-                return;
+        spin_lock_irqsave(&ioapic_lock, flags);
+        dest = set_desc_affinity(desc, mask);
+        if (dest != BAD_APICID) {
+                /* Only the high 8 bits are valid. */
+                dest = SET_APIC_LOGICAL_ID(dest);
+                __target_IO_APIC_irq(irq, dest, cfg);
+        }
+        spin_unlock_irqrestore(&ioapic_lock, flags);
+}
 
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
-        /*
-         * Only the high 8 bits are valid.
-         */
-        dest = SET_APIC_LOGICAL_ID(dest);
+static void
+set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
+{
+        struct irq_desc *desc;
 
         desc = irq_to_desc(irq);
-        spin_lock_irqsave(&ioapic_lock, flags);
-        __target_IO_APIC_irq(irq, dest, cfg->vector);
-        desc->affinity = mask;
-        spin_unlock_irqrestore(&ioapic_lock, flags);
+
+        set_ioapic_affinity_irq_desc(desc, mask);
 }
 #endif /* CONFIG_SMP */
 
@@ -397,16 +623,18 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
  * shared ISA-space IRQs, so we have to support them. We are super
  * fast in the common case, and fast for shared ISA-space IRQs.
  */
-static void add_pin_to_irq(unsigned int irq, int apic, int pin)
+static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
 {
-        struct irq_cfg *cfg;
         struct irq_pin_list *entry;
 
-        /* first time to refer irq_cfg, so with new */
-        cfg = irq_cfg_alloc(irq);
         entry = cfg->irq_2_pin;
         if (!entry) {
-                entry = get_one_free_irq_2_pin();
+                entry = get_one_free_irq_2_pin(cpu);
+                if (!entry) {
+                        printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
+                                        apic, pin);
+                        return;
+                }
                 cfg->irq_2_pin = entry;
                 entry->apic = apic;
                 entry->pin = pin;
@@ -421,7 +649,7 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin)
                 entry = entry->next;
         }
 
-        entry->next = get_one_free_irq_2_pin();
+        entry->next = get_one_free_irq_2_pin(cpu);
         entry = entry->next;
         entry->apic = apic;
         entry->pin = pin;
@@ -430,11 +658,10 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin)
 /*
  * Reroute an IRQ to a different pin.
  */
-static void __init replace_pin_at_irq(unsigned int irq,
+static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu,
                                       int oldapic, int oldpin,
                                       int newapic, int newpin)
 {
-        struct irq_cfg *cfg = irq_cfg(irq);
         struct irq_pin_list *entry = cfg->irq_2_pin;
         int replaced = 0;
 
@@ -451,18 +678,16 @@ static void __init replace_pin_at_irq(unsigned int irq,
 
         /* why? call replace before add? */
         if (!replaced)
-                add_pin_to_irq(irq, newapic, newpin);
+                add_pin_to_irq_cpu(cfg, cpu, newapic, newpin);
 }
 
-static inline void io_apic_modify_irq(unsigned int irq,
+static inline void io_apic_modify_irq(struct irq_cfg *cfg,
                                 int mask_and, int mask_or,
                                 void (*final)(struct irq_pin_list *entry))
 {
         int pin;
-        struct irq_cfg *cfg;
         struct irq_pin_list *entry;
 
-        cfg = irq_cfg(irq);
         for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) {
                 unsigned int reg;
                 pin = entry->pin;
@@ -475,13 +700,13 @@ static inline void io_apic_modify_irq(unsigned int irq,
         }
 }
 
-static void __unmask_IO_APIC_irq(unsigned int irq)
+static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
 {
-        io_apic_modify_irq(irq, ~IO_APIC_REDIR_MASKED, 0, NULL);
+        io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
 }
 
 #ifdef CONFIG_X86_64
-void io_apic_sync(struct irq_pin_list *entry)
+static void io_apic_sync(struct irq_pin_list *entry)
 {
         /*
          * Synchronize the IO-APIC and the CPU by doing
@@ -492,47 +717,64 @@ void io_apic_sync(struct irq_pin_list *entry)
         readl(&io_apic->data);
 }
 
-static void __mask_IO_APIC_irq(unsigned int irq)
+static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
 {
-        io_apic_modify_irq(irq, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
+        io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
 }
 #else /* CONFIG_X86_32 */
-static void __mask_IO_APIC_irq(unsigned int irq)
+static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
 {
-        io_apic_modify_irq(irq, ~0, IO_APIC_REDIR_MASKED, NULL);
+        io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, NULL);
 }
 
-static void __mask_and_edge_IO_APIC_irq(unsigned int irq)
+static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
 {
-        io_apic_modify_irq(irq, ~IO_APIC_REDIR_LEVEL_TRIGGER,
+        io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
                         IO_APIC_REDIR_MASKED, NULL);
 }
 
-static void __unmask_and_level_IO_APIC_irq(unsigned int irq)
+static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
 {
-        io_apic_modify_irq(irq, ~IO_APIC_REDIR_MASKED,
+        io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
                         IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
 }
 #endif /* CONFIG_X86_32 */
 
-static void mask_IO_APIC_irq (unsigned int irq)
+static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
 {
+        struct irq_cfg *cfg = desc->chip_data;
         unsigned long flags;
 
+        BUG_ON(!cfg);
+
         spin_lock_irqsave(&ioapic_lock, flags);
-        __mask_IO_APIC_irq(irq);
+        __mask_IO_APIC_irq(cfg);
         spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
-static void unmask_IO_APIC_irq (unsigned int irq)
+static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
 {
+        struct irq_cfg *cfg = desc->chip_data;
         unsigned long flags;
 
         spin_lock_irqsave(&ioapic_lock, flags);
-        __unmask_IO_APIC_irq(irq);
+        __unmask_IO_APIC_irq(cfg);
         spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
+static void mask_IO_APIC_irq(unsigned int irq)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+
+        mask_IO_APIC_irq_desc(desc);
+}
+static void unmask_IO_APIC_irq(unsigned int irq)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+
+        unmask_IO_APIC_irq_desc(desc);
+}
+
 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
 {
         struct IO_APIC_route_entry entry;
@@ -809,7 +1051,7 @@ EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
  */
 static int EISA_ELCR(unsigned int irq)
 {
-        if (irq < 16) {
+        if (irq < NR_IRQS_LEGACY) {
                 unsigned int port = 0x4d0 + (irq >> 3);
                 return (inb(port) >> (irq & 7)) & 1;
         }
@@ -1034,7 +1276,8 @@ void unlock_vector_lock(void)
         spin_unlock(&vector_lock);
 }
 
-static int __assign_irq_vector(int irq, cpumask_t mask)
+static int
+__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
 {
         /*
          * NOTE! The local APIC isn't very good at handling
@@ -1049,52 +1292,49 @@ static int __assign_irq_vector(int irq, cpumask_t mask)
          */
         static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
         unsigned int old_vector;
-        int cpu;
-        struct irq_cfg *cfg;
-
-        cfg = irq_cfg(irq);
-
-        /* Only try and allocate irqs on cpus that are present */
-        cpus_and(mask, mask, cpu_online_map);
+        int cpu, err;
+        cpumask_var_t tmp_mask;
 
         if ((cfg->move_in_progress) || cfg->move_cleanup_count)
                 return -EBUSY;
 
+        if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
+                return -ENOMEM;
+
         old_vector = cfg->vector;
         if (old_vector) {
-                cpumask_t tmp;
-                cpus_and(tmp, cfg->domain, mask);
-                if (!cpus_empty(tmp))
+                cpumask_and(tmp_mask, mask, cpu_online_mask);
+                cpumask_and(tmp_mask, cfg->domain, tmp_mask);
+                if (!cpumask_empty(tmp_mask)) {
+                        free_cpumask_var(tmp_mask);
                         return 0;
+                }
         }
 
-        for_each_cpu_mask_nr(cpu, mask) {
-                cpumask_t domain, new_mask;
+        /* Only try and allocate irqs on cpus that are present */
+        err = -ENOSPC;
+        for_each_cpu_and(cpu, mask, cpu_online_mask) {
                 int new_cpu;
                 int vector, offset;
 
-                domain = vector_allocation_domain(cpu);
-                cpus_and(new_mask, domain, cpu_online_map);
+                vector_allocation_domain(cpu, tmp_mask);
 
                 vector = current_vector;
                 offset = current_offset;
 next:
                 vector += 8;
                 if (vector >= first_system_vector) {
-                        /* If we run out of vectors on large boxen, must share them. */
+                        /* If out of vectors on large boxen, must share them. */
                         offset = (offset + 1) % 8;
                         vector = FIRST_DEVICE_VECTOR + offset;
                 }
                 if (unlikely(current_vector == vector))
                         continue;
-#ifdef CONFIG_X86_64
-                if (vector == IA32_SYSCALL_VECTOR)
-                        goto next;
-#else
-                if (vector == SYSCALL_VECTOR)
+
+                if (test_bit(vector, used_vectors))
                         goto next;
-#endif
-                for_each_cpu_mask_nr(new_cpu, new_mask)
+
+                for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
                         if (per_cpu(vector_irq, new_cpu)[vector] != -1)
                                 goto next;
                 /* Found one! */
@@ -1102,49 +1342,47 @@ next:
                 current_offset = offset;
                 if (old_vector) {
                         cfg->move_in_progress = 1;
-                        cfg->old_domain = cfg->domain;
+                        cpumask_copy(cfg->old_domain, cfg->domain);
                 }
-                for_each_cpu_mask_nr(new_cpu, new_mask)
+                for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
                         per_cpu(vector_irq, new_cpu)[vector] = irq;
                 cfg->vector = vector;
-                cfg->domain = domain;
-                return 0;
+                cpumask_copy(cfg->domain, tmp_mask);
+                err = 0;
+                break;
         }
-        return -ENOSPC;
+        free_cpumask_var(tmp_mask);
+        return err;
 }
 
-static int assign_irq_vector(int irq, cpumask_t mask)
+static int
+assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
 {
         int err;
         unsigned long flags;
 
         spin_lock_irqsave(&vector_lock, flags);
-        err = __assign_irq_vector(irq, mask);
+        err = __assign_irq_vector(irq, cfg, mask);
         spin_unlock_irqrestore(&vector_lock, flags);
         return err;
 }
 
-static void __clear_irq_vector(int irq)
+static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
 {
-        struct irq_cfg *cfg;
-        cpumask_t mask;
         int cpu, vector;
 
-        cfg = irq_cfg(irq);
         BUG_ON(!cfg->vector);
 
         vector = cfg->vector;
-        cpus_and(mask, cfg->domain, cpu_online_map);
-        for_each_cpu_mask_nr(cpu, mask)
+        for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
                 per_cpu(vector_irq, cpu)[vector] = -1;
 
         cfg->vector = 0;
-        cpus_clear(cfg->domain);
+        cpumask_clear(cfg->domain);
 
         if (likely(!cfg->move_in_progress))
                 return;
-        cpus_and(mask, cfg->old_domain, cpu_online_map);
-        for_each_cpu_mask_nr(cpu, mask) {
+        for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
                 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
                                                                 vector++) {
                         if (per_cpu(vector_irq, cpu)[vector] != irq)
@@ -1162,10 +1400,12 @@ void __setup_vector_irq(int cpu)
         /* This function must be called with vector_lock held */
         int irq, vector;
         struct irq_cfg *cfg;
+        struct irq_desc *desc;
 
         /* Mark the inuse vectors */
-        for_each_irq_cfg(irq, cfg) {
-                if (!cpu_isset(cpu, cfg->domain))
+        for_each_irq_desc(irq, desc) {
+                cfg = desc->chip_data;
+                if (!cpumask_test_cpu(cpu, cfg->domain))
                         continue;
                 vector = cfg->vector;
                 per_cpu(vector_irq, cpu)[vector] = irq;
@@ -1177,7 +1417,7 @@ void __setup_vector_irq(int cpu)
                         continue;
 
                 cfg = irq_cfg(irq);
-                if (!cpu_isset(cpu, cfg->domain))
+                if (!cpumask_test_cpu(cpu, cfg->domain))
                         per_cpu(vector_irq, cpu)[vector] = -1;
         }
 }
@@ -1215,11 +1455,8 @@ static inline int IO_APIC_irq_trigger(int irq)
 }
 #endif
 
-static void ioapic_register_intr(int irq, unsigned long trigger)
+static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
 {
-        struct irq_desc *desc;
-
-        desc = irq_to_desc(irq);
 
         if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
             trigger == IOAPIC_LEVEL)
@@ -1311,23 +1548,22 @@ static int setup_ioapic_entry(int apic, int irq,
         return 0;
 }
 
-static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
+static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_desc *desc,
                               int trigger, int polarity)
 {
         struct irq_cfg *cfg;
         struct IO_APIC_route_entry entry;
-        cpumask_t mask;
+        unsigned int dest;
 
         if (!IO_APIC_IRQ(irq))
                 return;
 
-        cfg = irq_cfg(irq);
+        cfg = desc->chip_data;
 
-        mask = TARGET_CPUS;
-        if (assign_irq_vector(irq, mask))
+        if (assign_irq_vector(irq, cfg, TARGET_CPUS))
                 return;
 
-        cpus_and(mask, cfg->domain, mask);
+        dest = cpu_mask_to_apicid_and(cfg->domain, TARGET_CPUS);
 
         apic_printk(APIC_VERBOSE,KERN_DEBUG
                     "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
@@ -1337,16 +1573,15 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
 
 
         if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
-                               cpu_mask_to_apicid(mask), trigger, polarity,
-                               cfg->vector)) {
+                               dest, trigger, polarity, cfg->vector)) {
                 printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
                        mp_ioapics[apic].mp_apicid, pin);
-                __clear_irq_vector(irq);
+                __clear_irq_vector(irq, cfg);
                 return;
         }
 
-        ioapic_register_intr(irq, trigger);
-        if (irq < 16)
+        ioapic_register_intr(irq, desc, trigger);
+        if (irq < NR_IRQS_LEGACY)
                 disable_8259A_irq(irq);
 
         ioapic_write_entry(apic, pin, entry);
@@ -1356,6 +1591,9 @@ static void __init setup_IO_APIC_irqs(void)
 {
         int apic, pin, idx, irq;
         int notcon = 0;
+        struct irq_desc *desc;
+        struct irq_cfg *cfg;
+        int cpu = boot_cpu_id;
 
         apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
 
@@ -1387,9 +1625,15 @@ static void __init setup_IO_APIC_irqs(void)
                         if (multi_timer_check(apic, irq))
                                 continue;
 #endif
-                        add_pin_to_irq(irq, apic, pin);
+                        desc = irq_to_desc_alloc_cpu(irq, cpu);
+                        if (!desc) {
+                                printk(KERN_INFO "can not get irq_desc for %d\n", irq);
+                                continue;
+                        }
+                        cfg = desc->chip_data;
+                        add_pin_to_irq_cpu(cfg, cpu, apic, pin);
 
-                        setup_IO_APIC_irq(apic, pin, irq,
+                        setup_IO_APIC_irq(apic, pin, irq, desc,
                                         irq_trigger(idx), irq_polarity(idx));
                 }
         }
@@ -1448,6 +1692,7 @@ __apicdebuginit(void) print_IO_APIC(void)
         union IO_APIC_reg_03 reg_03;
         unsigned long flags;
         struct irq_cfg *cfg;
+        struct irq_desc *desc;
         unsigned int irq;
 
         if (apic_verbosity == APIC_QUIET)
@@ -1537,8 +1782,11 @@ __apicdebuginit(void) print_IO_APIC(void)
         }
         }
         printk(KERN_DEBUG "IRQ to pin mappings:\n");
-        for_each_irq_cfg(irq, cfg) {
-                struct irq_pin_list *entry = cfg->irq_2_pin;
+        for_each_irq_desc(irq, desc) {
+                struct irq_pin_list *entry;
+
+                cfg = desc->chip_data;
+                entry = cfg->irq_2_pin;
                 if (!entry)
                         continue;
                 printk(KERN_DEBUG "IRQ%d ", irq);
@@ -2022,14 +2270,16 @@ static unsigned int startup_ioapic_irq(unsigned int irq)
 {
         int was_pending = 0;
         unsigned long flags;
+        struct irq_cfg *cfg;
 
         spin_lock_irqsave(&ioapic_lock, flags);
-        if (irq < 16) {
+        if (irq < NR_IRQS_LEGACY) {
                 disable_8259A_irq(irq);
                 if (i8259A_irq_pending(irq))
                         was_pending = 1;
         }
-        __unmask_IO_APIC_irq(irq);
+        cfg = irq_cfg(irq);
+        __unmask_IO_APIC_irq(cfg);
         spin_unlock_irqrestore(&ioapic_lock, flags);
 
         return was_pending;
@@ -2043,7 +2293,7 @@ static int ioapic_retrigger_irq(unsigned int irq)
         unsigned long flags;
 
         spin_lock_irqsave(&vector_lock, flags);
-        send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
+        send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
         spin_unlock_irqrestore(&vector_lock, flags);
 
         return 1;
@@ -2092,35 +2342,35 @@ static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
  * as simple as edge triggered migration and we can do the irq migration
  * with a simple atomic update to IO-APIC RTE.
  */
-static void migrate_ioapic_irq(int irq, cpumask_t mask)
+static void
+migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
 {
         struct irq_cfg *cfg;
-        struct irq_desc *desc;
-        cpumask_t tmp, cleanup_mask;
         struct irte irte;
         int modify_ioapic_rte;
         unsigned int dest;
         unsigned long flags;
+        unsigned int irq;
 
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
+        if (!cpumask_intersects(mask, cpu_online_mask))
                 return;
 
+        irq = desc->irq;
         if (get_irte(irq, &irte))
                 return;
 
-        if (assign_irq_vector(irq, mask))
+        cfg = desc->chip_data;
+        if (assign_irq_vector(irq, cfg, mask))
                 return;
 
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
+        set_extra_move_desc(desc, mask);
+
+        dest = cpu_mask_to_apicid_and(cfg->domain, mask);
 
-        desc = irq_to_desc(irq);
         modify_ioapic_rte = desc->status & IRQ_LEVEL;
         if (modify_ioapic_rte) {
                 spin_lock_irqsave(&ioapic_lock, flags);
-                __target_IO_APIC_irq(irq, dest, cfg->vector);
+                __target_IO_APIC_irq(irq, dest, cfg);
                 spin_unlock_irqrestore(&ioapic_lock, flags);
         }
 
@@ -2132,24 +2382,20 @@ static void migrate_ioapic_irq(int irq, cpumask_t mask)
          */
         modify_irte(irq, &irte);
 
-        if (cfg->move_in_progress) {
-                cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
-                cfg->move_cleanup_count = cpus_weight(cleanup_mask);
-                send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
-                cfg->move_in_progress = 0;
-        }
+        if (cfg->move_in_progress)
+                send_cleanup_vector(cfg);
 
-        desc->affinity = mask;
+        cpumask_copy(&desc->affinity, mask);
 }
 
-static int migrate_irq_remapped_level(int irq)
+static int migrate_irq_remapped_level_desc(struct irq_desc *desc)
 {
         int ret = -1;
-        struct irq_desc *desc = irq_to_desc(irq);
+        struct irq_cfg *cfg = desc->chip_data;
 
-        mask_IO_APIC_irq(irq);
+        mask_IO_APIC_irq_desc(desc);
 
-        if (io_apic_level_ack_pending(irq)) {
+        if (io_apic_level_ack_pending(cfg)) {
                 /*
                  * Interrupt in progress. Migrating irq now will change the
                  * vector information in the IO-APIC RTE and that will confuse
@@ -2161,14 +2407,15 @@ static int migrate_irq_remapped_level(int irq)
         }
 
         /* everthing is clear. we have right of way */
-        migrate_ioapic_irq(irq, desc->pending_mask);
+        migrate_ioapic_irq_desc(desc, &desc->pending_mask);
 
         ret = 0;
         desc->status &= ~IRQ_MOVE_PENDING;
-        cpus_clear(desc->pending_mask);
+        cpumask_clear(&desc->pending_mask);
 
 unmask:
-        unmask_IO_APIC_irq(irq);
+        unmask_IO_APIC_irq_desc(desc);
+
         return ret;
 }
 
@@ -2189,7 +2436,7 @@ static void ir_irq_migration(struct work_struct *work)
                                 continue;
                         }
 
-                        desc->chip->set_affinity(irq, desc->pending_mask);
+                        desc->chip->set_affinity(irq, &desc->pending_mask);
                         spin_unlock_irqrestore(&desc->lock, flags);
                 }
         }
@@ -2198,28 +2445,33 @@ static void ir_irq_migration(struct work_struct *work)
 /*
  * Migrates the IRQ destination in the process context.
  */
-static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
+static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
+                                            const struct cpumask *mask)
 {
-        struct irq_desc *desc = irq_to_desc(irq);
-
         if (desc->status & IRQ_LEVEL) {
                 desc->status |= IRQ_MOVE_PENDING;
-                desc->pending_mask = mask;
-                migrate_irq_remapped_level(irq);
+                cpumask_copy(&desc->pending_mask, mask);
+                migrate_irq_remapped_level_desc(desc);
                 return;
         }
 
-        migrate_ioapic_irq(irq, mask);
+        migrate_ioapic_irq_desc(desc, mask);
+}
+static void set_ir_ioapic_affinity_irq(unsigned int irq,
+                                       const struct cpumask *mask)
+{
+        struct irq_desc *desc = irq_to_desc(irq);
+
+        set_ir_ioapic_affinity_irq_desc(desc, mask);
 }
 #endif
 
 asmlinkage void smp_irq_move_cleanup_interrupt(void)
 {
         unsigned vector, me;
+
         ack_APIC_irq();
-#ifdef CONFIG_X86_64
         exit_idle();
-#endif
         irq_enter();
 
         me = smp_processor_id();
@@ -2229,6 +2481,9 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void)
                 struct irq_cfg *cfg;
                 irq = __get_cpu_var(vector_irq)[vector];
 
+                if (irq == -1)
+                        continue;
+
                 desc = irq_to_desc(irq);
                 if (!desc)
                         continue;
@@ -2238,7 +2493,7 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void)
                 if (!cfg->move_cleanup_count)
                         goto unlock;
 
-                if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
+                if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
                         goto unlock;
 
                 __get_cpu_var(vector_irq)[vector] = -1;
@@ -2250,28 +2505,44 @@ unlock:
         irq_exit();
 }
 
-static void irq_complete_move(unsigned int irq)
+static void irq_complete_move(struct irq_desc **descp)
 {
-        struct irq_cfg *cfg = irq_cfg(irq);
+        struct irq_desc *desc = *descp;
+        struct irq_cfg *cfg = desc->chip_data;
         unsigned vector, me;
 
-        if (likely(!cfg->move_in_progress))
+        if (likely(!cfg->move_in_progress)) {
+#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
+                if (likely(!cfg->move_desc_pending))
+                        return;
+
+                /* domain has not changed, but affinity did */
+                me = smp_processor_id();
+                if (cpu_isset(me, desc->affinity)) {
+                        *descp = desc = move_irq_desc(desc, me);
+                        /* get the new one */
+                        cfg = desc->chip_data;
+                        cfg->move_desc_pending = 0;
+                }
+#endif
                 return;
+        }
 
         vector = ~get_irq_regs()->orig_ax;
         me = smp_processor_id();
-        if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
-                cpumask_t cleanup_mask;
+#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
+                *descp = desc = move_irq_desc(desc, me);
+                /* get the new one */
+                cfg = desc->chip_data;
+#endif
 
-                cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
-                cfg->move_cleanup_count = cpus_weight(cleanup_mask);
-                send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
-                cfg->move_in_progress = 0;
-        }
+        if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
+                send_cleanup_vector(cfg);
 }
 #else
-static inline void irq_complete_move(unsigned int irq) {}
+static inline void irq_complete_move(struct irq_desc **descp) {}
 #endif
+
 #ifdef CONFIG_INTR_REMAP
 static void ack_x2apic_level(unsigned int irq)
 {
@@ -2282,11 +2553,14 @@ static void ack_x2apic_edge(unsigned int irq)
 {
         ack_x2APIC_irq();
 }
+
 #endif
 
 static void ack_apic_edge(unsigned int irq)
 {
-        irq_complete_move(irq);
+        struct irq_desc *desc = irq_to_desc(irq);
+
+        irq_complete_move(&desc);
         move_native_irq(irq);
         ack_APIC_irq();
 }
@@ -2295,18 +2569,21 @@ atomic_t irq_mis_count;
 
 static void ack_apic_level(unsigned int irq)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
+
 #ifdef CONFIG_X86_32
         unsigned long v;
         int i;
 #endif
+        struct irq_cfg *cfg;
         int do_unmask_irq = 0;
 
-        irq_complete_move(irq);
+        irq_complete_move(&desc);
 #ifdef CONFIG_GENERIC_PENDING_IRQ
         /* If we are moving the irq we need to mask it */
-        if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) {
+        if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
                 do_unmask_irq = 1;
-                mask_IO_APIC_irq(irq);
+                mask_IO_APIC_irq_desc(desc);
         }
 #endif
 
@@ -2330,7 +2607,8 @@ static void ack_apic_level(unsigned int irq)
         * operation to prevent an edge-triggered interrupt escaping meanwhile.
         * The idea is from Manfred Spraul.  --macro
         */
-        i = irq_cfg(irq)->vector;
+        cfg = desc->chip_data;
+        i = cfg->vector;
 
         v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
 #endif
@@ -2369,17 +2647,18 @@ static void ack_apic_level(unsigned int irq)
                  * accurate and is causing problems then it is a hardware bug
                  * and you can go talk to the chipset vendor about it.
                  */
-                if (!io_apic_level_ack_pending(irq))
+                cfg = desc->chip_data;
+                if (!io_apic_level_ack_pending(cfg))
                         move_masked_irq(irq);
-                unmask_IO_APIC_irq(irq);
+                unmask_IO_APIC_irq_desc(desc);
         }
 
 #ifdef CONFIG_X86_32
         if (!(v & (1 << (i & 0x1f)))) {
                 atomic_inc(&irq_mis_count);
                 spin_lock(&ioapic_lock);
-                __mask_and_edge_IO_APIC_irq(irq);
-                __unmask_and_level_IO_APIC_irq(irq);
+                __mask_and_edge_IO_APIC_irq(cfg);
+                __unmask_and_level_IO_APIC_irq(cfg);
                 spin_unlock(&ioapic_lock);
         }
 #endif
@@ -2430,20 +2709,19 @@ static inline void init_IO_APIC_traps(void)
          * Also, we've got to be careful not to trash gate
          * 0x80, because int 0x80 is hm, kind of importantish. ;)
          */
-        for_each_irq_cfg(irq, cfg) {
-                if (IO_APIC_IRQ(irq) && !cfg->vector) {
+        for_each_irq_desc(irq, desc) {
+                cfg = desc->chip_data;
+                if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
                         /*
                          * Hmm.. We don't have an entry for this,
                          * so default to an old-fashioned 8259
                          * interrupt if we can..
                          */
-                        if (irq < 16)
+                        if (irq < NR_IRQS_LEGACY)
                                 make_8259A_irq(irq);
-                        else {
-                                desc = irq_to_desc(irq);
+                        else
                                 /* Strange. Oh, well.. */
                                 desc->chip = &no_irq_chip;
-                        }
                 }
         }
 }
@@ -2468,7 +2746,7 @@ static void unmask_lapic_irq(unsigned int irq)
         apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
 }
 
-static void ack_lapic_irq (unsigned int irq)
+static void ack_lapic_irq(unsigned int irq)
 {
         ack_APIC_irq();
 }
@@ -2480,11 +2758,8 @@ static struct irq_chip lapic_chip __read_mostly = {
         .ack            = ack_lapic_irq,
 };
 
-static void lapic_register_intr(int irq)
+static void lapic_register_intr(int irq, struct irq_desc *desc)
 {
-        struct irq_desc *desc;
-
-        desc = irq_to_desc(irq);
         desc->status &= ~IRQ_LEVEL;
         set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
                                       "edge");
@@ -2588,7 +2863,9 @@ int timer_through_8259 __initdata;
  */
 static inline void __init check_timer(void)
 {
-        struct irq_cfg *cfg = irq_cfg(0);
+        struct irq_desc *desc = irq_to_desc(0);
+        struct irq_cfg *cfg = desc->chip_data;
+        int cpu = boot_cpu_id;
         int apic1, pin1, apic2, pin2;
         unsigned long flags;
         unsigned int ver;
@@ -2603,7 +2880,7 @@ static inline void __init check_timer(void)
          * get/set the timer IRQ vector:
          */
         disable_8259A_irq(0);
-        assign_irq_vector(0, TARGET_CPUS);
+        assign_irq_vector(0, cfg, TARGET_CPUS);
 
         /*
          * As IRQ0 is to be enabled in the 8259A, the virtual
@@ -2654,10 +2931,10 @@ static inline void __init check_timer(void)
                  * Ok, does IRQ0 through the IOAPIC work?
                  */
                 if (no_pin1) {
-                        add_pin_to_irq(0, apic1, pin1);
+                        add_pin_to_irq_cpu(cfg, cpu, apic1, pin1);
                         setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
                 }
-                unmask_IO_APIC_irq(0);
+                unmask_IO_APIC_irq_desc(desc);
                 if (timer_irq_works()) {
                         if (nmi_watchdog == NMI_IO_APIC) {
                                 setup_nmi();
@@ -2683,9 +2960,9 @@ static inline void __init check_timer(void)
                 /*
                  * legacy devices should be connected to IO APIC #0
                  */
-                replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
+                replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2);
                 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
-                unmask_IO_APIC_irq(0);
+                unmask_IO_APIC_irq_desc(desc);
                 enable_8259A_irq(0);
                 if (timer_irq_works()) {
                         apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
@@ -2717,7 +2994,7 @@ static inline void __init check_timer(void)
         apic_printk(APIC_QUIET, KERN_INFO
                     "...trying to set up timer as Virtual Wire IRQ...\n");
 
-        lapic_register_intr(0);
+        lapic_register_intr(0, desc);
         apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);     /* Fixed mode */
         enable_8259A_irq(0);
 
@@ -2902,22 +3179,26 @@ unsigned int create_irq_nr(unsigned int irq_want)
         unsigned int irq;
         unsigned int new;
         unsigned long flags;
-        struct irq_cfg *cfg_new;
-
-        irq_want = nr_irqs - 1;
+        struct irq_cfg *cfg_new = NULL;
+        int cpu = boot_cpu_id;
+        struct irq_desc *desc_new = NULL;
 
         irq = 0;
         spin_lock_irqsave(&vector_lock, flags);
-        for (new = irq_want; new > 0; new--) {
+        for (new = irq_want; new < NR_IRQS; new++) {
                 if (platform_legacy_irq(new))
                         continue;
-                cfg_new = irq_cfg(new);
-                if (cfg_new && cfg_new->vector != 0)
+
+                desc_new = irq_to_desc_alloc_cpu(new, cpu);
+                if (!desc_new) {
+                        printk(KERN_INFO "can not get irq_desc for %d\n", new);
                         continue;
-                /* check if need to create one */
-                if (!cfg_new)
-                        cfg_new = irq_cfg_alloc(new);
-                if (__assign_irq_vector(new, TARGET_CPUS) == 0)
+                }
+                cfg_new = desc_new->chip_data;
+
+                if (cfg_new->vector != 0)
+                        continue;
+                if (__assign_irq_vector(new, cfg_new, TARGET_CPUS) == 0)
                         irq = new;
                 break;
         }
@@ -2925,15 +3206,21 @@ unsigned int create_irq_nr(unsigned int irq_want)
 
         if (irq > 0) {
                 dynamic_irq_init(irq);
+                /* restore it, in case dynamic_irq_init clear it */
+                if (desc_new)
+                        desc_new->chip_data = cfg_new;
         }
         return irq;
 }
 
+static int nr_irqs_gsi = NR_IRQS_LEGACY;
 int create_irq(void)
 {
+        unsigned int irq_want;
         int irq;
 
-        irq = create_irq_nr(nr_irqs - 1);
+        irq_want = nr_irqs_gsi;
+        irq = create_irq_nr(irq_want);
 
         if (irq == 0)
                 irq = -1;
@@ -2944,14 +3231,22 @@ int create_irq(void)
 void destroy_irq(unsigned int irq)
 {
         unsigned long flags;
+        struct irq_cfg *cfg;
+        struct irq_desc *desc;
 
+        /* store it, in case dynamic_irq_cleanup clear it */
+        desc = irq_to_desc(irq);
+        cfg = desc->chip_data;
         dynamic_irq_cleanup(irq);
+        /* connect back irq_cfg */
+        if (desc)
+                desc->chip_data = cfg;
 
 #ifdef CONFIG_INTR_REMAP
         free_irte(irq);
 #endif
         spin_lock_irqsave(&vector_lock, flags);
-        __clear_irq_vector(irq);
+        __clear_irq_vector(irq, cfg);
         spin_unlock_irqrestore(&vector_lock, flags);
 }
 
@@ -2964,16 +3259,13 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
         struct irq_cfg *cfg;
         int err;
         unsigned dest;
-        cpumask_t tmp;
 
-        tmp = TARGET_CPUS;
-        err = assign_irq_vector(irq, tmp);
+        cfg = irq_cfg(irq);
+        err = assign_irq_vector(irq, cfg, TARGET_CPUS);
         if (err)
                 return err;
 
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, tmp);
-        dest = cpu_mask_to_apicid(tmp);
+        dest = cpu_mask_to_apicid_and(cfg->domain, TARGET_CPUS);
 
 #ifdef CONFIG_INTR_REMAP
         if (irq_remapped(irq)) {
@@ -3027,64 +3319,48 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
 }
 
 #ifdef CONFIG_SMP
-static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
+static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
         struct irq_cfg *cfg;
         struct msi_msg msg;
         unsigned int dest;
-        cpumask_t tmp;
-        struct irq_desc *desc;
 
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
+        dest = set_desc_affinity(desc, mask);
+        if (dest == BAD_APICID)
                 return;
 
-        if (assign_irq_vector(irq, mask))
-                return;
+        cfg = desc->chip_data;
 
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
-
-        read_msi_msg(irq, &msg);
+        read_msi_msg_desc(desc, &msg);
 
         msg.data &= ~MSI_DATA_VECTOR_MASK;
         msg.data |= MSI_DATA_VECTOR(cfg->vector);
         msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
         msg.address_lo |= MSI_ADDR_DEST_ID(dest);
 
-        write_msi_msg(irq, &msg);
-        desc = irq_to_desc(irq);
-        desc->affinity = mask;
+        write_msi_msg_desc(desc, &msg);
 }
-
 #ifdef CONFIG_INTR_REMAP
 /*
  * Migrate the MSI irq to another cpumask. This migration is
  * done in the process context using interrupt-remapping hardware.
  */
-static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
+static void
+ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
 {
-        struct irq_cfg *cfg;
+        struct irq_desc *desc = irq_to_desc(irq);
+        struct irq_cfg *cfg = desc->chip_data;
         unsigned int dest;
-        cpumask_t tmp, cleanup_mask;
         struct irte irte;
-        struct irq_desc *desc;
-
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
-                return;
 
         if (get_irte(irq, &irte))
                 return;
 
-        if (assign_irq_vector(irq, mask))
+        dest = set_desc_affinity(desc, mask);
+        if (dest == BAD_APICID)
                 return;
 
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
-
         irte.vector = cfg->vector;
         irte.dest_id = IRTE_DEST(dest);
 
@@ -3098,16 +3374,10 @@ static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
          * at the new destination. So, time to cleanup the previous
          * vector allocation.
          */
-        if (cfg->move_in_progress) {
-                cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
-                cfg->move_cleanup_count = cpus_weight(cleanup_mask);
-                send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
-                cfg->move_in_progress = 0;
-        }
-
-        desc = irq_to_desc(irq);
-        desc->affinity = mask;
+        if (cfg->move_in_progress)
+                send_cleanup_vector(cfg);
 }
+
 #endif
 #endif /* CONFIG_SMP */
 
@@ -3166,7 +3436,7 @@ static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
 }
 #endif
 
-static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
+static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
 {
         int ret;
         struct msi_msg msg;
@@ -3175,7 +3445,7 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
         if (ret < 0)
                 return ret;
 
-        set_irq_msi(irq, desc);
+        set_irq_msi(irq, msidesc);
         write_msi_msg(irq, &msg);
 
 #ifdef CONFIG_INTR_REMAP
@@ -3195,26 +3465,13 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
         return 0;
 }
 
-static unsigned int build_irq_for_pci_dev(struct pci_dev *dev)
-{
-        unsigned int irq;
-
-        irq = dev->bus->number;
-        irq <<= 8;
-        irq |= dev->devfn;
-        irq <<= 12;
-
-        return irq;
-}
-
-int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
+int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc)
 {
         unsigned int irq;
         int ret;
         unsigned int irq_want;
 
-        irq_want = build_irq_for_pci_dev(dev) + 0x100;
-
+        irq_want = nr_irqs_gsi;
         irq = create_irq_nr(irq_want);
         if (irq == 0)
                 return -1;
@@ -3228,7 +3485,7 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
                 goto error;
 no_ir:
 #endif
-        ret = setup_msi_irq(dev, desc, irq);
+        ret = setup_msi_irq(dev, msidesc, irq);
         if (ret < 0) {
                 destroy_irq(irq);
                 return ret;
@@ -3246,7 +3503,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
 {
         unsigned int irq;
         int ret, sub_handle;
-        struct msi_desc *desc;
+        struct msi_desc *msidesc;
         unsigned int irq_want;
 
 #ifdef CONFIG_INTR_REMAP
@@ -3254,10 +3511,11 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
         int index = 0;
 #endif
 
-        irq_want = build_irq_for_pci_dev(dev) + 0x100;
+        irq_want = nr_irqs_gsi;
         sub_handle = 0;
-        list_for_each_entry(desc, &dev->msi_list, list) {
-                irq = create_irq_nr(irq_want--);
+        list_for_each_entry(msidesc, &dev->msi_list, list) {
+                irq = create_irq_nr(irq_want);
+                irq_want++;
                 if (irq == 0)
                         return -1;
 #ifdef CONFIG_INTR_REMAP
@@ -3289,7 +3547,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
                 }
 no_ir:
 #endif
-                ret = setup_msi_irq(dev, desc, irq);
+                ret = setup_msi_irq(dev, msidesc, irq);
                 if (ret < 0)
                         goto error;
                 sub_handle++;
@@ -3308,24 +3566,18 @@ void arch_teardown_msi_irq(unsigned int irq)
 
 #ifdef CONFIG_DMAR
 #ifdef CONFIG_SMP
-static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
+static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
         struct irq_cfg *cfg;
         struct msi_msg msg;
         unsigned int dest;
-        cpumask_t tmp;
-        struct irq_desc *desc;
 
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
+        dest = set_desc_affinity(desc, mask);
+        if (dest == BAD_APICID)
                 return;
 
-        if (assign_irq_vector(irq, mask))
-                return;
-
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
+        cfg = desc->chip_data;
 
         dmar_msi_read(irq, &msg);
 
@@ -3335,9 +3587,8 @@ static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
         msg.address_lo |= MSI_ADDR_DEST_ID(dest);
 
         dmar_msi_write(irq, &msg);
-        desc = irq_to_desc(irq);
-        desc->affinity = mask;
 }
+
 #endif /* CONFIG_SMP */
 
 struct irq_chip dmar_msi_type = {
@@ -3369,24 +3620,18 @@ int arch_setup_dmar_msi(unsigned int irq)
 #ifdef CONFIG_HPET_TIMER
 
 #ifdef CONFIG_SMP
-static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
+static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
         struct irq_cfg *cfg;
-        struct irq_desc *desc;
         struct msi_msg msg;
         unsigned int dest;
-        cpumask_t tmp;
 
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
+        dest = set_desc_affinity(desc, mask);
+        if (dest == BAD_APICID)
                 return;
 
-        if (assign_irq_vector(irq, mask))
-                return;
-
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
+        cfg = desc->chip_data;
 
         hpet_msi_read(irq, &msg);
 
@@ -3396,9 +3641,8 @@ static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
         msg.address_lo |= MSI_ADDR_DEST_ID(dest);
 
         hpet_msi_write(irq, &msg);
-        desc = irq_to_desc(irq);
-        desc->affinity = mask;
 }
+
 #endif /* CONFIG_SMP */
 
 struct irq_chip hpet_msi_type = {
@@ -3451,28 +3695,21 @@ static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
         write_ht_irq_msg(irq, &msg);
 }
 
-static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
+static void set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
 {
+        struct irq_desc *desc = irq_to_desc(irq);
         struct irq_cfg *cfg;
         unsigned int dest;
-        cpumask_t tmp;
-        struct irq_desc *desc;
 
-        cpus_and(tmp, mask, cpu_online_map);
-        if (cpus_empty(tmp))
+        dest = set_desc_affinity(desc, mask);
+        if (dest == BAD_APICID)
                 return;
 
-        if (assign_irq_vector(irq, mask))
-                return;
-
-        cfg = irq_cfg(irq);
-        cpus_and(tmp, cfg->domain, mask);
-        dest = cpu_mask_to_apicid(tmp);
+        cfg = desc->chip_data;
 
         target_ht_irq(irq, dest, cfg->vector);
-        desc = irq_to_desc(irq);
-        desc->affinity = mask;
 }
+
 #endif
 
 static struct irq_chip ht_irq_chip = {
@@ -3490,17 +3727,14 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
 {
         struct irq_cfg *cfg;
         int err;
-        cpumask_t tmp;
 
-        tmp = TARGET_CPUS;
-        err = assign_irq_vector(irq, tmp);
+        cfg = irq_cfg(irq);
+        err = assign_irq_vector(irq, cfg, TARGET_CPUS);
         if (!err) {
                 struct ht_irq_msg msg;
                 unsigned dest;
 
-                cfg = irq_cfg(irq);
-                cpus_and(tmp, cfg->domain, tmp);
-                dest = cpu_mask_to_apicid(tmp);
+                dest = cpu_mask_to_apicid_and(cfg->domain, TARGET_CPUS);
 
                 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
 
@@ -3536,7 +3770,7 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
 int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
                        unsigned long mmr_offset)
 {
-        const cpumask_t *eligible_cpu = get_cpu_mask(cpu);
+        const struct cpumask *eligible_cpu = cpumask_of(cpu);
         struct irq_cfg *cfg;
         int mmr_pnode;
         unsigned long mmr_value;
@@ -3544,7 +3778,9 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
         unsigned long flags;
         int err;
 
-        err = assign_irq_vector(irq, *eligible_cpu);
+        cfg = irq_cfg(irq);
+
+        err = assign_irq_vector(irq, cfg, eligible_cpu);
         if (err != 0)
                 return err;
 
@@ -3553,8 +3789,6 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
                                       irq_name);
         spin_unlock_irqrestore(&vector_lock, flags);
 
-        cfg = irq_cfg(irq);
-
         mmr_value = 0;
         entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
         BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
@@ -3565,7 +3799,7 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
         entry->polarity = 0;
         entry->trigger = 0;
         entry->mask = 0;
-        entry->dest = cpu_mask_to_apicid(*eligible_cpu);
+        entry->dest = cpu_mask_to_apicid(eligible_cpu);
 
         mmr_pnode = uv_blade_to_pnode(mmr_blade);
         uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
@@ -3606,9 +3840,16 @@ int __init io_apic_get_redir_entries (int ioapic)
         return reg_01.bits.entries;
 }
 
-int __init probe_nr_irqs(void)
+void __init probe_nr_irqs_gsi(void)
 {
-        return NR_IRQS;
+        int idx;
+        int nr = 0;
+
+        for (idx = 0; idx < nr_ioapics; idx++)
+                nr += io_apic_get_redir_entries(idx) + 1;
+
+        if (nr > nr_irqs_gsi)
+                nr_irqs_gsi = nr;
 }
 
 /* --------------------------------------------------------------------------
@@ -3707,19 +3948,31 @@ int __init io_apic_get_version(int ioapic)
 
 int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
 {
+        struct irq_desc *desc;
+        struct irq_cfg *cfg;
+        int cpu = boot_cpu_id;
+
         if (!IO_APIC_IRQ(irq)) {
                 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
                         ioapic);
                 return -EINVAL;
         }
 
+        desc = irq_to_desc_alloc_cpu(irq, cpu);
+        if (!desc) {
+                printk(KERN_INFO "can not get irq_desc %d\n", irq);
+                return 0;
+        }
+
         /*
          * IRQs < 16 are already in the irq_2_pin[] map
          */
-        if (irq >= 16)
-                add_pin_to_irq(irq, ioapic, pin);
+        if (irq >= NR_IRQS_LEGACY) {
+                cfg = desc->chip_data;
+                add_pin_to_irq_cpu(cfg, cpu, ioapic, pin);
+        }
 
-        setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity);
+        setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity);
 
         return 0;
 }
@@ -3757,7 +4010,7 @@ void __init setup_ioapic_dest(void)
         int pin, ioapic, irq, irq_entry;
         struct irq_desc *desc;
         struct irq_cfg *cfg;
-        cpumask_t mask;
+        const struct cpumask *mask;
 
         if (skip_ioapic_setup == 1)
                 return;
@@ -3773,9 +4026,10 @@ void __init setup_ioapic_dest(void)
                          * when you have too many devices, because at that time only boot
                          * cpu is online.
                          */
-                        cfg = irq_cfg(irq);
+                        desc = irq_to_desc(irq);
+                        cfg = desc->chip_data;
                         if (!cfg->vector) {
-                                setup_IO_APIC_irq(ioapic, pin, irq,
+                                setup_IO_APIC_irq(ioapic, pin, irq, desc,
                                                   irq_trigger(irq_entry),
                                                   irq_polarity(irq_entry));
                                 continue;
@@ -3785,19 +4039,18 @@ void __init setup_ioapic_dest(void)
                         /*
                          * Honour affinities which have been set in early boot
                          */
-                        desc = irq_to_desc(irq);
                         if (desc->status &
                             (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
-                                mask = desc->affinity;
+                                mask = &desc->affinity;
                         else
                                 mask = TARGET_CPUS;
 
 #ifdef CONFIG_INTR_REMAP
                         if (intr_remapping_enabled)
-                                set_ir_ioapic_affinity_irq(irq, mask);
+                                set_ir_ioapic_affinity_irq_desc(desc, mask);
                         else
 #endif
-                                set_ioapic_affinity_irq(irq, mask);
+                                set_ioapic_affinity_irq_desc(desc, mask);
                 }
 
         }
@@ -3846,7 +4099,6 @@ void __init ioapic_init_mappings(void)
         struct resource *ioapic_res;
         int i;
 
-        irq_2_pin_init();
         ioapic_res = ioapic_setup_resources();
         for (i = 0; i < nr_ioapics; i++) {
                 if (smp_found_config) {
diff --git a/arch/x86/kernel/ipi.c b/arch/x86/kernel/ipi.c
index f1c688e46f35..285bbf8831fa 100644
--- a/arch/x86/kernel/ipi.c
+++ b/arch/x86/kernel/ipi.c
@@ -116,18 +116,18 @@ static inline void __send_IPI_dest_field(unsigned long mask, int vector)
 /*
  * This is only used on smaller machines.
  */
-void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
+void send_IPI_mask_bitmask(const struct cpumask *cpumask, int vector)
 {
-        unsigned long mask = cpus_addr(cpumask)[0];
+        unsigned long mask = cpumask_bits(cpumask)[0];
         unsigned long flags;
 
         local_irq_save(flags);
-        WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
+        WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
         __send_IPI_dest_field(mask, vector);
         local_irq_restore(flags);
 }
 
-void send_IPI_mask_sequence(cpumask_t mask, int vector)
+void send_IPI_mask_sequence(const struct cpumask *mask, int vector)
 {
         unsigned long flags;
         unsigned int query_cpu;
@@ -139,12 +139,24 @@ void send_IPI_mask_sequence(cpumask_t mask, int vector)
          */
 
         local_irq_save(flags);
-        for_each_possible_cpu(query_cpu) {
-                if (cpu_isset(query_cpu, mask)) {
+        for_each_cpu(query_cpu, mask)
+                __send_IPI_dest_field(cpu_to_logical_apicid(query_cpu), vector);
+        local_irq_restore(flags);
+}
+
+void send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
+{
+        unsigned long flags;
+        unsigned int query_cpu;
+        unsigned int this_cpu = smp_processor_id();
+
+        /* See Hack comment above */
+
+        local_irq_save(flags);
+        for_each_cpu(query_cpu, mask)
+                if (query_cpu != this_cpu)
                         __send_IPI_dest_field(cpu_to_logical_apicid(query_cpu),
                                               vector);
-                }
-        }
         local_irq_restore(flags);
 }
 
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index d1d4dc52f649..bce53e1352a0 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -9,6 +9,7 @@
 #include <asm/apic.h>
 #include <asm/io_apic.h>
 #include <asm/smp.h>
+#include <asm/irq.h>
 
 atomic_t irq_err_count;
 
@@ -118,6 +119,9 @@ int show_interrupts(struct seq_file *p, void *v)
         }
 
         desc = irq_to_desc(i);
+        if (!desc)
+                return 0;
+
         spin_lock_irqsave(&desc->lock, flags);
 #ifndef CONFIG_SMP
         any_count = kstat_irqs(i);
@@ -187,3 +191,5 @@ u64 arch_irq_stat(void)
 #endif
         return sum;
 }
+
+EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index a51382672de0..9dc5588f336a 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -233,25 +233,28 @@ unsigned int do_IRQ(struct pt_regs *regs)
 #ifdef CONFIG_HOTPLUG_CPU
 #include <mach_apic.h>
 
-void fixup_irqs(cpumask_t map)
+/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
+void fixup_irqs(void)
 {
         unsigned int irq;
         static int warned;
         struct irq_desc *desc;
 
         for_each_irq_desc(irq, desc) {
-                cpumask_t mask;
+                const struct cpumask *affinity;
 
+                if (!desc)
+                        continue;
                 if (irq == 2)
                         continue;
 
-                cpus_and(mask, desc->affinity, map);
-                if (any_online_cpu(mask) == NR_CPUS) {
+                affinity = &desc->affinity;
+                if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
                         printk("Breaking affinity for irq %i\n", irq);
-                        mask = map;
+                        affinity = cpu_all_mask;
                 }
                 if (desc->chip->set_affinity)
-                        desc->chip->set_affinity(irq, mask);
+                        desc->chip->set_affinity(irq, affinity);
                 else if (desc->action && !(warned++))
                         printk("Cannot set affinity for irq %i\n", irq);
         }
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 60eb84eb77a0..6383d50f82ea 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -13,12 +13,12 @@
 #include <linux/seq_file.h>
 #include <linux/module.h>
 #include <linux/delay.h>
+#include <linux/ftrace.h>
 #include <asm/uaccess.h>
 #include <asm/io_apic.h>
 #include <asm/idle.h>
 #include <asm/smp.h>
 
-#ifdef CONFIG_DEBUG_STACKOVERFLOW
 /*
  * Probabilistic stack overflow check:
  *
@@ -28,26 +28,25 @@
  */
 static inline void stack_overflow_check(struct pt_regs *regs)
 {
+#ifdef CONFIG_DEBUG_STACKOVERFLOW
         u64 curbase = (u64)task_stack_page(current);
-        static unsigned long warned = -60*HZ;
-
-        if (regs->sp >= curbase && regs->sp <= curbase + THREAD_SIZE &&
-            regs->sp <  curbase + sizeof(struct thread_info) + 128 &&
-            time_after(jiffies, warned + 60*HZ)) {
-                printk("do_IRQ: %s near stack overflow (cur:%Lx,sp:%lx)\n",
-                       current->comm, curbase, regs->sp);
-                show_stack(NULL,NULL);
-                warned = jiffies;
-        }
-}
+
+        WARN_ONCE(regs->sp >= curbase &&
+                  regs->sp <= curbase + THREAD_SIZE &&
+                  regs->sp <  curbase + sizeof(struct thread_info) +
+                                        sizeof(struct pt_regs) + 128,
+
+                  "do_IRQ: %s near stack overflow (cur:%Lx,sp:%lx)\n",
+                        current->comm, curbase, regs->sp);
 #endif
+}
 
 /*
  * do_IRQ handles all normal device IRQ's (the special
  * SMP cross-CPU interrupts have their own specific
  * handlers).
  */
-asmlinkage unsigned int do_IRQ(struct pt_regs *regs)
+asmlinkage unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
 {
         struct pt_regs *old_regs = set_irq_regs(regs);
         struct irq_desc *desc;
@@ -60,9 +59,7 @@ asmlinkage unsigned int do_IRQ(struct pt_regs *regs)
         irq_enter();
         irq = __get_cpu_var(vector_irq)[vector];
 
-#ifdef CONFIG_DEBUG_STACKOVERFLOW
         stack_overflow_check(regs);
-#endif
 
         desc = irq_to_desc(irq);
         if (likely(desc))
@@ -83,40 +80,43 @@ asmlinkage unsigned int do_IRQ(struct pt_regs *regs)
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
-void fixup_irqs(cpumask_t map)
+/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
+void fixup_irqs(void)
 {
         unsigned int irq;
         static int warned;
         struct irq_desc *desc;
 
         for_each_irq_desc(irq, desc) {
-                cpumask_t mask;
                 int break_affinity = 0;
                 int set_affinity = 1;
+                const struct cpumask *affinity;
 
+                if (!desc)
+                        continue;
                 if (irq == 2)
                         continue;
 
                 /* interrupt's are disabled at this point */
                 spin_lock(&desc->lock);
 
+                affinity = &desc->affinity;
                 if (!irq_has_action(irq) ||
-                    cpus_equal(desc->affinity, map)) {
+                    cpumask_equal(affinity, cpu_online_mask)) {
                         spin_unlock(&desc->lock);
                         continue;
                 }
 
-                cpus_and(mask, desc->affinity, map);
-                if (cpus_empty(mask)) {
+                if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
                         break_affinity = 1;
-                        mask = map;
+                        affinity = cpu_all_mask;
                 }
 
                 if (desc->chip->mask)
                         desc->chip->mask(irq);
 
                 if (desc->chip->set_affinity)
-                        desc->chip->set_affinity(irq, mask);
+                        desc->chip->set_affinity(irq, affinity);
                 else if (!(warned++))
                         set_affinity = 0;
 
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
index 845aa9803e80..84723295f88a 100644
--- a/arch/x86/kernel/irqinit_32.c
+++ b/arch/x86/kernel/irqinit_32.c
@@ -68,8 +68,7 @@ void __init init_ISA_irqs (void)
         /*
          * 16 old-style INTA-cycle interrupts:
          */
-        for (i = 0; i < 16; i++) {
-                /* first time call this irq_desc */
+        for (i = 0; i < NR_IRQS_LEGACY; i++) {
                 struct irq_desc *desc = irq_to_desc(i);
 
                 desc->status = IRQ_DISABLED;
@@ -111,6 +110,18 @@ DEFINE_PER_CPU(vector_irq_t, vector_irq) = {
         [IRQ15_VECTOR + 1 ... NR_VECTORS - 1] = -1
 };
 
+int vector_used_by_percpu_irq(unsigned int vector)
+{
+        int cpu;
+
+        for_each_online_cpu(cpu) {
+                if (per_cpu(vector_irq, cpu)[vector] != -1)
+                        return 1;
+        }
+
+        return 0;
+}
+
 /* Overridden in paravirt.c */
 void init_IRQ(void) __attribute__((weak, alias("native_init_IRQ")));
 
@@ -129,7 +140,7 @@ void __init native_init_IRQ(void)
         for (i =  FIRST_EXTERNAL_VECTOR; i < NR_VECTORS; i++) {
                 /* SYSCALL_VECTOR was reserved in trap_init. */
                 if (i != SYSCALL_VECTOR)
-                        set_intr_gate(i, interrupt[i]);
+                        set_intr_gate(i, interrupt[i-FIRST_EXTERNAL_VECTOR]);
         }
 
 
@@ -147,10 +158,12 @@ void __init native_init_IRQ(void)
         alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
 
         /* IPI for single call function */
-        set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt);
+        alloc_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
+                                 call_function_single_interrupt);
 
         /* Low priority IPI to cleanup after moving an irq */
         set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
+        set_bit(IRQ_MOVE_CLEANUP_VECTOR, used_vectors);
 #endif
 
 #ifdef CONFIG_X86_LOCAL_APIC
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c
index ff0235391285..31ebfe38e96c 100644
--- a/arch/x86/kernel/irqinit_64.c
+++ b/arch/x86/kernel/irqinit_64.c
@@ -24,41 +24,6 @@
 #include <asm/i8259.h>
 
 /*
- * Common place to define all x86 IRQ vectors
- *
- * This builds up the IRQ handler stubs using some ugly macros in irq.h
- *
- * These macros create the low-level assembly IRQ routines that save
- * register context and call do_IRQ(). do_IRQ() then does all the
- * operations that are needed to keep the AT (or SMP IOAPIC)
- * interrupt-controller happy.
- */
-
-#define IRQ_NAME2(nr) nr##_interrupt(void)
-#define IRQ_NAME(nr) IRQ_NAME2(IRQ##nr)
-
-/*
- *      SMP has a few special interrupts for IPI messages
- */
-
-#define BUILD_IRQ(nr)                           \
-        asmlinkage void IRQ_NAME(nr);           \
-        asm("\n.text\n.p2align\n"               \
-            "IRQ" #nr "_interrupt:\n\t"         \
-            "push $~(" #nr ") ; "               \
-            "jmp common_interrupt\n"            \
-            ".previous");
-
-#define BI(x,y) \
-        BUILD_IRQ(x##y)
-
-#define BUILD_16_IRQS(x) \
-        BI(x,0) BI(x,1) BI(x,2) BI(x,3) \
-        BI(x,4) BI(x,5) BI(x,6) BI(x,7) \
-        BI(x,8) BI(x,9) BI(x,a) BI(x,b) \
-        BI(x,c) BI(x,d) BI(x,e) BI(x,f)
-
-/*
  * ISA PIC or low IO-APIC triggered (INTA-cycle or APIC) interrupts:
  * (these are usually mapped to vectors 0x30-0x3f)
  */
@@ -73,37 +38,6 @@
  *
  * (these are usually mapped into the 0x30-0xff vector range)
  */
-                                      BUILD_16_IRQS(0x2) BUILD_16_IRQS(0x3)
-BUILD_16_IRQS(0x4) BUILD_16_IRQS(0x5) BUILD_16_IRQS(0x6) BUILD_16_IRQS(0x7)
-BUILD_16_IRQS(0x8) BUILD_16_IRQS(0x9) BUILD_16_IRQS(0xa) BUILD_16_IRQS(0xb)
-BUILD_16_IRQS(0xc) BUILD_16_IRQS(0xd) BUILD_16_IRQS(0xe) BUILD_16_IRQS(0xf)
-
-#undef BUILD_16_IRQS
-#undef BI
-
-
-#define IRQ(x,y) \
-        IRQ##x##y##_interrupt
-
-#define IRQLIST_16(x) \
-        IRQ(x,0), IRQ(x,1), IRQ(x,2), IRQ(x,3), \
-        IRQ(x,4), IRQ(x,5), IRQ(x,6), IRQ(x,7), \
-        IRQ(x,8), IRQ(x,9), IRQ(x,a), IRQ(x,b), \
-        IRQ(x,c), IRQ(x,d), IRQ(x,e), IRQ(x,f)
-
-/* for the irq vectors */
-static void (*__initdata interrupt[NR_VECTORS - FIRST_EXTERNAL_VECTOR])(void) = {
-                                          IRQLIST_16(0x2), IRQLIST_16(0x3),
-        IRQLIST_16(0x4), IRQLIST_16(0x5), IRQLIST_16(0x6), IRQLIST_16(0x7),
-        IRQLIST_16(0x8), IRQLIST_16(0x9), IRQLIST_16(0xa), IRQLIST_16(0xb),
-        IRQLIST_16(0xc), IRQLIST_16(0xd), IRQLIST_16(0xe), IRQLIST_16(0xf)
-};
-
-#undef IRQ
-#undef IRQLIST_16
-
-
-
 
 /*
  * IRQ2 is cascade interrupt to second interrupt controller
@@ -135,6 +69,18 @@ DEFINE_PER_CPU(vector_irq_t, vector_irq) = {
         [IRQ15_VECTOR + 1 ... NR_VECTORS - 1] = -1
 };
 
+int vector_used_by_percpu_irq(unsigned int vector)
+{
+        int cpu;
+
+        for_each_online_cpu(cpu) {
+                if (per_cpu(vector_irq, cpu)[vector] != -1)
+                        return 1;
+        }
+
+        return 0;
+}
+
 void __init init_ISA_irqs(void)
 {
         int i;
@@ -142,8 +88,7 @@ void __init init_ISA_irqs(void)
         init_bsp_APIC();
         init_8259A(0);
 
-        for (i = 0; i < 16; i++) {
-                /* first time call this irq_desc */
+        for (i = 0; i < NR_IRQS_LEGACY; i++) {
                 struct irq_desc *desc = irq_to_desc(i);
 
                 desc->status = IRQ_DISABLED;
@@ -188,6 +133,7 @@ static void __init smp_intr_init(void)
 
         /* Low priority IPI to cleanup after moving an irq */
         set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
+        set_bit(IRQ_MOVE_CLEANUP_VECTOR, used_vectors);
 #endif
 }
 
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index 6c27679ec6aa..884d985b8b82 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -376,9 +376,10 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p)
 
 void __kprobes arch_remove_kprobe(struct kprobe *p)
 {
-        mutex_lock(&kprobe_mutex);
-        free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
-        mutex_unlock(&kprobe_mutex);
+        if (p->ainsn.insn) {
+                free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
+                p->ainsn.insn = NULL;
+        }
 }
 
 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
@@ -694,7 +695,7 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
         /*
          * It is possible to have multiple instances associated with a given
          * task either because multiple functions in the call path have
-         * return probes installed on them, and/or more then one
+         * return probes installed on them, and/or more than one
          * return probe was registered for a target function.
          *
          * We can handle this because:
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index e169ae9b6a62..652fce6d2cce 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -89,17 +89,17 @@ static cycle_t kvm_clock_read(void)
  */
 static unsigned long kvm_get_tsc_khz(void)
 {
-        return preset_lpj;
+        struct pvclock_vcpu_time_info *src;
+        src = &per_cpu(hv_clock, 0);
+        return pvclock_tsc_khz(src);
 }
 
 static void kvm_get_preset_lpj(void)
 {
-        struct pvclock_vcpu_time_info *src;
         unsigned long khz;
         u64 lpj;
 
-        src = &per_cpu(hv_clock, 0);
-        khz = pvclock_tsc_khz(src);
+        khz = kvm_get_tsc_khz();
 
         lpj = ((u64)khz * 1000);
         do_div(lpj, HZ);
@@ -194,5 +194,7 @@ void __init kvmclock_init(void)
 #endif
                 kvm_get_preset_lpj();
                 clocksource_register(&kvm_clock);
+                pv_info.paravirt_enabled = 1;
+                pv_info.name = "KVM";
         }
 }
diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index eee32b43fee3..71f1d99a635d 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -12,8 +12,8 @@
 #include <linux/mm.h>
 #include <linux/smp.h>
 #include <linux/vmalloc.h>
+#include <linux/uaccess.h>
 
-#include <asm/uaccess.h>
 #include <asm/system.h>
 #include <asm/ldt.h>
 #include <asm/desc.h>
@@ -93,7 +93,7 @@ static inline int copy_ldt(mm_context_t *new, mm_context_t *old)
         if (err < 0)
                 return err;
 
-        for(i = 0; i < old->size; i++)
+        for (i = 0; i < old->size; i++)
                 write_ldt_entry(new->ldt, i, old->ldt + i * LDT_ENTRY_SIZE);
         return 0;
 }
diff --git a/arch/x86/kernel/machine_kexec_32.c b/arch/x86/kernel/machine_kexec_32.c
index 7a385746509a..37f420018a41 100644
--- a/arch/x86/kernel/machine_kexec_32.c
+++ b/arch/x86/kernel/machine_kexec_32.c
@@ -13,6 +13,7 @@
 #include <linux/numa.h>
 #include <linux/ftrace.h>
 #include <linux/suspend.h>
+#include <linux/gfp.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -25,15 +26,6 @@
 #include <asm/system.h>
 #include <asm/cacheflush.h>
 
-#define PAGE_ALIGNED __attribute__ ((__aligned__(PAGE_SIZE)))
-static u32 kexec_pgd[1024] PAGE_ALIGNED;
-#ifdef CONFIG_X86_PAE
-static u32 kexec_pmd0[1024] PAGE_ALIGNED;
-static u32 kexec_pmd1[1024] PAGE_ALIGNED;
-#endif
-static u32 kexec_pte0[1024] PAGE_ALIGNED;
-static u32 kexec_pte1[1024] PAGE_ALIGNED;
-
 static void set_idt(void *newidt, __u16 limit)
 {
         struct desc_ptr curidt;
@@ -76,6 +68,76 @@ static void load_segments(void)
 #undef __STR
 }
 
+static void machine_kexec_free_page_tables(struct kimage *image)
+{
+        free_page((unsigned long)image->arch.pgd);
+#ifdef CONFIG_X86_PAE
+        free_page((unsigned long)image->arch.pmd0);
+        free_page((unsigned long)image->arch.pmd1);
+#endif
+        free_page((unsigned long)image->arch.pte0);
+        free_page((unsigned long)image->arch.pte1);
+}
+
+static int machine_kexec_alloc_page_tables(struct kimage *image)
+{
+        image->arch.pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL);
+#ifdef CONFIG_X86_PAE
+        image->arch.pmd0 = (pmd_t *)get_zeroed_page(GFP_KERNEL);
+        image->arch.pmd1 = (pmd_t *)get_zeroed_page(GFP_KERNEL);
+#endif
+        image->arch.pte0 = (pte_t *)get_zeroed_page(GFP_KERNEL);
+        image->arch.pte1 = (pte_t *)get_zeroed_page(GFP_KERNEL);
+        if (!image->arch.pgd ||
+#ifdef CONFIG_X86_PAE
+            !image->arch.pmd0 || !image->arch.pmd1 ||
+#endif
+            !image->arch.pte0 || !image->arch.pte1) {
+                machine_kexec_free_page_tables(image);
+                return -ENOMEM;
+        }
+        return 0;
+}
+
+static void machine_kexec_page_table_set_one(
+        pgd_t *pgd, pmd_t *pmd, pte_t *pte,
+        unsigned long vaddr, unsigned long paddr)
+{
+        pud_t *pud;
+
+        pgd += pgd_index(vaddr);
+#ifdef CONFIG_X86_PAE
+        if (!(pgd_val(*pgd) & _PAGE_PRESENT))
+                set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT));
+#endif
+        pud = pud_offset(pgd, vaddr);
+        pmd = pmd_offset(pud, vaddr);
+        if (!(pmd_val(*pmd) & _PAGE_PRESENT))
+                set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
+        pte = pte_offset_kernel(pmd, vaddr);
+        set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
+}
+
+static void machine_kexec_prepare_page_tables(struct kimage *image)
+{
+        void *control_page;
+        pmd_t *pmd = 0;
+
+        control_page = page_address(image->control_code_page);
+#ifdef CONFIG_X86_PAE
+        pmd = image->arch.pmd0;
+#endif
+        machine_kexec_page_table_set_one(
+                image->arch.pgd, pmd, image->arch.pte0,
+                (unsigned long)control_page, __pa(control_page));
+#ifdef CONFIG_X86_PAE
+        pmd = image->arch.pmd1;
+#endif
+        machine_kexec_page_table_set_one(
+                image->arch.pgd, pmd, image->arch.pte1,
+                __pa(control_page), __pa(control_page));
+}
+
 /*
  * A architecture hook called to validate the
  * proposed image and prepare the control pages
@@ -87,12 +149,20 @@ static void load_segments(void)
  * reboot code buffer to allow us to avoid allocations
  * later.
  *
- * Make control page executable.
+ * - Make control page executable.
+ * - Allocate page tables
+ * - Setup page tables
  */
 int machine_kexec_prepare(struct kimage *image)
 {
+        int error;
+
         if (nx_enabled)
                 set_pages_x(image->control_code_page, 1);
+        error = machine_kexec_alloc_page_tables(image);
+        if (error)
+                return error;
+        machine_kexec_prepare_page_tables(image);
         return 0;
 }
 
@@ -104,6 +174,7 @@ void machine_kexec_cleanup(struct kimage *image)
 {
         if (nx_enabled)
                 set_pages_nx(image->control_code_page, 1);
+        machine_kexec_free_page_tables(image);
 }
 
 /*
@@ -150,18 +221,7 @@ void machine_kexec(struct kimage *image)
         relocate_kernel_ptr = control_page;
         page_list[PA_CONTROL_PAGE] = __pa(control_page);
         page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
-        page_list[PA_PGD] = __pa(kexec_pgd);
-        page_list[VA_PGD] = (unsigned long)kexec_pgd;
-#ifdef CONFIG_X86_PAE
-        page_list[PA_PMD_0] = __pa(kexec_pmd0);
-        page_list[VA_PMD_0] = (unsigned long)kexec_pmd0;
-        page_list[PA_PMD_1] = __pa(kexec_pmd1);
-        page_list[VA_PMD_1] = (unsigned long)kexec_pmd1;
-#endif
-        page_list[PA_PTE_0] = __pa(kexec_pte0);
-        page_list[VA_PTE_0] = (unsigned long)kexec_pte0;
-        page_list[PA_PTE_1] = __pa(kexec_pte1);
-        page_list[VA_PTE_1] = (unsigned long)kexec_pte1;
+        page_list[PA_PGD] = __pa(image->arch.pgd);
 
         if (image->type == KEXEC_TYPE_DEFAULT)
                 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
diff --git a/arch/x86/kernel/mfgpt_32.c b/arch/x86/kernel/mfgpt_32.c
index 3b599518c322..8815f3c7fec7 100644
--- a/arch/x86/kernel/mfgpt_32.c
+++ b/arch/x86/kernel/mfgpt_32.c
@@ -252,7 +252,7 @@ EXPORT_SYMBOL_GPL(geode_mfgpt_alloc_timer);
 /*
  * The MFPGT timers on the CS5536 provide us with suitable timers to use
  * as clock event sources - not as good as a HPET or APIC, but certainly
- * better then the PIT.  This isn't a general purpose MFGPT driver, but
+ * better than the PIT.  This isn't a general purpose MFGPT driver, but
  * a simplified one designed specifically to act as a clock event source.
  * For full details about the MFGPT, please consult the CS5536 data sheet.
  */
@@ -287,7 +287,7 @@ static struct clock_event_device mfgpt_clockevent = {
         .set_mode = mfgpt_set_mode,
         .set_next_event = mfgpt_next_event,
         .rating = 250,
-        .cpumask = CPU_MASK_ALL,
+        .cpumask = cpu_all_mask,
         .shift = 32
 };
 
diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c
index 5f8e5d75a254..c25fdb382292 100644
--- a/arch/x86/kernel/microcode_amd.c
+++ b/arch/x86/kernel/microcode_amd.c
@@ -10,7 +10,7 @@
  *  This driver allows to upgrade microcode on AMD
  *  family 0x10 and 0x11 processors.
  *
- *  Licensed unter the terms of the GNU General Public
+ *  Licensed under the terms of the GNU General Public
  *  License version 2. See file COPYING for details.
 */
 
@@ -32,9 +32,9 @@
 #include <linux/platform_device.h>
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
+#include <linux/uaccess.h>
 
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/processor.h>
 #include <asm/microcode.h>
 
@@ -47,43 +47,38 @@ MODULE_LICENSE("GPL v2");
 #define UCODE_UCODE_TYPE           0x00000001
 
 struct equiv_cpu_entry {
-        unsigned int installed_cpu;
-        unsigned int fixed_errata_mask;
-        unsigned int fixed_errata_compare;
-        unsigned int equiv_cpu;
-};
+        u32     installed_cpu;
+        u32     fixed_errata_mask;
+        u32     fixed_errata_compare;
+        u16     equiv_cpu;
+        u16     res;
+} __attribute__((packed));
 
 struct microcode_header_amd {
-        unsigned int  data_code;
-        unsigned int  patch_id;
-        unsigned char mc_patch_data_id[2];
-        unsigned char mc_patch_data_len;
-        unsigned char init_flag;
-        unsigned int  mc_patch_data_checksum;
-        unsigned int  nb_dev_id;
-        unsigned int  sb_dev_id;
-        unsigned char processor_rev_id[2];
-        unsigned char nb_rev_id;
-        unsigned char sb_rev_id;
-        unsigned char bios_api_rev;
-        unsigned char reserved1[3];
-        unsigned int  match_reg[8];
-};
+        u32     data_code;
+        u32     patch_id;
+        u16     mc_patch_data_id;
+        u8      mc_patch_data_len;
+        u8      init_flag;
+        u32     mc_patch_data_checksum;
+        u32     nb_dev_id;
+        u32     sb_dev_id;
+        u16     processor_rev_id;
+        u8      nb_rev_id;
+        u8      sb_rev_id;
+        u8      bios_api_rev;
+        u8      reserved1[3];
+        u32     match_reg[8];
+} __attribute__((packed));
 
 struct microcode_amd {
         struct microcode_header_amd hdr;
         unsigned int mpb[0];
 };
 
-#define UCODE_MAX_SIZE          (2048)
-#define DEFAULT_UCODE_DATASIZE  (896)
-#define MC_HEADER_SIZE          (sizeof(struct microcode_header_amd))
-#define DEFAULT_UCODE_TOTALSIZE (DEFAULT_UCODE_DATASIZE + MC_HEADER_SIZE)
-#define DWSIZE                  (sizeof(u32))
-/* For now we support a fixed ucode total size only */
-#define get_totalsize(mc) \
-        ((((struct microcode_amd *)mc)->hdr.mc_patch_data_len * 28) \
-         + MC_HEADER_SIZE)
+#define UCODE_MAX_SIZE                  2048
+#define UCODE_CONTAINER_SECTION_HDR     8
+#define UCODE_CONTAINER_HEADER_SIZE     12
 
 /* serialize access to the physical write */
 static DEFINE_SPINLOCK(microcode_update_lock);
@@ -93,31 +88,24 @@ static struct equiv_cpu_entry *equiv_cpu_table;
 static int collect_cpu_info_amd(int cpu, struct cpu_signature *csig)
 {
         struct cpuinfo_x86 *c = &cpu_data(cpu);
+        u32 dummy;
 
         memset(csig, 0, sizeof(*csig));
-
         if (c->x86_vendor != X86_VENDOR_AMD || c->x86 < 0x10) {
-                printk(KERN_ERR "microcode: CPU%d not a capable AMD processor\n",
-                       cpu);
+                printk(KERN_WARNING "microcode: CPU%d: AMD CPU family 0x%x not "
+                       "supported\n", cpu, c->x86);
                 return -1;
         }
-
-        asm volatile("movl %1, %%ecx; rdmsr"
-                     : "=a" (csig->rev)
-                     : "i" (0x0000008B) : "ecx");
-
-        printk(KERN_INFO "microcode: collect_cpu_info_amd : patch_id=0x%x\n",
-                csig->rev);
-
+        rdmsr(MSR_AMD64_PATCH_LEVEL, csig->rev, dummy);
+        printk(KERN_INFO "microcode: CPU%d: patch_level=0x%x\n", cpu, csig->rev);
         return 0;
 }
 
 static int get_matching_microcode(int cpu, void *mc, int rev)
 {
         struct microcode_header_amd *mc_header = mc;
-        struct pci_dev *nb_pci_dev, *sb_pci_dev;
         unsigned int current_cpu_id;
-        unsigned int equiv_cpu_id = 0x00;
+        u16 equiv_cpu_id = 0;
         unsigned int i = 0;
 
         BUG_ON(equiv_cpu_table == NULL);
@@ -132,57 +120,25 @@ static int get_matching_microcode(int cpu, void *mc, int rev)
         }
 
         if (!equiv_cpu_id) {
-                printk(KERN_ERR "microcode: CPU%d cpu_id "
-                       "not found in equivalent cpu table \n", cpu);
+                printk(KERN_WARNING "microcode: CPU%d: cpu revision "
+                       "not listed in equivalent cpu table\n", cpu);
                 return 0;
         }
 
-        if ((mc_header->processor_rev_id[0]) != (equiv_cpu_id & 0xff)) {
-                printk(KERN_ERR
-                        "microcode: CPU%d patch does not match "
-                        "(patch is %x, cpu extended is %x) \n",
-                        cpu, mc_header->processor_rev_id[0],
-                        (equiv_cpu_id & 0xff));
+        if (mc_header->processor_rev_id != equiv_cpu_id) {
+                printk(KERN_ERR "microcode: CPU%d: patch mismatch "
+                       "(processor_rev_id: %x, equiv_cpu_id: %x)\n",
+                       cpu, mc_header->processor_rev_id, equiv_cpu_id);
                 return 0;
         }
 
-        if ((mc_header->processor_rev_id[1]) != ((equiv_cpu_id >> 16) & 0xff)) {
-                printk(KERN_ERR "microcode: CPU%d patch does not match "
-                        "(patch is %x, cpu base id is %x) \n",
-                        cpu, mc_header->processor_rev_id[1],
-                        ((equiv_cpu_id >> 16) & 0xff));
-
+        /* ucode might be chipset specific -- currently we don't support this */
+        if (mc_header->nb_dev_id || mc_header->sb_dev_id) {
+                printk(KERN_ERR "microcode: CPU%d: loading of chipset "
+                       "specific code not yet supported\n", cpu);
                 return 0;
         }
 
-        /* ucode may be northbridge specific */
-        if (mc_header->nb_dev_id) {
-                nb_pci_dev = pci_get_device(PCI_VENDOR_ID_AMD,
-                                            (mc_header->nb_dev_id & 0xff),
-                                            NULL);
-                if ((!nb_pci_dev) ||
-                    (mc_header->nb_rev_id != nb_pci_dev->revision)) {
-                        printk(KERN_ERR "microcode: CPU%d NB mismatch \n", cpu);
-                        pci_dev_put(nb_pci_dev);
-                        return 0;
-                }
-                pci_dev_put(nb_pci_dev);
-        }
-
-        /* ucode may be southbridge specific */
-        if (mc_header->sb_dev_id) {
-                sb_pci_dev = pci_get_device(PCI_VENDOR_ID_AMD,
-                                            (mc_header->sb_dev_id & 0xff),
-                                            NULL);
-                if ((!sb_pci_dev) ||
-                    (mc_header->sb_rev_id != sb_pci_dev->revision)) {
-                        printk(KERN_ERR "microcode: CPU%d SB mismatch \n", cpu);
-                        pci_dev_put(sb_pci_dev);
-                        return 0;
-                }
-                pci_dev_put(sb_pci_dev);
-        }
-
         if (mc_header->patch_id <= rev)
                 return 0;
 
@@ -192,12 +148,10 @@ static int get_matching_microcode(int cpu, void *mc, int rev)
 static void apply_microcode_amd(int cpu)
 {
         unsigned long flags;
-        unsigned int eax, edx;
-        unsigned int rev;
+        u32 rev, dummy;
         int cpu_num = raw_smp_processor_id();
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu_num;
         struct microcode_amd *mc_amd = uci->mc;
-        unsigned long addr;
 
         /* We should bind the task to the CPU */
         BUG_ON(cpu_num != cpu);
@@ -206,42 +160,34 @@ static void apply_microcode_amd(int cpu)
                 return;
 
         spin_lock_irqsave(&microcode_update_lock, flags);
-
-        addr = (unsigned long)&mc_amd->hdr.data_code;
-        edx = (unsigned int)(((unsigned long)upper_32_bits(addr)));
-        eax = (unsigned int)(((unsigned long)lower_32_bits(addr)));
-
-        asm volatile("movl %0, %%ecx; wrmsr" :
-                     : "i" (0xc0010020), "a" (eax), "d" (edx) : "ecx");
-
+        wrmsrl(MSR_AMD64_PATCH_LOADER, (u64)(long)&mc_amd->hdr.data_code);
         /* get patch id after patching */
-        asm volatile("movl %1, %%ecx; rdmsr"
-                     : "=a" (rev)
-                     : "i" (0x0000008B) : "ecx");
-
+        rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
         spin_unlock_irqrestore(&microcode_update_lock, flags);
 
         /* check current patch id and patch's id for match */
         if (rev != mc_amd->hdr.patch_id) {
-                printk(KERN_ERR "microcode: CPU%d update from revision "
-                       "0x%x to 0x%x failed\n", cpu_num,
-                       mc_amd->hdr.patch_id, rev);
+                printk(KERN_ERR "microcode: CPU%d: update failed "
+                       "(for patch_level=0x%x)\n", cpu, mc_amd->hdr.patch_id);
                 return;
         }
 
-        printk(KERN_INFO "microcode: CPU%d updated from revision "
-               "0x%x to 0x%x \n",
-               cpu_num, uci->cpu_sig.rev, mc_amd->hdr.patch_id);
+        printk(KERN_INFO "microcode: CPU%d: updated (new patch_level=0x%x)\n",
+               cpu, rev);
 
         uci->cpu_sig.rev = rev;
 }
 
-static void * get_next_ucode(u8 *buf, unsigned int size,
-                        int (*get_ucode_data)(void *, const void *, size_t),
-                        unsigned int *mc_size)
+static int get_ucode_data(void *to, const u8 *from, size_t n)
+{
+        memcpy(to, from, n);
+        return 0;
+}
+
+static void *get_next_ucode(const u8 *buf, unsigned int size,
+                            unsigned int *mc_size)
 {
         unsigned int total_size;
-#define UCODE_CONTAINER_SECTION_HDR     8
         u8 section_hdr[UCODE_CONTAINER_SECTION_HDR];
         void *mc;
 
@@ -249,39 +195,37 @@ static void * get_next_ucode(u8 *buf, unsigned int size,
                 return NULL;
 
         if (section_hdr[0] != UCODE_UCODE_TYPE) {
-                printk(KERN_ERR "microcode: error! "
-                       "Wrong microcode payload type field\n");
+                printk(KERN_ERR "microcode: error: invalid type field in "
+                       "container file section header\n");
                 return NULL;
         }
 
         total_size = (unsigned long) (section_hdr[4] + (section_hdr[5] << 8));
 
-        printk(KERN_INFO "microcode: size %u, total_size %u\n",
-                size, total_size);
+        printk(KERN_DEBUG "microcode: size %u, total_size %u\n",
+               size, total_size);
 
         if (total_size > size || total_size > UCODE_MAX_SIZE) {
-                printk(KERN_ERR "microcode: error! Bad data in microcode data file\n");
+                printk(KERN_ERR "microcode: error: size mismatch\n");
                 return NULL;
         }
 
         mc = vmalloc(UCODE_MAX_SIZE);
         if (mc) {
                 memset(mc, 0, UCODE_MAX_SIZE);
-                if (get_ucode_data(mc, buf + UCODE_CONTAINER_SECTION_HDR, total_size)) {
+                if (get_ucode_data(mc, buf + UCODE_CONTAINER_SECTION_HDR,
+                                   total_size)) {
                         vfree(mc);
                         mc = NULL;
                 } else
                         *mc_size = total_size + UCODE_CONTAINER_SECTION_HDR;
         }
-#undef UCODE_CONTAINER_SECTION_HDR
         return mc;
 }
 
 
-static int install_equiv_cpu_table(u8 *buf,
-                int (*get_ucode_data)(void *, const void *, size_t))
+static int install_equiv_cpu_table(const u8 *buf)
 {
-#define UCODE_CONTAINER_HEADER_SIZE     12
         u8 *container_hdr[UCODE_CONTAINER_HEADER_SIZE];
         unsigned int *buf_pos = (unsigned int *)container_hdr;
         unsigned long size;
@@ -292,14 +236,15 @@ static int install_equiv_cpu_table(u8 *buf,
         size = buf_pos[2];
 
         if (buf_pos[1] != UCODE_EQUIV_CPU_TABLE_TYPE || !size) {
-                printk(KERN_ERR "microcode: error! "
-                       "Wrong microcode equivalnet cpu table\n");
+                printk(KERN_ERR "microcode: error: invalid type field in "
+                       "container file section header\n");
                 return 0;
         }
 
         equiv_cpu_table = (struct equiv_cpu_entry *) vmalloc(size);
         if (!equiv_cpu_table) {
-                printk(KERN_ERR "microcode: error, can't allocate memory for equiv CPU table\n");
+                printk(KERN_ERR "microcode: failed to allocate "
+                       "equivalent CPU table\n");
                 return 0;
         }
 
@@ -310,7 +255,6 @@ static int install_equiv_cpu_table(u8 *buf,
         }
 
         return size + UCODE_CONTAINER_HEADER_SIZE; /* add header length */
-#undef UCODE_CONTAINER_HEADER_SIZE
 }
 
 static void free_equiv_cpu_table(void)
@@ -321,18 +265,20 @@ static void free_equiv_cpu_table(void)
         }
 }
 
-static int generic_load_microcode(int cpu, void *data, size_t size,
-                int (*get_ucode_data)(void *, const void *, size_t))
+static int generic_load_microcode(int cpu, const u8 *data, size_t size)
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
-        u8 *ucode_ptr = data, *new_mc = NULL, *mc;
+        const u8 *ucode_ptr = data;
+        void *new_mc = NULL;
+        void *mc;
         int new_rev = uci->cpu_sig.rev;
         unsigned int leftover;
         unsigned long offset;
 
-        offset = install_equiv_cpu_table(ucode_ptr, get_ucode_data);
+        offset = install_equiv_cpu_table(ucode_ptr);
         if (!offset) {
-                printk(KERN_ERR "microcode: installing equivalent cpu table failed\n");
+                printk(KERN_ERR "microcode: failed to create "
+                       "equivalent cpu table\n");
                 return -EINVAL;
         }
 
@@ -343,7 +289,7 @@ static int generic_load_microcode(int cpu, void *data, size_t size,
                 unsigned int uninitialized_var(mc_size);
                 struct microcode_header_amd *mc_header;
 
-                mc = get_next_ucode(ucode_ptr, leftover, get_ucode_data, &mc_size);
+                mc = get_next_ucode(ucode_ptr, leftover, &mc_size);
                 if (!mc)
                         break;
 
@@ -353,7 +299,7 @@ static int generic_load_microcode(int cpu, void *data, size_t size,
                                 vfree(new_mc);
                         new_rev = mc_header->patch_id;
                         new_mc  = mc;
-                } else 
+                } else
                         vfree(mc);
 
                 ucode_ptr += mc_size;
@@ -365,9 +311,9 @@ static int generic_load_microcode(int cpu, void *data, size_t size,
                         if (uci->mc)
                                 vfree(uci->mc);
                         uci->mc = new_mc;
-                        pr_debug("microcode: CPU%d found a matching microcode update with"
-                                " version 0x%x (current=0x%x)\n",
-                                cpu, new_rev, uci->cpu_sig.rev);
+                        pr_debug("microcode: CPU%d found a matching microcode "
+                                 "update with version 0x%x (current=0x%x)\n",
+                                 cpu, new_rev, uci->cpu_sig.rev);
                 } else
                         vfree(new_mc);
         }
@@ -377,12 +323,6 @@ static int generic_load_microcode(int cpu, void *data, size_t size,
         return (int)leftover;
 }
 
-static int get_ucode_fw(void *to, const void *from, size_t n)
-{
-        memcpy(to, from, n);
-        return 0;
-}
-
 static int request_microcode_fw(int cpu, struct device *device)
 {
         const char *fw_name = "amd-ucode/microcode_amd.bin";
@@ -394,12 +334,11 @@ static int request_microcode_fw(int cpu, struct device *device)
 
         ret = request_firmware(&firmware, fw_name, device);
         if (ret) {
-                printk(KERN_ERR "microcode: ucode data file %s load failed\n", fw_name);
+                printk(KERN_ERR "microcode: failed to load file %s\n", fw_name);
                 return ret;
         }
 
-        ret = generic_load_microcode(cpu, (void*)firmware->data, firmware->size,
-                        &get_ucode_fw);
+        ret = generic_load_microcode(cpu, firmware->data, firmware->size);
 
         release_firmware(firmware);
 
@@ -408,8 +347,8 @@ static int request_microcode_fw(int cpu, struct device *device)
 
 static int request_microcode_user(int cpu, const void __user *buf, size_t size)
 {
-        printk(KERN_WARNING "microcode: AMD microcode update via /dev/cpu/microcode"
-                        "is not supported\n");
+        printk(KERN_INFO "microcode: AMD microcode update via "
+               "/dev/cpu/microcode not supported\n");
         return -1;
 }
 
@@ -433,3 +372,4 @@ struct microcode_ops * __init init_amd_microcode(void)
 {
         return &microcode_amd_ops;
 }
+
diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c
index c4b5b24e0217..c9b721ba968c 100644
--- a/arch/x86/kernel/microcode_core.c
+++ b/arch/x86/kernel/microcode_core.c
@@ -99,7 +99,7 @@ MODULE_LICENSE("GPL");
 
 #define MICROCODE_VERSION       "2.00"
 
-struct microcode_ops *microcode_ops;
+static struct microcode_ops *microcode_ops;
 
 /* no concurrent ->write()s are allowed on /dev/cpu/microcode */
 static DEFINE_MUTEX(microcode_mutex);
@@ -203,7 +203,7 @@ MODULE_ALIAS_MISCDEV(MICROCODE_MINOR);
 #endif
 
 /* fake device for request_firmware */
-struct platform_device *microcode_pdev;
+static struct platform_device *microcode_pdev;
 
 static ssize_t reload_store(struct sys_device *dev,
                             struct sysdev_attribute *attr,
@@ -328,7 +328,7 @@ static int microcode_resume_cpu(int cpu)
         return 0;
 }
 
-void microcode_update_cpu(int cpu)
+static void microcode_update_cpu(int cpu)
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
         int err = 0;
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index a8e62792d171..b7f4c929e615 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -471,7 +471,7 @@ static void microcode_fini_cpu(int cpu)
         uci->mc = NULL;
 }
 
-struct microcode_ops microcode_intel_ops = {
+static struct microcode_ops microcode_intel_ops = {
         .request_microcode_user           = request_microcode_user,
         .request_microcode_fw             = request_microcode_fw,
         .collect_cpu_info                 = collect_cpu_info,
diff --git a/arch/x86/kernel/mmconf-fam10h_64.c b/arch/x86/kernel/mmconf-fam10h_64.c
index efc2f361fe85..666e43df51f9 100644
--- a/arch/x86/kernel/mmconf-fam10h_64.c
+++ b/arch/x86/kernel/mmconf-fam10h_64.c
@@ -13,8 +13,7 @@
 #include <asm/msr.h>
 #include <asm/acpi.h>
 #include <asm/mmconfig.h>
-
-#include "../pci/pci.h"
+#include <asm/pci_x86.h>
 
 struct pci_hostbridge_probe {
         u32 bus;
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index 0f4c1fd5a1f4..c5c5b8df1dbc 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -16,14 +16,14 @@
 #include <linux/bitops.h>
 #include <linux/acpi.h>
 #include <linux/module.h>
+#include <linux/smp.h>
+#include <linux/acpi.h>
 
-#include <asm/smp.h>
 #include <asm/mtrr.h>
 #include <asm/mpspec.h>
 #include <asm/pgalloc.h>
 #include <asm/io_apic.h>
 #include <asm/proto.h>
-#include <asm/acpi.h>
 #include <asm/bios_ebda.h>
 #include <asm/e820.h>
 #include <asm/trampoline.h>
@@ -95,8 +95,8 @@ static void __init MP_bus_info(struct mpc_config_bus *m)
 #endif
 
         if (strncmp(str, BUSTYPE_ISA, sizeof(BUSTYPE_ISA) - 1) == 0) {
-                 set_bit(m->mpc_busid, mp_bus_not_pci);
-#if defined(CONFIG_EISA) || defined (CONFIG_MCA)
+                set_bit(m->mpc_busid, mp_bus_not_pci);
+#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
                 mp_bus_id_to_type[m->mpc_busid] = MP_BUS_ISA;
 #endif
         } else if (strncmp(str, BUSTYPE_PCI, sizeof(BUSTYPE_PCI) - 1) == 0) {
@@ -104,7 +104,7 @@ static void __init MP_bus_info(struct mpc_config_bus *m)
                         x86_quirks->mpc_oem_pci_bus(m);
 
                 clear_bit(m->mpc_busid, mp_bus_not_pci);
-#if defined(CONFIG_EISA) || defined (CONFIG_MCA)
+#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
                 mp_bus_id_to_type[m->mpc_busid] = MP_BUS_PCI;
         } else if (strncmp(str, BUSTYPE_EISA, sizeof(BUSTYPE_EISA) - 1) == 0) {
                 mp_bus_id_to_type[m->mpc_busid] = MP_BUS_EISA;
@@ -586,26 +586,23 @@ static void __init __get_smp_config(unsigned int early)
 {
         struct intel_mp_floating *mpf = mpf_found;
 
-        if (x86_quirks->mach_get_smp_config) {
-                if (x86_quirks->mach_get_smp_config(early))
-                        return;
-        }
+        if (!mpf)
+                return;
+
         if (acpi_lapic && early)
                 return;
+
         /*
-         * ACPI supports both logical (e.g. Hyper-Threading) and physical
-         * processors, where MPS only supports physical.
+         * MPS doesn't support hyperthreading, aka only have
+         * thread 0 apic id in MPS table
          */
-        if (acpi_lapic && acpi_ioapic) {
-                printk(KERN_INFO "Using ACPI (MADT) for SMP configuration "
-                       "information\n");
+        if (acpi_lapic && acpi_ioapic)
                 return;
-        } else if (acpi_lapic)
-                printk(KERN_INFO "Using ACPI for processor (LAPIC) "
-                       "configuration information\n");
 
-        if (!mpf)
-                return;
+        if (x86_quirks->mach_get_smp_config) {
+                if (x86_quirks->mach_get_smp_config(early))
+                        return;
+        }
 
         printk(KERN_INFO "Intel MultiProcessor Specification v1.%d\n",
                mpf->mpf_specification);
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 82a7c7ed6d45..726266695b2c 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -136,7 +136,7 @@ static int msr_open(struct inode *inode, struct file *file)
         lock_kernel();
         cpu = iminor(file->f_path.dentry->d_inode);
 
-        if (cpu >= NR_CPUS || !cpu_online(cpu)) {
+        if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
                 ret = -ENXIO;   /* No such CPU */
                 goto out;
         }
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 2c97f07f1c2c..45a09ccdc214 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -26,11 +26,10 @@
 #include <linux/kernel_stat.h>
 #include <linux/kdebug.h>
 #include <linux/smp.h>
+#include <linux/nmi.h>
 
 #include <asm/i8259.h>
 #include <asm/io_apic.h>
-#include <asm/smp.h>
-#include <asm/nmi.h>
 #include <asm/proto.h>
 #include <asm/timer.h>
 
@@ -131,6 +130,11 @@ static void report_broken_nmi(int cpu, int *prev_nmi_count)
         atomic_dec(&nmi_active);
 }
 
+static void __acpi_nmi_disable(void *__unused)
+{
+        apic_write(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED);
+}
+
 int __init check_nmi_watchdog(void)
 {
         unsigned int *prev_nmi_count;
@@ -179,8 +183,12 @@ int __init check_nmi_watchdog(void)
         kfree(prev_nmi_count);
         return 0;
 error:
-        if (nmi_watchdog == NMI_IO_APIC && !timer_through_8259)
-                disable_8259A_irq(0);
+        if (nmi_watchdog == NMI_IO_APIC) {
+                if (!timer_through_8259)
+                        disable_8259A_irq(0);
+                on_each_cpu(__acpi_nmi_disable, NULL, 1);
+        }
+
 #ifdef CONFIG_X86_32
         timer_ack = 0;
 #endif
@@ -199,12 +207,17 @@ static int __init setup_nmi_watchdog(char *str)
                 ++str;
         }
 
-        get_option(&str, &nmi);
-
-        if (nmi >= NMI_INVALID)
-                return 0;
+        if (!strncmp(str, "lapic", 5))
+                nmi_watchdog = NMI_LOCAL_APIC;
+        else if (!strncmp(str, "ioapic", 6))
+                nmi_watchdog = NMI_IO_APIC;
+        else {
+                get_option(&str, &nmi);
+                if (nmi >= NMI_INVALID)
+                        return 0;
+                nmi_watchdog = nmi;
+        }
 
-        nmi_watchdog = nmi;
         return 1;
 }
 __setup("nmi_watchdog=", setup_nmi_watchdog);
@@ -285,11 +298,6 @@ void acpi_nmi_enable(void)
                 on_each_cpu(__acpi_nmi_enable, NULL, 1);
 }
 
-static void __acpi_nmi_disable(void *__unused)
-{
-        apic_write(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED);
-}
-
 /*
  * Disable timer based NMIs on all CPUs:
  */
@@ -340,6 +348,8 @@ void stop_apic_nmi_watchdog(void *unused)
                 return;
         if (nmi_watchdog == NMI_LOCAL_APIC)
                 lapic_watchdog_stop();
+        else
+                __acpi_nmi_disable(NULL);
         __get_cpu_var(wd_enabled) = 0;
         atomic_dec(&nmi_active);
 }
@@ -465,6 +475,24 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
 
 #ifdef CONFIG_SYSCTL
 
+static void enable_ioapic_nmi_watchdog_single(void *unused)
+{
+        __get_cpu_var(wd_enabled) = 1;
+        atomic_inc(&nmi_active);
+        __acpi_nmi_enable(NULL);
+}
+
+static void enable_ioapic_nmi_watchdog(void)
+{
+        on_each_cpu(enable_ioapic_nmi_watchdog_single, NULL, 1);
+        touch_nmi_watchdog();
+}
+
+static void disable_ioapic_nmi_watchdog(void)
+{
+        on_each_cpu(stop_apic_nmi_watchdog, NULL, 1);
+}
+
 static int __init setup_unknown_nmi_panic(char *str)
 {
         unknown_nmi_panic = 1;
@@ -507,6 +535,11 @@ int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
                         enable_lapic_nmi_watchdog();
                 else
                         disable_lapic_nmi_watchdog();
+        } else if (nmi_watchdog == NMI_IO_APIC) {
+                if (nmi_watchdog_enabled)
+                        enable_ioapic_nmi_watchdog();
+                else
+                        disable_ioapic_nmi_watchdog();
         } else {
                 printk(KERN_WARNING
                         "NMI watchdog doesn't know what hardware to touch\n");
diff --git a/arch/x86/kernel/numaq_32.c b/arch/x86/kernel/numaq_32.c
index 4caff39078e0..0deea37a53cf 100644
--- a/arch/x86/kernel/numaq_32.c
+++ b/arch/x86/kernel/numaq_32.c
@@ -31,7 +31,7 @@
 #include <asm/numaq.h>
 #include <asm/topology.h>
 #include <asm/processor.h>
-#include <asm/mpspec.h>
+#include <asm/genapic.h>
 #include <asm/e820.h>
 #include <asm/setup.h>
 
@@ -235,6 +235,13 @@ static int __init numaq_setup_ioapic_ids(void)
         return 1;
 }
 
+static int __init numaq_update_genapic(void)
+{
+        genapic->wakeup_cpu = wakeup_secondary_cpu_via_nmi;
+
+        return 0;
+}
+
 static struct x86_quirks numaq_x86_quirks __initdata = {
         .arch_pre_time_init     = numaq_pre_time_init,
         .arch_time_init         = NULL,
@@ -250,6 +257,7 @@ static struct x86_quirks numaq_x86_quirks __initdata = {
         .mpc_oem_pci_bus        = mpc_oem_pci_bus,
         .smp_read_mpc_oem       = smp_read_mpc_oem,
         .setup_ioapic_ids       = numaq_setup_ioapic_ids,
+        .update_genapic         = numaq_update_genapic,
 };
 
 void numaq_mps_oem_check(struct mp_config_table *mpc, char *oem,
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 192624820217..b25428533141 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -6,6 +6,7 @@
 #include <asm/proto.h>
 #include <asm/dma.h>
 #include <asm/iommu.h>
+#include <asm/gart.h>
 #include <asm/calgary.h>
 #include <asm/amd_iommu.h>
 
@@ -30,11 +31,6 @@ int no_iommu __read_mostly;
 /* Set this to 1 if there is a HW IOMMU in the system */
 int iommu_detected __read_mostly = 0;
 
-/* This tells the BIO block layer to assume merging. Default to off
-   because we cannot guarantee merging later. */
-int iommu_bio_merge __read_mostly = 0;
-EXPORT_SYMBOL(iommu_bio_merge);
-
 dma_addr_t bad_dma_address __read_mostly = 0;
 EXPORT_SYMBOL(bad_dma_address);
 
@@ -42,7 +38,7 @@ EXPORT_SYMBOL(bad_dma_address);
    be probably a smaller DMA mask, but this is bug-to-bug compatible
    to older i386. */
 struct device x86_dma_fallback_dev = {
-        .bus_id = "fallback device",
+        .init_name = "fallback device",
         .coherent_dma_mask = DMA_32BIT_MASK,
         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
 };
@@ -105,11 +101,15 @@ static void __init dma32_free_bootmem(void)
         dma32_bootmem_ptr = NULL;
         dma32_bootmem_size = 0;
 }
+#endif
 
 void __init pci_iommu_alloc(void)
 {
+#ifdef CONFIG_X86_64
         /* free the range so iommu could get some range less than 4G */
         dma32_free_bootmem();
+#endif
+
         /*
          * The order of these functions is important for
          * fall-back/fail-over reasons
@@ -125,15 +125,6 @@ void __init pci_iommu_alloc(void)
         pci_swiotlb_init();
 }
 
-unsigned long iommu_nr_pages(unsigned long addr, unsigned long len)
-{
-        unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);
-
-        return size >> PAGE_SHIFT;
-}
-EXPORT_SYMBOL(iommu_nr_pages);
-#endif
-
 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
                                  dma_addr_t *dma_addr, gfp_t flag)
 {
@@ -188,7 +179,6 @@ static __init int iommu_setup(char *p)
                 }
 
                 if (!strncmp(p, "biomerge", 8)) {
-                        iommu_bio_merge = 4096;
                         iommu_merge = 1;
                         force_iommu = 1;
                 }
@@ -300,8 +290,8 @@ fs_initcall(pci_iommu_init);
 static __devinit void via_no_dac(struct pci_dev *dev)
 {
         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
-                printk(KERN_INFO "PCI: VIA PCI bridge detected."
-                                 "Disabling DAC.\n");
+                printk(KERN_INFO
+                        "PCI: VIA PCI bridge detected. Disabling DAC.\n");
                 forbid_dac = 1;
         }
 }
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index a35eaa379ff6..00c2bcd41463 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -52,7 +52,7 @@ static u32 *iommu_gatt_base;		/* Remapping table */
  * to trigger bugs with some popular PCI cards, in particular 3ware (but
  * has been also also seen with Qlogic at least).
  */
-int iommu_fullflush = 1;
+static int iommu_fullflush = 1;
 
 /* Allocation bitmap for the remapping area: */
 static DEFINE_SPINLOCK(iommu_bitmap_lock);
diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb_64.c
index 3c539d111abb..d59c91747665 100644
--- a/arch/x86/kernel/pci-swiotlb_64.c
+++ b/arch/x86/kernel/pci-swiotlb_64.c
@@ -3,6 +3,8 @@
 #include <linux/pci.h>
 #include <linux/cache.h>
 #include <linux/module.h>
+#include <linux/swiotlb.h>
+#include <linux/bootmem.h>
 #include <linux/dma-mapping.h>
 
 #include <asm/iommu.h>
@@ -11,6 +13,31 @@
 
 int swiotlb __read_mostly;
 
+void * __init swiotlb_alloc_boot(size_t size, unsigned long nslabs)
+{
+        return alloc_bootmem_low_pages(size);
+}
+
+void *swiotlb_alloc(unsigned order, unsigned long nslabs)
+{
+        return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order);
+}
+
+dma_addr_t swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr)
+{
+        return paddr;
+}
+
+phys_addr_t swiotlb_bus_to_phys(dma_addr_t baddr)
+{
+        return baddr;
+}
+
+int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
+{
+        return 0;
+}
+
 static dma_addr_t
 swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size,
                         int direction)
@@ -50,8 +77,10 @@ struct dma_mapping_ops swiotlb_dma_ops = {
 void __init pci_swiotlb_init(void)
 {
         /* don't initialize swiotlb if iommu=off (no_iommu=1) */
+#ifdef CONFIG_X86_64
         if (!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN)
                swiotlb = 1;
+#endif
         if (swiotlb_force)
                 swiotlb = 1;
         if (swiotlb) {
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index c622772744d8..e68bb9e30864 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -1,13 +1,16 @@
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
+#include <asm/idle.h>
 #include <linux/smp.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/module.h>
 #include <linux/pm.h>
 #include <linux/clockchips.h>
+#include <linux/ftrace.h>
 #include <asm/system.h>
+#include <asm/apic.h>
 
 unsigned long idle_halt;
 EXPORT_SYMBOL(idle_halt);
@@ -100,6 +103,9 @@ static inline int hlt_use_halt(void)
 void default_idle(void)
 {
         if (hlt_use_halt()) {
+                struct power_trace it;
+
+                trace_power_start(&it, POWER_CSTATE, 1);
                 current_thread_info()->status &= ~TS_POLLING;
                 /*
                  * TS_POLLING-cleared state must be visible before we
@@ -112,6 +118,7 @@ void default_idle(void)
                 else
                         local_irq_enable();
                 current_thread_info()->status |= TS_POLLING;
+                trace_power_end(&it);
         } else {
                 local_irq_enable();
                 /* loop is done by the caller */
@@ -122,6 +129,21 @@ void default_idle(void)
 EXPORT_SYMBOL(default_idle);
 #endif
 
+void stop_this_cpu(void *dummy)
+{
+        local_irq_disable();
+        /*
+         * Remove this CPU:
+         */
+        cpu_clear(smp_processor_id(), cpu_online_map);
+        disable_local_APIC();
+
+        for (;;) {
+                if (hlt_works(smp_processor_id()))
+                        halt();
+        }
+}
+
 static void do_nothing(void *unused)
 {
 }
@@ -154,24 +176,31 @@ EXPORT_SYMBOL_GPL(cpu_idle_wait);
  */
 void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
 {
+        struct power_trace it;
+
+        trace_power_start(&it, POWER_CSTATE, (ax>>4)+1);
         if (!need_resched()) {
                 __monitor((void *)&current_thread_info()->flags, 0, 0);
                 smp_mb();
                 if (!need_resched())
                         __mwait(ax, cx);
         }
+        trace_power_end(&it);
 }
 
 /* Default MONITOR/MWAIT with no hints, used for default C1 state */
 static void mwait_idle(void)
 {
+        struct power_trace it;
         if (!need_resched()) {
+                trace_power_start(&it, POWER_CSTATE, 1);
                 __monitor((void *)&current_thread_info()->flags, 0, 0);
                 smp_mb();
                 if (!need_resched())
                         __sti_mwait(0, 0);
                 else
                         local_irq_enable();
+                trace_power_end(&it);
         } else
                 local_irq_enable();
 }
@@ -183,9 +212,13 @@ static void mwait_idle(void)
  */
 static void poll_idle(void)
 {
+        struct power_trace it;
+
+        trace_power_start(&it, POWER_CSTATE, 0);
         local_irq_enable();
         while (!need_resched())
                 cpu_relax();
+        trace_power_end(&it);
 }
 
 /*
@@ -270,7 +303,7 @@ static void c1e_idle(void)
                 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
                 if (lo & K8_INTP_C1E_ACTIVE_MASK) {
                         c1e_detected = 1;
-                        if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
+                        if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
                                 mark_tsc_unstable("TSC halt in AMD C1E");
                         printk(KERN_INFO "System has AMD C1E enabled\n");
                         set_cpu_cap(&boot_cpu_data, X86_FEATURE_AMDC1E);
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 0a1302fe6d45..3ba155d24884 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -38,6 +38,7 @@
 #include <linux/percpu.h>
 #include <linux/prctl.h>
 #include <linux/dmi.h>
+#include <linux/ftrace.h>
 
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
@@ -59,6 +60,7 @@
 #include <asm/idle.h>
 #include <asm/syscalls.h>
 #include <asm/smp.h>
+#include <asm/ds.h>
 
 asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 
@@ -250,14 +252,8 @@ void exit_thread(void)
                 tss->x86_tss.io_bitmap_base = INVALID_IO_BITMAP_OFFSET;
                 put_cpu();
         }
-#ifdef CONFIG_X86_DS
-        /* Free any DS contexts that have not been properly released. */
-        if (unlikely(current->thread.ds_ctx)) {
-                /* we clear debugctl to make sure DS is not used. */
-                update_debugctlmsr(0);
-                ds_free(current->thread.ds_ctx);
-        }
-#endif /* CONFIG_X86_DS */
+
+        ds_exit_thread(current);
 }
 
 void flush_thread(void)
@@ -339,6 +335,12 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
                 kfree(p->thread.io_bitmap_ptr);
                 p->thread.io_bitmap_max = 0;
         }
+
+        ds_copy_thread(p, current);
+
+        clear_tsk_thread_flag(p, TIF_DEBUGCTLMSR);
+        p->thread.debugctlmsr = 0;
+
         return err;
 }
 
@@ -419,48 +421,19 @@ int set_tsc_mode(unsigned int val)
         return 0;
 }
 
-#ifdef CONFIG_X86_DS
-static int update_debugctl(struct thread_struct *prev,
-                        struct thread_struct *next, unsigned long debugctl)
-{
-        unsigned long ds_prev = 0;
-        unsigned long ds_next = 0;
-
-        if (prev->ds_ctx)
-                ds_prev = (unsigned long)prev->ds_ctx->ds;
-        if (next->ds_ctx)
-                ds_next = (unsigned long)next->ds_ctx->ds;
-
-        if (ds_next != ds_prev) {
-                /* we clear debugctl to make sure DS
-                 * is not in use when we change it */
-                debugctl = 0;
-                update_debugctlmsr(0);
-                wrmsr(MSR_IA32_DS_AREA, ds_next, 0);
-        }
-        return debugctl;
-}
-#else
-static int update_debugctl(struct thread_struct *prev,
-                        struct thread_struct *next, unsigned long debugctl)
-{
-        return debugctl;
-}
-#endif /* CONFIG_X86_DS */
-
 static noinline void
 __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
                  struct tss_struct *tss)
 {
         struct thread_struct *prev, *next;
-        unsigned long debugctl;
 
         prev = &prev_p->thread;
         next = &next_p->thread;
 
-        debugctl = update_debugctl(prev, next, prev->debugctlmsr);
-
-        if (next->debugctlmsr != debugctl)
+        if (test_tsk_thread_flag(next_p, TIF_DS_AREA_MSR) ||
+            test_tsk_thread_flag(prev_p, TIF_DS_AREA_MSR))
+                ds_switch_to(prev_p, next_p);
+        else if (next->debugctlmsr != prev->debugctlmsr)
                 update_debugctlmsr(next->debugctlmsr);
 
         if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
@@ -482,15 +455,6 @@ __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
                         hard_enable_TSC();
         }
 
-#ifdef CONFIG_X86_PTRACE_BTS
-        if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS))
-                ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS);
-
-        if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS))
-                ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES);
-#endif /* CONFIG_X86_PTRACE_BTS */
-
-
         if (!test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
                 /*
                  * Disable the bitmap via an invalid offset. We still cache
@@ -548,7 +512,8 @@ __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
  * the task-switch, and shows up in ret_from_fork in entry.S,
  * for example.
  */
-struct task_struct * __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
+__notrace_funcgraph struct task_struct *
+__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {
         struct thread_struct *prev = &prev_p->thread,
                                  *next = &next_p->thread;
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index c958120fb1b6..416fb9282f4f 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -39,6 +39,7 @@
 #include <linux/prctl.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/ftrace.h>
 
 #include <asm/pgtable.h>
 #include <asm/system.h>
@@ -52,6 +53,7 @@
 #include <asm/ia32.h>
 #include <asm/idle.h>
 #include <asm/syscalls.h>
+#include <asm/ds.h>
 
 asmlinkage extern void ret_from_fork(void);
 
@@ -235,14 +237,8 @@ void exit_thread(void)
                 t->io_bitmap_max = 0;
                 put_cpu();
         }
-#ifdef CONFIG_X86_DS
-        /* Free any DS contexts that have not been properly released. */
-        if (unlikely(t->ds_ctx)) {
-                /* we clear debugctl to make sure DS is not used. */
-                update_debugctlmsr(0);
-                ds_free(t->ds_ctx);
-        }
-#endif /* CONFIG_X86_DS */
+
+        ds_exit_thread(current);
 }
 
 void flush_thread(void)
@@ -372,6 +368,12 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
                 if (err)
                         goto out;
         }
+
+        ds_copy_thread(p, me);
+
+        clear_tsk_thread_flag(p, TIF_DEBUGCTLMSR);
+        p->thread.debugctlmsr = 0;
+
         err = 0;
 out:
         if (err && p->thread.io_bitmap_ptr) {
@@ -470,35 +472,14 @@ static inline void __switch_to_xtra(struct task_struct *prev_p,
                                     struct tss_struct *tss)
 {
         struct thread_struct *prev, *next;
-        unsigned long debugctl;
 
         prev = &prev_p->thread,
         next = &next_p->thread;
 
-        debugctl = prev->debugctlmsr;
-
-#ifdef CONFIG_X86_DS
-        {
-                unsigned long ds_prev = 0, ds_next = 0;
-
-                if (prev->ds_ctx)
-                        ds_prev = (unsigned long)prev->ds_ctx->ds;
-                if (next->ds_ctx)
-                        ds_next = (unsigned long)next->ds_ctx->ds;
-
-                if (ds_next != ds_prev) {
-                        /*
-                         * We clear debugctl to make sure DS
-                         * is not in use when we change it:
-                         */
-                        debugctl = 0;
-                        update_debugctlmsr(0);
-                        wrmsrl(MSR_IA32_DS_AREA, ds_next);
-                }
-        }
-#endif /* CONFIG_X86_DS */
-
-        if (next->debugctlmsr != debugctl)
+        if (test_tsk_thread_flag(next_p, TIF_DS_AREA_MSR) ||
+            test_tsk_thread_flag(prev_p, TIF_DS_AREA_MSR))
+                ds_switch_to(prev_p, next_p);
+        else if (next->debugctlmsr != prev->debugctlmsr)
                 update_debugctlmsr(next->debugctlmsr);
 
         if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
@@ -533,14 +514,6 @@ static inline void __switch_to_xtra(struct task_struct *prev_p,
                  */
                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
         }
-
-#ifdef CONFIG_X86_PTRACE_BTS
-        if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS))
-                ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS);
-
-        if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS))
-                ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES);
-#endif /* CONFIG_X86_PTRACE_BTS */
 }
 
 /*
@@ -551,8 +524,9 @@ static inline void __switch_to_xtra(struct task_struct *prev_p,
  * - could test fs/gs bitsliced
  *
  * Kprobes not supported here. Set the probe on schedule instead.
+ * Function graph tracer not supported too.
  */
-struct task_struct *
+__notrace_funcgraph struct task_struct *
 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {
         struct thread_struct *prev = &prev_p->thread;
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 0a6d8c12e10d..0a5df5f82fb9 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -581,158 +581,91 @@ static int ioperm_get(struct task_struct *target,
 }
 
 #ifdef CONFIG_X86_PTRACE_BTS
-/*
- * The configuration for a particular BTS hardware implementation.
- */
-struct bts_configuration {
-        /* the size of a BTS record in bytes; at most BTS_MAX_RECORD_SIZE */
-        unsigned char  sizeof_bts;
-        /* the size of a field in the BTS record in bytes */
-        unsigned char  sizeof_field;
-        /* a bitmask to enable/disable BTS in DEBUGCTL MSR */
-        unsigned long debugctl_mask;
-};
-static struct bts_configuration bts_cfg;
-
-#define BTS_MAX_RECORD_SIZE (8 * 3)
-
-
-/*
- * Branch Trace Store (BTS) uses the following format. Different
- * architectures vary in the size of those fields.
- * - source linear address
- * - destination linear address
- * - flags
- *
- * Later architectures use 64bit pointers throughout, whereas earlier
- * architectures use 32bit pointers in 32bit mode.
- *
- * We compute the base address for the first 8 fields based on:
- * - the field size stored in the DS configuration
- * - the relative field position
- *
- * In order to store additional information in the BTS buffer, we use
- * a special source address to indicate that the record requires
- * special interpretation.
- *
- * Netburst indicated via a bit in the flags field whether the branch
- * was predicted; this is ignored.
- */
-
-enum bts_field {
-        bts_from = 0,
-        bts_to,
-        bts_flags,
-
-        bts_escape = (unsigned long)-1,
-        bts_qual = bts_to,
-        bts_jiffies = bts_flags
-};
-
-static inline unsigned long bts_get(const char *base, enum bts_field field)
-{
-        base += (bts_cfg.sizeof_field * field);
-        return *(unsigned long *)base;
-}
-
-static inline void bts_set(char *base, enum bts_field field, unsigned long val)
-{
-        base += (bts_cfg.sizeof_field * field);;
-        (*(unsigned long *)base) = val;
-}
-
-/*
- * Translate a BTS record from the raw format into the bts_struct format
- *
- * out (out): bts_struct interpretation
- * raw: raw BTS record
- */
-static void ptrace_bts_translate_record(struct bts_struct *out, const void *raw)
-{
-        memset(out, 0, sizeof(*out));
-        if (bts_get(raw, bts_from) == bts_escape) {
-                out->qualifier       = bts_get(raw, bts_qual);
-                out->variant.jiffies = bts_get(raw, bts_jiffies);
-        } else {
-                out->qualifier = BTS_BRANCH;
-                out->variant.lbr.from_ip = bts_get(raw, bts_from);
-                out->variant.lbr.to_ip   = bts_get(raw, bts_to);
-        }
-}
-
 static int ptrace_bts_read_record(struct task_struct *child, size_t index,
                                   struct bts_struct __user *out)
 {
-        struct bts_struct ret;
-        const void *bts_record;
-        size_t bts_index, bts_end;
+        const struct bts_trace *trace;
+        struct bts_struct bts;
+        const unsigned char *at;
         int error;
 
-        error = ds_get_bts_end(child, &bts_end);
-        if (error < 0)
-                return error;
-
-        if (bts_end <= index)
-                return -EINVAL;
+        trace = ds_read_bts(child->bts);
+        if (!trace)
+                return -EPERM;
 
-        error = ds_get_bts_index(child, &bts_index);
-        if (error < 0)
-                return error;
+        at = trace->ds.top - ((index + 1) * trace->ds.size);
+        if ((void *)at < trace->ds.begin)
+                at += (trace->ds.n * trace->ds.size);
 
-        /* translate the ptrace bts index into the ds bts index */
-        bts_index += bts_end - (index + 1);
-        if (bts_end <= bts_index)
-                bts_index -= bts_end;
+        if (!trace->read)
+                return -EOPNOTSUPP;
 
-        error = ds_access_bts(child, bts_index, &bts_record);
+        error = trace->read(child->bts, at, &bts);
         if (error < 0)
                 return error;
 
-        ptrace_bts_translate_record(&ret, bts_record);
-
-        if (copy_to_user(out, &ret, sizeof(ret)))
+        if (copy_to_user(out, &bts, sizeof(bts)))
                 return -EFAULT;
 
-        return sizeof(ret);
+        return sizeof(bts);
 }
 
 static int ptrace_bts_drain(struct task_struct *child,
                             long size,
                             struct bts_struct __user *out)
 {
-        struct bts_struct ret;
-        const unsigned char *raw;
-        size_t end, i;
-        int error;
+        const struct bts_trace *trace;
+        const unsigned char *at;
+        int error, drained = 0;
 
-        error = ds_get_bts_index(child, &end);
-        if (error < 0)
-                return error;
+        trace = ds_read_bts(child->bts);
+        if (!trace)
+                return -EPERM;
 
-        if (size < (end * sizeof(struct bts_struct)))
+        if (!trace->read)
+                return -EOPNOTSUPP;
+
+        if (size < (trace->ds.top - trace->ds.begin))
                 return -EIO;
 
-        error = ds_access_bts(child, 0, (const void **)&raw);
-        if (error < 0)
-                return error;
+        for (at = trace->ds.begin; (void *)at < trace->ds.top;
+             out++, drained++, at += trace->ds.size) {
+                struct bts_struct bts;
+                int error;
 
-        for (i = 0; i < end; i++, out++, raw += bts_cfg.sizeof_bts) {
-                ptrace_bts_translate_record(&ret, raw);
+                error = trace->read(child->bts, at, &bts);
+                if (error < 0)
+                        return error;
 
-                if (copy_to_user(out, &ret, sizeof(ret)))
+                if (copy_to_user(out, &bts, sizeof(bts)))
                         return -EFAULT;
         }
 
-        error = ds_clear_bts(child);
+        memset(trace->ds.begin, 0, trace->ds.n * trace->ds.size);
+
+        error = ds_reset_bts(child->bts);
         if (error < 0)
                 return error;
 
-        return end;
+        return drained;
 }
 
-static void ptrace_bts_ovfl(struct task_struct *child)
+static int ptrace_bts_allocate_buffer(struct task_struct *child, size_t size)
 {
-        send_sig(child->thread.bts_ovfl_signal, child, 0);
+        child->bts_buffer = alloc_locked_buffer(size);
+        if (!child->bts_buffer)
+                return -ENOMEM;
+
+        child->bts_size = size;
+
+        return 0;
+}
+
+static void ptrace_bts_free_buffer(struct task_struct *child)
+{
+        free_locked_buffer(child->bts_buffer, child->bts_size);
+        child->bts_buffer = NULL;
+        child->bts_size = 0;
 }
 
 static int ptrace_bts_config(struct task_struct *child,
@@ -740,114 +673,86 @@ static int ptrace_bts_config(struct task_struct *child,
                              const struct ptrace_bts_config __user *ucfg)
 {
         struct ptrace_bts_config cfg;
-        int error = 0;
-
-        error = -EOPNOTSUPP;
-        if (!bts_cfg.sizeof_bts)
-                goto errout;
+        unsigned int flags = 0;
 
-        error = -EIO;
         if (cfg_size < sizeof(cfg))
-                goto errout;
+                return -EIO;
 
-        error = -EFAULT;
         if (copy_from_user(&cfg, ucfg, sizeof(cfg)))
-                goto errout;
+                return -EFAULT;
 
-        error = -EINVAL;
-        if ((cfg.flags & PTRACE_BTS_O_SIGNAL) &&
-            !(cfg.flags & PTRACE_BTS_O_ALLOC))
-                goto errout;
+        if (child->bts) {
+                ds_release_bts(child->bts);
+                child->bts = NULL;
+        }
 
-        if (cfg.flags & PTRACE_BTS_O_ALLOC) {
-                ds_ovfl_callback_t ovfl = NULL;
-                unsigned int sig = 0;
+        if (cfg.flags & PTRACE_BTS_O_SIGNAL) {
+                if (!cfg.signal)
+                        return -EINVAL;
 
-                /* we ignore the error in case we were not tracing child */
-                (void)ds_release_bts(child);
+                return -EOPNOTSUPP;
 
-                if (cfg.flags & PTRACE_BTS_O_SIGNAL) {
-                        if (!cfg.signal)
-                                goto errout;
+                child->thread.bts_ovfl_signal = cfg.signal;
+        }
 
-                        sig  = cfg.signal;
-                        ovfl = ptrace_bts_ovfl;
-                }
+        if ((cfg.flags & PTRACE_BTS_O_ALLOC) &&
+            (cfg.size != child->bts_size)) {
+                int error;
 
-                error = ds_request_bts(child, /* base = */ NULL, cfg.size, ovfl);
-                if (error < 0)
-                        goto errout;
+                ptrace_bts_free_buffer(child);
 
-                child->thread.bts_ovfl_signal = sig;
+                error = ptrace_bts_allocate_buffer(child, cfg.size);
+                if (error < 0)
+                        return error;
         }
 
-        error = -EINVAL;
-        if (!child->thread.ds_ctx && cfg.flags)
-                goto errout;
-
         if (cfg.flags & PTRACE_BTS_O_TRACE)
-                child->thread.debugctlmsr |= bts_cfg.debugctl_mask;
-        else
-                child->thread.debugctlmsr &= ~bts_cfg.debugctl_mask;
+                flags |= BTS_USER;
 
         if (cfg.flags & PTRACE_BTS_O_SCHED)
-                set_tsk_thread_flag(child, TIF_BTS_TRACE_TS);
-        else
-                clear_tsk_thread_flag(child, TIF_BTS_TRACE_TS);
+                flags |= BTS_TIMESTAMPS;
 
-        error = sizeof(cfg);
+        child->bts = ds_request_bts(child, child->bts_buffer, child->bts_size,
+                                    /* ovfl = */ NULL, /* th = */ (size_t)-1,
+                                    flags);
+        if (IS_ERR(child->bts)) {
+                int error = PTR_ERR(child->bts);
 
-out:
-        if (child->thread.debugctlmsr)
-                set_tsk_thread_flag(child, TIF_DEBUGCTLMSR);
-        else
-                clear_tsk_thread_flag(child, TIF_DEBUGCTLMSR);
+                ptrace_bts_free_buffer(child);
+                child->bts = NULL;
 
-        return error;
+                return error;
+        }
 
-errout:
-        child->thread.debugctlmsr &= ~bts_cfg.debugctl_mask;
-        clear_tsk_thread_flag(child, TIF_BTS_TRACE_TS);
-        goto out;
+        return sizeof(cfg);
 }
 
 static int ptrace_bts_status(struct task_struct *child,
                              long cfg_size,
                              struct ptrace_bts_config __user *ucfg)
 {
+        const struct bts_trace *trace;
         struct ptrace_bts_config cfg;
-        size_t end;
-        const void *base, *max;
-        int error;
 
         if (cfg_size < sizeof(cfg))
                 return -EIO;
 
-        error = ds_get_bts_end(child, &end);
-        if (error < 0)
-                return error;
-
-        error = ds_access_bts(child, /* index = */ 0, &base);
-        if (error < 0)
-                return error;
-
-        error = ds_access_bts(child, /* index = */ end, &max);
-        if (error < 0)
-                return error;
+        trace = ds_read_bts(child->bts);
+        if (!trace)
+                return -EPERM;
 
         memset(&cfg, 0, sizeof(cfg));
-        cfg.size = (max - base);
+        cfg.size = trace->ds.end - trace->ds.begin;
         cfg.signal = child->thread.bts_ovfl_signal;
         cfg.bts_size = sizeof(struct bts_struct);
 
         if (cfg.signal)
                 cfg.flags |= PTRACE_BTS_O_SIGNAL;
 
-        if (test_tsk_thread_flag(child, TIF_DEBUGCTLMSR) &&
-            child->thread.debugctlmsr & bts_cfg.debugctl_mask)
+        if (trace->ds.flags & BTS_USER)
                 cfg.flags |= PTRACE_BTS_O_TRACE;
 
-        if (test_tsk_thread_flag(child, TIF_BTS_TRACE_TS))
+        if (trace->ds.flags & BTS_TIMESTAMPS)
                 cfg.flags |= PTRACE_BTS_O_SCHED;
 
         if (copy_to_user(ucfg, &cfg, sizeof(cfg)))
@@ -856,110 +761,77 @@ static int ptrace_bts_status(struct task_struct *child,
         return sizeof(cfg);
 }
 
-static int ptrace_bts_write_record(struct task_struct *child,
-                                   const struct bts_struct *in)
+static int ptrace_bts_clear(struct task_struct *child)
 {
-        unsigned char bts_record[BTS_MAX_RECORD_SIZE];
+        const struct bts_trace *trace;
 
-        BUG_ON(BTS_MAX_RECORD_SIZE < bts_cfg.sizeof_bts);
+        trace = ds_read_bts(child->bts);
+        if (!trace)
+                return -EPERM;
 
-        memset(bts_record, 0, bts_cfg.sizeof_bts);
-        switch (in->qualifier) {
-        case BTS_INVALID:
-                break;
+        memset(trace->ds.begin, 0, trace->ds.n * trace->ds.size);
 
-        case BTS_BRANCH:
-                bts_set(bts_record, bts_from, in->variant.lbr.from_ip);
-                bts_set(bts_record, bts_to,   in->variant.lbr.to_ip);
-                break;
+        return ds_reset_bts(child->bts);
+}
 
-        case BTS_TASK_ARRIVES:
-        case BTS_TASK_DEPARTS:
-                bts_set(bts_record, bts_from,    bts_escape);
-                bts_set(bts_record, bts_qual,    in->qualifier);
-                bts_set(bts_record, bts_jiffies, in->variant.jiffies);
-                break;
+static int ptrace_bts_size(struct task_struct *child)
+{
+        const struct bts_trace *trace;
 
-        default:
-                return -EINVAL;
-        }
+        trace = ds_read_bts(child->bts);
+        if (!trace)
+                return -EPERM;
 
-        /* The writing task will be the switched-to task on a context
-         * switch. It needs to write into the switched-from task's BTS
-         * buffer. */
-        return ds_unchecked_write_bts(child, bts_record, bts_cfg.sizeof_bts);
+        return (trace->ds.top - trace->ds.begin) / trace->ds.size;
 }
 
-void ptrace_bts_take_timestamp(struct task_struct *tsk,
-                               enum bts_qualifier qualifier)
+static void ptrace_bts_fork(struct task_struct *tsk)
 {
-        struct bts_struct rec = {
-                .qualifier = qualifier,
-                .variant.jiffies = jiffies_64
-        };
-
-        ptrace_bts_write_record(tsk, &rec);
+        tsk->bts = NULL;
+        tsk->bts_buffer = NULL;
+        tsk->bts_size = 0;
+        tsk->thread.bts_ovfl_signal = 0;
 }
 
-static const struct bts_configuration bts_cfg_netburst = {
-        .sizeof_bts    = sizeof(long) * 3,
-        .sizeof_field  = sizeof(long),
-        .debugctl_mask = (1<<2)|(1<<3)|(1<<5)
-};
+static void ptrace_bts_untrace(struct task_struct *child)
+{
+        if (unlikely(child->bts)) {
+                ds_release_bts(child->bts);
+                child->bts = NULL;
+
+                /* We cannot update total_vm and locked_vm since
+                   child's mm is already gone. But we can reclaim the
+                   memory. */
+                kfree(child->bts_buffer);
+                child->bts_buffer = NULL;
+                child->bts_size = 0;
+        }
+}
 
-static const struct bts_configuration bts_cfg_pentium_m = {
-        .sizeof_bts    = sizeof(long) * 3,
-        .sizeof_field  = sizeof(long),
-        .debugctl_mask = (1<<6)|(1<<7)
-};
+static void ptrace_bts_detach(struct task_struct *child)
+{
+        if (unlikely(child->bts)) {
+                ds_release_bts(child->bts);
+                child->bts = NULL;
 
-static const struct bts_configuration bts_cfg_core2 = {
-        .sizeof_bts    = 8 * 3,
-        .sizeof_field  = 8,
-        .debugctl_mask = (1<<6)|(1<<7)|(1<<9)
-};
+                ptrace_bts_free_buffer(child);
+        }
+}
+#else
+static inline void ptrace_bts_fork(struct task_struct *tsk) {}
+static inline void ptrace_bts_detach(struct task_struct *child) {}
+static inline void ptrace_bts_untrace(struct task_struct *child) {}
+#endif /* CONFIG_X86_PTRACE_BTS */
 
-static inline void bts_configure(const struct bts_configuration *cfg)
+void x86_ptrace_fork(struct task_struct *child, unsigned long clone_flags)
 {
-        bts_cfg = *cfg;
+        ptrace_bts_fork(child);
 }
 
-void __cpuinit ptrace_bts_init_intel(struct cpuinfo_x86 *c)
+void x86_ptrace_untrace(struct task_struct *child)
 {
-        switch (c->x86) {
-        case 0x6:
-                switch (c->x86_model) {
-                case 0xD:
-                case 0xE: /* Pentium M */
-                        bts_configure(&bts_cfg_pentium_m);
-                        break;
-                case 0xF: /* Core2 */
-        case 0x1C: /* Atom */
-                        bts_configure(&bts_cfg_core2);
-                        break;
-                default:
-                        /* sorry, don't know about them */
-                        break;
-                }
-                break;
-        case 0xF:
-                switch (c->x86_model) {
-                case 0x0:
-                case 0x1:
-                case 0x2: /* Netburst */
-                        bts_configure(&bts_cfg_netburst);
-                        break;
-                default:
-                        /* sorry, don't know about them */
-                        break;
-                }
-                break;
-        default:
-                /* sorry, don't know about them */
-                break;
-        }
+        ptrace_bts_untrace(child);
 }
-#endif /* CONFIG_X86_PTRACE_BTS */
 
 /*
  * Called by kernel/ptrace.c when detaching..
@@ -972,15 +844,7 @@ void ptrace_disable(struct task_struct *child)
 #ifdef TIF_SYSCALL_EMU
         clear_tsk_thread_flag(child, TIF_SYSCALL_EMU);
 #endif
-#ifdef CONFIG_X86_PTRACE_BTS
-        (void)ds_release_bts(child);
-
-        child->thread.debugctlmsr &= ~bts_cfg.debugctl_mask;
-        if (!child->thread.debugctlmsr)
-                clear_tsk_thread_flag(child, TIF_DEBUGCTLMSR);
-
-        clear_tsk_thread_flag(child, TIF_BTS_TRACE_TS);
-#endif /* CONFIG_X86_PTRACE_BTS */
+        ptrace_bts_detach(child);
 }
 
 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
@@ -1112,7 +976,7 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data)
                 break;
 
         case PTRACE_BTS_SIZE:
-                ret = ds_get_bts_index(child, /* pos = */ NULL);
+                ret = ptrace_bts_size(child);
                 break;
 
         case PTRACE_BTS_GET:
@@ -1121,7 +985,7 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data)
                 break;
 
         case PTRACE_BTS_CLEAR:
-                ret = ds_clear_bts(child);
+                ret = ptrace_bts_clear(child);
                 break;
 
         case PTRACE_BTS_DRAIN:
@@ -1384,6 +1248,14 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 
         case PTRACE_GET_THREAD_AREA:
         case PTRACE_SET_THREAD_AREA:
+#ifdef CONFIG_X86_PTRACE_BTS
+        case PTRACE_BTS_CONFIG:
+        case PTRACE_BTS_STATUS:
+        case PTRACE_BTS_SIZE:
+        case PTRACE_BTS_GET:
+        case PTRACE_BTS_CLEAR:
+        case PTRACE_BTS_DRAIN:
+#endif /* CONFIG_X86_PTRACE_BTS */
                 return arch_ptrace(child, request, addr, data);
 
         default:
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index 67465ed89310..309949e9e1c1 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -168,6 +168,8 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH7_31,
                          ich_force_enable_hpet);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH8_1,
                          ich_force_enable_hpet);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH8_4,
+                         ich_force_enable_hpet);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH9_7,
                          ich_force_enable_hpet);
 
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index cc5a2545dd41..2b46eb41643b 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -12,6 +12,8 @@
 #include <asm/proto.h>
 #include <asm/reboot_fixups.h>
 #include <asm/reboot.h>
+#include <asm/pci_x86.h>
+#include <asm/virtext.h>
 
 #ifdef CONFIG_X86_32
 # include <linux/dmi.h>
@@ -21,6 +23,8 @@
 # include <asm/iommu.h>
 #endif
 
+#include <mach_ipi.h>
+
 /*
  * Power off function, if any
  */
@@ -36,7 +40,16 @@ int reboot_force;
 static int reboot_cpu = -1;
 #endif
 
-/* reboot=b[ios] | s[mp] | t[riple] | k[bd] | e[fi] [, [w]arm | [c]old]
+/* This is set if we need to go through the 'emergency' path.
+ * When machine_emergency_restart() is called, we may be on
+ * an inconsistent state and won't be able to do a clean cleanup
+ */
+static int reboot_emergency;
+
+/* This is set by the PCI code if either type 1 or type 2 PCI is detected */
+bool port_cf9_safe = false;
+
+/* reboot=b[ios] | s[mp] | t[riple] | k[bd] | e[fi] [, [w]arm | [c]old] | p[ci]
    warm   Don't set the cold reboot flag
    cold   Set the cold reboot flag
    bios   Reboot by jumping through the BIOS (only for X86_32)
@@ -45,6 +58,7 @@ static int reboot_cpu = -1;
    kbd    Use the keyboard controller. cold reset (default)
    acpi   Use the RESET_REG in the FADT
    efi    Use efi reset_system runtime service
+   pci    Use the so-called "PCI reset register", CF9
    force  Avoid anything that could hang.
  */
 static int __init reboot_setup(char *str)
@@ -79,6 +93,7 @@ static int __init reboot_setup(char *str)
                 case 'k':
                 case 't':
                 case 'e':
+                case 'p':
                         reboot_type = *str;
                         break;
 
@@ -360,6 +375,48 @@ static inline void kb_wait(void)
         }
 }
 
+static void vmxoff_nmi(int cpu, struct die_args *args)
+{
+        cpu_emergency_vmxoff();
+}
+
+/* Use NMIs as IPIs to tell all CPUs to disable virtualization
+ */
+static void emergency_vmx_disable_all(void)
+{
+        /* Just make sure we won't change CPUs while doing this */
+        local_irq_disable();
+
+        /* We need to disable VMX on all CPUs before rebooting, otherwise
+         * we risk hanging up the machine, because the CPU ignore INIT
+         * signals when VMX is enabled.
+         *
+         * We can't take any locks and we may be on an inconsistent
+         * state, so we use NMIs as IPIs to tell the other CPUs to disable
+         * VMX and halt.
+         *
+         * For safety, we will avoid running the nmi_shootdown_cpus()
+         * stuff unnecessarily, but we don't have a way to check
+         * if other CPUs have VMX enabled. So we will call it only if the
+         * CPU we are running on has VMX enabled.
+         *
+         * We will miss cases where VMX is not enabled on all CPUs. This
+         * shouldn't do much harm because KVM always enable VMX on all
+         * CPUs anyway. But we can miss it on the small window where KVM
+         * is still enabling VMX.
+         */
+        if (cpu_has_vmx() && cpu_vmx_enabled()) {
+                /* Disable VMX on this CPU.
+                 */
+                cpu_vmxoff();
+
+                /* Halt and disable VMX on the other CPUs */
+                nmi_shootdown_cpus(vmxoff_nmi);
+
+        }
+}
+
+
 void __attribute__((weak)) mach_reboot_fixups(void)
 {
 }
@@ -368,6 +425,9 @@ static void native_machine_emergency_restart(void)
 {
         int i;
 
+        if (reboot_emergency)
+                emergency_vmx_disable_all();
+
         /* Tell the BIOS if we want cold or warm reboot */
         *((unsigned short *)__va(0x472)) = reboot_mode;
 
@@ -404,12 +464,27 @@ static void native_machine_emergency_restart(void)
                         reboot_type = BOOT_KBD;
                         break;
 
-
                 case BOOT_EFI:
                         if (efi_enabled)
-                                efi.reset_system(reboot_mode ? EFI_RESET_WARM : EFI_RESET_COLD,
+                                efi.reset_system(reboot_mode ?
+                                                 EFI_RESET_WARM :
+                                                 EFI_RESET_COLD,
                                                  EFI_SUCCESS, 0, NULL);
+                        reboot_type = BOOT_KBD;
+                        break;
+
+                case BOOT_CF9:
+                        port_cf9_safe = true;
+                        /* fall through */
 
+                case BOOT_CF9_COND:
+                        if (port_cf9_safe) {
+                                u8 cf9 = inb(0xcf9) & ~6;
+                                outb(cf9|2, 0xcf9); /* Request hard reset */
+                                udelay(50);
+                                outb(cf9|6, 0xcf9); /* Actually do the reset */
+                                udelay(50);
+                        }
                         reboot_type = BOOT_KBD;
                         break;
                 }
@@ -426,7 +501,7 @@ void native_machine_shutdown(void)
 
 #ifdef CONFIG_X86_32
         /* See if there has been given a command line override */
-        if ((reboot_cpu != -1) && (reboot_cpu < NR_CPUS) &&
+        if ((reboot_cpu != -1) && (reboot_cpu < nr_cpu_ids) &&
                 cpu_online(reboot_cpu))
                 reboot_cpu_id = reboot_cpu;
 #endif
@@ -436,7 +511,7 @@ void native_machine_shutdown(void)
                 reboot_cpu_id = smp_processor_id();
 
         /* Make certain I only run on the appropriate processor */
-        set_cpus_allowed_ptr(current, &cpumask_of_cpu(reboot_cpu_id));
+        set_cpus_allowed_ptr(current, cpumask_of(reboot_cpu_id));
 
         /* O.K Now that I'm on the appropriate processor,
          * stop all of the others.
@@ -459,17 +534,28 @@ void native_machine_shutdown(void)
 #endif
 }
 
+static void __machine_emergency_restart(int emergency)
+{
+        reboot_emergency = emergency;
+        machine_ops.emergency_restart();
+}
+
 static void native_machine_restart(char *__unused)
 {
         printk("machine restart\n");
 
         if (!reboot_force)
                 machine_shutdown();
-        machine_emergency_restart();
+        __machine_emergency_restart(0);
 }
 
 static void native_machine_halt(void)
 {
+        /* stop other cpus and apics */
+        machine_shutdown();
+
+        /* stop this cpu */
+        stop_this_cpu(NULL);
 }
 
 static void native_machine_power_off(void)
@@ -504,7 +590,7 @@ void machine_shutdown(void)
 
 void machine_emergency_restart(void)
 {
-        machine_ops.emergency_restart();
+        __machine_emergency_restart(1);
 }
 
 void machine_restart(char *cmd)
@@ -523,3 +609,92 @@ void machine_crash_shutdown(struct pt_regs *regs)
         machine_ops.crash_shutdown(regs);
 }
 #endif
+
+
+#if defined(CONFIG_SMP)
+
+/* This keeps a track of which one is crashing cpu. */
+static int crashing_cpu;
+static nmi_shootdown_cb shootdown_callback;
+
+static atomic_t waiting_for_crash_ipi;
+
+static int crash_nmi_callback(struct notifier_block *self,
+                        unsigned long val, void *data)
+{
+        int cpu;
+
+        if (val != DIE_NMI_IPI)
+                return NOTIFY_OK;
+
+        cpu = raw_smp_processor_id();
+
+        /* Don't do anything if this handler is invoked on crashing cpu.
+         * Otherwise, system will completely hang. Crashing cpu can get
+         * an NMI if system was initially booted with nmi_watchdog parameter.
+         */
+        if (cpu == crashing_cpu)
+                return NOTIFY_STOP;
+        local_irq_disable();
+
+        shootdown_callback(cpu, (struct die_args *)data);
+
+        atomic_dec(&waiting_for_crash_ipi);
+        /* Assume hlt works */
+        halt();
+        for (;;)
+                cpu_relax();
+
+        return 1;
+}
+
+static void smp_send_nmi_allbutself(void)
+{
+        send_IPI_allbutself(NMI_VECTOR);
+}
+
+static struct notifier_block crash_nmi_nb = {
+        .notifier_call = crash_nmi_callback,
+};
+
+/* Halt all other CPUs, calling the specified function on each of them
+ *
+ * This function can be used to halt all other CPUs on crash
+ * or emergency reboot time. The function passed as parameter
+ * will be called inside a NMI handler on all CPUs.
+ */
+void nmi_shootdown_cpus(nmi_shootdown_cb callback)
+{
+        unsigned long msecs;
+        local_irq_disable();
+
+        /* Make a note of crashing cpu. Will be used in NMI callback.*/
+        crashing_cpu = safe_smp_processor_id();
+
+        shootdown_callback = callback;
+
+        atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1);
+        /* Would it be better to replace the trap vector here? */
+        if (register_die_notifier(&crash_nmi_nb))
+                return;         /* return what? */
+        /* Ensure the new callback function is set before sending
+         * out the NMI
+         */
+        wmb();
+
+        smp_send_nmi_allbutself();
+
+        msecs = 1000; /* Wait at most a second for the other cpus to stop */
+        while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) {
+                mdelay(1);
+                msecs--;
+        }
+
+        /* Leave the nmi callback set */
+}
+#else /* !CONFIG_SMP */
+void nmi_shootdown_cpus(nmi_shootdown_cb callback)
+{
+        /* No other CPUs to shoot down */
+}
+#endif
diff --git a/arch/x86/kernel/relocate_kernel_32.S b/arch/x86/kernel/relocate_kernel_32.S
index 6f50664b2ba5..a160f3119725 100644
--- a/arch/x86/kernel/relocate_kernel_32.S
+++ b/arch/x86/kernel/relocate_kernel_32.S
@@ -10,15 +10,12 @@
 #include <asm/page.h>
 #include <asm/kexec.h>
 #include <asm/processor-flags.h>
-#include <asm/pgtable.h>
 
 /*
  * Must be relocatable PIC code callable as a C function
  */
 
 #define PTR(x) (x << 2)
-#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)
-#define PAE_PGD_ATTR (_PAGE_PRESENT)
 
 /* control_page + KEXEC_CONTROL_CODE_MAX_SIZE
  * ~ control_page + PAGE_SIZE are used as data storage and stack for
@@ -39,7 +36,6 @@
 #define CP_PA_BACKUP_PAGES_MAP  DATA(0x1c)
 
         .text
-        .align PAGE_SIZE
         .globl relocate_kernel
 relocate_kernel:
         /* Save the CPU context, used for jumping back */
@@ -60,117 +56,6 @@ relocate_kernel:
         movl    %cr4, %eax
         movl    %eax, CR4(%edi)
 
-#ifdef CONFIG_X86_PAE
-        /* map the control page at its virtual address */
-
-        movl    PTR(VA_PGD)(%ebp), %edi
-        movl    PTR(VA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0xc0000000, %eax
-        shrl    $27, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_PMD_0)(%ebp), %edx
-        orl     $PAE_PGD_ATTR, %edx
-        movl    %edx, (%eax)
-
-        movl    PTR(VA_PMD_0)(%ebp), %edi
-        movl    PTR(VA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0x3fe00000, %eax
-        shrl    $18, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_PTE_0)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-
-        movl    PTR(VA_PTE_0)(%ebp), %edi
-        movl    PTR(VA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0x001ff000, %eax
-        shrl    $9, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-
-        /* identity map the control page at its physical address */
-
-        movl    PTR(VA_PGD)(%ebp), %edi
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0xc0000000, %eax
-        shrl    $27, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_PMD_1)(%ebp), %edx
-        orl     $PAE_PGD_ATTR, %edx
-        movl    %edx, (%eax)
-
-        movl    PTR(VA_PMD_1)(%ebp), %edi
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0x3fe00000, %eax
-        shrl    $18, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_PTE_1)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-
-        movl    PTR(VA_PTE_1)(%ebp), %edi
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0x001ff000, %eax
-        shrl    $9, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-#else
-        /* map the control page at its virtual address */
-
-        movl    PTR(VA_PGD)(%ebp), %edi
-        movl    PTR(VA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0xffc00000, %eax
-        shrl    $20, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_PTE_0)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-
-        movl    PTR(VA_PTE_0)(%ebp), %edi
-        movl    PTR(VA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0x003ff000, %eax
-        shrl    $10, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-
-        /* identity map the control page at its physical address */
-
-        movl    PTR(VA_PGD)(%ebp), %edi
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0xffc00000, %eax
-        shrl    $20, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_PTE_1)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-
-        movl    PTR(VA_PTE_1)(%ebp), %edi
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %eax
-        andl    $0x003ff000, %eax
-        shrl    $10, %eax
-        addl    %edi, %eax
-
-        movl    PTR(PA_CONTROL_PAGE)(%ebp), %edx
-        orl     $PAGE_ATTR, %edx
-        movl    %edx, (%eax)
-#endif
-
-relocate_new_kernel:
         /* read the arguments and say goodbye to the stack */
         movl  20+4(%esp), %ebx /* page_list */
         movl  20+8(%esp), %ebp /* list of pages */
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index bdec76e55594..ae0d8042cf69 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -93,11 +93,13 @@
 #include <asm/desc.h>
 #include <asm/dma.h>
 #include <asm/iommu.h>
+#include <asm/gart.h>
 #include <asm/mmu_context.h>
 #include <asm/proto.h>
 
 #include <mach_apic.h>
 #include <asm/paravirt.h>
+#include <asm/hypervisor.h>
 
 #include <asm/percpu.h>
 #include <asm/topology.h>
@@ -448,6 +450,7 @@ static void __init reserve_early_setup_data(void)
  * @size: Size of the crashkernel memory to reserve.
  * Returns the base address on success, and -1ULL on failure.
  */
+static
 unsigned long long __init find_and_reserve_crashkernel(unsigned long long size)
 {
         const unsigned long long alignment = 16<<20;    /* 16M */
@@ -583,161 +586,24 @@ static int __init setup_elfcorehdr(char *arg)
 early_param("elfcorehdr", setup_elfcorehdr);
 #endif
 
-static struct x86_quirks default_x86_quirks __initdata;
-
-struct x86_quirks *x86_quirks __initdata = &default_x86_quirks;
-
-/*
- * Some BIOSes seem to corrupt the low 64k of memory during events
- * like suspend/resume and unplugging an HDMI cable.  Reserve all
- * remaining free memory in that area and fill it with a distinct
- * pattern.
- */
-#ifdef CONFIG_X86_CHECK_BIOS_CORRUPTION
-#define MAX_SCAN_AREAS  8
-
-static int __read_mostly memory_corruption_check = -1;
-
-static unsigned __read_mostly corruption_check_size = 64*1024;
-static unsigned __read_mostly corruption_check_period = 60; /* seconds */
-
-static struct e820entry scan_areas[MAX_SCAN_AREAS];
-static int num_scan_areas;
-
-
-static int set_corruption_check(char *arg)
-{
-        char *end;
-
-        memory_corruption_check = simple_strtol(arg, &end, 10);
-
-        return (*end == 0) ? 0 : -EINVAL;
-}
-early_param("memory_corruption_check", set_corruption_check);
-
-static int set_corruption_check_period(char *arg)
-{
-        char *end;
-
-        corruption_check_period = simple_strtoul(arg, &end, 10);
-
-        return (*end == 0) ? 0 : -EINVAL;
-}
-early_param("memory_corruption_check_period", set_corruption_check_period);
-
-static int set_corruption_check_size(char *arg)
+static int __init default_update_genapic(void)
 {
-        char *end;
-        unsigned size;
-
-        size = memparse(arg, &end);
-
-        if (*end == '\0')
-                corruption_check_size = size;
-
-        return (size == corruption_check_size) ? 0 : -EINVAL;
-}
-early_param("memory_corruption_check_size", set_corruption_check_size);
-
-
-static void __init setup_bios_corruption_check(void)
-{
-        u64 addr = PAGE_SIZE;   /* assume first page is reserved anyway */
-
-        if (memory_corruption_check == -1) {
-                memory_corruption_check =
-#ifdef CONFIG_X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK
-                        1
-#else
-                        0
+#ifdef CONFIG_X86_SMP
+# if defined(CONFIG_X86_GENERICARCH) || defined(CONFIG_X86_64)
+        genapic->wakeup_cpu = wakeup_secondary_cpu_via_init;
+# endif
 #endif
-                        ;
-        }
-
-        if (corruption_check_size == 0)
-                memory_corruption_check = 0;
-
-        if (!memory_corruption_check)
-                return;
-
-        corruption_check_size = round_up(corruption_check_size, PAGE_SIZE);
 
-        while(addr < corruption_check_size && num_scan_areas < MAX_SCAN_AREAS) {
-                u64 size;
-                addr = find_e820_area_size(addr, &size, PAGE_SIZE);
-
-                if (addr == 0)
-                        break;
-
-                if ((addr + size) > corruption_check_size)
-                        size = corruption_check_size - addr;
-
-                if (size == 0)
-                        break;
-
-                e820_update_range(addr, size, E820_RAM, E820_RESERVED);
-                scan_areas[num_scan_areas].addr = addr;
-                scan_areas[num_scan_areas].size = size;
-                num_scan_areas++;
-
-                /* Assume we've already mapped this early memory */
-                memset(__va(addr), 0, size);
-
-                addr += size;
-        }
-
-        printk(KERN_INFO "Scanning %d areas for low memory corruption\n",
-               num_scan_areas);
-        update_e820();
-}
-
-static struct timer_list periodic_check_timer;
-
-void check_for_bios_corruption(void)
-{
-        int i;
-        int corruption = 0;
-
-        if (!memory_corruption_check)
-                return;
-
-        for(i = 0; i < num_scan_areas; i++) {
-                unsigned long *addr = __va(scan_areas[i].addr);
-                unsigned long size = scan_areas[i].size;
-
-                for(; size; addr++, size -= sizeof(unsigned long)) {
-                        if (!*addr)
-                                continue;
-                        printk(KERN_ERR "Corrupted low memory at %p (%lx phys) = %08lx\n",
-                               addr, __pa(addr), *addr);
-                        corruption = 1;
-                        *addr = 0;
-                }
-        }
-
-        WARN(corruption, KERN_ERR "Memory corruption detected in low memory\n");
-}
-
-static void periodic_check_for_corruption(unsigned long data)
-{
-        check_for_bios_corruption();
-        mod_timer(&periodic_check_timer, round_jiffies(jiffies + corruption_check_period*HZ));
+        return 0;
 }
 
-void start_periodic_check_for_corruption(void)
-{
-        if (!memory_corruption_check || corruption_check_period == 0)
-                return;
-
-        printk(KERN_INFO "Scanning for low memory corruption every %d seconds\n",
-               corruption_check_period);
+static struct x86_quirks default_x86_quirks __initdata = {
+        .update_genapic         = default_update_genapic,
+};
 
-        init_timer(&periodic_check_timer);
-        periodic_check_timer.function = &periodic_check_for_corruption;
-        periodic_check_for_corruption(0);
-}
-#endif
+struct x86_quirks *x86_quirks __initdata = &default_x86_quirks;
 
+#ifdef CONFIG_X86_RESERVE_LOW_64K
 static int __init dmi_low_memory_corruption(const struct dmi_system_id *d)
 {
         printk(KERN_NOTICE
@@ -749,6 +615,7 @@ static int __init dmi_low_memory_corruption(const struct dmi_system_id *d)
 
         return 0;
 }
+#endif
 
 /* List of systems that have known low memory corruption BIOS problems */
 static struct dmi_system_id __initdata bad_bios_dmi_table[] = {
@@ -907,6 +774,12 @@ void __init setup_arch(char **cmdline_p)
 
         dmi_check_system(bad_bios_dmi_table);
 
+        /*
+         * VMware detection requires dmi to be available, so this
+         * needs to be done after dmi_scan_machine, for the BP.
+         */
+        init_hypervisor(&boot_cpu_data);
+
 #ifdef CONFIG_X86_32
         probe_roms();
 #endif
@@ -1080,7 +953,7 @@ void __init setup_arch(char **cmdline_p)
         ioapic_init_mappings();
 
         /* need to wait for io_apic is mapped */
-        nr_irqs = probe_nr_irqs();
+        probe_nr_irqs_gsi();
 
         kvm_guest_init();
 
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index ae0c0d3bb770..a4b619c33106 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -152,8 +152,11 @@ void __init setup_per_cpu_areas(void)
         old_size = PERCPU_ENOUGH_ROOM;
         align = max_t(unsigned long, PAGE_SIZE, align);
         size = roundup(old_size, align);
-        printk(KERN_INFO "PERCPU: Allocating %zd bytes of per cpu data\n",
-                          size);
+
+        pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
+                NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
+
+        pr_info("PERCPU: Allocating %zd bytes of per cpu data\n", size);
 
         for_each_possible_cpu(cpu) {
 #ifndef CONFIG_NEED_MULTIPLE_NODES
@@ -164,28 +167,21 @@ void __init setup_per_cpu_areas(void)
                 if (!node_online(node) || !NODE_DATA(node)) {
                         ptr = __alloc_bootmem(size, align,
                                          __pa(MAX_DMA_ADDRESS));
-                        printk(KERN_INFO
-                               "cpu %d has no node %d or node-local memory\n",
+                        pr_info("cpu %d has no node %d or node-local memory\n",
                                 cpu, node);
-                        if (ptr)
-                                printk(KERN_DEBUG "per cpu data for cpu%d at %016lx\n",
-                                         cpu, __pa(ptr));
-                }
-                else {
+                        pr_debug("per cpu data for cpu%d at %016lx\n",
+                                 cpu, __pa(ptr));
+                } else {
                         ptr = __alloc_bootmem_node(NODE_DATA(node), size, align,
                                                         __pa(MAX_DMA_ADDRESS));
-                        if (ptr)
-                                printk(KERN_DEBUG "per cpu data for cpu%d on node%d at %016lx\n",
-                                         cpu, node, __pa(ptr));
+                        pr_debug("per cpu data for cpu%d on node%d at %016lx\n",
+                                cpu, node, __pa(ptr));
                 }
 #endif
                 per_cpu_offset(cpu) = ptr - __per_cpu_start;
                 memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
         }
 
-        printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d, nr_node_ids %d\n",
-                NR_CPUS, nr_cpu_ids, nr_node_ids);
-
         /* Setup percpu data maps */
         setup_per_cpu_maps();
 
@@ -282,7 +278,7 @@ static void __cpuinit numa_set_cpumask(int cpu, int enable)
         else
                 cpu_clear(cpu, *mask);
 
-        cpulist_scnprintf(buf, sizeof(buf), *mask);
+        cpulist_scnprintf(buf, sizeof(buf), mask);
         printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
                 enable? "numa_add_cpu":"numa_remove_cpu", cpu, node, buf);
  }
@@ -334,25 +330,25 @@ static const cpumask_t cpu_mask_none;
 /*
  * Returns a pointer to the bitmask of CPUs on Node 'node'.
  */
-const cpumask_t *_node_to_cpumask_ptr(int node)
+const cpumask_t *cpumask_of_node(int node)
 {
         if (node_to_cpumask_map == NULL) {
                 printk(KERN_WARNING
-                        "_node_to_cpumask_ptr(%d): no node_to_cpumask_map!\n",
+                        "cpumask_of_node(%d): no node_to_cpumask_map!\n",
                         node);
                 dump_stack();
                 return (const cpumask_t *)&cpu_online_map;
         }
         if (node >= nr_node_ids) {
                 printk(KERN_WARNING
-                        "_node_to_cpumask_ptr(%d): node > nr_node_ids(%d)\n",
+                        "cpumask_of_node(%d): node > nr_node_ids(%d)\n",
                         node, nr_node_ids);
                 dump_stack();
                 return &cpu_mask_none;
         }
         return &node_to_cpumask_map[node];
 }
-EXPORT_SYMBOL(_node_to_cpumask_ptr);
+EXPORT_SYMBOL(cpumask_of_node);
 
 /*
  * Returns a bitmask of CPUs on Node 'node'.
diff --git a/arch/x86/kernel/sigframe.h b/arch/x86/kernel/sigframe.h
deleted file mode 100644
index cc673aa55ce4..000000000000
--- a/arch/x86/kernel/sigframe.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifdef CONFIG_X86_32
-struct sigframe {
-        char __user *pretcode;
-        int sig;
-        struct sigcontext sc;
-        /*
-         * fpstate is unused. fpstate is moved/allocated after
-         * retcode[] below. This movement allows to have the FP state and the
-         * future state extensions (xsave) stay together.
-         * And at the same time retaining the unused fpstate, prevents changing
-         * the offset of extramask[] in the sigframe and thus prevent any
-         * legacy application accessing/modifying it.
-         */
-        struct _fpstate fpstate_unused;
-        unsigned long extramask[_NSIG_WORDS-1];
-        char retcode[8];
-        /* fp state follows here */
-};
-
-struct rt_sigframe {
-        char __user *pretcode;
-        int sig;
-        struct siginfo __user *pinfo;
-        void __user *puc;
-        struct siginfo info;
-        struct ucontext uc;
-        char retcode[8];
-        /* fp state follows here */
-};
-#else
-struct rt_sigframe {
-        char __user *pretcode;
-        struct ucontext uc;
-        struct siginfo info;
-        /* fp state follows here */
-};
-
-int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                sigset_t *set, struct pt_regs *regs);
-int ia32_setup_frame(int sig, struct k_sigaction *ka,
-                sigset_t *set, struct pt_regs *regs);
-#endif
diff --git a/arch/x86/kernel/signal_32.c b/arch/x86/kernel/signal.c
index d6dd057d0f22..89bb7668041d 100644
--- a/arch/x86/kernel/signal_32.c
+++ b/arch/x86/kernel/signal.c
@@ -1,36 +1,41 @@
 /*
  *  Copyright (C) 1991, 1992  Linus Torvalds
+ *  Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
  *
  *  1997-11-28  Modified for POSIX.1b signals by Richard Henderson
  *  2000-06-20  Pentium III FXSR, SSE support by Gareth Hughes
+ *  2000-2002   x86-64 support by Andi Kleen
  */
-#include <linux/list.h>
 
-#include <linux/personality.h>
-#include <linux/binfmts.h>
-#include <linux/suspend.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/smp.h>
 #include <linux/kernel.h>
-#include <linux/ptrace.h>
 #include <linux/signal.h>
-#include <linux/stddef.h>
-#include <linux/unistd.h>
 #include <linux/errno.h>
-#include <linux/sched.h>
 #include <linux/wait.h>
+#include <linux/ptrace.h>
 #include <linux/tracehook.h>
-#include <linux/elf.h>
-#include <linux/smp.h>
-#include <linux/mm.h>
+#include <linux/unistd.h>
+#include <linux/stddef.h>
+#include <linux/personality.h>
+#include <linux/uaccess.h>
 
 #include <asm/processor.h>
 #include <asm/ucontext.h>
-#include <asm/uaccess.h>
 #include <asm/i387.h>
 #include <asm/vdso.h>
+
+#ifdef CONFIG_X86_64
+#include <asm/proto.h>
+#include <asm/ia32_unistd.h>
+#include <asm/mce.h>
+#endif /* CONFIG_X86_64 */
+
 #include <asm/syscall.h>
 #include <asm/syscalls.h>
 
-#include "sigframe.h"
+#include <asm/sigframe.h>
 
 #define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP)))
 
@@ -45,74 +50,6 @@
 # define FIX_EFLAGS     __FIX_EFLAGS
 #endif
 
-/*
- * Atomically swap in the new signal mask, and wait for a signal.
- */
-asmlinkage int
-sys_sigsuspend(int history0, int history1, old_sigset_t mask)
-{
-        mask &= _BLOCKABLE;
-        spin_lock_irq(&current->sighand->siglock);
-        current->saved_sigmask = current->blocked;
-        siginitset(&current->blocked, mask);
-        recalc_sigpending();
-        spin_unlock_irq(&current->sighand->siglock);
-
-        current->state = TASK_INTERRUPTIBLE;
-        schedule();
-        set_restore_sigmask();
-
-        return -ERESTARTNOHAND;
-}
-
-asmlinkage int
-sys_sigaction(int sig, const struct old_sigaction __user *act,
-              struct old_sigaction __user *oact)
-{
-        struct k_sigaction new_ka, old_ka;
-        int ret;
-
-        if (act) {
-                old_sigset_t mask;
-
-                if (!access_ok(VERIFY_READ, act, sizeof(*act)) ||
-                    __get_user(new_ka.sa.sa_handler, &act->sa_handler) ||
-                    __get_user(new_ka.sa.sa_restorer, &act->sa_restorer))
-                        return -EFAULT;
-
-                __get_user(new_ka.sa.sa_flags, &act->sa_flags);
-                __get_user(mask, &act->sa_mask);
-                siginitset(&new_ka.sa.sa_mask, mask);
-        }
-
-        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
-
-        if (!ret && oact) {
-                if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) ||
-                    __put_user(old_ka.sa.sa_handler, &oact->sa_handler) ||
-                    __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer))
-                        return -EFAULT;
-
-                __put_user(old_ka.sa.sa_flags, &oact->sa_flags);
-                __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask);
-        }
-
-        return ret;
-}
-
-asmlinkage int sys_sigaltstack(unsigned long bx)
-{
-        /*
-         * This is needed to make gcc realize it doesn't own the
-         * "struct pt_regs"
-         */
-        struct pt_regs *regs = (struct pt_regs *)&bx;
-        const stack_t __user *uss = (const stack_t __user *)bx;
-        stack_t __user *uoss = (stack_t __user *)regs->cx;
-
-        return do_sigaltstack(uss, uoss, regs->sp);
-}
-
 #define COPY(x)                 {               \
         err |= __get_user(regs->x, &sc->x);     \
 }
@@ -123,7 +60,7 @@ asmlinkage int sys_sigaltstack(unsigned long bx)
                 regs->seg = tmp;                        \
 }
 
-#define COPY_SEG_STRICT(seg)    {                       \
+#define COPY_SEG_CPL3(seg)      {                       \
                 unsigned short tmp;                     \
                 err |= __get_user(tmp, &sc->seg);       \
                 regs->seg = tmp | 3;                    \
@@ -135,9 +72,6 @@ asmlinkage int sys_sigaltstack(unsigned long bx)
                 loadsegment(seg, tmp);                  \
 }
 
-/*
- * Do a signal return; undo the signal stack.
- */
 static int
 restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
                    unsigned long *pax)
@@ -149,14 +83,36 @@ restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
         /* Always make any pending restarted system calls return -EINTR */
         current_thread_info()->restart_block.fn = do_no_restart_syscall;
 
+#ifdef CONFIG_X86_32
         GET_SEG(gs);
         COPY_SEG(fs);
         COPY_SEG(es);
         COPY_SEG(ds);
+#endif /* CONFIG_X86_32 */
+
         COPY(di); COPY(si); COPY(bp); COPY(sp); COPY(bx);
         COPY(dx); COPY(cx); COPY(ip);
-        COPY_SEG_STRICT(cs);
-        COPY_SEG_STRICT(ss);
+
+#ifdef CONFIG_X86_64
+        COPY(r8);
+        COPY(r9);
+        COPY(r10);
+        COPY(r11);
+        COPY(r12);
+        COPY(r13);
+        COPY(r14);
+        COPY(r15);
+#endif /* CONFIG_X86_64 */
+
+#ifdef CONFIG_X86_32
+        COPY_SEG_CPL3(cs);
+        COPY_SEG_CPL3(ss);
+#else /* !CONFIG_X86_32 */
+        /* Kernel saves and restores only the CS segment register on signals,
+         * which is the bare minimum needed to allow mixed 32/64-bit code.
+         * App's signal handler can save/restore other segments if needed. */
+        COPY_SEG_CPL3(cs);
+#endif /* CONFIG_X86_32 */
 
         err |= __get_user(tmpflags, &sc->flags);
         regs->flags = (regs->flags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS);
@@ -169,102 +125,24 @@ restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
         return err;
 }
 
-asmlinkage unsigned long sys_sigreturn(unsigned long __unused)
-{
-        struct sigframe __user *frame;
-        struct pt_regs *regs;
-        unsigned long ax;
-        sigset_t set;
-
-        regs = (struct pt_regs *) &__unused;
-        frame = (struct sigframe __user *)(regs->sp - 8);
-
-        if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
-                goto badframe;
-        if (__get_user(set.sig[0], &frame->sc.oldmask) || (_NSIG_WORDS > 1
-                && __copy_from_user(&set.sig[1], &frame->extramask,
-                                    sizeof(frame->extramask))))
-                goto badframe;
-
-        sigdelsetmask(&set, ~_BLOCKABLE);
-        spin_lock_irq(&current->sighand->siglock);
-        current->blocked = set;
-        recalc_sigpending();
-        spin_unlock_irq(&current->sighand->siglock);
-
-        if (restore_sigcontext(regs, &frame->sc, &ax))
-                goto badframe;
-        return ax;
-
-badframe:
-        if (show_unhandled_signals && printk_ratelimit()) {
-                printk("%s%s[%d] bad frame in sigreturn frame:"
-                        "%p ip:%lx sp:%lx oeax:%lx",
-                    task_pid_nr(current) > 1 ? KERN_INFO : KERN_EMERG,
-                    current->comm, task_pid_nr(current), frame, regs->ip,
-                    regs->sp, regs->orig_ax);
-                print_vma_addr(" in ", regs->ip);
-                printk(KERN_CONT "\n");
-        }
-
-        force_sig(SIGSEGV, current);
-
-        return 0;
-}
-
-static long do_rt_sigreturn(struct pt_regs *regs)
-{
-        struct rt_sigframe __user *frame;
-        unsigned long ax;
-        sigset_t set;
-
-        frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long));
-        if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
-                goto badframe;
-        if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
-                goto badframe;
-
-        sigdelsetmask(&set, ~_BLOCKABLE);
-        spin_lock_irq(&current->sighand->siglock);
-        current->blocked = set;
-        recalc_sigpending();
-        spin_unlock_irq(&current->sighand->siglock);
-
-        if (restore_sigcontext(regs, &frame->uc.uc_mcontext, &ax))
-                goto badframe;
-
-        if (do_sigaltstack(&frame->uc.uc_stack, NULL, regs->sp) == -EFAULT)
-                goto badframe;
-
-        return ax;
-
-badframe:
-        signal_fault(regs, frame, "rt_sigreturn");
-        return 0;
-}
-
-asmlinkage int sys_rt_sigreturn(unsigned long __unused)
-{
-        struct pt_regs *regs = (struct pt_regs *)&__unused;
-
-        return do_rt_sigreturn(regs);
-}
-
-/*
- * Set up a signal frame.
- */
 static int
 setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
                  struct pt_regs *regs, unsigned long mask)
 {
-        int tmp, err = 0;
+        int err = 0;
 
-        err |= __put_user(regs->fs, (unsigned int __user *)&sc->fs);
-        savesegment(gs, tmp);
-        err |= __put_user(tmp, (unsigned int __user *)&sc->gs);
+#ifdef CONFIG_X86_32
+        {
+                unsigned int tmp;
 
+                savesegment(gs, tmp);
+                err |= __put_user(tmp, (unsigned int __user *)&sc->gs);
+        }
+        err |= __put_user(regs->fs, (unsigned int __user *)&sc->fs);
         err |= __put_user(regs->es, (unsigned int __user *)&sc->es);
         err |= __put_user(regs->ds, (unsigned int __user *)&sc->ds);
+#endif /* CONFIG_X86_32 */
+
         err |= __put_user(regs->di, &sc->di);
         err |= __put_user(regs->si, &sc->si);
         err |= __put_user(regs->bp, &sc->bp);
@@ -273,19 +151,33 @@ setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
         err |= __put_user(regs->dx, &sc->dx);
         err |= __put_user(regs->cx, &sc->cx);
         err |= __put_user(regs->ax, &sc->ax);
+#ifdef CONFIG_X86_64
+        err |= __put_user(regs->r8, &sc->r8);
+        err |= __put_user(regs->r9, &sc->r9);
+        err |= __put_user(regs->r10, &sc->r10);
+        err |= __put_user(regs->r11, &sc->r11);
+        err |= __put_user(regs->r12, &sc->r12);
+        err |= __put_user(regs->r13, &sc->r13);
+        err |= __put_user(regs->r14, &sc->r14);
+        err |= __put_user(regs->r15, &sc->r15);
+#endif /* CONFIG_X86_64 */
+
         err |= __put_user(current->thread.trap_no, &sc->trapno);
         err |= __put_user(current->thread.error_code, &sc->err);
         err |= __put_user(regs->ip, &sc->ip);
+#ifdef CONFIG_X86_32
         err |= __put_user(regs->cs, (unsigned int __user *)&sc->cs);
         err |= __put_user(regs->flags, &sc->flags);
         err |= __put_user(regs->sp, &sc->sp_at_signal);
         err |= __put_user(regs->ss, (unsigned int __user *)&sc->ss);
+#else /* !CONFIG_X86_32 */
+        err |= __put_user(regs->flags, &sc->flags);
+        err |= __put_user(regs->cs, &sc->cs);
+        err |= __put_user(0, &sc->gs);
+        err |= __put_user(0, &sc->fs);
+#endif /* CONFIG_X86_32 */
 
-        tmp = save_i387_xstate(fpstate);
-        if (tmp < 0)
-                err = 1;
-        else
-                err |= __put_user(tmp ? fpstate : NULL, &sc->fpstate);
+        err |= __put_user(fpstate, &sc->fpstate);
 
         /* non-iBCS2 extensions.. */
         err |= __put_user(mask, &sc->oldmask);
@@ -295,6 +187,32 @@ setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
 }
 
 /*
+ * Set up a signal frame.
+ */
+#ifdef CONFIG_X86_32
+static const struct {
+        u16 poplmovl;
+        u32 val;
+        u16 int80;
+} __attribute__((packed)) retcode = {
+        0xb858,         /* popl %eax; movl $..., %eax */
+        __NR_sigreturn,
+        0x80cd,         /* int $0x80 */
+};
+
+static const struct {
+        u8  movl;
+        u32 val;
+        u16 int80;
+        u8  pad;
+} __attribute__((packed)) rt_retcode = {
+        0xb8,           /* movl $..., %eax */
+        __NR_rt_sigreturn,
+        0x80cd,         /* int $0x80 */
+        0
+};
+
+/*
  * Determine which stack to use..
  */
 static inline void __user *
@@ -328,6 +246,8 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
         if (used_math()) {
                 sp = sp - sig_xstate_size;
                 *fpstate = (struct _fpstate *) sp;
+                if (save_i387_xstate(*fpstate) < 0)
+                        return (void __user *)-1L;
         }
 
         sp -= frame_size;
@@ -383,9 +303,7 @@ __setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
          * reasons and because gdb uses it as a signature to notice
          * signal handler stack frames.
          */
-        err |= __put_user(0xb858, (short __user *)(frame->retcode+0));
-        err |= __put_user(__NR_sigreturn, (int __user *)(frame->retcode+2));
-        err |= __put_user(0x80cd, (short __user *)(frame->retcode+6));
+        err |= __put_user(*((u64 *)&retcode), (u64 *)frame->retcode);
 
         if (err)
                 return -EFAULT;
@@ -454,9 +372,7 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
          * reasons and because gdb uses it as a signature to notice
          * signal handler stack frames.
          */
-        err |= __put_user(0xb8, (char __user *)(frame->retcode+0));
-        err |= __put_user(__NR_rt_sigreturn, (int __user *)(frame->retcode+1));
-        err |= __put_user(0x80cd, (short __user *)(frame->retcode+5));
+        err |= __put_user(*((u64 *)&rt_retcode), (u64 *)frame->retcode);
 
         if (err)
                 return -EFAULT;
@@ -475,23 +391,293 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         return 0;
 }
+#else /* !CONFIG_X86_32 */
+/*
+ * Determine which stack to use..
+ */
+static void __user *
+get_stack(struct k_sigaction *ka, unsigned long sp, unsigned long size)
+{
+        /* Default to using normal stack - redzone*/
+        sp -= 128;
+
+        /* This is the X/Open sanctioned signal stack switching.  */
+        if (ka->sa.sa_flags & SA_ONSTACK) {
+                if (sas_ss_flags(sp) == 0)
+                        sp = current->sas_ss_sp + current->sas_ss_size;
+        }
+
+        return (void __user *)round_down(sp - size, 64);
+}
+
+static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
+                            sigset_t *set, struct pt_regs *regs)
+{
+        struct rt_sigframe __user *frame;
+        void __user *fp = NULL;
+        int err = 0;
+        struct task_struct *me = current;
+
+        if (used_math()) {
+                fp = get_stack(ka, regs->sp, sig_xstate_size);
+                frame = (void __user *)round_down(
+                        (unsigned long)fp - sizeof(struct rt_sigframe), 16) - 8;
+
+                if (save_i387_xstate(fp) < 0)
+                        return -EFAULT;
+        } else
+                frame = get_stack(ka, regs->sp, sizeof(struct rt_sigframe)) - 8;
+
+        if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
+                return -EFAULT;
+
+        if (ka->sa.sa_flags & SA_SIGINFO) {
+                if (copy_siginfo_to_user(&frame->info, info))
+                        return -EFAULT;
+        }
+
+        /* Create the ucontext.  */
+        if (cpu_has_xsave)
+                err |= __put_user(UC_FP_XSTATE, &frame->uc.uc_flags);
+        else
+                err |= __put_user(0, &frame->uc.uc_flags);
+        err |= __put_user(0, &frame->uc.uc_link);
+        err |= __put_user(me->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
+        err |= __put_user(sas_ss_flags(regs->sp),
+                          &frame->uc.uc_stack.ss_flags);
+        err |= __put_user(me->sas_ss_size, &frame->uc.uc_stack.ss_size);
+        err |= setup_sigcontext(&frame->uc.uc_mcontext, fp, regs, set->sig[0]);
+        err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
+
+        /* Set up to return from userspace.  If provided, use a stub
+           already in userspace.  */
+        /* x86-64 should always use SA_RESTORER. */
+        if (ka->sa.sa_flags & SA_RESTORER) {
+                err |= __put_user(ka->sa.sa_restorer, &frame->pretcode);
+        } else {
+                /* could use a vstub here */
+                return -EFAULT;
+        }
+
+        if (err)
+                return -EFAULT;
+
+        /* Set up registers for signal handler */
+        regs->di = sig;
+        /* In case the signal handler was declared without prototypes */
+        regs->ax = 0;
+
+        /* This also works for non SA_SIGINFO handlers because they expect the
+           next argument after the signal number on the stack. */
+        regs->si = (unsigned long)&frame->info;
+        regs->dx = (unsigned long)&frame->uc;
+        regs->ip = (unsigned long) ka->sa.sa_handler;
+
+        regs->sp = (unsigned long)frame;
+
+        /* Set up the CS register to run signal handlers in 64-bit mode,
+           even if the handler happens to be interrupting 32-bit code. */
+        regs->cs = __USER_CS;
+
+        return 0;
+}
+#endif /* CONFIG_X86_32 */
+
+#ifdef CONFIG_X86_32
+/*
+ * Atomically swap in the new signal mask, and wait for a signal.
+ */
+asmlinkage int
+sys_sigsuspend(int history0, int history1, old_sigset_t mask)
+{
+        mask &= _BLOCKABLE;
+        spin_lock_irq(&current->sighand->siglock);
+        current->saved_sigmask = current->blocked;
+        siginitset(&current->blocked, mask);
+        recalc_sigpending();
+        spin_unlock_irq(&current->sighand->siglock);
+
+        current->state = TASK_INTERRUPTIBLE;
+        schedule();
+        set_restore_sigmask();
+
+        return -ERESTARTNOHAND;
+}
+
+asmlinkage int
+sys_sigaction(int sig, const struct old_sigaction __user *act,
+              struct old_sigaction __user *oact)
+{
+        struct k_sigaction new_ka, old_ka;
+        int ret;
+
+        if (act) {
+                old_sigset_t mask;
+
+                if (!access_ok(VERIFY_READ, act, sizeof(*act)) ||
+                    __get_user(new_ka.sa.sa_handler, &act->sa_handler) ||
+                    __get_user(new_ka.sa.sa_restorer, &act->sa_restorer))
+                        return -EFAULT;
+
+                __get_user(new_ka.sa.sa_flags, &act->sa_flags);
+                __get_user(mask, &act->sa_mask);
+                siginitset(&new_ka.sa.sa_mask, mask);
+        }
+
+        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
+
+        if (!ret && oact) {
+                if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) ||
+                    __put_user(old_ka.sa.sa_handler, &oact->sa_handler) ||
+                    __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer))
+                        return -EFAULT;
+
+                __put_user(old_ka.sa.sa_flags, &oact->sa_flags);
+                __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask);
+        }
+
+        return ret;
+}
+#endif /* CONFIG_X86_32 */
+
+#ifdef CONFIG_X86_32
+asmlinkage int sys_sigaltstack(unsigned long bx)
+{
+        /*
+         * This is needed to make gcc realize it doesn't own the
+         * "struct pt_regs"
+         */
+        struct pt_regs *regs = (struct pt_regs *)&bx;
+        const stack_t __user *uss = (const stack_t __user *)bx;
+        stack_t __user *uoss = (stack_t __user *)regs->cx;
+
+        return do_sigaltstack(uss, uoss, regs->sp);
+}
+#else /* !CONFIG_X86_32 */
+asmlinkage long
+sys_sigaltstack(const stack_t __user *uss, stack_t __user *uoss,
+                struct pt_regs *regs)
+{
+        return do_sigaltstack(uss, uoss, regs->sp);
+}
+#endif /* CONFIG_X86_32 */
+
+/*
+ * Do a signal return; undo the signal stack.
+ */
+#ifdef CONFIG_X86_32
+asmlinkage unsigned long sys_sigreturn(unsigned long __unused)
+{
+        struct sigframe __user *frame;
+        struct pt_regs *regs;
+        unsigned long ax;
+        sigset_t set;
+
+        regs = (struct pt_regs *) &__unused;
+        frame = (struct sigframe __user *)(regs->sp - 8);
+
+        if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
+                goto badframe;
+        if (__get_user(set.sig[0], &frame->sc.oldmask) || (_NSIG_WORDS > 1
+                && __copy_from_user(&set.sig[1], &frame->extramask,
+                                    sizeof(frame->extramask))))
+                goto badframe;
+
+        sigdelsetmask(&set, ~_BLOCKABLE);
+        spin_lock_irq(&current->sighand->siglock);
+        current->blocked = set;
+        recalc_sigpending();
+        spin_unlock_irq(&current->sighand->siglock);
+
+        if (restore_sigcontext(regs, &frame->sc, &ax))
+                goto badframe;
+        return ax;
+
+badframe:
+        signal_fault(regs, frame, "sigreturn");
+
+        return 0;
+}
+#endif /* CONFIG_X86_32 */
+
+static long do_rt_sigreturn(struct pt_regs *regs)
+{
+        struct rt_sigframe __user *frame;
+        unsigned long ax;
+        sigset_t set;
+
+        frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long));
+        if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
+                goto badframe;
+        if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
+                goto badframe;
+
+        sigdelsetmask(&set, ~_BLOCKABLE);
+        spin_lock_irq(&current->sighand->siglock);
+        current->blocked = set;
+        recalc_sigpending();
+        spin_unlock_irq(&current->sighand->siglock);
+
+        if (restore_sigcontext(regs, &frame->uc.uc_mcontext, &ax))
+                goto badframe;
+
+        if (do_sigaltstack(&frame->uc.uc_stack, NULL, regs->sp) == -EFAULT)
+                goto badframe;
+
+        return ax;
+
+badframe:
+        signal_fault(regs, frame, "rt_sigreturn");
+        return 0;
+}
+
+#ifdef CONFIG_X86_32
+asmlinkage int sys_rt_sigreturn(struct pt_regs regs)
+{
+        return do_rt_sigreturn(&regs);
+}
+#else /* !CONFIG_X86_32 */
+asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
+{
+        return do_rt_sigreturn(regs);
+}
+#endif /* CONFIG_X86_32 */
 
 /*
  * OK, we're invoking a handler:
  */
 static int signr_convert(int sig)
 {
+#ifdef CONFIG_X86_32
         struct thread_info *info = current_thread_info();
 
         if (info->exec_domain && info->exec_domain->signal_invmap && sig < 32)
                 return info->exec_domain->signal_invmap[sig];
+#endif /* CONFIG_X86_32 */
         return sig;
 }
 
+#ifdef CONFIG_X86_32
+
 #define is_ia32 1
 #define ia32_setup_frame        __setup_frame
 #define ia32_setup_rt_frame     __setup_rt_frame
 
+#else /* !CONFIG_X86_32 */
+
+#ifdef CONFIG_IA32_EMULATION
+#define is_ia32 test_thread_flag(TIF_IA32)
+#else /* !CONFIG_IA32_EMULATION */
+#define is_ia32 0
+#endif /* CONFIG_IA32_EMULATION */
+
+int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
+                sigset_t *set, struct pt_regs *regs);
+int ia32_setup_frame(int sig, struct k_sigaction *ka,
+                sigset_t *set, struct pt_regs *regs);
+
+#endif /* CONFIG_X86_32 */
+
 static int
 setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                sigset_t *set, struct pt_regs *regs)
@@ -592,7 +778,13 @@ handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
         return 0;
 }
 
+#ifdef CONFIG_X86_32
 #define NR_restart_syscall      __NR_restart_syscall
+#else /* !CONFIG_X86_32 */
+#define NR_restart_syscall      \
+        test_thread_flag(TIF_IA32) ? __NR_ia32_restart_syscall : __NR_restart_syscall
+#endif /* CONFIG_X86_32 */
+
 /*
  * Note that 'init' is a special process: it doesn't get signals it doesn't
  * want to handle. Thus you cannot kill init even with a SIGKILL even by
@@ -704,8 +896,9 @@ void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
         struct task_struct *me = current;
 
         if (show_unhandled_signals && printk_ratelimit()) {
-                printk(KERN_INFO
+                printk("%s"
                        "%s[%d] bad frame in %s frame:%p ip:%lx sp:%lx orax:%lx",
+                       task_pid_nr(current) > 1 ? KERN_INFO : KERN_EMERG,
                        me->comm, me->pid, where, frame,
                        regs->ip, regs->sp, regs->orig_ax);
                 print_vma_addr(" in ", regs->ip);
diff --git a/arch/x86/kernel/signal_64.c b/arch/x86/kernel/signal_64.c
deleted file mode 100644
index a5c9627f4db9..000000000000
--- a/arch/x86/kernel/signal_64.c
+++ /dev/null
@@ -1,516 +0,0 @@
-/*
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *  Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
- *
- *  1997-11-28  Modified for POSIX.1b signals by Richard Henderson
- *  2000-06-20  Pentium III FXSR, SSE support by Gareth Hughes
- *  2000-2002   x86-64 support by Andi Kleen
- */
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-#include <linux/smp.h>
-#include <linux/kernel.h>
-#include <linux/signal.h>
-#include <linux/errno.h>
-#include <linux/wait.h>
-#include <linux/ptrace.h>
-#include <linux/tracehook.h>
-#include <linux/unistd.h>
-#include <linux/stddef.h>
-#include <linux/personality.h>
-#include <linux/compiler.h>
-#include <linux/uaccess.h>
-
-#include <asm/processor.h>
-#include <asm/ucontext.h>
-#include <asm/i387.h>
-#include <asm/proto.h>
-#include <asm/ia32_unistd.h>
-#include <asm/mce.h>
-#include <asm/syscall.h>
-#include <asm/syscalls.h>
-#include "sigframe.h"
-
-#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP)))
-
-#define __FIX_EFLAGS    (X86_EFLAGS_AC | X86_EFLAGS_OF | \
-                         X86_EFLAGS_DF | X86_EFLAGS_TF | X86_EFLAGS_SF | \
-                         X86_EFLAGS_ZF | X86_EFLAGS_AF | X86_EFLAGS_PF | \
-                         X86_EFLAGS_CF)
-
-#ifdef CONFIG_X86_32
-# define FIX_EFLAGS     (__FIX_EFLAGS | X86_EFLAGS_RF)
-#else
-# define FIX_EFLAGS     __FIX_EFLAGS
-#endif
-
-asmlinkage long
-sys_sigaltstack(const stack_t __user *uss, stack_t __user *uoss,
-                struct pt_regs *regs)
-{
-        return do_sigaltstack(uss, uoss, regs->sp);
-}
-
-#define COPY(x)                 {               \
-        err |= __get_user(regs->x, &sc->x);     \
-}
-
-#define COPY_SEG_STRICT(seg)    {                       \
-                unsigned short tmp;                     \
-                err |= __get_user(tmp, &sc->seg);       \
-                regs->seg = tmp | 3;                    \
-}
-
-/*
- * Do a signal return; undo the signal stack.
- */
-static int
-restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
-                   unsigned long *pax)
-{
-        void __user *buf;
-        unsigned int tmpflags;
-        unsigned int err = 0;
-
-        /* Always make any pending restarted system calls return -EINTR */
-        current_thread_info()->restart_block.fn = do_no_restart_syscall;
-
-        COPY(di); COPY(si); COPY(bp); COPY(sp); COPY(bx);
-        COPY(dx); COPY(cx); COPY(ip);
-        COPY(r8);
-        COPY(r9);
-        COPY(r10);
-        COPY(r11);
-        COPY(r12);
-        COPY(r13);
-        COPY(r14);
-        COPY(r15);
-
-        /* Kernel saves and restores only the CS segment register on signals,
-         * which is the bare minimum needed to allow mixed 32/64-bit code.
-         * App's signal handler can save/restore other segments if needed. */
-        COPY_SEG_STRICT(cs);
-
-        err |= __get_user(tmpflags, &sc->flags);
-        regs->flags = (regs->flags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS);
-        regs->orig_ax = -1;             /* disable syscall checks */
-
-        err |= __get_user(buf, &sc->fpstate);
-        err |= restore_i387_xstate(buf);
-
-        err |= __get_user(*pax, &sc->ax);
-        return err;
-}
-
-static long do_rt_sigreturn(struct pt_regs *regs)
-{
-        struct rt_sigframe __user *frame;
-        unsigned long ax;
-        sigset_t set;
-
-        frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long));
-        if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
-                goto badframe;
-        if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
-                goto badframe;
-
-        sigdelsetmask(&set, ~_BLOCKABLE);
-        spin_lock_irq(&current->sighand->siglock);
-        current->blocked = set;
-        recalc_sigpending();
-        spin_unlock_irq(&current->sighand->siglock);
-
-        if (restore_sigcontext(regs, &frame->uc.uc_mcontext, &ax))
-                goto badframe;
-
-        if (do_sigaltstack(&frame->uc.uc_stack, NULL, regs->sp) == -EFAULT)
-                goto badframe;
-
-        return ax;
-
-badframe:
-        signal_fault(regs, frame, "rt_sigreturn");
-        return 0;
-}
-
-asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
-{
-        return do_rt_sigreturn(regs);
-}
-
-/*
- * Set up a signal frame.
- */
-
-static inline int
-setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
-                unsigned long mask, struct task_struct *me)
-{
-        int err = 0;
-
-        err |= __put_user(regs->cs, &sc->cs);
-        err |= __put_user(0, &sc->gs);
-        err |= __put_user(0, &sc->fs);
-
-        err |= __put_user(regs->di, &sc->di);
-        err |= __put_user(regs->si, &sc->si);
-        err |= __put_user(regs->bp, &sc->bp);
-        err |= __put_user(regs->sp, &sc->sp);
-        err |= __put_user(regs->bx, &sc->bx);
-        err |= __put_user(regs->dx, &sc->dx);
-        err |= __put_user(regs->cx, &sc->cx);
-        err |= __put_user(regs->ax, &sc->ax);
-        err |= __put_user(regs->r8, &sc->r8);
-        err |= __put_user(regs->r9, &sc->r9);
-        err |= __put_user(regs->r10, &sc->r10);
-        err |= __put_user(regs->r11, &sc->r11);
-        err |= __put_user(regs->r12, &sc->r12);
-        err |= __put_user(regs->r13, &sc->r13);
-        err |= __put_user(regs->r14, &sc->r14);
-        err |= __put_user(regs->r15, &sc->r15);
-        err |= __put_user(me->thread.trap_no, &sc->trapno);
-        err |= __put_user(me->thread.error_code, &sc->err);
-        err |= __put_user(regs->ip, &sc->ip);
-        err |= __put_user(regs->flags, &sc->flags);
-        err |= __put_user(mask, &sc->oldmask);
-        err |= __put_user(me->thread.cr2, &sc->cr2);
-
-        return err;
-}
-
-/*
- * Determine which stack to use..
- */
-
-static void __user *
-get_stack(struct k_sigaction *ka, struct pt_regs *regs, unsigned long size)
-{
-        unsigned long sp;
-
-        /* Default to using normal stack - redzone*/
-        sp = regs->sp - 128;
-
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (sas_ss_flags(sp) == 0)
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
-
-        return (void __user *)round_down(sp - size, 64);
-}
-
-static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                            sigset_t *set, struct pt_regs *regs)
-{
-        struct rt_sigframe __user *frame;
-        void __user *fp = NULL;
-        int err = 0;
-        struct task_struct *me = current;
-
-        if (used_math()) {
-                fp = get_stack(ka, regs, sig_xstate_size);
-                frame = (void __user *)round_down(
-                        (unsigned long)fp - sizeof(struct rt_sigframe), 16) - 8;
-
-                if (save_i387_xstate(fp) < 0)
-                        return -EFAULT;
-        } else
-                frame = get_stack(ka, regs, sizeof(struct rt_sigframe)) - 8;
-
-        if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                return -EFAULT;
-
-        if (ka->sa.sa_flags & SA_SIGINFO) {
-                if (copy_siginfo_to_user(&frame->info, info))
-                        return -EFAULT;
-        }
-
-        /* Create the ucontext.  */
-        if (cpu_has_xsave)
-                err |= __put_user(UC_FP_XSTATE, &frame->uc.uc_flags);
-        else
-                err |= __put_user(0, &frame->uc.uc_flags);
-        err |= __put_user(0, &frame->uc.uc_link);
-        err |= __put_user(me->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
-        err |= __put_user(sas_ss_flags(regs->sp),
-                          &frame->uc.uc_stack.ss_flags);
-        err |= __put_user(me->sas_ss_size, &frame->uc.uc_stack.ss_size);
-        err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0], me);
-        err |= __put_user(fp, &frame->uc.uc_mcontext.fpstate);
-        if (sizeof(*set) == 16) {
-                __put_user(set->sig[0], &frame->uc.uc_sigmask.sig[0]);
-                __put_user(set->sig[1], &frame->uc.uc_sigmask.sig[1]);
-        } else
-                err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
-
-        /* Set up to return from userspace.  If provided, use a stub
-           already in userspace.  */
-        /* x86-64 should always use SA_RESTORER. */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                err |= __put_user(ka->sa.sa_restorer, &frame->pretcode);
-        } else {
-                /* could use a vstub here */
-                return -EFAULT;
-        }
-
-        if (err)
-                return -EFAULT;
-
-        /* Set up registers for signal handler */
-        regs->di = sig;
-        /* In case the signal handler was declared without prototypes */
-        regs->ax = 0;
-
-        /* This also works for non SA_SIGINFO handlers because they expect the
-           next argument after the signal number on the stack. */
-        regs->si = (unsigned long)&frame->info;
-        regs->dx = (unsigned long)&frame->uc;
-        regs->ip = (unsigned long) ka->sa.sa_handler;
-
-        regs->sp = (unsigned long)frame;
-
-        /* Set up the CS register to run signal handlers in 64-bit mode,
-           even if the handler happens to be interrupting 32-bit code. */
-        regs->cs = __USER_CS;
-
-        return 0;
-}
-
-/*
- * OK, we're invoking a handler
- */
-static int signr_convert(int sig)
-{
-        return sig;
-}
-
-#ifdef CONFIG_IA32_EMULATION
-#define is_ia32 test_thread_flag(TIF_IA32)
-#else
-#define is_ia32 0
-#endif
-
-static int
-setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-               sigset_t *set, struct pt_regs *regs)
-{
-        int usig = signr_convert(sig);
-        int ret;
-
-        /* Set up the stack frame */
-        if (is_ia32) {
-                if (ka->sa.sa_flags & SA_SIGINFO)
-                        ret = ia32_setup_rt_frame(usig, ka, info, set, regs);
-                else
-                        ret = ia32_setup_frame(usig, ka, set, regs);
-        } else
-                ret = __setup_rt_frame(sig, ka, info, set, regs);
-
-        if (ret) {
-                force_sigsegv(sig, current);
-                return -EFAULT;
-        }
-
-        return ret;
-}
-
-static int
-handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
-              sigset_t *oldset, struct pt_regs *regs)
-{
-        int ret;
-
-        /* Are we from a system call? */
-        if (syscall_get_nr(current, regs) >= 0) {
-                /* If so, check system call restarting.. */
-                switch (syscall_get_error(current, regs)) {
-                case -ERESTART_RESTARTBLOCK:
-                case -ERESTARTNOHAND:
-                        regs->ax = -EINTR;
-                        break;
-
-                case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
-                                regs->ax = -EINTR;
-                                break;
-                        }
-                /* fallthrough */
-                case -ERESTARTNOINTR:
-                        regs->ax = regs->orig_ax;
-                        regs->ip -= 2;
-                        break;
-                }
-        }
-
-        /*
-         * If TF is set due to a debugger (TIF_FORCED_TF), clear the TF
-         * flag so that register information in the sigcontext is correct.
-         */
-        if (unlikely(regs->flags & X86_EFLAGS_TF) &&
-            likely(test_and_clear_thread_flag(TIF_FORCED_TF)))
-                regs->flags &= ~X86_EFLAGS_TF;
-
-        ret = setup_rt_frame(sig, ka, info, oldset, regs);
-
-        if (ret)
-                return ret;
-
-#ifdef CONFIG_X86_64
-        /*
-         * This has nothing to do with segment registers,
-         * despite the name.  This magic affects uaccess.h
-         * macros' behavior.  Reset it to the normal setting.
-         */
-        set_fs(USER_DS);
-#endif
-
-        /*
-         * Clear the direction flag as per the ABI for function entry.
-         */
-        regs->flags &= ~X86_EFLAGS_DF;
-
-        /*
-         * Clear TF when entering the signal handler, but
-         * notify any tracer that was single-stepping it.
-         * The tracer may want to single-step inside the
-         * handler too.
-         */
-        regs->flags &= ~X86_EFLAGS_TF;
-
-        spin_lock_irq(&current->sighand->siglock);
-        sigorsets(&current->blocked, &current->blocked, &ka->sa.sa_mask);
-        if (!(ka->sa.sa_flags & SA_NODEFER))
-                sigaddset(&current->blocked, sig);
-        recalc_sigpending();
-        spin_unlock_irq(&current->sighand->siglock);
-
-        tracehook_signal_handler(sig, info, ka, regs,
-                                 test_thread_flag(TIF_SINGLESTEP));
-
-        return 0;
-}
-
-#define NR_restart_syscall      \
-        test_thread_flag(TIF_IA32) ? __NR_ia32_restart_syscall : __NR_restart_syscall
-/*
- * Note that 'init' is a special process: it doesn't get signals it doesn't
- * want to handle. Thus you cannot kill init even with a SIGKILL even by
- * mistake.
- */
-static void do_signal(struct pt_regs *regs)
-{
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
-        sigset_t *oldset;
-
-        /*
-         * We want the common case to go fast, which is why we may in certain
-         * cases get here from kernel mode. Just return without doing anything
-         * if so.
-         * X86_32: vm86 regs switched out by assembly code before reaching
-         * here, so testing against kernel CS suffices.
-         */
-        if (!user_mode(regs))
-                return;
-
-        if (current_thread_info()->status & TS_RESTORE_SIGMASK)
-                oldset = &current->saved_sigmask;
-        else
-                oldset = &current->blocked;
-
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
-                /*
-                 * Re-enable any watchpoints before delivering the
-                 * signal to user space. The processor register will
-                 * have been cleared if the watchpoint triggered
-                 * inside the kernel.
-                 */
-                if (current->thread.debugreg7)
-                        set_debugreg(current->thread.debugreg7, 7);
-
-                /* Whee! Actually deliver the signal.  */
-                if (handle_signal(signr, &info, &ka, oldset, regs) == 0) {
-                        /*
-                         * A signal was successfully delivered; the saved
-                         * sigmask will have been stored in the signal frame,
-                         * and will be restored by sigreturn, so we can simply
-                         * clear the TS_RESTORE_SIGMASK flag.
-                         */
-                        current_thread_info()->status &= ~TS_RESTORE_SIGMASK;
-                }
-                return;
-        }
-
-        /* Did we come from a system call? */
-        if (syscall_get_nr(current, regs) >= 0) {
-                /* Restart the system call - no handlers present */
-                switch (syscall_get_error(current, regs)) {
-                case -ERESTARTNOHAND:
-                case -ERESTARTSYS:
-                case -ERESTARTNOINTR:
-                        regs->ax = regs->orig_ax;
-                        regs->ip -= 2;
-                        break;
-
-                case -ERESTART_RESTARTBLOCK:
-                        regs->ax = NR_restart_syscall;
-                        regs->ip -= 2;
-                        break;
-                }
-        }
-
-        /*
-         * If there's no signal to deliver, we just put the saved sigmask
-         * back.
-         */
-        if (current_thread_info()->status & TS_RESTORE_SIGMASK) {
-                current_thread_info()->status &= ~TS_RESTORE_SIGMASK;
-                sigprocmask(SIG_SETMASK, &current->saved_sigmask, NULL);
-        }
-}
-
-/*
- * notification of userspace execution resumption
- * - triggered by the TIF_WORK_MASK flags
- */
-void
-do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
-{
-#if defined(CONFIG_X86_64) && defined(CONFIG_X86_MCE)
-        /* notify userspace of pending MCEs */
-        if (thread_info_flags & _TIF_MCE_NOTIFY)
-                mce_notify_user();
-#endif /* CONFIG_X86_64 && CONFIG_X86_MCE */
-
-        /* deal with pending signal delivery */
-        if (thread_info_flags & _TIF_SIGPENDING)
-                do_signal(regs);
-
-        if (thread_info_flags & _TIF_NOTIFY_RESUME) {
-                clear_thread_flag(TIF_NOTIFY_RESUME);
-                tracehook_notify_resume(regs);
-        }
-
-#ifdef CONFIG_X86_32
-        clear_thread_flag(TIF_IRET);
-#endif /* CONFIG_X86_32 */
-}
-
-void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
-{
-        struct task_struct *me = current;
-
-        if (show_unhandled_signals && printk_ratelimit()) {
-                printk(KERN_INFO
-                       "%s[%d] bad frame in %s frame:%p ip:%lx sp:%lx orax:%lx",
-                       me->comm, me->pid, where, frame,
-                       regs->ip, regs->sp, regs->orig_ax);
-                print_vma_addr(" in ", regs->ip);
-                printk(KERN_CONT "\n");
-        }
-
-        force_sig(SIGSEGV, me);
-}
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index 18f9b19f5f8f..beea2649a240 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -118,41 +118,28 @@ static void native_smp_send_reschedule(int cpu)
                 WARN_ON(1);
                 return;
         }
-        send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
+        send_IPI_mask(cpumask_of(cpu), RESCHEDULE_VECTOR);
 }
 
 void native_send_call_func_single_ipi(int cpu)
 {
-        send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_SINGLE_VECTOR);
+        send_IPI_mask(cpumask_of(cpu), CALL_FUNCTION_SINGLE_VECTOR);
 }
 
-void native_send_call_func_ipi(cpumask_t mask)
+void native_send_call_func_ipi(const struct cpumask *mask)
 {
         cpumask_t allbutself;
 
         allbutself = cpu_online_map;
         cpu_clear(smp_processor_id(), allbutself);
 
-        if (cpus_equal(mask, allbutself) &&
+        if (cpus_equal(*mask, allbutself) &&
             cpus_equal(cpu_online_map, cpu_callout_map))
                 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
         else
                 send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
 }
 
-static void stop_this_cpu(void *dummy)
-{
-        local_irq_disable();
-        /*
-         * Remove this CPU:
-         */
-        cpu_clear(smp_processor_id(), cpu_online_map);
-        disable_local_APIC();
-        if (hlt_works(smp_processor_id()))
-                for (;;) halt();
-        for (;;);
-}
-
 /*
  * this function calls the 'stop' function on all other CPUs in the system.
  */
@@ -178,11 +165,7 @@ static void native_smp_send_stop(void)
 void smp_reschedule_interrupt(struct pt_regs *regs)
 {
         ack_APIC_irq();
-#ifdef CONFIG_X86_32
-        __get_cpu_var(irq_stat).irq_resched_count++;
-#else
-        add_pda(irq_resched_count, 1);
-#endif
+        inc_irq_stat(irq_resched_count);
 }
 
 void smp_call_function_interrupt(struct pt_regs *regs)
@@ -190,11 +173,7 @@ void smp_call_function_interrupt(struct pt_regs *regs)
         ack_APIC_irq();
         irq_enter();
         generic_smp_call_function_interrupt();
-#ifdef CONFIG_X86_32
-        __get_cpu_var(irq_stat).irq_call_count++;
-#else
-        add_pda(irq_call_count, 1);
-#endif
+        inc_irq_stat(irq_call_count);
         irq_exit();
 }
 
@@ -203,11 +182,7 @@ void smp_call_function_single_interrupt(struct pt_regs *regs)
         ack_APIC_irq();
         irq_enter();
         generic_smp_call_function_single_interrupt();
-#ifdef CONFIG_X86_32
-        __get_cpu_var(irq_stat).irq_call_count++;
-#else
-        add_pda(irq_call_count, 1);
-#endif
+        inc_irq_stat(irq_call_count);
         irq_exit();
 }
 
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index f71f96fc9e62..6bd4d9b73870 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -62,6 +62,7 @@
 #include <asm/mtrr.h>
 #include <asm/vmi.h>
 #include <asm/genapic.h>
+#include <asm/setup.h>
 #include <linux/mc146818rtc.h>
 
 #include <mach_apic.h>
@@ -101,14 +102,8 @@ EXPORT_SYMBOL(smp_num_siblings);
 /* Last level cache ID of each logical CPU */
 DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
 
-/* bitmap of online cpus */
-cpumask_t cpu_online_map __read_mostly;
-EXPORT_SYMBOL(cpu_online_map);
-
 cpumask_t cpu_callin_map;
 cpumask_t cpu_callout_map;
-cpumask_t cpu_possible_map;
-EXPORT_SYMBOL(cpu_possible_map);
 
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
@@ -287,7 +282,7 @@ static int __cpuinitdata unsafe_smp;
 /*
  * Activate a secondary processor.
  */
-static void __cpuinit start_secondary(void *unused)
+notrace static void __cpuinit start_secondary(void *unused)
 {
         /*
          * Don't put *anything* before cpu_init(), SMP booting is too
@@ -501,7 +496,7 @@ void __cpuinit set_cpu_sibling_map(int cpu)
 }
 
 /* maps the cpu to the sched domain representing multi-core */
-cpumask_t cpu_coregroup_map(int cpu)
+const struct cpumask *cpu_coregroup_mask(int cpu)
 {
         struct cpuinfo_x86 *c = &cpu_data(cpu);
         /*
@@ -509,9 +504,14 @@ cpumask_t cpu_coregroup_map(int cpu)
          * And for power savings, we return cpu_core_map
          */
         if (sched_mc_power_savings || sched_smt_power_savings)
-                return per_cpu(cpu_core_map, cpu);
+                return &per_cpu(cpu_core_map, cpu);
         else
-                return c->llc_shared_map;
+                return &c->llc_shared_map;
+}
+
+cpumask_t cpu_coregroup_map(int cpu)
+{
+        return *cpu_coregroup_mask(cpu);
 }
 
 static void impress_friends(void)
@@ -534,7 +534,7 @@ static void impress_friends(void)
         pr_debug("Before bogocount - setting activated=1.\n");
 }
 
-static inline void __inquire_remote_apic(int apicid)
+void __inquire_remote_apic(int apicid)
 {
         unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
         char *names[] = { "ID", "VERSION", "SPIV" };
@@ -573,14 +573,13 @@ static inline void __inquire_remote_apic(int apicid)
         }
 }
 
-#ifdef WAKE_SECONDARY_VIA_NMI
 /*
  * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
  * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
  * won't ... remember to clear down the APIC, etc later.
  */
-static int __devinit
-wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
+int __devinit
+wakeup_secondary_cpu_via_nmi(int logical_apicid, unsigned long start_eip)
 {
         unsigned long send_status, accept_status = 0;
         int maxlvt;
@@ -597,7 +596,7 @@ wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
          * Give the other CPU some time to accept the IPI.
          */
         udelay(200);
-        if (APIC_INTEGRATED(apic_version[phys_apicid])) {
+        if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
                 maxlvt = lapic_get_maxlvt();
                 if (maxlvt > 3)                 /* Due to the Pentium erratum 3AP.  */
                         apic_write(APIC_ESR, 0);
@@ -612,11 +611,9 @@ wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
 
         return (send_status | accept_status);
 }
-#endif  /* WAKE_SECONDARY_VIA_NMI */
 
-#ifdef WAKE_SECONDARY_VIA_INIT
-static int __devinit
-wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
+int __devinit
+wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
 {
         unsigned long send_status, accept_status = 0;
         int maxlvt, num_starts, j;
@@ -735,7 +732,6 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
 
         return (send_status | accept_status);
 }
-#endif  /* WAKE_SECONDARY_VIA_INIT */
 
 struct create_idle {
         struct work_struct work;
@@ -1084,8 +1080,10 @@ static int __init smp_sanity_check(unsigned max_cpus)
 #endif
 
         if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
-                printk(KERN_WARNING "weird, boot CPU (#%d) not listed"
-                                    "by the BIOS.\n", hard_smp_processor_id());
+                printk(KERN_WARNING
+                        "weird, boot CPU (#%d) not listed by the BIOS.\n",
+                        hard_smp_processor_id());
+
                 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
         }
 
@@ -1156,7 +1154,7 @@ static void __init smp_cpu_index_default(void)
         for_each_possible_cpu(i) {
                 c = &cpu_data(i);
                 /* mark all to hotplug */
-                c->cpu_index = NR_CPUS;
+                c->cpu_index = nr_cpu_ids;
         }
 }
 
@@ -1261,6 +1259,15 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
         check_nmi_watchdog();
 }
 
+static int __initdata setup_possible_cpus = -1;
+static int __init _setup_possible_cpus(char *str)
+{
+        get_option(&str, &setup_possible_cpus);
+        return 0;
+}
+early_param("possible_cpus", _setup_possible_cpus);
+
+
 /*
  * cpu_possible_map should be static, it cannot change as cpu's
  * are onlined, or offlined. The reason is per-cpu data-structures
@@ -1273,7 +1280,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
  *
  * Three ways to find out the number of additional hotplug CPUs:
  * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
- * - The user can overwrite it with additional_cpus=NUM
+ * - The user can overwrite it with possible_cpus=NUM
  * - Otherwise don't reserve additional CPUs.
  * We do this because additional CPUs waste a lot of memory.
  * -AK
@@ -1286,9 +1293,19 @@ __init void prefill_possible_map(void)
         if (!num_processors)
                 num_processors = 1;
 
-        possible = num_processors + disabled_cpus;
-        if (possible > NR_CPUS)
-                possible = NR_CPUS;
+        if (setup_possible_cpus == -1)
+                possible = num_processors + disabled_cpus;
+        else
+                possible = setup_possible_cpus;
+
+        total_cpus = max_t(int, possible, num_processors + disabled_cpus);
+
+        if (possible > CONFIG_NR_CPUS) {
+                printk(KERN_WARNING
+                        "%d Processors exceeds NR_CPUS limit of %d\n",
+                        possible, CONFIG_NR_CPUS);
+                possible = CONFIG_NR_CPUS;
+        }
 
         printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
                 possible, max_t(int, possible - num_processors, 0));
@@ -1353,7 +1370,7 @@ void cpu_disable_common(void)
         lock_vector_lock();
         remove_cpu_from_maps(cpu);
         unlock_vector_lock();
-        fixup_irqs(cpu_online_map);
+        fixup_irqs();
 }
 
 int native_cpu_disable(void)
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index a03e7f6d90c3..10786af95545 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -6,6 +6,7 @@
 #include <linux/sched.h>
 #include <linux/stacktrace.h>
 #include <linux/module.h>
+#include <linux/uaccess.h>
 #include <asm/stacktrace.h>
 
 static void save_stack_warning(void *data, char *msg)
@@ -83,3 +84,66 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
                 trace->entries[trace->nr_entries++] = ULONG_MAX;
 }
 EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
+
+/* Userspace stacktrace - based on kernel/trace/trace_sysprof.c */
+
+struct stack_frame {
+        const void __user       *next_fp;
+        unsigned long           ret_addr;
+};
+
+static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
+{
+        int ret;
+
+        if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
+                return 0;
+
+        ret = 1;
+        pagefault_disable();
+        if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
+                ret = 0;
+        pagefault_enable();
+
+        return ret;
+}
+
+static inline void __save_stack_trace_user(struct stack_trace *trace)
+{
+        const struct pt_regs *regs = task_pt_regs(current);
+        const void __user *fp = (const void __user *)regs->bp;
+
+        if (trace->nr_entries < trace->max_entries)
+                trace->entries[trace->nr_entries++] = regs->ip;
+
+        while (trace->nr_entries < trace->max_entries) {
+                struct stack_frame frame;
+
+                frame.next_fp = NULL;
+                frame.ret_addr = 0;
+                if (!copy_stack_frame(fp, &frame))
+                        break;
+                if ((unsigned long)fp < regs->sp)
+                        break;
+                if (frame.ret_addr) {
+                        trace->entries[trace->nr_entries++] =
+                                frame.ret_addr;
+                }
+                if (fp == frame.next_fp)
+                        break;
+                fp = frame.next_fp;
+        }
+}
+
+void save_stack_trace_user(struct stack_trace *trace)
+{
+        /*
+         * Trace user stack if we are not a kernel thread
+         */
+        if (current->mm) {
+                __save_stack_trace_user(trace);
+        }
+        if (trace->nr_entries < trace->max_entries)
+                trace->entries[trace->nr_entries++] = ULONG_MAX;
+}
+
diff --git a/arch/x86/kernel/time_32.c b/arch/x86/kernel/time_32.c
index 77b400f06ea2..65309e4cb1c0 100644
--- a/arch/x86/kernel/time_32.c
+++ b/arch/x86/kernel/time_32.c
@@ -75,7 +75,7 @@ EXPORT_SYMBOL(profile_pc);
 irqreturn_t timer_interrupt(int irq, void *dev_id)
 {
         /* Keep nmi watchdog up to date */
-        per_cpu(irq_stat, smp_processor_id()).irq0_irqs++;
+        inc_irq_stat(irq0_irqs);
 
 #ifdef CONFIG_X86_IO_APIC
         if (timer_ack) {
diff --git a/arch/x86/kernel/time_64.c b/arch/x86/kernel/time_64.c
index cb19d650c216..891e7a7c4334 100644
--- a/arch/x86/kernel/time_64.c
+++ b/arch/x86/kernel/time_64.c
@@ -49,9 +49,9 @@ unsigned long profile_pc(struct pt_regs *regs)
 }
 EXPORT_SYMBOL(profile_pc);
 
-irqreturn_t timer_interrupt(int irq, void *dev_id)
+static irqreturn_t timer_interrupt(int irq, void *dev_id)
 {
-        add_pda(irq0_irqs, 1);
+        inc_irq_stat(irq0_irqs);
 
         global_clock_event->event_handler(global_clock_event);
 
@@ -80,6 +80,8 @@ unsigned long __init calibrate_cpu(void)
                         break;
         no_ctr_free = (i == 4);
         if (no_ctr_free) {
+                WARN(1, KERN_WARNING "Warning: AMD perfctrs busy ... "
+                     "cpu_khz value may be incorrect.\n");
                 i = 3;
                 rdmsrl(MSR_K7_EVNTSEL3, evntsel3);
                 wrmsrl(MSR_K7_EVNTSEL3, 0);
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
index f4049f3513b6..ce5054642247 100644
--- a/arch/x86/kernel/tlb_32.c
+++ b/arch/x86/kernel/tlb_32.c
@@ -34,9 +34,8 @@ static DEFINE_SPINLOCK(tlbstate_lock);
  */
 void leave_mm(int cpu)
 {
-        if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
-                BUG();
-        cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
+        BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK);
+        cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask);
         load_cr3(swapper_pg_dir);
 }
 EXPORT_SYMBOL_GPL(leave_mm);
@@ -104,8 +103,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
                  * BUG();
                  */
 
-        if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
-                if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
+        if (flush_mm == x86_read_percpu(cpu_tlbstate.active_mm)) {
+                if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK) {
                         if (flush_va == TLB_FLUSH_ALL)
                                 local_flush_tlb();
                         else
@@ -119,7 +118,7 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
         smp_mb__after_clear_bit();
 out:
         put_cpu_no_resched();
-        __get_cpu_var(irq_stat).irq_tlb_count++;
+        inc_irq_stat(irq_tlb_count);
 }
 
 void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
@@ -164,7 +163,7 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
          * We have to send the IPI only to
          * CPUs affected.
          */
-        send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
+        send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR);
 
         while (!cpus_empty(flush_cpumask))
                 /* nothing. lockup detection does not belong here */
@@ -238,7 +237,7 @@ static void do_flush_tlb_all(void *info)
         unsigned long cpu = smp_processor_id();
 
         __flush_tlb_all();
-        if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
+        if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_LAZY)
                 leave_mm(cpu);
 }
 
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c
index 8f919ca69494..f8be6f1d2e48 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/kernel/tlb_64.c
@@ -154,7 +154,7 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
 out:
         ack_APIC_irq();
         cpu_clear(cpu, f->flush_cpumask);
-        add_pda(irq_tlb_count, 1);
+        inc_irq_stat(irq_tlb_count);
 }
 
 void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
@@ -191,7 +191,7 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
          * We have to send the IPI only to
          * CPUs affected.
          */
-        send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender);
+        send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR_START + sender);
 
         while (!cpus_empty(f->flush_cpumask))
                 cpu_relax();
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index 04431f34fd16..f885023167e0 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -566,14 +566,10 @@ static int __init uv_ptc_init(void)
         if (!is_uv_system())
                 return 0;
 
-        if (!proc_mkdir("sgi_uv", NULL))
-                return -EINVAL;
-
         proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL);
         if (!proc_uv_ptc) {
                 printk(KERN_ERR "unable to create %s proc entry\n",
                        UV_PTC_BASENAME);
-                remove_proc_entry("sgi_uv", NULL);
                 return -EINVAL;
         }
         proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
@@ -586,7 +582,6 @@ static int __init uv_ptc_init(void)
 static struct bau_control * __init uv_table_bases_init(int blade, int node)
 {
         int i;
-        int *ip;
         struct bau_msg_status *msp;
         struct bau_control *bau_tabp;
 
@@ -603,13 +598,6 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node)
                 bau_cpubits_clear(&msp->seen_by, (int)
                                   uv_blade_nr_possible_cpus(blade));
 
-        bau_tabp->watching =
-            kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node);
-        BUG_ON(!bau_tabp->watching);
-
-        for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++)
-                *ip = 0;
-
         uv_bau_table_bases[blade] = bau_tabp;
 
         return bau_tabp;
@@ -632,7 +620,6 @@ uv_table_bases_finish(int blade, int node, int cur_cpu,
                 bcp->bau_msg_head       = bau_tablesp->va_queue_first;
                 bcp->va_queue_first     = bau_tablesp->va_queue_first;
                 bcp->va_queue_last      = bau_tablesp->va_queue_last;
-                bcp->watching           = bau_tablesp->watching;
                 bcp->msg_statuses       = bau_tablesp->msg_statuses;
                 bcp->descriptor_base    = adp;
         }
diff --git a/arch/x86/kernel/trampoline.c b/arch/x86/kernel/trampoline.c
index 1106fac6024d..808031a5ba19 100644
--- a/arch/x86/kernel/trampoline.c
+++ b/arch/x86/kernel/trampoline.c
@@ -1,10 +1,26 @@
 #include <linux/io.h>
 
 #include <asm/trampoline.h>
+#include <asm/e820.h>
 
 /* ready for x86_64 and x86 */
 unsigned char *trampoline_base = __va(TRAMPOLINE_BASE);
 
+void __init reserve_trampoline_memory(void)
+{
+#ifdef CONFIG_X86_32
+        /*
+         * But first pinch a few for the stack/trampoline stuff
+         * FIXME: Don't need the extra page at 4K, but need to fix
+         * trampoline before removing it. (see the GDT stuff)
+         */
+        reserve_early(PAGE_SIZE, PAGE_SIZE + PAGE_SIZE, "EX TRAMPOLINE");
+#endif
+        /* Has to be in very low memory so we can execute real-mode AP code. */
+        reserve_early(TRAMPOLINE_BASE, TRAMPOLINE_BASE + TRAMPOLINE_SIZE,
+                        "TRAMPOLINE");
+}
+
 /*
  * Currently trivial. Write the real->protected mode
  * bootstrap into the page concerned. The caller
@@ -12,7 +28,6 @@ unsigned char *trampoline_base = __va(TRAMPOLINE_BASE);
  */
 unsigned long setup_trampoline(void)
 {
-        memcpy(trampoline_base, trampoline_data,
-               trampoline_end - trampoline_data);
+        memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
         return virt_to_phys(trampoline_base);
 }
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 04d242ab0161..c9a666cdd3db 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -20,7 +20,6 @@
 #include <linux/module.h>
 #include <linux/ptrace.h>
 #include <linux/string.h>
-#include <linux/unwind.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/kexec.h>
@@ -51,7 +50,6 @@
 #include <asm/debugreg.h>
 #include <asm/atomic.h>
 #include <asm/system.h>
-#include <asm/unwind.h>
 #include <asm/traps.h>
 #include <asm/desc.h>
 #include <asm/i387.h>
@@ -72,9 +70,6 @@
 
 #include "cpu/mcheck/mce.h"
 
-DECLARE_BITMAP(used_vectors, NR_VECTORS);
-EXPORT_SYMBOL_GPL(used_vectors);
-
 asmlinkage int system_call(void);
 
 /* Do we ignore FPU interrupts ? */
@@ -89,6 +84,9 @@ gate_desc idt_table[256]
         __attribute__((__section__(".data.idt"))) = { { { { 0, 0 } } }, };
 #endif
 
+DECLARE_BITMAP(used_vectors, NR_VECTORS);
+EXPORT_SYMBOL_GPL(used_vectors);
+
 static int ignore_nmis;
 
 static inline void conditional_sti(struct pt_regs *regs)
@@ -292,8 +290,10 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
         tsk->thread.error_code = error_code;
         tsk->thread.trap_no = 8;
 
-        /* This is always a kernel trap and never fixable (and thus must
-           never return). */
+        /*
+         * This is always a kernel trap and never fixable (and thus must
+         * never return).
+         */
         for (;;)
                 die(str, regs, error_code);
 }
@@ -481,11 +481,7 @@ do_nmi(struct pt_regs *regs, long error_code)
 {
         nmi_enter();
 
-#ifdef CONFIG_X86_32
-        { int cpu; cpu = smp_processor_id(); ++nmi_count(cpu); }
-#else
-        add_pda(__nmi_count, 1);
-#endif
+        inc_irq_stat(__nmi_count);
 
         if (!ignore_nmis)
                 default_do_nmi(regs);
@@ -524,9 +520,11 @@ dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
 }
 
 #ifdef CONFIG_X86_64
-/* Help handler running on IST stack to switch back to user stack
-   for scheduling or signal handling. The actual stack switch is done in
-   entry.S */
+/*
+ * Help handler running on IST stack to switch back to user stack
+ * for scheduling or signal handling. The actual stack switch is done in
+ * entry.S
+ */
 asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
 {
         struct pt_regs *regs = eregs;
@@ -536,8 +534,10 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
         /* Exception from user space */
         else if (user_mode(eregs))
                 regs = task_pt_regs(current);
-        /* Exception from kernel and interrupts are enabled. Move to
-           kernel process stack. */
+        /*
+         * Exception from kernel and interrupts are enabled. Move to
+         * kernel process stack.
+         */
         else if (eregs->flags & X86_EFLAGS_IF)
                 regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs));
         if (eregs != regs)
@@ -664,7 +664,7 @@ void math_error(void __user *ip)
 {
         struct task_struct *task;
         siginfo_t info;
-        unsigned short cwd, swd;
+        unsigned short cwd, swd, err;
 
         /*
          * Save the info for the exception handler and clear the error.
@@ -675,7 +675,6 @@ void math_error(void __user *ip)
         task->thread.error_code = 0;
         info.si_signo = SIGFPE;
         info.si_errno = 0;
-        info.si_code = __SI_FAULT;
         info.si_addr = ip;
         /*
          * (~cwd & swd) will mask out exceptions that are not set to unmasked
@@ -689,34 +688,30 @@ void math_error(void __user *ip)
          */
         cwd = get_fpu_cwd(task);
         swd = get_fpu_swd(task);
-        switch (swd & ~cwd & 0x3f) {
-        case 0x000: /* No unmasked exception */
-#ifdef CONFIG_X86_32
-                return;
-#endif
-        default: /* Multiple exceptions */
-                break;
-        case 0x001: /* Invalid Op */
+
+        err = swd & ~cwd;
+
+        if (err & 0x001) {      /* Invalid op */
                 /*
                  * swd & 0x240 == 0x040: Stack Underflow
                  * swd & 0x240 == 0x240: Stack Overflow
                  * User must clear the SF bit (0x40) if set
                  */
                 info.si_code = FPE_FLTINV;
-                break;
-        case 0x002: /* Denormalize */
-        case 0x010: /* Underflow */
-                info.si_code = FPE_FLTUND;
-                break;
-        case 0x004: /* Zero Divide */
+        } else if (err & 0x004) { /* Divide by Zero */
                 info.si_code = FPE_FLTDIV;
-                break;
-        case 0x008: /* Overflow */
+        } else if (err & 0x008) { /* Overflow */
                 info.si_code = FPE_FLTOVF;
-                break;
-        case 0x020: /* Precision */
+        } else if (err & 0x012) { /* Denormal, Underflow */
+                info.si_code = FPE_FLTUND;
+        } else if (err & 0x020) { /* Precision */
                 info.si_code = FPE_FLTRES;
-                break;
+        } else {
+                /*
+                 * If we're using IRQ 13, or supposedly even some trap 16
+                 * implementations, it's possible we get a spurious trap...
+                 */
+                return;         /* Spurious trap, no error */
         }
         force_sig_info(SIGFPE, &info, task);
 }
@@ -949,9 +944,7 @@ dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
 
 void __init trap_init(void)
 {
-#ifdef CONFIG_X86_32
         int i;
-#endif
 
 #ifdef CONFIG_EISA
         void __iomem *p = early_ioremap(0x0FFFD9, 4);
@@ -1008,11 +1001,15 @@ void __init trap_init(void)
         }
 
         set_system_trap_gate(SYSCALL_VECTOR, &system_call);
+#endif
 
         /* Reserve all the builtin and the syscall vector: */
         for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
                 set_bit(i, used_vectors);
 
+#ifdef CONFIG_X86_64
+        set_bit(IA32_SYSCALL_VECTOR, used_vectors);
+#else
         set_bit(SYSCALL_VECTOR, used_vectors);
 #endif
         /*
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 424093b157d3..599e58168631 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -15,6 +15,7 @@
 #include <asm/vgtod.h>
 #include <asm/time.h>
 #include <asm/delay.h>
+#include <asm/hypervisor.h>
 
 unsigned int cpu_khz;           /* TSC clocks / usec, not used here */
 EXPORT_SYMBOL(cpu_khz);
@@ -31,6 +32,7 @@ static int tsc_unstable;
    erroneous rdtsc usage on !cpu_has_tsc processors */
 static int tsc_disabled = -1;
 
+static int tsc_clocksource_reliable;
 /*
  * Scheduler clock - returns current time in nanosec units.
  */
@@ -98,6 +100,15 @@ int __init notsc_setup(char *str)
 
 __setup("notsc", notsc_setup);
 
+static int __init tsc_setup(char *str)
+{
+        if (!strcmp(str, "reliable"))
+                tsc_clocksource_reliable = 1;
+        return 1;
+}
+
+__setup("tsc=", tsc_setup);
+
 #define MAX_RETRIES     5
 #define SMI_TRESHOLD    50000
 
@@ -352,9 +363,15 @@ unsigned long native_calibrate_tsc(void)
 {
         u64 tsc1, tsc2, delta, ref1, ref2;
         unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
-        unsigned long flags, latch, ms, fast_calibrate;
+        unsigned long flags, latch, ms, fast_calibrate, tsc_khz;
         int hpet = is_hpet_enabled(), i, loopmin;
 
+        tsc_khz = get_hypervisor_tsc_freq();
+        if (tsc_khz) {
+                printk(KERN_INFO "TSC: Frequency read from the hypervisor\n");
+                return tsc_khz;
+        }
+
         local_irq_save(flags);
         fast_calibrate = quick_pit_calibrate();
         local_irq_restore(flags);
@@ -731,24 +748,21 @@ static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
         {}
 };
 
-/*
- * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
- */
+static void __init check_system_tsc_reliable(void)
+{
 #ifdef CONFIG_MGEODE_LX
-/* RTSC counts during suspend */
+        /* RTSC counts during suspend */
 #define RTSC_SUSP 0x100
-
-static void __init check_geode_tsc_reliable(void)
-{
         unsigned long res_low, res_high;
 
         rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
+        /* Geode_LX - the OLPC CPU has a possibly a very reliable TSC */
         if (res_low & RTSC_SUSP)
-                clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
-}
-#else
-static inline void check_geode_tsc_reliable(void) { }
+                tsc_clocksource_reliable = 1;
 #endif
+        if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
+                tsc_clocksource_reliable = 1;
+}
 
 /*
  * Make an educated guess if the TSC is trustworthy and synchronized
@@ -783,6 +797,8 @@ static void __init init_tsc_clocksource(void)
 {
         clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
                         clocksource_tsc.shift);
+        if (tsc_clocksource_reliable)
+                clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
         /* lower the rating if we already know its unstable: */
         if (check_tsc_unstable()) {
                 clocksource_tsc.rating = 0;
@@ -843,7 +859,7 @@ void __init tsc_init(void)
         if (unsynchronized_tsc())
                 mark_tsc_unstable("TSCs unsynchronized");
 
-        check_geode_tsc_reliable();
+        check_system_tsc_reliable();
         init_tsc_clocksource();
 }
 
diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c
index 1c0dfbca87c1..bf36328f6ef9 100644
--- a/arch/x86/kernel/tsc_sync.c
+++ b/arch/x86/kernel/tsc_sync.c
@@ -112,6 +112,12 @@ void __cpuinit check_tsc_sync_source(int cpu)
         if (unsynchronized_tsc())
                 return;
 
+        if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE)) {
+                printk(KERN_INFO
+                       "Skipping synchronization checks as TSC is reliable.\n");
+                return;
+        }
+
         printk(KERN_INFO "checking TSC synchronization [CPU#%d -> CPU#%d]:",
                           smp_processor_id(), cpu);
 
@@ -165,7 +171,7 @@ void __cpuinit check_tsc_sync_target(void)
 {
         int cpus = 2;
 
-        if (unsynchronized_tsc())
+        if (unsynchronized_tsc() || boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
                 return;
 
         /*
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
index 22fd6577156a..23206ba16874 100644
--- a/arch/x86/kernel/vmi_32.c
+++ b/arch/x86/kernel/vmi_32.c
@@ -266,109 +266,6 @@ static void vmi_nop(void)
 {
 }
 
-#ifdef CONFIG_DEBUG_PAGE_TYPE
-
-#ifdef CONFIG_X86_PAE
-#define MAX_BOOT_PTS (2048+4+1)
-#else
-#define MAX_BOOT_PTS (1024+1)
-#endif
-
-/*
- * During boot, mem_map is not yet available in paging_init, so stash
- * all the boot page allocations here.
- */
-static struct {
-        u32 pfn;
-        int type;
-} boot_page_allocations[MAX_BOOT_PTS];
-static int num_boot_page_allocations;
-static int boot_allocations_applied;
-
-void vmi_apply_boot_page_allocations(void)
-{
-        int i;
-        BUG_ON(!mem_map);
-        for (i = 0; i < num_boot_page_allocations; i++) {
-                struct page *page = pfn_to_page(boot_page_allocations[i].pfn);
-                page->type = boot_page_allocations[i].type;
-                page->type = boot_page_allocations[i].type &
-                                ~(VMI_PAGE_ZEROED | VMI_PAGE_CLONE);
-        }
-        boot_allocations_applied = 1;
-}
-
-static void record_page_type(u32 pfn, int type)
-{
-        BUG_ON(num_boot_page_allocations >= MAX_BOOT_PTS);
-        boot_page_allocations[num_boot_page_allocations].pfn = pfn;
-        boot_page_allocations[num_boot_page_allocations].type = type;
-        num_boot_page_allocations++;
-}
-
-static void check_zeroed_page(u32 pfn, int type, struct page *page)
-{
-        u32 *ptr;
-        int i;
-        int limit = PAGE_SIZE / sizeof(int);
-
-        if (page_address(page))
-                ptr = (u32 *)page_address(page);
-        else
-                ptr = (u32 *)__va(pfn << PAGE_SHIFT);
-        /*
-         * When cloning the root in non-PAE mode, only the userspace
-         * pdes need to be zeroed.
-         */
-        if (type & VMI_PAGE_CLONE)
-                limit = KERNEL_PGD_BOUNDARY;
-        for (i = 0; i < limit; i++)
-                BUG_ON(ptr[i]);
-}
-
-/*
- * We stash the page type into struct page so we can verify the page
- * types are used properly.
- */
-static void vmi_set_page_type(u32 pfn, int type)
-{
-        /* PAE can have multiple roots per page - don't track */
-        if (PTRS_PER_PMD > 1 && (type & VMI_PAGE_PDP))
-                return;
-
-        if (boot_allocations_applied) {
-                struct page *page = pfn_to_page(pfn);
-                if (type != VMI_PAGE_NORMAL)
-                        BUG_ON(page->type);
-                else
-                        BUG_ON(page->type == VMI_PAGE_NORMAL);
-                page->type = type & ~(VMI_PAGE_ZEROED | VMI_PAGE_CLONE);
-                if (type & VMI_PAGE_ZEROED)
-                        check_zeroed_page(pfn, type, page);
-        } else {
-                record_page_type(pfn, type);
-        }
-}
-
-static void vmi_check_page_type(u32 pfn, int type)
-{
-        /* PAE can have multiple roots per page - skip checks */
-        if (PTRS_PER_PMD > 1 && (type & VMI_PAGE_PDP))
-                return;
-
-        type &= ~(VMI_PAGE_ZEROED | VMI_PAGE_CLONE);
-        if (boot_allocations_applied) {
-                struct page *page = pfn_to_page(pfn);
-                BUG_ON((page->type ^ type) & VMI_PAGE_PAE);
-                BUG_ON(type == VMI_PAGE_NORMAL && page->type);
-                BUG_ON((type & page->type) == 0);
-        }
-}
-#else
-#define vmi_set_page_type(p,t) do { } while (0)
-#define vmi_check_page_type(p,t) do { } while (0)
-#endif
-
 #ifdef CONFIG_HIGHPTE
 static void *vmi_kmap_atomic_pte(struct page *page, enum km_type type)
 {
@@ -395,7 +292,6 @@ static void *vmi_kmap_atomic_pte(struct page *page, enum km_type type)
 
 static void vmi_allocate_pte(struct mm_struct *mm, unsigned long pfn)
 {
-        vmi_set_page_type(pfn, VMI_PAGE_L1);
         vmi_ops.allocate_page(pfn, VMI_PAGE_L1, 0, 0, 0);
 }
 
@@ -406,27 +302,22 @@ static void vmi_allocate_pmd(struct mm_struct *mm, unsigned long pfn)
          * It is called only for swapper_pg_dir, which already has
          * data on it.
          */
-        vmi_set_page_type(pfn, VMI_PAGE_L2);
         vmi_ops.allocate_page(pfn, VMI_PAGE_L2, 0, 0, 0);
 }
 
 static void vmi_allocate_pmd_clone(unsigned long pfn, unsigned long clonepfn, unsigned long start, unsigned long count)
 {
-        vmi_set_page_type(pfn, VMI_PAGE_L2 | VMI_PAGE_CLONE);
-        vmi_check_page_type(clonepfn, VMI_PAGE_L2);
         vmi_ops.allocate_page(pfn, VMI_PAGE_L2 | VMI_PAGE_CLONE, clonepfn, start, count);
 }
 
 static void vmi_release_pte(unsigned long pfn)
 {
         vmi_ops.release_page(pfn, VMI_PAGE_L1);
-        vmi_set_page_type(pfn, VMI_PAGE_NORMAL);
 }
 
 static void vmi_release_pmd(unsigned long pfn)
 {
         vmi_ops.release_page(pfn, VMI_PAGE_L2);
-        vmi_set_page_type(pfn, VMI_PAGE_NORMAL);
 }
 
 /*
@@ -450,26 +341,22 @@ static void vmi_release_pmd(unsigned long pfn)
 
 static void vmi_update_pte(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
-        vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
         vmi_ops.update_pte(ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
 }
 
 static void vmi_update_pte_defer(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
-        vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
         vmi_ops.update_pte(ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 0));
 }
 
 static void vmi_set_pte(pte_t *ptep, pte_t pte)
 {
         /* XXX because of set_pmd_pte, this can be called on PT or PD layers */
-        vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE | VMI_PAGE_PD);
         vmi_ops.set_pte(pte, ptep, VMI_PAGE_PT);
 }
 
 static void vmi_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte)
 {
-        vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
         vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
 }
 
@@ -477,10 +364,8 @@ static void vmi_set_pmd(pmd_t *pmdp, pmd_t pmdval)
 {
 #ifdef CONFIG_X86_PAE
         const pte_t pte = { .pte = pmdval.pmd };
-        vmi_check_page_type(__pa(pmdp) >> PAGE_SHIFT, VMI_PAGE_PMD);
 #else
         const pte_t pte = { pmdval.pud.pgd.pgd };
-        vmi_check_page_type(__pa(pmdp) >> PAGE_SHIFT, VMI_PAGE_PGD);
 #endif
         vmi_ops.set_pte(pte, (pte_t *)pmdp, VMI_PAGE_PD);
 }
@@ -502,7 +387,6 @@ static void vmi_set_pte_atomic(pte_t *ptep, pte_t pteval)
 
 static void vmi_set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte)
 {
-        vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
         vmi_ops.set_pte(pte, ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 1));
 }
 
@@ -510,21 +394,18 @@ static void vmi_set_pud(pud_t *pudp, pud_t pudval)
 {
         /* Um, eww */
         const pte_t pte = { .pte = pudval.pgd.pgd };
-        vmi_check_page_type(__pa(pudp) >> PAGE_SHIFT, VMI_PAGE_PGD);
         vmi_ops.set_pte(pte, (pte_t *)pudp, VMI_PAGE_PDP);
 }
 
 static void vmi_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
         const pte_t pte = { .pte = 0 };
-        vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
         vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
 }
 
 static void vmi_pmd_clear(pmd_t *pmd)
 {
         const pte_t pte = { .pte = 0 };
-        vmi_check_page_type(__pa(pmd) >> PAGE_SHIFT, VMI_PAGE_PMD);
         vmi_ops.set_pte(pte, (pte_t *)pmd, VMI_PAGE_PD);
 }
 #endif
diff --git a/arch/x86/kernel/vmiclock_32.c b/arch/x86/kernel/vmiclock_32.c
index 254ee07f8635..c4c1f9e09402 100644
--- a/arch/x86/kernel/vmiclock_32.c
+++ b/arch/x86/kernel/vmiclock_32.c
@@ -226,7 +226,7 @@ static void __devinit vmi_time_init_clockevent(void)
         /* Upper bound is clockevent's use of ulong for cycle deltas. */
         evt->max_delta_ns = clockevent_delta2ns(ULONG_MAX, evt);
         evt->min_delta_ns = clockevent_delta2ns(1, evt);
-        evt->cpumask = cpumask_of_cpu(cpu);
+        evt->cpumask = cpumask_of(cpu);
 
         printk(KERN_WARNING "vmi: registering clock event %s. mult=%lu shift=%u\n",
                evt->name, evt->mult, evt->shift);
diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S
index a9b8560adbc2..82c67559dde7 100644
--- a/arch/x86/kernel/vmlinux_32.lds.S
+++ b/arch/x86/kernel/vmlinux_32.lds.S
@@ -44,6 +44,7 @@ SECTIONS
         SCHED_TEXT
         LOCK_TEXT
         KPROBES_TEXT
+        IRQENTRY_TEXT
         *(.fixup)
         *(.gnu.warning)
         _etext = .;                     /* End of text section */
diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S
index 46e05447405b..1a614c0e6bef 100644
--- a/arch/x86/kernel/vmlinux_64.lds.S
+++ b/arch/x86/kernel/vmlinux_64.lds.S
@@ -35,6 +35,7 @@ SECTIONS
         SCHED_TEXT
         LOCK_TEXT
         KPROBES_TEXT
+        IRQENTRY_TEXT
         *(.fixup)
         *(.gnu.warning)
         _etext = .;             /* End of text section */
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 0b8b6690a86d..44153afc9067 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -17,6 +17,9 @@
  *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  */
 
+/* Disable profiling for userspace code: */
+#define DISABLE_BRANCH_PROFILING
+
 #include <linux/time.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -128,7 +131,16 @@ static __always_inline void do_vgettimeofday(struct timeval * tv)
                         gettimeofday(tv,NULL);
                         return;
                 }
+
+                /*
+                 * Surround the RDTSC by barriers, to make sure it's not
+                 * speculated to outside the seqlock critical section and
+                 * does not cause time warps:
+                 */
+                rdtsc_barrier();
                 now = vread();
+                rdtsc_barrier();
+
                 base = __vsyscall_gtod_data.clock.cycle_last;
                 mask = __vsyscall_gtod_data.clock.mask;
                 mult = __vsyscall_gtod_data.clock.mult;
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 15c3e6999182..2b54fe002e94 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -159,7 +159,7 @@ int save_i387_xstate(void __user *buf)
  * Restore the extended state if present. Otherwise, restore the FP/SSE
  * state.
  */
-int restore_user_xstate(void __user *buf)
+static int restore_user_xstate(void __user *buf)
 {
         struct _fpx_sw_bytes fx_sw_user;
         u64 mask;