43 files changed, 1876 insertions, 1029 deletions

diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index cd40aba6aa95..9a5ed58f09dc 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -94,6 +94,53 @@ enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_PIC;
 
 
 /*
+ * ISA irqs by default are the first 16 gsis but can be
+ * any gsi as specified by an interrupt source override.
+ */
+static u32 isa_irq_to_gsi[NR_IRQS_LEGACY] __read_mostly = {
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+};
+
+static unsigned int gsi_to_irq(unsigned int gsi)
+{
+        unsigned int irq = gsi + NR_IRQS_LEGACY;
+        unsigned int i;
+
+        for (i = 0; i < NR_IRQS_LEGACY; i++) {
+                if (isa_irq_to_gsi[i] == gsi) {
+                        return i;
+                }
+        }
+
+        /* Provide an identity mapping of gsi == irq
+         * except on truly weird platforms that have
+         * non isa irqs in the first 16 gsis.
+         */
+        if (gsi >= NR_IRQS_LEGACY)
+                irq = gsi;
+        else
+                irq = gsi_end + 1 + gsi;
+
+        return irq;
+}
+
+static u32 irq_to_gsi(int irq)
+{
+        unsigned int gsi;
+
+        if (irq < NR_IRQS_LEGACY)
+                gsi = isa_irq_to_gsi[irq];
+        else if (irq <= gsi_end)
+                gsi = irq;
+        else if (irq <= (gsi_end + NR_IRQS_LEGACY))
+                gsi = irq - gsi_end;
+        else
+                gsi = 0xffffffff;
+
+        return gsi;
+}
+
+/*
  * Temporarily use the virtual area starting from FIX_IO_APIC_BASE_END,
  * to map the target physical address. The problem is that set_fixmap()
  * provides a single page, and it is possible that the page is not
@@ -313,7 +360,7 @@ acpi_parse_ioapic(struct acpi_subtable_header * header, const unsigned long end)
 /*
  * Parse Interrupt Source Override for the ACPI SCI
  */
-static void __init acpi_sci_ioapic_setup(u32 gsi, u16 polarity, u16 trigger)
+static void __init acpi_sci_ioapic_setup(u8 bus_irq, u16 polarity, u16 trigger, u32 gsi)
 {
         if (trigger == 0)       /* compatible SCI trigger is level */
                 trigger = 3;
@@ -333,7 +380,7 @@ static void __init acpi_sci_ioapic_setup(u32 gsi, u16 polarity, u16 trigger)
          * If GSI is < 16, this will update its flags,
          * else it will create a new mp_irqs[] entry.
          */
-        mp_override_legacy_irq(gsi, polarity, trigger, gsi);
+        mp_override_legacy_irq(bus_irq, polarity, trigger, gsi);
 
         /*
          * stash over-ride to indicate we've been here
@@ -357,9 +404,10 @@ acpi_parse_int_src_ovr(struct acpi_subtable_header * header,
         acpi_table_print_madt_entry(header);
 
         if (intsrc->source_irq == acpi_gbl_FADT.sci_interrupt) {
-                acpi_sci_ioapic_setup(intsrc->global_irq,
+                acpi_sci_ioapic_setup(intsrc->source_irq,
                                       intsrc->inti_flags & ACPI_MADT_POLARITY_MASK,
-                                      (intsrc->inti_flags & ACPI_MADT_TRIGGER_MASK) >> 2);
+                                      (intsrc->inti_flags & ACPI_MADT_TRIGGER_MASK) >> 2,
+                                      intsrc->global_irq);
                 return 0;
         }
 
@@ -448,7 +496,7 @@ void __init acpi_pic_sci_set_trigger(unsigned int irq, u16 trigger)
 
 int acpi_gsi_to_irq(u32 gsi, unsigned int *irq)
 {
-        *irq = gsi;
+        *irq = gsi_to_irq(gsi);
 
 #ifdef CONFIG_X86_IO_APIC
         if (acpi_irq_model == ACPI_IRQ_MODEL_IOAPIC)
@@ -458,6 +506,14 @@ int acpi_gsi_to_irq(u32 gsi, unsigned int *irq)
         return 0;
 }
 
+int acpi_isa_irq_to_gsi(unsigned isa_irq, u32 *gsi)
+{
+        if (isa_irq >= 16)
+                return -1;
+        *gsi = irq_to_gsi(isa_irq);
+        return 0;
+}
+
 /*
  * success: return IRQ number (>=0)
  * failure: return < 0
@@ -482,7 +538,7 @@ int acpi_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity)
                 plat_gsi = mp_register_gsi(dev, gsi, trigger, polarity);
         }
 #endif
-        irq = plat_gsi;
+        irq = gsi_to_irq(plat_gsi);
 
         return irq;
 }
@@ -867,29 +923,6 @@ static int __init acpi_parse_madt_lapic_entries(void)
 extern int es7000_plat;
 #endif
 
-int __init acpi_probe_gsi(void)
-{
-        int idx;
-        int gsi;
-        int max_gsi = 0;
-
-        if (acpi_disabled)
-                return 0;
-
-        if (!acpi_ioapic)
-                return 0;
-
-        max_gsi = 0;
-        for (idx = 0; idx < nr_ioapics; idx++) {
-                gsi = mp_gsi_routing[idx].gsi_end;
-
-                if (gsi > max_gsi)
-                        max_gsi = gsi;
-        }
-
-        return max_gsi + 1;
-}
-
 static void assign_to_mp_irq(struct mpc_intsrc *m,
                                     struct mpc_intsrc *mp_irq)
 {
@@ -947,13 +980,13 @@ void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi)
         mp_irq.dstirq = pin;    /* INTIN# */
 
         save_mp_irq(&mp_irq);
+
+        isa_irq_to_gsi[bus_irq] = gsi;
 }
 
 void __init mp_config_acpi_legacy_irqs(void)
 {
         int i;
-        int ioapic;
-        unsigned int dstapic;
         struct mpc_intsrc mp_irq;
 
 #if defined (CONFIG_MCA) || defined (CONFIG_EISA)
@@ -974,19 +1007,27 @@ void __init mp_config_acpi_legacy_irqs(void)
 #endif
 
         /*
-         * Locate the IOAPIC that manages the ISA IRQs (0-15).
-         */
-        ioapic = mp_find_ioapic(0);
-        if (ioapic < 0)
-                return;
-        dstapic = mp_ioapics[ioapic].apicid;
-
-        /*
          * Use the default configuration for the IRQs 0-15.  Unless
          * overridden by (MADT) interrupt source override entries.
          */
         for (i = 0; i < 16; i++) {
+                int ioapic, pin;
+                unsigned int dstapic;
                 int idx;
+                u32 gsi;
+
+                /* Locate the gsi that irq i maps to. */
+                if (acpi_isa_irq_to_gsi(i, &gsi))
+                        continue;
+
+                /*
+                 * Locate the IOAPIC that manages the ISA IRQ.
+                 */
+                ioapic = mp_find_ioapic(gsi);
+                if (ioapic < 0)
+                        continue;
+                pin = mp_find_ioapic_pin(ioapic, gsi);
+                dstapic = mp_ioapics[ioapic].apicid;
 
                 for (idx = 0; idx < mp_irq_entries; idx++) {
                         struct mpc_intsrc *irq = mp_irqs + idx;
@@ -996,7 +1037,7 @@ void __init mp_config_acpi_legacy_irqs(void)
                                 break;
 
                         /* Do we already have a mapping for this IOAPIC pin */
-                        if (irq->dstapic == dstapic && irq->dstirq == i)
+                        if (irq->dstapic == dstapic && irq->dstirq == pin)
                                 break;
                 }
 
@@ -1011,7 +1052,7 @@ void __init mp_config_acpi_legacy_irqs(void)
                 mp_irq.dstapic = dstapic;
                 mp_irq.irqtype = mp_INT;
                 mp_irq.srcbusirq = i; /* Identity mapped */
-                mp_irq.dstirq = i;
+                mp_irq.dstirq = pin;
 
                 save_mp_irq(&mp_irq);
         }
@@ -1076,11 +1117,6 @@ int mp_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity)
 
         ioapic_pin = mp_find_ioapic_pin(ioapic, gsi);
 
-#ifdef CONFIG_X86_32
-        if (ioapic_renumber_irq)
-                gsi = ioapic_renumber_irq(ioapic, gsi);
-#endif
-
         if (ioapic_pin > MP_MAX_IOAPIC_PIN) {
                 printk(KERN_ERR "Invalid reference to IOAPIC pin "
                        "%d-%d\n", mp_ioapics[ioapic].apicid,
@@ -1094,7 +1130,7 @@ int mp_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity)
         set_io_apic_irq_attr(&irq_attr, ioapic, ioapic_pin,
                              trigger == ACPI_EDGE_SENSITIVE ? 0 : 1,
                              polarity == ACPI_ACTIVE_HIGH ? 0 : 1);
-        io_apic_set_pci_routing(dev, gsi, &irq_attr);
+        io_apic_set_pci_routing(dev, gsi_to_irq(gsi), &irq_attr);
 
         return gsi;
 }
@@ -1154,7 +1190,8 @@ static int __init acpi_parse_madt_ioapic_entries(void)
          * pretend we got one so we can set the SCI flags.
          */
         if (!acpi_sci_override_gsi)
-                acpi_sci_ioapic_setup(acpi_gbl_FADT.sci_interrupt, 0, 0);
+                acpi_sci_ioapic_setup(acpi_gbl_FADT.sci_interrupt, 0, 0,
+                                      acpi_gbl_FADT.sci_interrupt);
 
         /* Fill in identity legacy mappings where no override */
         mp_config_acpi_legacy_irqs();
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 1a160d5d44d0..70237732a6c7 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -194,7 +194,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)
 }
 
 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
-extern u8 *__smp_locks[], *__smp_locks_end[];
+extern s32 __smp_locks[], __smp_locks_end[];
 static void *text_poke_early(void *addr, const void *opcode, size_t len);
 
 /* Replace instructions with better alternatives for this CPU type.
@@ -235,37 +235,41 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
 
 #ifdef CONFIG_SMP
 
-static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
+static void alternatives_smp_lock(const s32 *start, const s32 *end,
+                                  u8 *text, u8 *text_end)
 {
-        u8 **ptr;
+        const s32 *poff;
 
         mutex_lock(&text_mutex);
-        for (ptr = start; ptr < end; ptr++) {
-                if (*ptr < text)
-                        continue;
-                if (*ptr > text_end)
+        for (poff = start; poff < end; poff++) {
+                u8 *ptr = (u8 *)poff + *poff;
+
+                if (!*poff || ptr < text || ptr >= text_end)
                         continue;
                 /* turn DS segment override prefix into lock prefix */
-                text_poke(*ptr, ((unsigned char []){0xf0}), 1);
+                if (*ptr == 0x3e)
+                        text_poke(ptr, ((unsigned char []){0xf0}), 1);
         };
         mutex_unlock(&text_mutex);
 }
 
-static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
+static void alternatives_smp_unlock(const s32 *start, const s32 *end,
+                                    u8 *text, u8 *text_end)
 {
-        u8 **ptr;
+        const s32 *poff;
 
         if (noreplace_smp)
                 return;
 
         mutex_lock(&text_mutex);
-        for (ptr = start; ptr < end; ptr++) {
-                if (*ptr < text)
-                        continue;
-                if (*ptr > text_end)
+        for (poff = start; poff < end; poff++) {
+                u8 *ptr = (u8 *)poff + *poff;
+
+                if (!*poff || ptr < text || ptr >= text_end)
                         continue;
                 /* turn lock prefix into DS segment override prefix */
-                text_poke(*ptr, ((unsigned char []){0x3E}), 1);
+                if (*ptr == 0xf0)
+                        text_poke(ptr, ((unsigned char []){0x3E}), 1);
         };
         mutex_unlock(&text_mutex);
 }
@@ -276,8 +280,8 @@ struct smp_alt_module {
         char            *name;
 
         /* ptrs to lock prefixes */
-        u8              **locks;
-        u8              **locks_end;
+        const s32       *locks;
+        const s32       *locks_end;
 
         /* .text segment, needed to avoid patching init code ;) */
         u8              *text;
@@ -398,16 +402,19 @@ void alternatives_smp_switch(int smp)
 int alternatives_text_reserved(void *start, void *end)
 {
         struct smp_alt_module *mod;
-        u8 **ptr;
+        const s32 *poff;
         u8 *text_start = start;
         u8 *text_end = end;
 
         list_for_each_entry(mod, &smp_alt_modules, next) {
                 if (mod->text > text_end || mod->text_end < text_start)
                         continue;
-                for (ptr = mod->locks; ptr < mod->locks_end; ptr++)
-                        if (text_start <= *ptr && text_end >= *ptr)
+                for (poff = mod->locks; poff < mod->locks_end; poff++) {
+                        const u8 *ptr = (const u8 *)poff + *poff;
+
+                        if (text_start <= ptr && text_end > ptr)
                                 return 1;
+                }
         }
 
         return 0;
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index f854d89b7edf..fa5a1474cd18 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -731,18 +731,22 @@ static bool increase_address_space(struct protection_domain *domain,
 
 static u64 *alloc_pte(struct protection_domain *domain,
                       unsigned long address,
-                      int end_lvl,
+                      unsigned long page_size,
                       u64 **pte_page,
                       gfp_t gfp)
 {
+        int level, end_lvl;
         u64 *pte, *page;
-        int level;
+
+        BUG_ON(!is_power_of_2(page_size));
 
         while (address > PM_LEVEL_SIZE(domain->mode))
                 increase_address_space(domain, gfp);
 
-        level =  domain->mode - 1;
-        pte   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
+        level   = domain->mode - 1;
+        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
+        address = PAGE_SIZE_ALIGN(address, page_size);
+        end_lvl = PAGE_SIZE_LEVEL(page_size);
 
         while (level > end_lvl) {
                 if (!IOMMU_PTE_PRESENT(*pte)) {
@@ -752,6 +756,10 @@ static u64 *alloc_pte(struct protection_domain *domain,
                         *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
                 }
 
+                /* No level skipping support yet */
+                if (PM_PTE_LEVEL(*pte) != level)
+                        return NULL;
+
                 level -= 1;
 
                 pte = IOMMU_PTE_PAGE(*pte);
@@ -769,28 +777,47 @@ static u64 *alloc_pte(struct protection_domain *domain,
  * This function checks if there is a PTE for a given dma address. If
  * there is one, it returns the pointer to it.
  */
-static u64 *fetch_pte(struct protection_domain *domain,
-                      unsigned long address, int map_size)
+static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
 {
         int level;
         u64 *pte;
 
-        level =  domain->mode - 1;
-        pte   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
+        if (address > PM_LEVEL_SIZE(domain->mode))
+                return NULL;
+
+        level   =  domain->mode - 1;
+        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
 
-        while (level > map_size) {
+        while (level > 0) {
+
+                /* Not Present */
                 if (!IOMMU_PTE_PRESENT(*pte))
                         return NULL;
 
+                /* Large PTE */
+                if (PM_PTE_LEVEL(*pte) == 0x07) {
+                        unsigned long pte_mask, __pte;
+
+                        /*
+                         * If we have a series of large PTEs, make
+                         * sure to return a pointer to the first one.
+                         */
+                        pte_mask = PTE_PAGE_SIZE(*pte);
+                        pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
+                        __pte    = ((unsigned long)pte) & pte_mask;
+
+                        return (u64 *)__pte;
+                }
+
+                /* No level skipping support yet */
+                if (PM_PTE_LEVEL(*pte) != level)
+                        return NULL;
+
                 level -= 1;
 
+                /* Walk to the next level */
                 pte = IOMMU_PTE_PAGE(*pte);
                 pte = &pte[PM_LEVEL_INDEX(level, address)];
-
-                if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) {
-                        pte = NULL;
-                        break;
-                }
         }
 
         return pte;
@@ -807,44 +834,84 @@ static int iommu_map_page(struct protection_domain *dom,
                           unsigned long bus_addr,
                           unsigned long phys_addr,
                           int prot,
-                          int map_size)
+                          unsigned long page_size)
 {
         u64 __pte, *pte;
-
-        bus_addr  = PAGE_ALIGN(bus_addr);
-        phys_addr = PAGE_ALIGN(phys_addr);
-
-        BUG_ON(!PM_ALIGNED(map_size, bus_addr));
-        BUG_ON(!PM_ALIGNED(map_size, phys_addr));
+        int i, count;
 
         if (!(prot & IOMMU_PROT_MASK))
                 return -EINVAL;
 
-        pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL);
+        bus_addr  = PAGE_ALIGN(bus_addr);
+        phys_addr = PAGE_ALIGN(phys_addr);
+        count     = PAGE_SIZE_PTE_COUNT(page_size);
+        pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
+
+        for (i = 0; i < count; ++i)
+                if (IOMMU_PTE_PRESENT(pte[i]))
+                        return -EBUSY;
 
-        if (IOMMU_PTE_PRESENT(*pte))
-                return -EBUSY;
+        if (page_size > PAGE_SIZE) {
+                __pte = PAGE_SIZE_PTE(phys_addr, page_size);
+                __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
+        } else
+                __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
 
-        __pte = phys_addr | IOMMU_PTE_P;
         if (prot & IOMMU_PROT_IR)
                 __pte |= IOMMU_PTE_IR;
         if (prot & IOMMU_PROT_IW)
                 __pte |= IOMMU_PTE_IW;
 
-        *pte = __pte;
+        for (i = 0; i < count; ++i)
+                pte[i] = __pte;
 
         update_domain(dom);
 
         return 0;
 }
 
-static void iommu_unmap_page(struct protection_domain *dom,
-                             unsigned long bus_addr, int map_size)
+static unsigned long iommu_unmap_page(struct protection_domain *dom,
+                                      unsigned long bus_addr,
+                                      unsigned long page_size)
 {
-        u64 *pte = fetch_pte(dom, bus_addr, map_size);
+        unsigned long long unmap_size, unmapped;
+        u64 *pte;
+
+        BUG_ON(!is_power_of_2(page_size));
+
+        unmapped = 0;
 
-        if (pte)
-                *pte = 0;
+        while (unmapped < page_size) {
+
+                pte = fetch_pte(dom, bus_addr);
+
+                if (!pte) {
+                        /*
+                         * No PTE for this address
+                         * move forward in 4kb steps
+                         */
+                        unmap_size = PAGE_SIZE;
+                } else if (PM_PTE_LEVEL(*pte) == 0) {
+                        /* 4kb PTE found for this address */
+                        unmap_size = PAGE_SIZE;
+                        *pte       = 0ULL;
+                } else {
+                        int count, i;
+
+                        /* Large PTE found which maps this address */
+                        unmap_size = PTE_PAGE_SIZE(*pte);
+                        count      = PAGE_SIZE_PTE_COUNT(unmap_size);
+                        for (i = 0; i < count; i++)
+                                pte[i] = 0ULL;
+                }
+
+                bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
+                unmapped += unmap_size;
+        }
+
+        BUG_ON(!is_power_of_2(unmapped));
+
+        return unmapped;
 }
 
 /*
@@ -878,7 +945,7 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
         for (addr = e->address_start; addr < e->address_end;
              addr += PAGE_SIZE) {
                 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
-                                     PM_MAP_4k);
+                                     PAGE_SIZE);
                 if (ret)
                         return ret;
                 /*
@@ -1006,7 +1073,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
                 u64 *pte, *pte_page;
 
                 for (i = 0; i < num_ptes; ++i) {
-                        pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k,
+                        pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
                                         &pte_page, gfp);
                         if (!pte)
                                 goto out_free;
@@ -1042,7 +1109,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
         for (i = dma_dom->aperture[index]->offset;
              i < dma_dom->aperture_size;
              i += PAGE_SIZE) {
-                u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k);
+                u64 *pte = fetch_pte(&dma_dom->domain, i);
                 if (!pte || !IOMMU_PTE_PRESENT(*pte))
                         continue;
 
@@ -1712,7 +1779,7 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
 
         pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
         if (!pte) {
-                pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page,
+                pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
                                 GFP_ATOMIC);
                 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
         } else
@@ -2439,12 +2506,11 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
         return ret;
 }
 
-static int amd_iommu_map_range(struct iommu_domain *dom,
-                               unsigned long iova, phys_addr_t paddr,
-                               size_t size, int iommu_prot)
+static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
+                         phys_addr_t paddr, int gfp_order, int iommu_prot)
 {
+        unsigned long page_size = 0x1000UL << gfp_order;
         struct protection_domain *domain = dom->priv;
-        unsigned long i,  npages = iommu_num_pages(paddr, size, PAGE_SIZE);
         int prot = 0;
         int ret;
 
@@ -2453,61 +2519,50 @@ static int amd_iommu_map_range(struct iommu_domain *dom,
         if (iommu_prot & IOMMU_WRITE)
                 prot |= IOMMU_PROT_IW;
 
-        iova  &= PAGE_MASK;
-        paddr &= PAGE_MASK;
-
         mutex_lock(&domain->api_lock);
-
-        for (i = 0; i < npages; ++i) {
-                ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k);
-                if (ret)
-                        return ret;
-
-                iova  += PAGE_SIZE;
-                paddr += PAGE_SIZE;
-        }
-
+        ret = iommu_map_page(domain, iova, paddr, prot, page_size);
         mutex_unlock(&domain->api_lock);
 
-        return 0;
+        return ret;
 }
 
-static void amd_iommu_unmap_range(struct iommu_domain *dom,
-                                  unsigned long iova, size_t size)
+static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
+                           int gfp_order)
 {
-
         struct protection_domain *domain = dom->priv;
-        unsigned long i,  npages = iommu_num_pages(iova, size, PAGE_SIZE);
+        unsigned long page_size, unmap_size;
 
-        iova  &= PAGE_MASK;
+        page_size  = 0x1000UL << gfp_order;
 
         mutex_lock(&domain->api_lock);
-
-        for (i = 0; i < npages; ++i) {
-                iommu_unmap_page(domain, iova, PM_MAP_4k);
-                iova  += PAGE_SIZE;
-        }
+        unmap_size = iommu_unmap_page(domain, iova, page_size);
+        mutex_unlock(&domain->api_lock);
 
         iommu_flush_tlb_pde(domain);
 
-        mutex_unlock(&domain->api_lock);
+        return get_order(unmap_size);
 }
 
 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
                                           unsigned long iova)
 {
         struct protection_domain *domain = dom->priv;
-        unsigned long offset = iova & ~PAGE_MASK;
+        unsigned long offset_mask;
         phys_addr_t paddr;
-        u64 *pte;
+        u64 *pte, __pte;
 
-        pte = fetch_pte(domain, iova, PM_MAP_4k);
+        pte = fetch_pte(domain, iova);
 
         if (!pte || !IOMMU_PTE_PRESENT(*pte))
                 return 0;
 
-        paddr  = *pte & IOMMU_PAGE_MASK;
-        paddr |= offset;
+        if (PM_PTE_LEVEL(*pte) == 0)
+                offset_mask = PAGE_SIZE - 1;
+        else
+                offset_mask = PTE_PAGE_SIZE(*pte) - 1;
+
+        __pte = *pte & PM_ADDR_MASK;
+        paddr = (__pte & ~offset_mask) | (iova & offset_mask);
 
         return paddr;
 }
@@ -2523,8 +2578,8 @@ static struct iommu_ops amd_iommu_ops = {
         .domain_destroy = amd_iommu_domain_destroy,
         .attach_dev = amd_iommu_attach_device,
         .detach_dev = amd_iommu_detach_device,
-        .map = amd_iommu_map_range,
-        .unmap = amd_iommu_unmap_range,
+        .map = amd_iommu_map,
+        .unmap = amd_iommu_unmap,
         .iova_to_phys = amd_iommu_iova_to_phys,
         .domain_has_cap = amd_iommu_domain_has_cap,
 };
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index 6360abf993d4..3bacb4d0844c 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -120,6 +120,7 @@ struct ivmd_header {
 bool amd_iommu_dump;
 
 static int __initdata amd_iommu_detected;
+static bool __initdata amd_iommu_disabled;
 
 u16 amd_iommu_last_bdf;                 /* largest PCI device id we have
                                            to handle */
@@ -1372,6 +1373,9 @@ void __init amd_iommu_detect(void)
         if (no_iommu || (iommu_detected && !gart_iommu_aperture))
                 return;
 
+        if (amd_iommu_disabled)
+                return;
+
         if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {
                 iommu_detected = 1;
                 amd_iommu_detected = 1;
@@ -1401,6 +1405,8 @@ static int __init parse_amd_iommu_options(char *str)
         for (; *str; ++str) {
                 if (strncmp(str, "fullflush", 9) == 0)
                         amd_iommu_unmap_flush = true;
+                if (strncmp(str, "off", 3) == 0)
+                        amd_iommu_disabled = true;
         }
 
         return 1;
diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c
index 03ba1b895f5e..425e53a87feb 100644
--- a/arch/x86/kernel/apic/es7000_32.c
+++ b/arch/x86/kernel/apic/es7000_32.c
@@ -131,24 +131,6 @@ int					es7000_plat;
 
 static unsigned int                     base;
 
-static int
-es7000_rename_gsi(int ioapic, int gsi)
-{
-        if (es7000_plat == ES7000_ZORRO)
-                return gsi;
-
-        if (!base) {
-                int i;
-                for (i = 0; i < nr_ioapics; i++)
-                        base += nr_ioapic_registers[i];
-        }
-
-        if (!ioapic && (gsi < 16))
-                gsi += base;
-
-        return gsi;
-}
-
 static int __cpuinit wakeup_secondary_cpu_via_mip(int cpu, unsigned long eip)
 {
         unsigned long vect = 0, psaival = 0;
@@ -190,7 +172,6 @@ static void setup_unisys(void)
                 es7000_plat = ES7000_ZORRO;
         else
                 es7000_plat = ES7000_CLASSIC;
-        ioapic_renumber_irq = es7000_rename_gsi;
 }
 
 /*
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index eb2789c3f721..33f3563a2a52 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -89,6 +89,9 @@ int nr_ioapics;
 /* IO APIC gsi routing info */
 struct mp_ioapic_gsi  mp_gsi_routing[MAX_IO_APICS];
 
+/* The last gsi number used */
+u32 gsi_end;
+
 /* MP IRQ source entries */
 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
 
@@ -1013,10 +1016,9 @@ static inline int irq_trigger(int idx)
         return MPBIOS_trigger(idx);
 }
 
-int (*ioapic_renumber_irq)(int ioapic, int irq);
 static int pin_2_irq(int idx, int apic, int pin)
 {
-        int irq, i;
+        int irq;
         int bus = mp_irqs[idx].srcbus;
 
         /*
@@ -1028,18 +1030,12 @@ static int pin_2_irq(int idx, int apic, int pin)
         if (test_bit(bus, mp_bus_not_pci)) {
                 irq = mp_irqs[idx].srcbusirq;
         } else {
-                /*
-                 * PCI IRQs are mapped in order
-                 */
-                i = irq = 0;
-                while (i < apic)
-                        irq += nr_ioapic_registers[i++];
-                irq += pin;
-                /*
-                 * For MPS mode, so far only needed by ES7000 platform
-                 */
-                if (ioapic_renumber_irq)
-                        irq = ioapic_renumber_irq(apic, irq);
+                u32 gsi = mp_gsi_routing[apic].gsi_base + pin;
+
+                if (gsi >= NR_IRQS_LEGACY)
+                        irq = gsi;
+                else
+                        irq = gsi_end + 1 + gsi;
         }
 
 #ifdef CONFIG_X86_32
@@ -1950,20 +1946,8 @@ static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
 
 void __init enable_IO_APIC(void)
 {
-        union IO_APIC_reg_01 reg_01;
         int i8259_apic, i8259_pin;
         int apic;
-        unsigned long flags;
-
-        /*
-         * The number of IO-APIC IRQ registers (== #pins):
-         */
-        for (apic = 0; apic < nr_ioapics; apic++) {
-                raw_spin_lock_irqsave(&ioapic_lock, flags);
-                reg_01.raw = io_apic_read(apic, 1);
-                raw_spin_unlock_irqrestore(&ioapic_lock, flags);
-                nr_ioapic_registers[apic] = reg_01.bits.entries+1;
-        }
 
         if (!legacy_pic->nr_legacy_irqs)
                 return;
@@ -3858,27 +3842,20 @@ int __init io_apic_get_redir_entries (int ioapic)
         reg_01.raw = io_apic_read(ioapic, 1);
         raw_spin_unlock_irqrestore(&ioapic_lock, flags);
 
-        return reg_01.bits.entries;
+        /* The register returns the maximum index redir index
+         * supported, which is one less than the total number of redir
+         * entries.
+         */
+        return reg_01.bits.entries + 1;
 }
 
 void __init probe_nr_irqs_gsi(void)
 {
-        int nr = 0;
+        int nr;
 
-        nr = acpi_probe_gsi();
-        if (nr > nr_irqs_gsi) {
+        nr = gsi_end + 1 + NR_IRQS_LEGACY;
+        if (nr > nr_irqs_gsi)
                 nr_irqs_gsi = nr;
-        } else {
-                /* for acpi=off or acpi is not compiled in */
-                int idx;
-
-                nr = 0;
-                for (idx = 0; idx < nr_ioapics; idx++)
-                        nr += io_apic_get_redir_entries(idx) + 1;
-
-                if (nr > nr_irqs_gsi)
-                        nr_irqs_gsi = nr;
-        }
 
         printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
 }
@@ -4085,22 +4062,27 @@ int __init io_apic_get_version(int ioapic)
         return reg_01.bits.version;
 }
 
-int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
+int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
 {
-        int i;
+        int ioapic, pin, idx;
 
         if (skip_ioapic_setup)
                 return -1;
 
-        for (i = 0; i < mp_irq_entries; i++)
-                if (mp_irqs[i].irqtype == mp_INT &&
-                    mp_irqs[i].srcbusirq == bus_irq)
-                        break;
-        if (i >= mp_irq_entries)
+        ioapic = mp_find_ioapic(gsi);
+        if (ioapic < 0)
                 return -1;
 
-        *trigger = irq_trigger(i);
-        *polarity = irq_polarity(i);
+        pin = mp_find_ioapic_pin(ioapic, gsi);
+        if (pin < 0)
+                return -1;
+
+        idx = find_irq_entry(ioapic, pin, mp_INT);
+        if (idx < 0)
+                return -1;
+
+        *trigger = irq_trigger(idx);
+        *polarity = irq_polarity(idx);
         return 0;
 }
 
@@ -4241,7 +4223,7 @@ void __init ioapic_insert_resources(void)
         }
 }
 
-int mp_find_ioapic(int gsi)
+int mp_find_ioapic(u32 gsi)
 {
         int i = 0;
 
@@ -4256,7 +4238,7 @@ int mp_find_ioapic(int gsi)
         return -1;
 }
 
-int mp_find_ioapic_pin(int ioapic, int gsi)
+int mp_find_ioapic_pin(int ioapic, u32 gsi)
 {
         if (WARN_ON(ioapic == -1))
                 return -1;
@@ -4284,6 +4266,7 @@ static int bad_ioapic(unsigned long address)
 void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
 {
         int idx = 0;
+        int entries;
 
         if (bad_ioapic(address))
                 return;
@@ -4302,9 +4285,17 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
          * Build basic GSI lookup table to facilitate gsi->io_apic lookups
          * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
          */
+        entries = io_apic_get_redir_entries(idx);
         mp_gsi_routing[idx].gsi_base = gsi_base;
-        mp_gsi_routing[idx].gsi_end = gsi_base +
-            io_apic_get_redir_entries(idx);
+        mp_gsi_routing[idx].gsi_end = gsi_base + entries - 1;
+
+        /*
+         * The number of IO-APIC IRQ registers (== #pins):
+         */
+        nr_ioapic_registers[idx] = entries;
+
+        if (mp_gsi_routing[idx].gsi_end > gsi_end)
+                gsi_end = mp_gsi_routing[idx].gsi_end;
 
         printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
                "GSI %d-%d\n", idx, mp_ioapics[idx].apicid,
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index c085d52dbaf2..e46f98f36e31 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -735,9 +735,6 @@ void __init uv_system_init(void)
                 uv_node_to_blade[nid] = blade;
                 uv_cpu_to_blade[cpu] = blade;
                 max_pnode = max(pnode, max_pnode);
-
-                printk(KERN_DEBUG "UV: cpu %d, apicid 0x%x, pnode %d, nid %d, lcpu %d, blade %d\n",
-                        cpu, apicid, pnode, nid, lcpu, blade);
         }
 
         /* Add blade/pnode info for nodes without cpus */
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index 031aa887b0eb..c4f9182ca3ac 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -1224,7 +1224,7 @@ static void reinit_timer(void)
 #ifdef INIT_TIMER_AFTER_SUSPEND
         unsigned long flags;
 
-        spin_lock_irqsave(&i8253_lock, flags);
+        raw_spin_lock_irqsave(&i8253_lock, flags);
         /* set the clock to HZ */
         outb_pit(0x34, PIT_MODE);               /* binary, mode 2, LSB/MSB, ch 0 */
         udelay(10);
@@ -1232,7 +1232,7 @@ static void reinit_timer(void)
         udelay(10);
         outb_pit(LATCH >> 8, PIT_CH0);  /* MSB */
         udelay(10);
-        spin_unlock_irqrestore(&i8253_lock, flags);
+        raw_spin_unlock_irqrestore(&i8253_lock, flags);
 #endif
 }
 
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index c202b62f3671..3a785da34b6f 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -14,7 +14,7 @@ CFLAGS_common.o		:= $(nostackp)
 
 obj-y                   := intel_cacheinfo.o addon_cpuid_features.o
 obj-y                   += proc.o capflags.o powerflags.o common.o
-obj-y                   += vmware.o hypervisor.o sched.o
+obj-y                   += vmware.o hypervisor.o sched.o mshyperv.o
 
 obj-$(CONFIG_X86_32)    += bugs.o cmpxchg.o
 obj-$(CONFIG_X86_64)    += bugs_64.o
diff --git a/arch/x86/kernel/cpu/addon_cpuid_features.c b/arch/x86/kernel/cpu/addon_cpuid_features.c
index 97ad79cdf688..10fa5684a662 100644
--- a/arch/x86/kernel/cpu/addon_cpuid_features.c
+++ b/arch/x86/kernel/cpu/addon_cpuid_features.c
@@ -30,12 +30,14 @@ void __cpuinit init_scattered_cpuid_features(struct cpuinfo_x86 *c)
         const struct cpuid_bit *cb;
 
         static const struct cpuid_bit __cpuinitconst cpuid_bits[] = {
-                { X86_FEATURE_IDA, CR_EAX, 1, 0x00000006 },
-                { X86_FEATURE_ARAT, CR_EAX, 2, 0x00000006 },
-                { X86_FEATURE_NPT,   CR_EDX, 0, 0x8000000a },
-                { X86_FEATURE_LBRV,  CR_EDX, 1, 0x8000000a },
-                { X86_FEATURE_SVML,  CR_EDX, 2, 0x8000000a },
-                { X86_FEATURE_NRIPS, CR_EDX, 3, 0x8000000a },
+                { X86_FEATURE_IDA,              CR_EAX, 1, 0x00000006 },
+                { X86_FEATURE_ARAT,             CR_EAX, 2, 0x00000006 },
+                { X86_FEATURE_APERFMPERF,       CR_ECX, 0, 0x00000006 },
+                { X86_FEATURE_CPB,              CR_EDX, 9, 0x80000007 },
+                { X86_FEATURE_NPT,              CR_EDX, 0, 0x8000000a },
+                { X86_FEATURE_LBRV,             CR_EDX, 1, 0x8000000a },
+                { X86_FEATURE_SVML,             CR_EDX, 2, 0x8000000a },
+                { X86_FEATURE_NRIPS,            CR_EDX, 3, 0x8000000a },
                 { 0, 0, 0, 0 }
         };
 
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 01a265212395..c39576cb3018 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -86,7 +86,7 @@ static void __init check_fpu(void)
 
 static void __init check_hlt(void)
 {
-        if (paravirt_enabled())
+        if (boot_cpu_data.x86 >= 5 || paravirt_enabled())
                 return;
 
         printk(KERN_INFO "Checking 'hlt' instruction... ");
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 4868e4a951ee..c1c00d0b1692 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1243,10 +1243,7 @@ void __cpuinit cpu_init(void)
         /*
          * Force FPU initialization:
          */
-        if (cpu_has_xsave)
-                current_thread_info()->status = TS_XSAVE;
-        else
-                current_thread_info()->status = 0;
+        current_thread_info()->status = 0;
         clear_used_math();
         mxcsr_feature_mask_init();
 
diff --git a/arch/x86/kernel/cpu/cpufreq/Makefile b/arch/x86/kernel/cpu/cpufreq/Makefile
index 1840c0a5170b..bd54bf67e6fb 100644
--- a/arch/x86/kernel/cpu/cpufreq/Makefile
+++ b/arch/x86/kernel/cpu/cpufreq/Makefile
@@ -2,8 +2,8 @@
 # K8 systems. ACPI is preferred to all other hardware-specific drivers.
 # speedstep-* is preferred over p4-clockmod.
 
-obj-$(CONFIG_X86_POWERNOW_K8)           += powernow-k8.o
-obj-$(CONFIG_X86_ACPI_CPUFREQ)          += acpi-cpufreq.o
+obj-$(CONFIG_X86_POWERNOW_K8)           += powernow-k8.o mperf.o
+obj-$(CONFIG_X86_ACPI_CPUFREQ)          += acpi-cpufreq.o mperf.o
 obj-$(CONFIG_X86_PCC_CPUFREQ)           += pcc-cpufreq.o
 obj-$(CONFIG_X86_POWERNOW_K6)           += powernow-k6.o
 obj-$(CONFIG_X86_POWERNOW_K7)           += powernow-k7.o
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
index 459168083b77..1d3cddaa40ee 100644
--- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
+++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
@@ -46,6 +46,7 @@
 #include <asm/msr.h>
 #include <asm/processor.h>
 #include <asm/cpufeature.h>
+#include "mperf.h"
 
 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \
                 "acpi-cpufreq", msg)
@@ -71,8 +72,6 @@ struct acpi_cpufreq_data {
 
 static DEFINE_PER_CPU(struct acpi_cpufreq_data *, acfreq_data);
 
-static DEFINE_PER_CPU(struct aperfmperf, acfreq_old_perf);
-
 /* acpi_perf_data is a pointer to percpu data. */
 static struct acpi_processor_performance *acpi_perf_data;
 
@@ -240,45 +239,6 @@ static u32 get_cur_val(const struct cpumask *mask)
         return cmd.val;
 }
 
-/* Called via smp_call_function_single(), on the target CPU */
-static void read_measured_perf_ctrs(void *_cur)
-{
-        struct aperfmperf *am = _cur;
-
-        get_aperfmperf(am);
-}
-
-/*
- * Return the measured active (C0) frequency on this CPU since last call
- * to this function.
- * Input: cpu number
- * Return: Average CPU frequency in terms of max frequency (zero on error)
- *
- * We use IA32_MPERF and IA32_APERF MSRs to get the measured performance
- * over a period of time, while CPU is in C0 state.
- * IA32_MPERF counts at the rate of max advertised frequency
- * IA32_APERF counts at the rate of actual CPU frequency
- * Only IA32_APERF/IA32_MPERF ratio is architecturally defined and
- * no meaning should be associated with absolute values of these MSRs.
- */
-static unsigned int get_measured_perf(struct cpufreq_policy *policy,
-                                      unsigned int cpu)
-{
-        struct aperfmperf perf;
-        unsigned long ratio;
-        unsigned int retval;
-
-        if (smp_call_function_single(cpu, read_measured_perf_ctrs, &perf, 1))
-                return 0;
-
-        ratio = calc_aperfmperf_ratio(&per_cpu(acfreq_old_perf, cpu), &perf);
-        per_cpu(acfreq_old_perf, cpu) = perf;
-
-        retval = (policy->cpuinfo.max_freq * ratio) >> APERFMPERF_SHIFT;
-
-        return retval;
-}
-
 static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
 {
         struct acpi_cpufreq_data *data = per_cpu(acfreq_data, cpu);
@@ -702,7 +662,7 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
 
         /* Check for APERF/MPERF support in hardware */
         if (cpu_has(c, X86_FEATURE_APERFMPERF))
-                acpi_cpufreq_driver.getavg = get_measured_perf;
+                acpi_cpufreq_driver.getavg = cpufreq_get_measured_perf;
 
         dprintk("CPU%u - ACPI performance management activated.\n", cpu);
         for (i = 0; i < perf->state_count; i++)
diff --git a/arch/x86/kernel/cpu/cpufreq/mperf.c b/arch/x86/kernel/cpu/cpufreq/mperf.c
new file mode 100644
index 000000000000..911e193018ae
--- /dev/null
+++ b/arch/x86/kernel/cpu/cpufreq/mperf.c
@@ -0,0 +1,51 @@
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/slab.h>
+
+#include "mperf.h"
+
+static DEFINE_PER_CPU(struct aperfmperf, acfreq_old_perf);
+
+/* Called via smp_call_function_single(), on the target CPU */
+static void read_measured_perf_ctrs(void *_cur)
+{
+        struct aperfmperf *am = _cur;
+
+        get_aperfmperf(am);
+}
+
+/*
+ * Return the measured active (C0) frequency on this CPU since last call
+ * to this function.
+ * Input: cpu number
+ * Return: Average CPU frequency in terms of max frequency (zero on error)
+ *
+ * We use IA32_MPERF and IA32_APERF MSRs to get the measured performance
+ * over a period of time, while CPU is in C0 state.
+ * IA32_MPERF counts at the rate of max advertised frequency
+ * IA32_APERF counts at the rate of actual CPU frequency
+ * Only IA32_APERF/IA32_MPERF ratio is architecturally defined and
+ * no meaning should be associated with absolute values of these MSRs.
+ */
+unsigned int cpufreq_get_measured_perf(struct cpufreq_policy *policy,
+                                        unsigned int cpu)
+{
+        struct aperfmperf perf;
+        unsigned long ratio;
+        unsigned int retval;
+
+        if (smp_call_function_single(cpu, read_measured_perf_ctrs, &perf, 1))
+                return 0;
+
+        ratio = calc_aperfmperf_ratio(&per_cpu(acfreq_old_perf, cpu), &perf);
+        per_cpu(acfreq_old_perf, cpu) = perf;
+
+        retval = (policy->cpuinfo.max_freq * ratio) >> APERFMPERF_SHIFT;
+
+        return retval;
+}
+EXPORT_SYMBOL_GPL(cpufreq_get_measured_perf);
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/kernel/cpu/cpufreq/mperf.h b/arch/x86/kernel/cpu/cpufreq/mperf.h
new file mode 100644
index 000000000000..5dbf2950dc22
--- /dev/null
+++ b/arch/x86/kernel/cpu/cpufreq/mperf.h
@@ -0,0 +1,9 @@
+/*
+ *  (c) 2010 Advanced Micro Devices, Inc.
+ *  Your use of this code is subject to the terms and conditions of the
+ *  GNU general public license version 2. See "COPYING" or
+ *  http://www.gnu.org/licenses/gpl.html
+ */
+
+unsigned int cpufreq_get_measured_perf(struct cpufreq_policy *policy,
+                                        unsigned int cpu);
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
index b6215b9798e2..6f3dc8fbbfdc 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
@@ -1,6 +1,5 @@
-
 /*
- *   (c) 2003-2006 Advanced Micro Devices, Inc.
+ *   (c) 2003-2010 Advanced Micro Devices, Inc.
  *  Your use of this code is subject to the terms and conditions of the
  *  GNU general public license version 2. See "COPYING" or
  *  http://www.gnu.org/licenses/gpl.html
@@ -46,6 +45,7 @@
 #define PFX "powernow-k8: "
 #define VERSION "version 2.20.00"
 #include "powernow-k8.h"
+#include "mperf.h"
 
 /* serialize freq changes  */
 static DEFINE_MUTEX(fidvid_mutex);
@@ -54,6 +54,12 @@ static DEFINE_PER_CPU(struct powernow_k8_data *, powernow_data);
 
 static int cpu_family = CPU_OPTERON;
 
+/* core performance boost */
+static bool cpb_capable, cpb_enabled;
+static struct msr __percpu *msrs;
+
+static struct cpufreq_driver cpufreq_amd64_driver;
+
 #ifndef CONFIG_SMP
 static inline const struct cpumask *cpu_core_mask(int cpu)
 {
@@ -1249,6 +1255,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
         struct powernow_k8_data *data;
         struct init_on_cpu init_on_cpu;
         int rc;
+        struct cpuinfo_x86 *c = &cpu_data(pol->cpu);
 
         if (!cpu_online(pol->cpu))
                 return -ENODEV;
@@ -1323,6 +1330,10 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
                 return -EINVAL;
         }
 
+        /* Check for APERF/MPERF support in hardware */
+        if (cpu_has(c, X86_FEATURE_APERFMPERF))
+                cpufreq_amd64_driver.getavg = cpufreq_get_measured_perf;
+
         cpufreq_frequency_table_get_attr(data->powernow_table, pol->cpu);
 
         if (cpu_family == CPU_HW_PSTATE)
@@ -1394,8 +1405,77 @@ out:
         return khz;
 }
 
+static void _cpb_toggle_msrs(bool t)
+{
+        int cpu;
+
+        get_online_cpus();
+
+        rdmsr_on_cpus(cpu_online_mask, MSR_K7_HWCR, msrs);
+
+        for_each_cpu(cpu, cpu_online_mask) {
+                struct msr *reg = per_cpu_ptr(msrs, cpu);
+                if (t)
+                        reg->l &= ~BIT(25);
+                else
+                        reg->l |= BIT(25);
+        }
+        wrmsr_on_cpus(cpu_online_mask, MSR_K7_HWCR, msrs);
+
+        put_online_cpus();
+}
+
+/*
+ * Switch on/off core performance boosting.
+ *
+ * 0=disable
+ * 1=enable.
+ */
+static void cpb_toggle(bool t)
+{
+        if (!cpb_capable)
+                return;
+
+        if (t && !cpb_enabled) {
+                cpb_enabled = true;
+                _cpb_toggle_msrs(t);
+                printk(KERN_INFO PFX "Core Boosting enabled.\n");
+        } else if (!t && cpb_enabled) {
+                cpb_enabled = false;
+                _cpb_toggle_msrs(t);
+                printk(KERN_INFO PFX "Core Boosting disabled.\n");
+        }
+}
+
+static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,
+                                 size_t count)
+{
+        int ret = -EINVAL;
+        unsigned long val = 0;
+
+        ret = strict_strtoul(buf, 10, &val);
+        if (!ret && (val == 0 || val == 1) && cpb_capable)
+                cpb_toggle(val);
+        else
+                return -EINVAL;
+
+        return count;
+}
+
+static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf)
+{
+        return sprintf(buf, "%u\n", cpb_enabled);
+}
+
+#define define_one_rw(_name) \
+static struct freq_attr _name = \
+__ATTR(_name, 0644, show_##_name, store_##_name)
+
+define_one_rw(cpb);
+
 static struct freq_attr *powernow_k8_attr[] = {
         &cpufreq_freq_attr_scaling_available_freqs,
+        &cpb,
         NULL,
 };
 
@@ -1411,10 +1491,51 @@ static struct cpufreq_driver cpufreq_amd64_driver = {
         .attr           = powernow_k8_attr,
 };
 
+/*
+ * Clear the boost-disable flag on the CPU_DOWN path so that this cpu
+ * cannot block the remaining ones from boosting. On the CPU_UP path we
+ * simply keep the boost-disable flag in sync with the current global
+ * state.
+ */
+static int __cpuinit cpb_notify(struct notifier_block *nb, unsigned long action,
+                                void *hcpu)
+{
+        unsigned cpu = (long)hcpu;
+        u32 lo, hi;
+
+        switch (action) {
+        case CPU_UP_PREPARE:
+        case CPU_UP_PREPARE_FROZEN:
+
+                if (!cpb_enabled) {
+                        rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
+                        lo |= BIT(25);
+                        wrmsr_on_cpu(cpu, MSR_K7_HWCR, lo, hi);
+                }
+                break;
+
+        case CPU_DOWN_PREPARE:
+        case CPU_DOWN_PREPARE_FROZEN:
+                rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
+                lo &= ~BIT(25);
+                wrmsr_on_cpu(cpu, MSR_K7_HWCR, lo, hi);
+                break;
+
+        default:
+                break;
+        }
+
+        return NOTIFY_OK;
+}
+
+static struct notifier_block __cpuinitdata cpb_nb = {
+        .notifier_call          = cpb_notify,
+};
+
 /* driver entry point for init */
 static int __cpuinit powernowk8_init(void)
 {
-        unsigned int i, supported_cpus = 0;
+        unsigned int i, supported_cpus = 0, cpu;
 
         for_each_online_cpu(i) {
                 int rc;
@@ -1423,15 +1544,36 @@ static int __cpuinit powernowk8_init(void)
                         supported_cpus++;
         }
 
-        if (supported_cpus == num_online_cpus()) {
-                printk(KERN_INFO PFX "Found %d %s "
-                        "processors (%d cpu cores) (" VERSION ")\n",
-                        num_online_nodes(),
-                        boot_cpu_data.x86_model_id, supported_cpus);
-                return cpufreq_register_driver(&cpufreq_amd64_driver);
+        if (supported_cpus != num_online_cpus())
+                return -ENODEV;
+
+        printk(KERN_INFO PFX "Found %d %s (%d cpu cores) (" VERSION ")\n",
+                num_online_nodes(), boot_cpu_data.x86_model_id, supported_cpus);
+
+        if (boot_cpu_has(X86_FEATURE_CPB)) {
+
+                cpb_capable = true;
+
+                register_cpu_notifier(&cpb_nb);
+
+                msrs = msrs_alloc();
+                if (!msrs) {
+                        printk(KERN_ERR "%s: Error allocating msrs!\n", __func__);
+                        return -ENOMEM;
+                }
+
+                rdmsr_on_cpus(cpu_online_mask, MSR_K7_HWCR, msrs);
+
+                for_each_cpu(cpu, cpu_online_mask) {
+                        struct msr *reg = per_cpu_ptr(msrs, cpu);
+                        cpb_enabled |= !(!!(reg->l & BIT(25)));
+                }
+
+                printk(KERN_INFO PFX "Core Performance Boosting: %s.\n",
+                        (cpb_enabled ? "on" : "off"));
         }
 
-        return -ENODEV;
+        return cpufreq_register_driver(&cpufreq_amd64_driver);
 }
 
 /* driver entry point for term */
@@ -1439,6 +1581,13 @@ static void __exit powernowk8_exit(void)
 {
         dprintk("exit\n");
 
+        if (boot_cpu_has(X86_FEATURE_CPB)) {
+                msrs_free(msrs);
+                msrs = NULL;
+
+                unregister_cpu_notifier(&cpb_nb);
+        }
+
         cpufreq_unregister_driver(&cpufreq_amd64_driver);
 }
 
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.h b/arch/x86/kernel/cpu/cpufreq/powernow-k8.h
index 02ce824073cb..df3529b1c02d 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.h
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.h
@@ -5,7 +5,6 @@
  *  http://www.gnu.org/licenses/gpl.html
  */
 
-
 enum pstate {
         HW_PSTATE_INVALID = 0xff,
         HW_PSTATE_0 = 0,
@@ -55,7 +54,6 @@ struct powernow_k8_data {
         struct cpumask *available_cores;
 };
 
-
 /* processor's cpuid instruction support */
 #define CPUID_PROCESSOR_SIGNATURE       1       /* function 1 */
 #define CPUID_XFAM                      0x0ff00000      /* extended family */
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index 08be922de33a..dd531cc56a8f 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -21,37 +21,55 @@
  *
  */
 
+#include <linux/module.h>
 #include <asm/processor.h>
-#include <asm/vmware.h>
 #include <asm/hypervisor.h>
 
-static inline void __cpuinit
-detect_hypervisor_vendor(struct cpuinfo_x86 *c)
+/*
+ * Hypervisor detect order.  This is specified explicitly here because
+ * some hypervisors might implement compatibility modes for other
+ * hypervisors and therefore need to be detected in specific sequence.
+ */
+static const __initconst struct hypervisor_x86 * const hypervisors[] =
 {
-        if (vmware_platform())
-                c->x86_hyper_vendor = X86_HYPER_VENDOR_VMWARE;
-        else
-                c->x86_hyper_vendor = X86_HYPER_VENDOR_NONE;
-}
+        &x86_hyper_vmware,
+        &x86_hyper_ms_hyperv,
+};
 
-static inline void __cpuinit
-hypervisor_set_feature_bits(struct cpuinfo_x86 *c)
+const struct hypervisor_x86 *x86_hyper;
+EXPORT_SYMBOL(x86_hyper);
+
+static inline void __init
+detect_hypervisor_vendor(void)
 {
-        if (boot_cpu_data.x86_hyper_vendor == X86_HYPER_VENDOR_VMWARE) {
-                vmware_set_feature_bits(c);
-                return;
+        const struct hypervisor_x86 *h, * const *p;
+
+        for (p = hypervisors; p < hypervisors + ARRAY_SIZE(hypervisors); p++) {
+                h = *p;
+                if (h->detect()) {
+                        x86_hyper = h;
+                        printk(KERN_INFO "Hypervisor detected: %s\n", h->name);
+                        break;
+                }
         }
 }
 
 void __cpuinit init_hypervisor(struct cpuinfo_x86 *c)
 {
-        detect_hypervisor_vendor(c);
-        hypervisor_set_feature_bits(c);
+        if (x86_hyper && x86_hyper->set_cpu_features)
+                x86_hyper->set_cpu_features(c);
 }
 
 void __init init_hypervisor_platform(void)
 {
+
+        detect_hypervisor_vendor();
+
+        if (!x86_hyper)
+                return;
+
         init_hypervisor(&boot_cpu_data);
-        if (boot_cpu_data.x86_hyper_vendor == X86_HYPER_VENDOR_VMWARE)
-                vmware_platform_setup();
+
+        if (x86_hyper->init_platform)
+                x86_hyper->init_platform();
 }
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index f5e5390d3459..85f69cdeae10 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -372,12 +372,6 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
                         set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON);
         }
 
-        if (c->cpuid_level > 6) {
-                unsigned ecx = cpuid_ecx(6);
-                if (ecx & 0x01)
-                        set_cpu_cap(c, X86_FEATURE_APERFMPERF);
-        }
-
         if (cpu_has_xmm2)
                 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
         if (cpu_has_ds) {
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index b3eeb66c0a51..33eae2062cf5 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -148,13 +148,19 @@ union _cpuid4_leaf_ecx {
         u32 full;
 };
 
+struct amd_l3_cache {
+        struct   pci_dev *dev;
+        bool     can_disable;
+        unsigned indices;
+        u8       subcaches[4];
+};
+
 struct _cpuid4_info {
         union _cpuid4_leaf_eax eax;
         union _cpuid4_leaf_ebx ebx;
         union _cpuid4_leaf_ecx ecx;
         unsigned long size;
-        bool can_disable;
-        unsigned int l3_indices;
+        struct amd_l3_cache *l3;
         DECLARE_BITMAP(shared_cpu_map, NR_CPUS);
 };
 
@@ -164,8 +170,7 @@ struct _cpuid4_info_regs {
         union _cpuid4_leaf_ebx ebx;
         union _cpuid4_leaf_ecx ecx;
         unsigned long size;
-        bool can_disable;
-        unsigned int l3_indices;
+        struct amd_l3_cache *l3;
 };
 
 unsigned short                  num_cache_leaves;
@@ -302,87 +307,163 @@ struct _cache_attr {
 };
 
 #ifdef CONFIG_CPU_SUP_AMD
-static unsigned int __cpuinit amd_calc_l3_indices(void)
+
+/*
+ * L3 cache descriptors
+ */
+static struct amd_l3_cache **__cpuinitdata l3_caches;
+
+static void __cpuinit amd_calc_l3_indices(struct amd_l3_cache *l3)
 {
-        /*
-         * We're called over smp_call_function_single() and therefore
-         * are on the correct cpu.
-         */
-        int cpu = smp_processor_id();
-        int node = cpu_to_node(cpu);
-        struct pci_dev *dev = node_to_k8_nb_misc(node);
         unsigned int sc0, sc1, sc2, sc3;
         u32 val = 0;
 
-        pci_read_config_dword(dev, 0x1C4, &val);
+        pci_read_config_dword(l3->dev, 0x1C4, &val);
 
         /* calculate subcache sizes */
-        sc0 = !(val & BIT(0));
-        sc1 = !(val & BIT(4));
-        sc2 = !(val & BIT(8))  + !(val & BIT(9));
-        sc3 = !(val & BIT(12)) + !(val & BIT(13));
+        l3->subcaches[0] = sc0 = !(val & BIT(0));
+        l3->subcaches[1] = sc1 = !(val & BIT(4));
+        l3->subcaches[2] = sc2 = !(val & BIT(8))  + !(val & BIT(9));
+        l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13));
 
-        return (max(max(max(sc0, sc1), sc2), sc3) << 10) - 1;
+        l3->indices = (max(max(max(sc0, sc1), sc2), sc3) << 10) - 1;
+}
+
+static struct amd_l3_cache * __cpuinit amd_init_l3_cache(int node)
+{
+        struct amd_l3_cache *l3;
+        struct pci_dev *dev = node_to_k8_nb_misc(node);
+
+        l3 = kzalloc(sizeof(struct amd_l3_cache), GFP_ATOMIC);
+        if (!l3) {
+                printk(KERN_WARNING "Error allocating L3 struct\n");
+                return NULL;
+        }
+
+        l3->dev = dev;
+
+        amd_calc_l3_indices(l3);
+
+        return l3;
 }
 
 static void __cpuinit
 amd_check_l3_disable(int index, struct _cpuid4_info_regs *this_leaf)
 {
-        if (index < 3)
+        int node;
+
+        if (boot_cpu_data.x86 != 0x10)
                 return;
 
-        if (boot_cpu_data.x86 == 0x11)
+        if (index < 3)
                 return;
 
         /* see errata #382 and #388 */
-        if ((boot_cpu_data.x86 == 0x10) &&
-            ((boot_cpu_data.x86_model < 0x8) ||
-             (boot_cpu_data.x86_mask  < 0x1)))
+        if (boot_cpu_data.x86_model < 0x8)
+                return;
+
+        if ((boot_cpu_data.x86_model == 0x8 ||
+             boot_cpu_data.x86_model == 0x9)
+                &&
+             boot_cpu_data.x86_mask < 0x1)
+                        return;
+
+        /* not in virtualized environments */
+        if (num_k8_northbridges == 0)
                 return;
 
-        this_leaf->can_disable = true;
-        this_leaf->l3_indices  = amd_calc_l3_indices();
+        /*
+         * Strictly speaking, the amount in @size below is leaked since it is
+         * never freed but this is done only on shutdown so it doesn't matter.
+         */
+        if (!l3_caches) {
+                int size = num_k8_northbridges * sizeof(struct amd_l3_cache *);
+
+                l3_caches = kzalloc(size, GFP_ATOMIC);
+                if (!l3_caches)
+                        return;
+        }
+
+        node = amd_get_nb_id(smp_processor_id());
+
+        if (!l3_caches[node]) {
+                l3_caches[node] = amd_init_l3_cache(node);
+                l3_caches[node]->can_disable = true;
+        }
+
+        WARN_ON(!l3_caches[node]);
+
+        this_leaf->l3 = l3_caches[node];
 }
 
 static ssize_t show_cache_disable(struct _cpuid4_info *this_leaf, char *buf,
-                                  unsigned int index)
+                                  unsigned int slot)
 {
-        int cpu = cpumask_first(to_cpumask(this_leaf->shared_cpu_map));
-        int node = amd_get_nb_id(cpu);
-        struct pci_dev *dev = node_to_k8_nb_misc(node);
+        struct pci_dev *dev = this_leaf->l3->dev;
         unsigned int reg = 0;
 
-        if (!this_leaf->can_disable)
+        if (!this_leaf->l3 || !this_leaf->l3->can_disable)
                 return -EINVAL;
 
         if (!dev)
                 return -EINVAL;
 
-        pci_read_config_dword(dev, 0x1BC + index * 4, &reg);
+        pci_read_config_dword(dev, 0x1BC + slot * 4, &reg);
         return sprintf(buf, "0x%08x\n", reg);
 }
 
-#define SHOW_CACHE_DISABLE(index)                                       \
+#define SHOW_CACHE_DISABLE(slot)                                        \
 static ssize_t                                                          \
-show_cache_disable_##index(struct _cpuid4_info *this_leaf, char *buf)   \
+show_cache_disable_##slot(struct _cpuid4_info *this_leaf, char *buf)    \
 {                                                                       \
-        return show_cache_disable(this_leaf, buf, index);               \
+        return show_cache_disable(this_leaf, buf, slot);                \
 }
 SHOW_CACHE_DISABLE(0)
 SHOW_CACHE_DISABLE(1)
 
+static void amd_l3_disable_index(struct amd_l3_cache *l3, int cpu,
+                                 unsigned slot, unsigned long idx)
+{
+        int i;
+
+        idx |= BIT(30);
+
+        /*
+         *  disable index in all 4 subcaches
+         */
+        for (i = 0; i < 4; i++) {
+                u32 reg = idx | (i << 20);
+
+                if (!l3->subcaches[i])
+                        continue;
+
+                pci_write_config_dword(l3->dev, 0x1BC + slot * 4, reg);
+
+                /*
+                 * We need to WBINVD on a core on the node containing the L3
+                 * cache which indices we disable therefore a simple wbinvd()
+                 * is not sufficient.
+                 */
+                wbinvd_on_cpu(cpu);
+
+                reg |= BIT(31);
+                pci_write_config_dword(l3->dev, 0x1BC + slot * 4, reg);
+        }
+}
+
+
 static ssize_t store_cache_disable(struct _cpuid4_info *this_leaf,
-        const char *buf, size_t count, unsigned int index)
+                                   const char *buf, size_t count,
+                                   unsigned int slot)
 {
+        struct pci_dev *dev = this_leaf->l3->dev;
         int cpu = cpumask_first(to_cpumask(this_leaf->shared_cpu_map));
-        int node = amd_get_nb_id(cpu);
-        struct pci_dev *dev = node_to_k8_nb_misc(node);
         unsigned long val = 0;
 
 #define SUBCACHE_MASK   (3UL << 20)
 #define SUBCACHE_INDEX  0xfff
 
-        if (!this_leaf->can_disable)
+        if (!this_leaf->l3 || !this_leaf->l3->can_disable)
                 return -EINVAL;
 
         if (!capable(CAP_SYS_ADMIN))
@@ -396,26 +477,20 @@ static ssize_t store_cache_disable(struct _cpuid4_info *this_leaf,
 
         /* do not allow writes outside of allowed bits */
         if ((val & ~(SUBCACHE_MASK | SUBCACHE_INDEX)) ||
-            ((val & SUBCACHE_INDEX) > this_leaf->l3_indices))
+            ((val & SUBCACHE_INDEX) > this_leaf->l3->indices))
                 return -EINVAL;
 
-        val |= BIT(30);
-        pci_write_config_dword(dev, 0x1BC + index * 4, val);
-        /*
-         * We need to WBINVD on a core on the node containing the L3 cache which
-         * indices we disable therefore a simple wbinvd() is not sufficient.
-         */
-        wbinvd_on_cpu(cpu);
-        pci_write_config_dword(dev, 0x1BC + index * 4, val | BIT(31));
+        amd_l3_disable_index(this_leaf->l3, cpu, slot, val);
+
         return count;
 }
 
-#define STORE_CACHE_DISABLE(index)                                      \
+#define STORE_CACHE_DISABLE(slot)                                       \
 static ssize_t                                                          \
-store_cache_disable_##index(struct _cpuid4_info *this_leaf,             \
+store_cache_disable_##slot(struct _cpuid4_info *this_leaf,              \
                             const char *buf, size_t count)              \
 {                                                                       \
-        return store_cache_disable(this_leaf, buf, count, index);       \
+        return store_cache_disable(this_leaf, buf, count, slot);        \
 }
 STORE_CACHE_DISABLE(0)
 STORE_CACHE_DISABLE(1)
@@ -443,8 +518,7 @@ __cpuinit cpuid4_cache_lookup_regs(int index,
 
         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
                 amd_cpuid4(index, &eax, &ebx, &ecx);
-                if (boot_cpu_data.x86 >= 0x10)
-                        amd_check_l3_disable(index, this_leaf);
+                amd_check_l3_disable(index, this_leaf);
         } else {
                 cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx);
         }
@@ -701,6 +775,7 @@ static void __cpuinit free_cache_attributes(unsigned int cpu)
         for (i = 0; i < num_cache_leaves; i++)
                 cache_remove_shared_cpu_map(cpu, i);
 
+        kfree(per_cpu(ici_cpuid4_info, cpu)->l3);
         kfree(per_cpu(ici_cpuid4_info, cpu));
         per_cpu(ici_cpuid4_info, cpu) = NULL;
 }
@@ -985,7 +1060,7 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev)
 
                 this_leaf = CPUID4_INFO_IDX(cpu, i);
 
-                if (this_leaf->can_disable)
+                if (this_leaf->l3 && this_leaf->l3->can_disable)
                         ktype_cache.default_attrs = default_l3_attrs;
                 else
                         ktype_cache.default_attrs = default_attrs;
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 8a6f0afa767e..7a355ddcc64b 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -539,7 +539,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
         struct mce m;
         int i;
 
-        __get_cpu_var(mce_poll_count)++;
+        percpu_inc(mce_poll_count);
 
         mce_setup(&m);
 
@@ -934,7 +934,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
 
         atomic_inc(&mce_entry);
 
-        __get_cpu_var(mce_exception_count)++;
+        percpu_inc(mce_exception_count);
 
         if (notify_die(DIE_NMI, "machine check", regs, error_code,
                            18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
new file mode 100644
index 000000000000..16f41bbe46b6
--- /dev/null
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -0,0 +1,55 @@
+/*
+ * HyperV  Detection code.
+ *
+ * Copyright (C) 2010, Novell, Inc.
+ * Author : K. Y. Srinivasan <ksrinivasan@novell.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <asm/processor.h>
+#include <asm/hypervisor.h>
+#include <asm/hyperv.h>
+#include <asm/mshyperv.h>
+
+struct ms_hyperv_info ms_hyperv;
+
+static bool __init ms_hyperv_platform(void)
+{
+        u32 eax;
+        u32 hyp_signature[3];
+
+        if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
+                return false;
+
+        cpuid(HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS,
+              &eax, &hyp_signature[0], &hyp_signature[1], &hyp_signature[2]);
+
+        return eax >= HYPERV_CPUID_MIN &&
+                eax <= HYPERV_CPUID_MAX &&
+                !memcmp("Microsoft Hv", hyp_signature, 12);
+}
+
+static void __init ms_hyperv_init_platform(void)
+{
+        /*
+         * Extract the features and hints
+         */
+        ms_hyperv.features = cpuid_eax(HYPERV_CPUID_FEATURES);
+        ms_hyperv.hints    = cpuid_eax(HYPERV_CPUID_ENLIGHTMENT_INFO);
+
+        printk(KERN_INFO "HyperV: features 0x%x, hints 0x%x\n",
+               ms_hyperv.features, ms_hyperv.hints);
+}
+
+const __refconst struct hypervisor_x86 x86_hyper_ms_hyperv = {
+        .name                   = "Microsoft HyperV",
+        .detect                 = ms_hyperv_platform,
+        .init_platform          = ms_hyperv_init_platform,
+};
+EXPORT_SYMBOL(x86_hyper_ms_hyperv);
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index dfdb4dba2320..b9d1ff588445 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -24,8 +24,8 @@
 #include <linux/dmi.h>
 #include <linux/module.h>
 #include <asm/div64.h>
-#include <asm/vmware.h>
 #include <asm/x86_init.h>
+#include <asm/hypervisor.h>
 
 #define CPUID_VMWARE_INFO_LEAF  0x40000000
 #define VMWARE_HYPERVISOR_MAGIC 0x564D5868
@@ -65,7 +65,7 @@ static unsigned long vmware_get_tsc_khz(void)
         return tsc_hz;
 }
 
-void __init vmware_platform_setup(void)
+static void __init vmware_platform_setup(void)
 {
         uint32_t eax, ebx, ecx, edx;
 
@@ -83,26 +83,22 @@ void __init vmware_platform_setup(void)
  * serial key should be enough, as this will always have a VMware
  * specific string when running under VMware hypervisor.
  */
-int vmware_platform(void)
+static bool __init vmware_platform(void)
 {
         if (cpu_has_hypervisor) {
-                unsigned int eax, ebx, ecx, edx;
-                char hyper_vendor_id[13];
-
-                cpuid(CPUID_VMWARE_INFO_LEAF, &eax, &ebx, &ecx, &edx);
-                memcpy(hyper_vendor_id + 0, &ebx, 4);
-                memcpy(hyper_vendor_id + 4, &ecx, 4);
-                memcpy(hyper_vendor_id + 8, &edx, 4);
-                hyper_vendor_id[12] = '\0';
-                if (!strcmp(hyper_vendor_id, "VMwareVMware"))
-                        return 1;
+                unsigned int eax;
+                unsigned int hyper_vendor_id[3];
+
+                cpuid(CPUID_VMWARE_INFO_LEAF, &eax, &hyper_vendor_id[0],
+                      &hyper_vendor_id[1], &hyper_vendor_id[2]);
+                if (!memcmp(hyper_vendor_id, "VMwareVMware", 12))
+                        return true;
         } else if (dmi_available && dmi_name_in_serial("VMware") &&
                    __vmware_platform())
-                return 1;
+                return true;
 
-        return 0;
+        return false;
 }
-EXPORT_SYMBOL(vmware_platform);
 
 /*
  * VMware hypervisor takes care of exporting a reliable TSC to the guest.
@@ -116,8 +112,16 @@ EXPORT_SYMBOL(vmware_platform);
  * so that the kernel could just trust the hypervisor with providing a
  * reliable virtual TSC that is suitable for timekeeping.
  */
-void __cpuinit vmware_set_feature_bits(struct cpuinfo_x86 *c)
+static void __cpuinit vmware_set_cpu_features(struct cpuinfo_x86 *c)
 {
         set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
         set_cpu_cap(c, X86_FEATURE_TSC_RELIABLE);
 }
+
+const __refconst struct hypervisor_x86 x86_hyper_vmware = {
+        .name                   = "VMware",
+        .detect                 = vmware_platform,
+        .set_cpu_features       = vmware_set_cpu_features,
+        .init_platform          = vmware_platform_setup,
+};
+EXPORT_SYMBOL(x86_hyper_vmware);
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 44a8e0dc6737..cd49141cf153 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -53,6 +53,7 @@
 #include <asm/processor-flags.h>
 #include <asm/ftrace.h>
 #include <asm/irq_vectors.h>
+#include <asm/cpufeature.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -905,7 +906,25 @@ ENTRY(simd_coprocessor_error)
         RING0_INT_FRAME
         pushl $0
         CFI_ADJUST_CFA_OFFSET 4
+#ifdef CONFIG_X86_INVD_BUG
+        /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
+661:    pushl $do_general_protection
+662:
+.section .altinstructions,"a"
+        .balign 4
+        .long 661b
+        .long 663f
+        .byte X86_FEATURE_XMM
+        .byte 662b-661b
+        .byte 664f-663f
+.previous
+.section .altinstr_replacement,"ax"
+663:    pushl $do_simd_coprocessor_error
+664:
+.previous
+#else
         pushl $do_simd_coprocessor_error
+#endif
         CFI_ADJUST_CFA_OFFSET 4
         jmp error_code
         CFI_ENDPROC
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index 54c31c285488..86cef6b32253 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -102,65 +102,62 @@ void __cpuinit fpu_init(void)
 
         mxcsr_feature_mask_init();
         /* clean state in init */
-        if (cpu_has_xsave)
-                current_thread_info()->status = TS_XSAVE;
-        else
-                current_thread_info()->status = 0;
+        current_thread_info()->status = 0;
         clear_used_math();
 }
 #endif  /* CONFIG_X86_64 */
 
-/*
- * The _current_ task is using the FPU for the first time
- * so initialize it and set the mxcsr to its default
- * value at reset if we support XMM instructions and then
- * remeber the current task has used the FPU.
- */
-int init_fpu(struct task_struct *tsk)
+static void fpu_finit(struct fpu *fpu)
 {
-        if (tsk_used_math(tsk)) {
-                if (HAVE_HWFP && tsk == current)
-                        unlazy_fpu(tsk);
-                return 0;
-        }
-
-        /*
-         * Memory allocation at the first usage of the FPU and other state.
-         */
-        if (!tsk->thread.xstate) {
-                tsk->thread.xstate = kmem_cache_alloc(task_xstate_cachep,
-                                                      GFP_KERNEL);
-                if (!tsk->thread.xstate)
-                        return -ENOMEM;
-        }
-
 #ifdef CONFIG_X86_32
         if (!HAVE_HWFP) {
-                memset(tsk->thread.xstate, 0, xstate_size);
-                finit_task(tsk);
-                set_stopped_child_used_math(tsk);
-                return 0;
+                finit_soft_fpu(&fpu->state->soft);
+                return;
         }
 #endif
 
         if (cpu_has_fxsr) {
-                struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave;
+                struct i387_fxsave_struct *fx = &fpu->state->fxsave;
 
                 memset(fx, 0, xstate_size);
                 fx->cwd = 0x37f;
                 if (cpu_has_xmm)
                         fx->mxcsr = MXCSR_DEFAULT;
         } else {
-                struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave;
+                struct i387_fsave_struct *fp = &fpu->state->fsave;
                 memset(fp, 0, xstate_size);
                 fp->cwd = 0xffff037fu;
                 fp->swd = 0xffff0000u;
                 fp->twd = 0xffffffffu;
                 fp->fos = 0xffff0000u;
         }
+}
+
+/*
+ * The _current_ task is using the FPU for the first time
+ * so initialize it and set the mxcsr to its default
+ * value at reset if we support XMM instructions and then
+ * remeber the current task has used the FPU.
+ */
+int init_fpu(struct task_struct *tsk)
+{
+        int ret;
+
+        if (tsk_used_math(tsk)) {
+                if (HAVE_HWFP && tsk == current)
+                        unlazy_fpu(tsk);
+                return 0;
+        }
+
         /*
-         * Only the device not available exception or ptrace can call init_fpu.
+         * Memory allocation at the first usage of the FPU and other state.
          */
+        ret = fpu_alloc(&tsk->thread.fpu);
+        if (ret)
+                return ret;
+
+        fpu_finit(&tsk->thread.fpu);
+
         set_stopped_child_used_math(tsk);
         return 0;
 }
@@ -194,7 +191,7 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
                 return ret;
 
         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
-                                   &target->thread.xstate->fxsave, 0, -1);
+                                   &target->thread.fpu.state->fxsave, 0, -1);
 }
 
 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
@@ -211,19 +208,19 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
                 return ret;
 
         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
-                                 &target->thread.xstate->fxsave, 0, -1);
+                                 &target->thread.fpu.state->fxsave, 0, -1);
 
         /*
          * mxcsr reserved bits must be masked to zero for security reasons.
          */
-        target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
+        target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 
         /*
          * update the header bits in the xsave header, indicating the
          * presence of FP and SSE state.
          */
         if (cpu_has_xsave)
-                target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
+                target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
 
         return ret;
 }
@@ -246,14 +243,14 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
          * memory layout in the thread struct, so that we can copy the entire
          * xstateregs to the user using one user_regset_copyout().
          */
-        memcpy(&target->thread.xstate->fxsave.sw_reserved,
+        memcpy(&target->thread.fpu.state->fxsave.sw_reserved,
                xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
 
         /*
          * Copy the xstate memory layout.
          */
         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
-                                  &target->thread.xstate->xsave, 0, -1);
+                                  &target->thread.fpu.state->xsave, 0, -1);
         return ret;
 }
 
@@ -272,14 +269,14 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
                 return ret;
 
         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
-                                 &target->thread.xstate->xsave, 0, -1);
+                                 &target->thread.fpu.state->xsave, 0, -1);
 
         /*
          * mxcsr reserved bits must be masked to zero for security reasons.
          */
-        target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
+        target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 
-        xsave_hdr = &target->thread.xstate->xsave.xsave_hdr;
+        xsave_hdr = &target->thread.fpu.state->xsave.xsave_hdr;
 
         xsave_hdr->xstate_bv &= pcntxt_mask;
         /*
@@ -365,7 +362,7 @@ static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
 static void
 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
 {
-        struct i387_fxsave_struct *fxsave = &tsk->thread.xstate->fxsave;
+        struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
         struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
         struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
         int i;
@@ -405,7 +402,7 @@ static void convert_to_fxsr(struct task_struct *tsk,
                             const struct user_i387_ia32_struct *env)
 
 {
-        struct i387_fxsave_struct *fxsave = &tsk->thread.xstate->fxsave;
+        struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
         struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
         struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
         int i;
@@ -445,7 +442,7 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 
         if (!cpu_has_fxsr) {
                 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
-                                           &target->thread.xstate->fsave, 0,
+                                           &target->thread.fpu.state->fsave, 0,
                                            -1);
         }
 
@@ -475,7 +472,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 
         if (!cpu_has_fxsr) {
                 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
-                                          &target->thread.xstate->fsave, 0, -1);
+                                          &target->thread.fpu.state->fsave, 0, -1);
         }
 
         if (pos > 0 || count < sizeof(env))
@@ -490,7 +487,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
          * presence of FP.
          */
         if (cpu_has_xsave)
-                target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
+                target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
         return ret;
 }
 
@@ -501,7 +498,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
 {
         struct task_struct *tsk = current;
-        struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave;
+        struct i387_fsave_struct *fp = &tsk->thread.fpu.state->fsave;
 
         fp->status = fp->swd;
         if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
@@ -512,7 +509,7 @@ static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
 static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
 {
         struct task_struct *tsk = current;
-        struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave;
+        struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
         struct user_i387_ia32_struct env;
         int err = 0;
 
@@ -547,7 +544,7 @@ static int save_i387_xsave(void __user *buf)
          * header as well as change any contents in the memory layout.
          * xrestore as part of sigreturn will capture all the changes.
          */
-        tsk->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
+        tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
 
         if (save_i387_fxsave(fx) < 0)
                 return -1;
@@ -599,7 +596,7 @@ static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
 {
         struct task_struct *tsk = current;
 
-        return __copy_from_user(&tsk->thread.xstate->fsave, buf,
+        return __copy_from_user(&tsk->thread.fpu.state->fsave, buf,
                                 sizeof(struct i387_fsave_struct));
 }
 
@@ -610,10 +607,10 @@ static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
         struct user_i387_ia32_struct env;
         int err;
 
-        err = __copy_from_user(&tsk->thread.xstate->fxsave, &buf->_fxsr_env[0],
+        err = __copy_from_user(&tsk->thread.fpu.state->fxsave, &buf->_fxsr_env[0],
                                size);
         /* mxcsr reserved bits must be masked to zero for security reasons */
-        tsk->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask;
+        tsk->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
         if (err || __copy_from_user(&env, buf, sizeof(env)))
                 return 1;
         convert_to_fxsr(tsk, &env);
@@ -629,7 +626,7 @@ static int restore_i387_xsave(void __user *buf)
         struct i387_fxsave_struct __user *fx =
                 (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
         struct xsave_hdr_struct *xsave_hdr =
-                                &current->thread.xstate->xsave.xsave_hdr;
+                                &current->thread.fpu.state->xsave.xsave_hdr;
         u64 mask;
         int err;
 
diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c
index 23c167925a5c..2dfd31597443 100644
--- a/arch/x86/kernel/i8253.c
+++ b/arch/x86/kernel/i8253.c
@@ -16,7 +16,7 @@
 #include <asm/hpet.h>
 #include <asm/smp.h>
 
-DEFINE_SPINLOCK(i8253_lock);
+DEFINE_RAW_SPINLOCK(i8253_lock);
 EXPORT_SYMBOL(i8253_lock);
 
 /*
@@ -33,7 +33,7 @@ struct clock_event_device *global_clock_event;
 static void init_pit_timer(enum clock_event_mode mode,
                            struct clock_event_device *evt)
 {
-        spin_lock(&i8253_lock);
+        raw_spin_lock(&i8253_lock);
 
         switch (mode) {
         case CLOCK_EVT_MODE_PERIODIC:
@@ -62,7 +62,7 @@ static void init_pit_timer(enum clock_event_mode mode,
                 /* Nothing to do here */
                 break;
         }
-        spin_unlock(&i8253_lock);
+        raw_spin_unlock(&i8253_lock);
 }
 
 /*
@@ -72,10 +72,10 @@ static void init_pit_timer(enum clock_event_mode mode,
  */
 static int pit_next_event(unsigned long delta, struct clock_event_device *evt)
 {
-        spin_lock(&i8253_lock);
+        raw_spin_lock(&i8253_lock);
         outb_pit(delta & 0xff , PIT_CH0);       /* LSB */
         outb_pit(delta >> 8 , PIT_CH0);         /* MSB */
-        spin_unlock(&i8253_lock);
+        raw_spin_unlock(&i8253_lock);
 
         return 0;
 }
@@ -130,7 +130,7 @@ static cycle_t pit_read(struct clocksource *cs)
         int count;
         u32 jifs;
 
-        spin_lock_irqsave(&i8253_lock, flags);
+        raw_spin_lock_irqsave(&i8253_lock, flags);
         /*
          * Although our caller may have the read side of xtime_lock,
          * this is now a seqlock, and we are cheating in this routine
@@ -176,7 +176,7 @@ static cycle_t pit_read(struct clocksource *cs)
         old_count = count;
         old_jifs = jifs;
 
-        spin_unlock_irqrestore(&i8253_lock, flags);
+        raw_spin_unlock_irqrestore(&i8253_lock, flags);
 
         count = (LATCH - 1) - count;
 
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 0ed2d300cd46..990ae7cfc578 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -60,7 +60,7 @@ static irqreturn_t math_error_irq(int cpl, void *dev_id)
         outb(0, 0xF0);
         if (ignore_fpu_irq || !boot_cpu_data.hard_math)
                 return IRQ_NONE;
-        math_error((void __user *)get_irq_regs()->ip);
+        math_error(get_irq_regs(), 0, 16);
         return IRQ_HANDLED;
 }
 
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index f2f56c0967b6..345a4b1fe144 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -542,20 +542,6 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
         struct kprobe_ctlblk *kcb;
 
         addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
-        if (*addr != BREAKPOINT_INSTRUCTION) {
-                /*
-                 * The breakpoint instruction was removed right
-                 * after we hit it.  Another cpu has removed
-                 * either a probepoint or a debugger breakpoint
-                 * at this address.  In either case, no further
-                 * handling of this interrupt is appropriate.
-                 * Back up over the (now missing) int3 and run
-                 * the original instruction.
-                 */
-                regs->ip = (unsigned long)addr;
-                return 1;
-        }
-
         /*
          * We don't want to be preempted for the entire
          * duration of kprobe processing. We conditionally
@@ -587,6 +573,19 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
                                 setup_singlestep(p, regs, kcb, 0);
                         return 1;
                 }
+        } else if (*addr != BREAKPOINT_INSTRUCTION) {
+                /*
+                 * The breakpoint instruction was removed right
+                 * after we hit it.  Another cpu has removed
+                 * either a probepoint or a debugger breakpoint
+                 * at this address.  In either case, no further
+                 * handling of this interrupt is appropriate.
+                 * Back up over the (now missing) int3 and run
+                 * the original instruction.
+                 */
+                regs->ip = (unsigned long)addr;
+                preempt_enable_no_resched();
+                return 1;
         } else if (kprobe_running()) {
                 p = __get_cpu_var(current_kprobe);
                 if (p->break_handler && p->break_handler(p, regs)) {
diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c
index cceb5bc3c3c2..2cd8c544e41a 100644
--- a/arch/x86/kernel/microcode_core.c
+++ b/arch/x86/kernel/microcode_core.c
@@ -201,9 +201,9 @@ static int do_microcode_update(const void __user *buf, size_t size)
         return error;
 }
 
-static int microcode_open(struct inode *unused1, struct file *unused2)
+static int microcode_open(struct inode *inode, struct file *file)
 {
-        return capable(CAP_SYS_RAWIO) ? 0 : -EPERM;
+        return capable(CAP_SYS_RAWIO) ? nonseekable_open(inode, file) : -EPERM;
 }
 
 static ssize_t microcode_write(struct file *file, const char __user *buf,
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index 85a343e28937..356170262a93 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -343,10 +343,11 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
                                 int (*get_ucode_data)(void *, const void *, size_t))
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
-        u8 *ucode_ptr = data, *new_mc = NULL, *mc;
+        u8 *ucode_ptr = data, *new_mc = NULL, *mc = NULL;
         int new_rev = uci->cpu_sig.rev;
         unsigned int leftover = size;
         enum ucode_state state = UCODE_OK;
+        unsigned int curr_mc_size = 0;
 
         while (leftover) {
                 struct microcode_header_intel mc_header;
@@ -361,9 +362,15 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
                         break;
                 }
 
-                mc = vmalloc(mc_size);
-                if (!mc)
-                        break;
+                /* For performance reasons, reuse mc area when possible */
+                if (!mc || mc_size > curr_mc_size) {
+                        if (mc)
+                                vfree(mc);
+                        mc = vmalloc(mc_size);
+                        if (!mc)
+                                break;
+                        curr_mc_size = mc_size;
+                }
 
                 if (get_ucode_data(mc, ucode_ptr, mc_size) ||
                     microcode_sanity_check(mc) < 0) {
@@ -376,13 +383,16 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
                                 vfree(new_mc);
                         new_rev = mc_header.rev;
                         new_mc  = mc;
-                } else
-                        vfree(mc);
+                        mc = NULL;      /* trigger new vmalloc */
+                }
 
                 ucode_ptr += mc_size;
                 leftover  -= mc_size;
         }
 
+        if (mc)
+                vfree(mc);
+
         if (leftover) {
                 if (new_mc)
                         vfree(new_mc);
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index e81030f71a8f..5ae5d2426edf 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -115,21 +115,6 @@ static void __init MP_bus_info(struct mpc_bus *m)
                 printk(KERN_WARNING "Unknown bustype %s - ignoring\n", str);
 }
 
-static int bad_ioapic(unsigned long address)
-{
-        if (nr_ioapics >= MAX_IO_APICS) {
-                printk(KERN_ERR "ERROR: Max # of I/O APICs (%d) exceeded "
-                       "(found %d)\n", MAX_IO_APICS, nr_ioapics);
-                panic("Recompile kernel with bigger MAX_IO_APICS!\n");
-        }
-        if (!address) {
-                printk(KERN_ERR "WARNING: Bogus (zero) I/O APIC address"
-                       " found in table, skipping!\n");
-                return 1;
-        }
-        return 0;
-}
-
 static void __init MP_ioapic_info(struct mpc_ioapic *m)
 {
         if (!(m->flags & MPC_APIC_USABLE))
@@ -138,15 +123,7 @@ static void __init MP_ioapic_info(struct mpc_ioapic *m)
         printk(KERN_INFO "I/O APIC #%d Version %d at 0x%X.\n",
                m->apicid, m->apicver, m->apicaddr);
 
-        if (bad_ioapic(m->apicaddr))
-                return;
-
-        mp_ioapics[nr_ioapics].apicaddr = m->apicaddr;
-        mp_ioapics[nr_ioapics].apicid = m->apicid;
-        mp_ioapics[nr_ioapics].type = m->type;
-        mp_ioapics[nr_ioapics].apicver = m->apicver;
-        mp_ioapics[nr_ioapics].flags = m->flags;
-        nr_ioapics++;
+        mp_register_ioapic(m->apicid, m->apicaddr, gsi_end + 1);
 }
 
 static void print_MP_intsrc_info(struct mpc_intsrc *m)
diff --git a/arch/x86/kernel/mrst.c b/arch/x86/kernel/mrst.c
index 0aad8670858e..e796448f0eb5 100644
--- a/arch/x86/kernel/mrst.c
+++ b/arch/x86/kernel/mrst.c
@@ -237,4 +237,9 @@ void __init x86_mrst_early_setup(void)
         x86_init.pci.fixup_irqs = x86_init_noop;
 
         legacy_pic = &null_legacy_pic;
+
+        /* Avoid searching for BIOS MP tables */
+        x86_init.mpparse.find_smp_config = x86_init_noop;
+        x86_init.mpparse.get_smp_config = x86_init_uint_noop;
+
 }
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index eccdb57094e3..e7e35219b32f 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -31,24 +31,22 @@ struct kmem_cache *task_xstate_cachep;
 
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 {
+        int ret;
+
         *dst = *src;
-        if (src->thread.xstate) {
-                dst->thread.xstate = kmem_cache_alloc(task_xstate_cachep,
-                                                      GFP_KERNEL);
-                if (!dst->thread.xstate)
-                        return -ENOMEM;
-                WARN_ON((unsigned long)dst->thread.xstate & 15);
-                memcpy(dst->thread.xstate, src->thread.xstate, xstate_size);
+        if (fpu_allocated(&src->thread.fpu)) {
+                memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu));
+                ret = fpu_alloc(&dst->thread.fpu);
+                if (ret)
+                        return ret;
+                fpu_copy(&dst->thread.fpu, &src->thread.fpu);
         }
         return 0;
 }
 
 void free_thread_xstate(struct task_struct *tsk)
 {
-        if (tsk->thread.xstate) {
-                kmem_cache_free(task_xstate_cachep, tsk->thread.xstate);
-                tsk->thread.xstate = NULL;
-        }
+        fpu_free(&tsk->thread.fpu);
 }
 
 void free_thread_info(struct thread_info *ti)
@@ -548,11 +546,13 @@ static int __cpuinit check_c1e_idle(const struct cpuinfo_x86 *c)
                  * check OSVW bit for CPUs that are not affected
                  * by erratum #400
                  */
-                rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, val);
-                if (val >= 2) {
-                        rdmsrl(MSR_AMD64_OSVW_STATUS, val);
-                        if (!(val & BIT(1)))
-                                goto no_c1e_idle;
+                if (cpu_has(c, X86_FEATURE_OSVW)) {
+                        rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, val);
+                        if (val >= 2) {
+                                rdmsrl(MSR_AMD64_OSVW_STATUS, val);
+                                if (!(val & BIT(1)))
+                                        goto no_c1e_idle;
+                        }
                 }
                 return 1;
         }
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 75090c589b7a..8d128783af47 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -309,7 +309,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
         /* we're going to use this soon, after a few expensive things */
         if (preload_fpu)
-                prefetch(next->xstate);
+                prefetch(next->fpu.state);
 
         /*
          * Reload esp0.
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 50cc84ac0a0d..3c2422a99f1f 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -388,7 +388,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
         /* we're going to use this soon, after a few expensive things */
         if (preload_fpu)
-                prefetch(next->xstate);
+                prefetch(next->fpu.state);
 
         /*
          * Reload esp0, LDT and the page table pointer:
diff --git a/arch/x86/kernel/sfi.c b/arch/x86/kernel/sfi.c
index 34e099382651..7ded57896c0a 100644
--- a/arch/x86/kernel/sfi.c
+++ b/arch/x86/kernel/sfi.c
@@ -81,7 +81,6 @@ static int __init sfi_parse_cpus(struct sfi_table_header *table)
 #endif /* CONFIG_X86_LOCAL_APIC */
 
 #ifdef CONFIG_X86_IO_APIC
-static u32 gsi_base;
 
 static int __init sfi_parse_ioapic(struct sfi_table_header *table)
 {
@@ -94,8 +93,7 @@ static int __init sfi_parse_ioapic(struct sfi_table_header *table)
         pentry = (struct sfi_apic_table_entry *)sb->pentry;
 
         for (i = 0; i < num; i++) {
-                mp_register_ioapic(i, pentry->phys_addr, gsi_base);
-                gsi_base += io_apic_get_redir_entries(i);
+                mp_register_ioapic(i, pentry->phys_addr, gsi_end + 1);
                 pentry++;
         }
 
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index 86c9f91b48ae..cc2c60474fd0 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -175,6 +175,9 @@ static void add_mac_region(phys_addr_t start, unsigned long size)
         struct tboot_mac_region *mr;
         phys_addr_t end = start + size;
 
+        if (tboot->num_mac_regions >= MAX_TB_MAC_REGIONS)
+                panic("tboot: Too many MAC regions\n");
+
         if (start && size) {
                 mr = &tboot->mac_regions[tboot->num_mac_regions++];
                 mr->start = round_down(start, PAGE_SIZE);
@@ -184,18 +187,17 @@ static void add_mac_region(phys_addr_t start, unsigned long size)
 
 static int tboot_setup_sleep(void)
 {
+        int i;
+
         tboot->num_mac_regions = 0;
 
-        /* S3 resume code */
-        add_mac_region(acpi_wakeup_address, WAKEUP_SIZE);
+        for (i = 0; i < e820.nr_map; i++) {
+                if ((e820.map[i].type != E820_RAM)
+                 && (e820.map[i].type != E820_RESERVED_KERN))
+                        continue;
 
-#ifdef CONFIG_X86_TRAMPOLINE
-        /* AP trampoline code */
-        add_mac_region(virt_to_phys(trampoline_base), TRAMPOLINE_SIZE);
-#endif
-
-        /* kernel code + data + bss */
-        add_mac_region(virt_to_phys(_text), _end - _text);
+                add_mac_region(e820.map[i].addr, e820.map[i].size);
+        }
 
         tboot->acpi_sinfo.kernel_s3_resume_vector = acpi_wakeup_address;
 
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index 17b03dd3a6b5..7fea555929e2 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -1,7 +1,7 @@
 /*
  *      SGI UltraViolet TLB flush routines.
  *
- *      (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI.
+ *      (c) 2008-2010 Cliff Wickman <cpw@sgi.com>, SGI.
  *
  *      This code is released under the GNU General Public License version 2 or
  *      later.
@@ -20,42 +20,67 @@
 #include <asm/idle.h>
 #include <asm/tsc.h>
 #include <asm/irq_vectors.h>
+#include <asm/timer.h>
 
-static struct bau_control       **uv_bau_table_bases __read_mostly;
-static int                      uv_bau_retry_limit __read_mostly;
+struct msg_desc {
+        struct bau_payload_queue_entry *msg;
+        int msg_slot;
+        int sw_ack_slot;
+        struct bau_payload_queue_entry *va_queue_first;
+        struct bau_payload_queue_entry *va_queue_last;
+};
 
-/* base pnode in this partition */
-static int                      uv_partition_base_pnode __read_mostly;
+#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD 0x000000000bUL
+
+static int uv_bau_max_concurrent __read_mostly;
+
+static int nobau;
+static int __init setup_nobau(char *arg)
+{
+        nobau = 1;
+        return 0;
+}
+early_param("nobau", setup_nobau);
 
-static unsigned long            uv_mmask __read_mostly;
+/* base pnode in this partition */
+static int uv_partition_base_pnode __read_mostly;
+/* position of pnode (which is nasid>>1): */
+static int uv_nshift __read_mostly;
+static unsigned long uv_mmask __read_mostly;
 
 static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
 static DEFINE_PER_CPU(struct bau_control, bau_control);
+static DEFINE_PER_CPU(cpumask_var_t, uv_flush_tlb_mask);
+
+struct reset_args {
+        int sender;
+};
 
 /*
- * Determine the first node on a blade.
+ * Determine the first node on a uvhub. 'Nodes' are used for kernel
+ * memory allocation.
  */
-static int __init blade_to_first_node(int blade)
+static int __init uvhub_to_first_node(int uvhub)
 {
         int node, b;
 
         for_each_online_node(node) {
                 b = uv_node_to_blade_id(node);
-                if (blade == b)
+                if (uvhub == b)
                         return node;
         }
-        return -1; /* shouldn't happen */
+        return -1;
 }
 
 /*
- * Determine the apicid of the first cpu on a blade.
+ * Determine the apicid of the first cpu on a uvhub.
  */
-static int __init blade_to_first_apicid(int blade)
+static int __init uvhub_to_first_apicid(int uvhub)
 {
         int cpu;
 
         for_each_present_cpu(cpu)
-                if (blade == uv_cpu_to_blade_id(cpu))
+                if (uvhub == uv_cpu_to_blade_id(cpu))
                         return per_cpu(x86_cpu_to_apicid, cpu);
         return -1;
 }
@@ -68,195 +93,459 @@ static int __init blade_to_first_apicid(int blade)
  * clear of the Timeout bit (as well) will free the resource. No reply will
  * be sent (the hardware will only do one reply per message).
  */
-static void uv_reply_to_message(int resource,
-                                struct bau_payload_queue_entry *msg,
-                                struct bau_msg_status *msp)
+static inline void uv_reply_to_message(struct msg_desc *mdp,
+                                       struct bau_control *bcp)
 {
         unsigned long dw;
+        struct bau_payload_queue_entry *msg;
 
-        dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource);
+        msg = mdp->msg;
+        if (!msg->canceled) {
+                dw = (msg->sw_ack_vector << UV_SW_ACK_NPENDING) |
+                                                msg->sw_ack_vector;
+                uv_write_local_mmr(
+                                UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
+        }
         msg->replied_to = 1;
         msg->sw_ack_vector = 0;
-        if (msp)
-                msp->seen_by.bits = 0;
-        uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
 }
 
 /*
- * Do all the things a cpu should do for a TLB shootdown message.
- * Other cpu's may come here at the same time for this message.
+ * Process the receipt of a RETRY message
  */
-static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
-                                   int msg_slot, int sw_ack_slot)
+static inline void uv_bau_process_retry_msg(struct msg_desc *mdp,
+                                            struct bau_control *bcp)
 {
-        unsigned long this_cpu_mask;
-        struct bau_msg_status *msp;
-        int cpu;
+        int i;
+        int cancel_count = 0;
+        int slot2;
+        unsigned long msg_res;
+        unsigned long mmr = 0;
+        struct bau_payload_queue_entry *msg;
+        struct bau_payload_queue_entry *msg2;
+        struct ptc_stats *stat;
 
-        msp = __get_cpu_var(bau_control).msg_statuses + msg_slot;
-        cpu = uv_blade_processor_id();
-        msg->number_of_cpus =
-                uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
-        this_cpu_mask = 1UL << cpu;
-        if (msp->seen_by.bits & this_cpu_mask)
-                return;
-        atomic_or_long(&msp->seen_by.bits, this_cpu_mask);
+        msg = mdp->msg;
+        stat = &per_cpu(ptcstats, bcp->cpu);
+        stat->d_retries++;
+        /*
+         * cancel any message from msg+1 to the retry itself
+         */
+        for (msg2 = msg+1, i = 0; i < DEST_Q_SIZE; msg2++, i++) {
+                if (msg2 > mdp->va_queue_last)
+                        msg2 = mdp->va_queue_first;
+                if (msg2 == msg)
+                        break;
+
+                /* same conditions for cancellation as uv_do_reset */
+                if ((msg2->replied_to == 0) && (msg2->canceled == 0) &&
+                    (msg2->sw_ack_vector) && ((msg2->sw_ack_vector &
+                        msg->sw_ack_vector) == 0) &&
+                    (msg2->sending_cpu == msg->sending_cpu) &&
+                    (msg2->msg_type != MSG_NOOP)) {
+                        slot2 = msg2 - mdp->va_queue_first;
+                        mmr = uv_read_local_mmr
+                                (UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
+                        msg_res = ((msg2->sw_ack_vector << 8) |
+                                   msg2->sw_ack_vector);
+                        /*
+                         * This is a message retry; clear the resources held
+                         * by the previous message only if they timed out.
+                         * If it has not timed out we have an unexpected
+                         * situation to report.
+                         */
+                        if (mmr & (msg_res << 8)) {
+                                /*
+                                 * is the resource timed out?
+                                 * make everyone ignore the cancelled message.
+                                 */
+                                msg2->canceled = 1;
+                                stat->d_canceled++;
+                                cancel_count++;
+                                uv_write_local_mmr(
+                                    UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS,
+                                        (msg_res << 8) | msg_res);
+                        } else
+                                printk(KERN_INFO "note bau retry: no effect\n");
+                }
+        }
+        if (!cancel_count)
+                stat->d_nocanceled++;
+}
 
-        if (msg->replied_to == 1)
-                return;
+/*
+ * Do all the things a cpu should do for a TLB shootdown message.
+ * Other cpu's may come here at the same time for this message.
+ */
+static void uv_bau_process_message(struct msg_desc *mdp,
+                                   struct bau_control *bcp)
+{
+        int msg_ack_count;
+        short socket_ack_count = 0;
+        struct ptc_stats *stat;
+        struct bau_payload_queue_entry *msg;
+        struct bau_control *smaster = bcp->socket_master;
 
+        /*
+         * This must be a normal message, or retry of a normal message
+         */
+        msg = mdp->msg;
+        stat = &per_cpu(ptcstats, bcp->cpu);
         if (msg->address == TLB_FLUSH_ALL) {
                 local_flush_tlb();
-                __get_cpu_var(ptcstats).alltlb++;
+                stat->d_alltlb++;
         } else {
                 __flush_tlb_one(msg->address);
-                __get_cpu_var(ptcstats).onetlb++;
+                stat->d_onetlb++;
         }
+        stat->d_requestee++;
+
+        /*
+         * One cpu on each uvhub has the additional job on a RETRY
+         * of releasing the resource held by the message that is
+         * being retried.  That message is identified by sending
+         * cpu number.
+         */
+        if (msg->msg_type == MSG_RETRY && bcp == bcp->uvhub_master)
+                uv_bau_process_retry_msg(mdp, bcp);
 
-        __get_cpu_var(ptcstats).requestee++;
+        /*
+         * This is a sw_ack message, so we have to reply to it.
+         * Count each responding cpu on the socket. This avoids
+         * pinging the count's cache line back and forth between
+         * the sockets.
+         */
+        socket_ack_count = atomic_add_short_return(1, (struct atomic_short *)
+                        &smaster->socket_acknowledge_count[mdp->msg_slot]);
+        if (socket_ack_count == bcp->cpus_in_socket) {
+                /*
+                 * Both sockets dump their completed count total into
+                 * the message's count.
+                 */
+                smaster->socket_acknowledge_count[mdp->msg_slot] = 0;
+                msg_ack_count = atomic_add_short_return(socket_ack_count,
+                                (struct atomic_short *)&msg->acknowledge_count);
+
+                if (msg_ack_count == bcp->cpus_in_uvhub) {
+                        /*
+                         * All cpus in uvhub saw it; reply
+                         */
+                        uv_reply_to_message(mdp, bcp);
+                }
+        }
 
-        atomic_inc_short(&msg->acknowledge_count);
-        if (msg->number_of_cpus == msg->acknowledge_count)
-                uv_reply_to_message(sw_ack_slot, msg, msp);
+        return;
 }
 
 /*
- * Examine the payload queue on one distribution node to see
- * which messages have not been seen, and which cpu(s) have not seen them.
+ * Determine the first cpu on a uvhub.
+ */
+static int uvhub_to_first_cpu(int uvhub)
+{
+        int cpu;
+        for_each_present_cpu(cpu)
+                if (uvhub == uv_cpu_to_blade_id(cpu))
+                        return cpu;
+        return -1;
+}
+
+/*
+ * Last resort when we get a large number of destination timeouts is
+ * to clear resources held by a given cpu.
+ * Do this with IPI so that all messages in the BAU message queue
+ * can be identified by their nonzero sw_ack_vector field.
  *
- * Returns the number of cpu's that have not responded.
+ * This is entered for a single cpu on the uvhub.
+ * The sender want's this uvhub to free a specific message's
+ * sw_ack resources.
  */
-static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)
+static void
+uv_do_reset(void *ptr)
 {
-        struct bau_payload_queue_entry *msg;
-        struct bau_msg_status *msp;
-        int count = 0;
         int i;
-        int j;
+        int slot;
+        int count = 0;
+        unsigned long mmr;
+        unsigned long msg_res;
+        struct bau_control *bcp;
+        struct reset_args *rap;
+        struct bau_payload_queue_entry *msg;
+        struct ptc_stats *stat;
 
-        for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;
-             msg++, i++) {
-                if ((msg->sending_cpu == sender) && (!msg->replied_to)) {
-                        msp = bau_tablesp->msg_statuses + i;
-                        printk(KERN_DEBUG
-                               "blade %d: address:%#lx %d of %d, not cpu(s): ",
-                               i, msg->address, msg->acknowledge_count,
-                               msg->number_of_cpus);
-                        for (j = 0; j < msg->number_of_cpus; j++) {
-                                if (!((1L << j) & msp->seen_by.bits)) {
-                                        count++;
-                                        printk("%d ", j);
-                                }
+        bcp = &per_cpu(bau_control, smp_processor_id());
+        rap = (struct reset_args *)ptr;
+        stat = &per_cpu(ptcstats, bcp->cpu);
+        stat->d_resets++;
+
+        /*
+         * We're looking for the given sender, and
+         * will free its sw_ack resource.
+         * If all cpu's finally responded after the timeout, its
+         * message 'replied_to' was set.
+         */
+        for (msg = bcp->va_queue_first, i = 0; i < DEST_Q_SIZE; msg++, i++) {
+                /* uv_do_reset: same conditions for cancellation as
+                   uv_bau_process_retry_msg() */
+                if ((msg->replied_to == 0) &&
+                    (msg->canceled == 0) &&
+                    (msg->sending_cpu == rap->sender) &&
+                    (msg->sw_ack_vector) &&
+                    (msg->msg_type != MSG_NOOP)) {
+                        /*
+                         * make everyone else ignore this message
+                         */
+                        msg->canceled = 1;
+                        slot = msg - bcp->va_queue_first;
+                        count++;
+                        /*
+                         * only reset the resource if it is still pending
+                         */
+                        mmr = uv_read_local_mmr
+                                        (UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
+                        msg_res = ((msg->sw_ack_vector << 8) |
+                                                   msg->sw_ack_vector);
+                        if (mmr & msg_res) {
+                                stat->d_rcanceled++;
+                                uv_write_local_mmr(
+                                    UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS,
+                                                        msg_res);
                         }
-                        printk("\n");
                 }
         }
-        return count;
+        return;
 }
 
 /*
- * Examine the payload queue on all the distribution nodes to see
- * which messages have not been seen, and which cpu(s) have not seen them.
- *
- * Returns the number of cpu's that have not responded.
+ * Use IPI to get all target uvhubs to release resources held by
+ * a given sending cpu number.
  */
-static int uv_examine_destinations(struct bau_target_nodemask *distribution)
+static void uv_reset_with_ipi(struct bau_target_uvhubmask *distribution,
+                              int sender)
 {
-        int sender;
-        int i;
-        int count = 0;
+        int uvhub;
+        int cpu;
+        cpumask_t mask;
+        struct reset_args reset_args;
+
+        reset_args.sender = sender;
 
-        sender = smp_processor_id();
-        for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) {
-                if (!bau_node_isset(i, distribution))
+        cpus_clear(mask);
+        /* find a single cpu for each uvhub in this distribution mask */
+        for (uvhub = 0;
+                    uvhub < sizeof(struct bau_target_uvhubmask) * BITSPERBYTE;
+                    uvhub++) {
+                if (!bau_uvhub_isset(uvhub, distribution))
                         continue;
-                count += uv_examine_destination(uv_bau_table_bases[i], sender);
+                /* find a cpu for this uvhub */
+                cpu = uvhub_to_first_cpu(uvhub);
+                cpu_set(cpu, mask);
         }
-        return count;
+        /* IPI all cpus; Preemption is already disabled */
+        smp_call_function_many(&mask, uv_do_reset, (void *)&reset_args, 1);
+        return;
+}
+
+static inline unsigned long
+cycles_2_us(unsigned long long cyc)
+{
+        unsigned long long ns;
+        unsigned long us;
+        ns =  (cyc * per_cpu(cyc2ns, smp_processor_id()))
+                                                >> CYC2NS_SCALE_FACTOR;
+        us = ns / 1000;
+        return us;
 }
 
 /*
- * wait for completion of a broadcast message
- *
- * return COMPLETE, RETRY or GIVEUP
+ * wait for all cpus on this hub to finish their sends and go quiet
+ * leaves uvhub_quiesce set so that no new broadcasts are started by
+ * bau_flush_send_and_wait()
+ */
+static inline void
+quiesce_local_uvhub(struct bau_control *hmaster)
+{
+        atomic_add_short_return(1, (struct atomic_short *)
+                 &hmaster->uvhub_quiesce);
+}
+
+/*
+ * mark this quiet-requestor as done
+ */
+static inline void
+end_uvhub_quiesce(struct bau_control *hmaster)
+{
+        atomic_add_short_return(-1, (struct atomic_short *)
+                &hmaster->uvhub_quiesce);
+}
+
+/*
+ * Wait for completion of a broadcast software ack message
+ * return COMPLETE, RETRY(PLUGGED or TIMEOUT) or GIVEUP
  */
 static int uv_wait_completion(struct bau_desc *bau_desc,
-                              unsigned long mmr_offset, int right_shift)
+        unsigned long mmr_offset, int right_shift, int this_cpu,
+        struct bau_control *bcp, struct bau_control *smaster, long try)
 {
-        int exams = 0;
-        long destination_timeouts = 0;
-        long source_timeouts = 0;
+        int relaxes = 0;
         unsigned long descriptor_status;
+        unsigned long mmr;
+        unsigned long mask;
+        cycles_t ttime;
+        cycles_t timeout_time;
+        struct ptc_stats *stat = &per_cpu(ptcstats, this_cpu);
+        struct bau_control *hmaster;
+
+        hmaster = bcp->uvhub_master;
+        timeout_time = get_cycles() + bcp->timeout_interval;
 
+        /* spin on the status MMR, waiting for it to go idle */
         while ((descriptor_status = (((unsigned long)
                 uv_read_local_mmr(mmr_offset) >>
                         right_shift) & UV_ACT_STATUS_MASK)) !=
                         DESC_STATUS_IDLE) {
-                if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
-                        source_timeouts++;
-                        if (source_timeouts > SOURCE_TIMEOUT_LIMIT)
-                                source_timeouts = 0;
-                        __get_cpu_var(ptcstats).s_retry++;
-                        return FLUSH_RETRY;
-                }
                 /*
-                 * spin here looking for progress at the destinations
+                 * Our software ack messages may be blocked because there are
+                 * no swack resources available.  As long as none of them
+                 * has timed out hardware will NACK our message and its
+                 * state will stay IDLE.
                  */
-                if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) {
-                        destination_timeouts++;
-                        if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) {
-                                /*
-                                 * returns number of cpus not responding
-                                 */
-                                if (uv_examine_destinations
-                                    (&bau_desc->distribution) == 0) {
-                                        __get_cpu_var(ptcstats).d_retry++;
-                                        return FLUSH_RETRY;
-                                }
-                                exams++;
-                                if (exams >= uv_bau_retry_limit) {
-                                        printk(KERN_DEBUG
-                                               "uv_flush_tlb_others");
-                                        printk("giving up on cpu %d\n",
-                                               smp_processor_id());
+                if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
+                        stat->s_stimeout++;
+                        return FLUSH_GIVEUP;
+                } else if (descriptor_status ==
+                                        DESC_STATUS_DESTINATION_TIMEOUT) {
+                        stat->s_dtimeout++;
+                        ttime = get_cycles();
+
+                        /*
+                         * Our retries may be blocked by all destination
+                         * swack resources being consumed, and a timeout
+                         * pending.  In that case hardware returns the
+                         * ERROR that looks like a destination timeout.
+                         */
+                        if (cycles_2_us(ttime - bcp->send_message) < BIOS_TO) {
+                                bcp->conseccompletes = 0;
+                                return FLUSH_RETRY_PLUGGED;
+                        }
+
+                        bcp->conseccompletes = 0;
+                        return FLUSH_RETRY_TIMEOUT;
+                } else {
+                        /*
+                         * descriptor_status is still BUSY
+                         */
+                        cpu_relax();
+                        relaxes++;
+                        if (relaxes >= 10000) {
+                                relaxes = 0;
+                                if (get_cycles() > timeout_time) {
+                                        quiesce_local_uvhub(hmaster);
+
+                                        /* single-thread the register change */
+                                        spin_lock(&hmaster->masks_lock);
+                                        mmr = uv_read_local_mmr(mmr_offset);
+                                        mask = 0UL;
+                                        mask |= (3UL < right_shift);
+                                        mask = ~mask;
+                                        mmr &= mask;
+                                        uv_write_local_mmr(mmr_offset, mmr);
+                                        spin_unlock(&hmaster->masks_lock);
+                                        end_uvhub_quiesce(hmaster);
+                                        stat->s_busy++;
                                         return FLUSH_GIVEUP;
                                 }
-                                /*
-                                 * delays can hang the simulator
-                                   udelay(1000);
-                                 */
-                                destination_timeouts = 0;
                         }
                 }
-                cpu_relax();
         }
+        bcp->conseccompletes++;
         return FLUSH_COMPLETE;
 }
 
+static inline cycles_t
+sec_2_cycles(unsigned long sec)
+{
+        unsigned long ns;
+        cycles_t cyc;
+
+        ns = sec * 1000000000;
+        cyc = (ns << CYC2NS_SCALE_FACTOR)/(per_cpu(cyc2ns, smp_processor_id()));
+        return cyc;
+}
+
+/*
+ * conditionally add 1 to *v, unless *v is >= u
+ * return 0 if we cannot add 1 to *v because it is >= u
+ * return 1 if we can add 1 to *v because it is < u
+ * the add is atomic
+ *
+ * This is close to atomic_add_unless(), but this allows the 'u' value
+ * to be lowered below the current 'v'.  atomic_add_unless can only stop
+ * on equal.
+ */
+static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
+{
+        spin_lock(lock);
+        if (atomic_read(v) >= u) {
+                spin_unlock(lock);
+                return 0;
+        }
+        atomic_inc(v);
+        spin_unlock(lock);
+        return 1;
+}
+
 /**
  * uv_flush_send_and_wait
  *
- * Send a broadcast and wait for a broadcast message to complete.
+ * Send a broadcast and wait for it to complete.
  *
- * The flush_mask contains the cpus the broadcast was sent to.
+ * The flush_mask contains the cpus the broadcast is to be sent to, plus
+ * cpus that are on the local uvhub.
  *
- * Returns NULL if all remote flushing was done. The mask is zeroed.
+ * Returns NULL if all flushing represented in the mask was done. The mask
+ * is zeroed.
  * Returns @flush_mask if some remote flushing remains to be done. The
- * mask will have some bits still set.
+ * mask will have some bits still set, representing any cpus on the local
+ * uvhub (not current cpu) and any on remote uvhubs if the broadcast failed.
  */
-const struct cpumask *uv_flush_send_and_wait(int cpu, int this_pnode,
-                                             struct bau_desc *bau_desc,
-                                             struct cpumask *flush_mask)
+const struct cpumask *uv_flush_send_and_wait(struct bau_desc *bau_desc,
+                                             struct cpumask *flush_mask,
+                                             struct bau_control *bcp)
 {
-        int completion_status = 0;
         int right_shift;
-        int tries = 0;
-        int pnode;
+        int uvhub;
         int bit;
+        int completion_status = 0;
+        int seq_number = 0;
+        long try = 0;
+        int cpu = bcp->uvhub_cpu;
+        int this_cpu = bcp->cpu;
+        int this_uvhub = bcp->uvhub;
         unsigned long mmr_offset;
         unsigned long index;
         cycles_t time1;
         cycles_t time2;
+        struct ptc_stats *stat = &per_cpu(ptcstats, bcp->cpu);
+        struct bau_control *smaster = bcp->socket_master;
+        struct bau_control *hmaster = bcp->uvhub_master;
+
+        /*
+         * Spin here while there are hmaster->max_concurrent or more active
+         * descriptors. This is the per-uvhub 'throttle'.
+         */
+        if (!atomic_inc_unless_ge(&hmaster->uvhub_lock,
+                        &hmaster->active_descriptor_count,
+                        hmaster->max_concurrent)) {
+                stat->s_throttles++;
+                do {
+                        cpu_relax();
+                } while (!atomic_inc_unless_ge(&hmaster->uvhub_lock,
+                        &hmaster->active_descriptor_count,
+                        hmaster->max_concurrent));
+        }
+
+        while (hmaster->uvhub_quiesce)
+                cpu_relax();
 
         if (cpu < UV_CPUS_PER_ACT_STATUS) {
                 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
@@ -268,24 +557,108 @@ const struct cpumask *uv_flush_send_and_wait(int cpu, int this_pnode,
         }
         time1 = get_cycles();
         do {
-                tries++;
+                /*
+                 * Every message from any given cpu gets a unique message
+                 * sequence number. But retries use that same number.
+                 * Our message may have timed out at the destination because
+                 * all sw-ack resources are in use and there is a timeout
+                 * pending there.  In that case, our last send never got
+                 * placed into the queue and we need to persist until it
+                 * does.
+                 *
+                 * Make any retry a type MSG_RETRY so that the destination will
+                 * free any resource held by a previous message from this cpu.
+                 */
+                if (try == 0) {
+                        /* use message type set by the caller the first time */
+                        seq_number = bcp->message_number++;
+                } else {
+                        /* use RETRY type on all the rest; same sequence */
+                        bau_desc->header.msg_type = MSG_RETRY;
+                        stat->s_retry_messages++;
+                }
+                bau_desc->header.sequence = seq_number;
                 index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |
-                        cpu;
+                        bcp->uvhub_cpu;
+                bcp->send_message = get_cycles();
+
                 uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
+
+                try++;
                 completion_status = uv_wait_completion(bau_desc, mmr_offset,
-                                        right_shift);
-        } while (completion_status == FLUSH_RETRY);
+                        right_shift, this_cpu, bcp, smaster, try);
+
+                if (completion_status == FLUSH_RETRY_PLUGGED) {
+                        /*
+                         * Our retries may be blocked by all destination swack
+                         * resources being consumed, and a timeout pending. In
+                         * that case hardware immediately returns the ERROR
+                         * that looks like a destination timeout.
+                         */
+                        udelay(TIMEOUT_DELAY);
+                        bcp->plugged_tries++;
+                        if (bcp->plugged_tries >= PLUGSB4RESET) {
+                                bcp->plugged_tries = 0;
+                                quiesce_local_uvhub(hmaster);
+                                spin_lock(&hmaster->queue_lock);
+                                uv_reset_with_ipi(&bau_desc->distribution,
+                                                        this_cpu);
+                                spin_unlock(&hmaster->queue_lock);
+                                end_uvhub_quiesce(hmaster);
+                                bcp->ipi_attempts++;
+                                stat->s_resets_plug++;
+                        }
+                } else if (completion_status == FLUSH_RETRY_TIMEOUT) {
+                        hmaster->max_concurrent = 1;
+                        bcp->timeout_tries++;
+                        udelay(TIMEOUT_DELAY);
+                        if (bcp->timeout_tries >= TIMEOUTSB4RESET) {
+                                bcp->timeout_tries = 0;
+                                quiesce_local_uvhub(hmaster);
+                                spin_lock(&hmaster->queue_lock);
+                                uv_reset_with_ipi(&bau_desc->distribution,
+                                                                this_cpu);
+                                spin_unlock(&hmaster->queue_lock);
+                                end_uvhub_quiesce(hmaster);
+                                bcp->ipi_attempts++;
+                                stat->s_resets_timeout++;
+                        }
+                }
+                if (bcp->ipi_attempts >= 3) {
+                        bcp->ipi_attempts = 0;
+                        completion_status = FLUSH_GIVEUP;
+                        break;
+                }
+                cpu_relax();
+        } while ((completion_status == FLUSH_RETRY_PLUGGED) ||
+                 (completion_status == FLUSH_RETRY_TIMEOUT));
         time2 = get_cycles();
-        __get_cpu_var(ptcstats).sflush += (time2 - time1);
-        if (tries > 1)
-                __get_cpu_var(ptcstats).retriesok++;
 
-        if (completion_status == FLUSH_GIVEUP) {
+        if ((completion_status == FLUSH_COMPLETE) && (bcp->conseccompletes > 5)
+            && (hmaster->max_concurrent < hmaster->max_concurrent_constant))
+                        hmaster->max_concurrent++;
+
+        /*
+         * hold any cpu not timing out here; no other cpu currently held by
+         * the 'throttle' should enter the activation code
+         */
+        while (hmaster->uvhub_quiesce)
+                cpu_relax();
+        atomic_dec(&hmaster->active_descriptor_count);
+
+        /* guard against cycles wrap */
+        if (time2 > time1)
+                stat->s_time += (time2 - time1);
+        else
+                stat->s_requestor--; /* don't count this one */
+        if (completion_status == FLUSH_COMPLETE && try > 1)
+                stat->s_retriesok++;
+        else if (completion_status == FLUSH_GIVEUP) {
                 /*
                  * Cause the caller to do an IPI-style TLB shootdown on
-                 * the cpu's, all of which are still in the mask.
+                 * the target cpu's, all of which are still in the mask.
                  */
-                __get_cpu_var(ptcstats).ptc_i++;
+                stat->s_giveup++;
                 return flush_mask;
         }
 
@@ -294,18 +667,17 @@ const struct cpumask *uv_flush_send_and_wait(int cpu, int this_pnode,
          * use the IPI method of shootdown on them.
          */
         for_each_cpu(bit, flush_mask) {
-                pnode = uv_cpu_to_pnode(bit);
-                if (pnode == this_pnode)
+                uvhub = uv_cpu_to_blade_id(bit);
+                if (uvhub == this_uvhub)
                         continue;
                 cpumask_clear_cpu(bit, flush_mask);
         }
         if (!cpumask_empty(flush_mask))
                 return flush_mask;
+
         return NULL;
 }
 
-static DEFINE_PER_CPU(cpumask_var_t, uv_flush_tlb_mask);
-
 /**
  * uv_flush_tlb_others - globally purge translation cache of a virtual
  * address or all TLB's
@@ -322,8 +694,8 @@ static DEFINE_PER_CPU(cpumask_var_t, uv_flush_tlb_mask);
  * The caller has derived the cpumask from the mm_struct.  This function
  * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
  *
- * The cpumask is converted into a nodemask of the nodes containing
- * the cpus.
+ * The cpumask is converted into a uvhubmask of the uvhubs containing
+ * those cpus.
  *
  * Note that this function should be called with preemption disabled.
  *
@@ -335,52 +707,82 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
                                           struct mm_struct *mm,
                                           unsigned long va, unsigned int cpu)
 {
-        struct cpumask *flush_mask = __get_cpu_var(uv_flush_tlb_mask);
-        int i;
-        int bit;
-        int pnode;
-        int uv_cpu;
-        int this_pnode;
+        int remotes;
+        int tcpu;
+        int uvhub;
         int locals = 0;
         struct bau_desc *bau_desc;
+        struct cpumask *flush_mask;
+        struct ptc_stats *stat;
+        struct bau_control *bcp;
 
-        cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));
+        if (nobau)
+                return cpumask;
 
-        uv_cpu = uv_blade_processor_id();
-        this_pnode = uv_hub_info->pnode;
-        bau_desc = __get_cpu_var(bau_control).descriptor_base;
-        bau_desc += UV_ITEMS_PER_DESCRIPTOR * uv_cpu;
+        bcp = &per_cpu(bau_control, cpu);
+        /*
+         * Each sending cpu has a per-cpu mask which it fills from the caller's
+         * cpu mask.  Only remote cpus are converted to uvhubs and copied.
+         */
+        flush_mask = (struct cpumask *)per_cpu(uv_flush_tlb_mask, cpu);
+        /*
+         * copy cpumask to flush_mask, removing current cpu
+         * (current cpu should already have been flushed by the caller and
+         *  should never be returned if we return flush_mask)
+         */
+        cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));
+        if (cpu_isset(cpu, *cpumask))
+                locals++;  /* current cpu was targeted */
 
-        bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
+        bau_desc = bcp->descriptor_base;
+        bau_desc += UV_ITEMS_PER_DESCRIPTOR * bcp->uvhub_cpu;
 
-        i = 0;
-        for_each_cpu(bit, flush_mask) {
-                pnode = uv_cpu_to_pnode(bit);
-                BUG_ON(pnode > (UV_DISTRIBUTION_SIZE - 1));
-                if (pnode == this_pnode) {
+        bau_uvhubs_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
+        remotes = 0;
+        for_each_cpu(tcpu, flush_mask) {
+                uvhub = uv_cpu_to_blade_id(tcpu);
+                if (uvhub == bcp->uvhub) {
                         locals++;
                         continue;
                 }
-                bau_node_set(pnode - uv_partition_base_pnode,
-                                &bau_desc->distribution);
-                i++;
+                bau_uvhub_set(uvhub, &bau_desc->distribution);
+                remotes++;
         }
-        if (i == 0) {
+        if (remotes == 0) {
                 /*
-                 * no off_node flushing; return status for local node
+                 * No off_hub flushing; return status for local hub.
+                 * Return the caller's mask if all were local (the current
+                 * cpu may be in that mask).
                  */
                 if (locals)
-                        return flush_mask;
+                        return cpumask;
                 else
                         return NULL;
         }
-        __get_cpu_var(ptcstats).requestor++;
-        __get_cpu_var(ptcstats).ntargeted += i;
+        stat = &per_cpu(ptcstats, cpu);
+        stat->s_requestor++;
+        stat->s_ntargcpu += remotes;
+        remotes = bau_uvhub_weight(&bau_desc->distribution);
+        stat->s_ntarguvhub += remotes;
+        if (remotes >= 16)
+                stat->s_ntarguvhub16++;
+        else if (remotes >= 8)
+                stat->s_ntarguvhub8++;
+        else if (remotes >= 4)
+                stat->s_ntarguvhub4++;
+        else if (remotes >= 2)
+                stat->s_ntarguvhub2++;
+        else
+                stat->s_ntarguvhub1++;
 
         bau_desc->payload.address = va;
         bau_desc->payload.sending_cpu = cpu;
 
-        return uv_flush_send_and_wait(uv_cpu, this_pnode, bau_desc, flush_mask);
+        /*
+         * uv_flush_send_and_wait returns null if all cpu's were messaged, or
+         * the adjusted flush_mask if any cpu's were not messaged.
+         */
+        return uv_flush_send_and_wait(bau_desc, flush_mask, bcp);
 }
 
 /*
@@ -389,87 +791,70 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
  *
  * We received a broadcast assist message.
  *
- * Interrupts may have been disabled; this interrupt could represent
+ * Interrupts are disabled; this interrupt could represent
  * the receipt of several messages.
  *
- * All cores/threads on this node get this interrupt.
- * The last one to see it does the s/w ack.
+ * All cores/threads on this hub get this interrupt.
+ * The last one to see it does the software ack.
  * (the resource will not be freed until noninterruptable cpus see this
- *  interrupt; hardware will timeout the s/w ack and reply ERROR)
+ *  interrupt; hardware may timeout the s/w ack and reply ERROR)
  */
 void uv_bau_message_interrupt(struct pt_regs *regs)
 {
-        struct bau_payload_queue_entry *va_queue_first;
-        struct bau_payload_queue_entry *va_queue_last;
-        struct bau_payload_queue_entry *msg;
-        struct pt_regs *old_regs = set_irq_regs(regs);
-        cycles_t time1;
-        cycles_t time2;
-        int msg_slot;
-        int sw_ack_slot;
-        int fw;
         int count = 0;
-        unsigned long local_pnode;
-
-        ack_APIC_irq();
-        exit_idle();
-        irq_enter();
-
-        time1 = get_cycles();
-
-        local_pnode = uv_blade_to_pnode(uv_numa_blade_id());
-
-        va_queue_first = __get_cpu_var(bau_control).va_queue_first;
-        va_queue_last = __get_cpu_var(bau_control).va_queue_last;
-
-        msg = __get_cpu_var(bau_control).bau_msg_head;
+        cycles_t time_start;
+        struct bau_payload_queue_entry *msg;
+        struct bau_control *bcp;
+        struct ptc_stats *stat;
+        struct msg_desc msgdesc;
+
+        time_start = get_cycles();
+        bcp = &per_cpu(bau_control, smp_processor_id());
+        stat = &per_cpu(ptcstats, smp_processor_id());
+        msgdesc.va_queue_first = bcp->va_queue_first;
+        msgdesc.va_queue_last = bcp->va_queue_last;
+        msg = bcp->bau_msg_head;
         while (msg->sw_ack_vector) {
                 count++;
-                fw = msg->sw_ack_vector;
-                msg_slot = msg - va_queue_first;
-                sw_ack_slot = ffs(fw) - 1;
-
-                uv_bau_process_message(msg, msg_slot, sw_ack_slot);
-
+                msgdesc.msg_slot = msg - msgdesc.va_queue_first;
+                msgdesc.sw_ack_slot = ffs(msg->sw_ack_vector) - 1;
+                msgdesc.msg = msg;
+                uv_bau_process_message(&msgdesc, bcp);
                 msg++;
-                if (msg > va_queue_last)
-                        msg = va_queue_first;
-                __get_cpu_var(bau_control).bau_msg_head = msg;
+                if (msg > msgdesc.va_queue_last)
+                        msg = msgdesc.va_queue_first;
+                bcp->bau_msg_head = msg;
         }
+        stat->d_time += (get_cycles() - time_start);
         if (!count)
-                __get_cpu_var(ptcstats).nomsg++;
+                stat->d_nomsg++;
         else if (count > 1)
-                __get_cpu_var(ptcstats).multmsg++;
-
-        time2 = get_cycles();
-        __get_cpu_var(ptcstats).dflush += (time2 - time1);
-
-        irq_exit();
-        set_irq_regs(old_regs);
+                stat->d_multmsg++;
+        ack_APIC_irq();
 }
 
 /*
  * uv_enable_timeouts
  *
- * Each target blade (i.e. blades that have cpu's) needs to have
+ * Each target uvhub (i.e. a uvhub that has no cpu's) needs to have
  * shootdown message timeouts enabled.  The timeout does not cause
  * an interrupt, but causes an error message to be returned to
  * the sender.
  */
 static void uv_enable_timeouts(void)
 {
-        int blade;
-        int nblades;
+        int uvhub;
+        int nuvhubs;
         int pnode;
         unsigned long mmr_image;
 
-        nblades = uv_num_possible_blades();
+        nuvhubs = uv_num_possible_blades();
 
-        for (blade = 0; blade < nblades; blade++) {
-                if (!uv_blade_nr_possible_cpus(blade))
+        for (uvhub = 0; uvhub < nuvhubs; uvhub++) {
+                if (!uv_blade_nr_possible_cpus(uvhub))
                         continue;
 
-                pnode = uv_blade_to_pnode(blade);
+                pnode = uv_blade_to_pnode(uvhub);
                 mmr_image =
                     uv_read_global_mmr64(pnode, UVH_LB_BAU_MISC_CONTROL);
                 /*
@@ -479,16 +864,16 @@ static void uv_enable_timeouts(void)
                  * To program the period, the SOFT_ACK_MODE must be off.
                  */
                 mmr_image &= ~((unsigned long)1 <<
-                               UV_ENABLE_INTD_SOFT_ACK_MODE_SHIFT);
+                    UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT);
                 uv_write_global_mmr64
                     (pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image);
                 /*
                  * Set the 4-bit period.
                  */
                 mmr_image &= ~((unsigned long)0xf <<
-                        UV_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHIFT);
+                     UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT);
                 mmr_image |= (UV_INTD_SOFT_ACK_TIMEOUT_PERIOD <<
-                             UV_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHIFT);
+                     UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT);
                 uv_write_global_mmr64
                     (pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image);
                 /*
@@ -497,7 +882,7 @@ static void uv_enable_timeouts(void)
                  * indicated in bits 2:0 (7 causes all of them to timeout).
                  */
                 mmr_image |= ((unsigned long)1 <<
-                              UV_ENABLE_INTD_SOFT_ACK_MODE_SHIFT);
+                    UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT);
                 uv_write_global_mmr64
                     (pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image);
         }
@@ -522,9 +907,20 @@ static void uv_ptc_seq_stop(struct seq_file *file, void *data)
 {
 }
 
+static inline unsigned long long
+millisec_2_cycles(unsigned long millisec)
+{
+        unsigned long ns;
+        unsigned long long cyc;
+
+        ns = millisec * 1000;
+        cyc = (ns << CYC2NS_SCALE_FACTOR)/(per_cpu(cyc2ns, smp_processor_id()));
+        return cyc;
+}
+
 /*
- * Display the statistics thru /proc
- * data points to the cpu number
+ * Display the statistics thru /proc.
+ * 'data' points to the cpu number
  */
 static int uv_ptc_seq_show(struct seq_file *file, void *data)
 {
@@ -535,78 +931,155 @@ static int uv_ptc_seq_show(struct seq_file *file, void *data)
 
         if (!cpu) {
                 seq_printf(file,
-                "# cpu requestor requestee one all sretry dretry ptc_i ");
+                        "# cpu sent stime numuvhubs numuvhubs16 numuvhubs8 ");
                 seq_printf(file,
-                "sw_ack sflush dflush sok dnomsg dmult starget\n");
+                        "numuvhubs4 numuvhubs2 numuvhubs1 numcpus dto ");
+                seq_printf(file,
+                        "retries rok resetp resett giveup sto bz throt ");
+                seq_printf(file,
+                        "sw_ack recv rtime all ");
+                seq_printf(file,
+                        "one mult none retry canc nocan reset rcan\n");
         }
         if (cpu < num_possible_cpus() && cpu_online(cpu)) {
                 stat = &per_cpu(ptcstats, cpu);
-                seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ",
-                           cpu, stat->requestor,
-                           stat->requestee, stat->onetlb, stat->alltlb,
-                           stat->s_retry, stat->d_retry, stat->ptc_i);
-                seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n",
+                /* source side statistics */
+                seq_printf(file,
+                        "cpu %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld ",
+                           cpu, stat->s_requestor, cycles_2_us(stat->s_time),
+                           stat->s_ntarguvhub, stat->s_ntarguvhub16,
+                           stat->s_ntarguvhub8, stat->s_ntarguvhub4,
+                           stat->s_ntarguvhub2, stat->s_ntarguvhub1,
+                           stat->s_ntargcpu, stat->s_dtimeout);
+                seq_printf(file, "%ld %ld %ld %ld %ld %ld %ld %ld ",
+                           stat->s_retry_messages, stat->s_retriesok,
+                           stat->s_resets_plug, stat->s_resets_timeout,
+                           stat->s_giveup, stat->s_stimeout,
+                           stat->s_busy, stat->s_throttles);
+                /* destination side statistics */
+                seq_printf(file,
+                           "%lx %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n",
                            uv_read_global_mmr64(uv_cpu_to_pnode(cpu),
                                         UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE),
-                           stat->sflush, stat->dflush,
-                           stat->retriesok, stat->nomsg,
-                           stat->multmsg, stat->ntargeted);
+                           stat->d_requestee, cycles_2_us(stat->d_time),
+                           stat->d_alltlb, stat->d_onetlb, stat->d_multmsg,
+                           stat->d_nomsg, stat->d_retries, stat->d_canceled,
+                           stat->d_nocanceled, stat->d_resets,
+                           stat->d_rcanceled);
         }
 
         return 0;
 }
 
 /*
+ * -1: resetf the statistics
  *  0: display meaning of the statistics
- * >0: retry limit
+ * >0: maximum concurrent active descriptors per uvhub (throttle)
  */
 static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
                                  size_t count, loff_t *data)
 {
-        long newmode;
+        int cpu;
+        long input_arg;
         char optstr[64];
+        struct ptc_stats *stat;
+        struct bau_control *bcp;
 
         if (count == 0 || count > sizeof(optstr))
                 return -EINVAL;
         if (copy_from_user(optstr, user, count))
                 return -EFAULT;
         optstr[count - 1] = '\0';
-        if (strict_strtoul(optstr, 10, &newmode) < 0) {
+        if (strict_strtol(optstr, 10, &input_arg) < 0) {
                 printk(KERN_DEBUG "%s is invalid\n", optstr);
                 return -EINVAL;
         }
 
-        if (newmode == 0) {
+        if (input_arg == 0) {
                 printk(KERN_DEBUG "# cpu:      cpu number\n");
+                printk(KERN_DEBUG "Sender statistics:\n");
+                printk(KERN_DEBUG
+                "sent:     number of shootdown messages sent\n");
+                printk(KERN_DEBUG
+                "stime:    time spent sending messages\n");
+                printk(KERN_DEBUG
+                "numuvhubs: number of hubs targeted with shootdown\n");
+                printk(KERN_DEBUG
+                "numuvhubs16: number times 16 or more hubs targeted\n");
+                printk(KERN_DEBUG
+                "numuvhubs8: number times 8 or more hubs targeted\n");
+                printk(KERN_DEBUG
+                "numuvhubs4: number times 4 or more hubs targeted\n");
+                printk(KERN_DEBUG
+                "numuvhubs2: number times 2 or more hubs targeted\n");
+                printk(KERN_DEBUG
+                "numuvhubs1: number times 1 hub targeted\n");
+                printk(KERN_DEBUG
+                "numcpus:  number of cpus targeted with shootdown\n");
+                printk(KERN_DEBUG
+                "dto:      number of destination timeouts\n");
+                printk(KERN_DEBUG
+                "retries:  destination timeout retries sent\n");
+                printk(KERN_DEBUG
+                "rok:   :  destination timeouts successfully retried\n");
+                printk(KERN_DEBUG
+                "resetp:   ipi-style resource resets for plugs\n");
+                printk(KERN_DEBUG
+                "resett:   ipi-style resource resets for timeouts\n");
+                printk(KERN_DEBUG
+                "giveup:   fall-backs to ipi-style shootdowns\n");
+                printk(KERN_DEBUG
+                "sto:      number of source timeouts\n");
+                printk(KERN_DEBUG
+                "bz:       number of stay-busy's\n");
+                printk(KERN_DEBUG
+                "throt:    number times spun in throttle\n");
+                printk(KERN_DEBUG "Destination side statistics:\n");
                 printk(KERN_DEBUG
-                "requestor:  times this cpu was the flush requestor\n");
+                "sw_ack:   image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
                 printk(KERN_DEBUG
-                "requestee:  times this cpu was requested to flush its TLBs\n");
+                "recv:     shootdown messages received\n");
                 printk(KERN_DEBUG
-                "one:        times requested to flush a single address\n");
+                "rtime:    time spent processing messages\n");
                 printk(KERN_DEBUG
-                "all:        times requested to flush all TLB's\n");
+                "all:      shootdown all-tlb messages\n");
                 printk(KERN_DEBUG
-                "sretry:     number of retries of source-side timeouts\n");
+                "one:      shootdown one-tlb messages\n");
                 printk(KERN_DEBUG
-                "dretry:     number of retries of destination-side timeouts\n");
+                "mult:     interrupts that found multiple messages\n");
                 printk(KERN_DEBUG
-                "ptc_i:      times UV fell through to IPI-style flushes\n");
+                "none:     interrupts that found no messages\n");
                 printk(KERN_DEBUG
-                "sw_ack:     image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
+                "retry:    number of retry messages processed\n");
                 printk(KERN_DEBUG
-                "sflush_us:  cycles spent in uv_flush_tlb_others()\n");
+                "canc:     number messages canceled by retries\n");
                 printk(KERN_DEBUG
-                "dflush_us:  cycles spent in handling flush requests\n");
-                printk(KERN_DEBUG "sok:        successes on retry\n");
-                printk(KERN_DEBUG "dnomsg:     interrupts with no message\n");
+                "nocan:    number retries that found nothing to cancel\n");
                 printk(KERN_DEBUG
-                "dmult:      interrupts with multiple messages\n");
-                printk(KERN_DEBUG "starget:    nodes targeted\n");
+                "reset:    number of ipi-style reset requests processed\n");
+                printk(KERN_DEBUG
+                "rcan:     number messages canceled by reset requests\n");
+        } else if (input_arg == -1) {
+                for_each_present_cpu(cpu) {
+                        stat = &per_cpu(ptcstats, cpu);
+                        memset(stat, 0, sizeof(struct ptc_stats));
+                }
         } else {
-                uv_bau_retry_limit = newmode;
-                printk(KERN_DEBUG "timeout retry limit:%d\n",
-                       uv_bau_retry_limit);
+                uv_bau_max_concurrent = input_arg;
+                bcp = &per_cpu(bau_control, smp_processor_id());
+                if (uv_bau_max_concurrent < 1 ||
+                    uv_bau_max_concurrent > bcp->cpus_in_uvhub) {
+                        printk(KERN_DEBUG
+                                "Error: BAU max concurrent %d; %d is invalid\n",
+                                bcp->max_concurrent, uv_bau_max_concurrent);
+                        return -EINVAL;
+                }
+                printk(KERN_DEBUG "Set BAU max concurrent:%d\n",
+                       uv_bau_max_concurrent);
+                for_each_present_cpu(cpu) {
+                        bcp = &per_cpu(bau_control, cpu);
+                        bcp->max_concurrent = uv_bau_max_concurrent;
+                }
         }
 
         return count;
@@ -650,79 +1123,30 @@ static int __init uv_ptc_init(void)
 }
 
 /*
- * begin the initialization of the per-blade control structures
- */
-static struct bau_control * __init uv_table_bases_init(int blade, int node)
-{
-        int i;
-        struct bau_msg_status *msp;
-        struct bau_control *bau_tabp;
-
-        bau_tabp =
-            kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
-        BUG_ON(!bau_tabp);
-
-        bau_tabp->msg_statuses =
-            kmalloc_node(sizeof(struct bau_msg_status) *
-                         DEST_Q_SIZE, GFP_KERNEL, node);
-        BUG_ON(!bau_tabp->msg_statuses);
-
-        for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)
-                bau_cpubits_clear(&msp->seen_by, (int)
-                                  uv_blade_nr_possible_cpus(blade));
-
-        uv_bau_table_bases[blade] = bau_tabp;
-
-        return bau_tabp;
-}
-
-/*
- * finish the initialization of the per-blade control structures
- */
-static void __init
-uv_table_bases_finish(int blade,
-                      struct bau_control *bau_tablesp,
-                      struct bau_desc *adp)
-{
-        struct bau_control *bcp;
-        int cpu;
-
-        for_each_present_cpu(cpu) {
-                if (blade != uv_cpu_to_blade_id(cpu))
-                        continue;
-
-                bcp = (struct bau_control *)&per_cpu(bau_control, cpu);
-                bcp->bau_msg_head       = bau_tablesp->va_queue_first;
-                bcp->va_queue_first     = bau_tablesp->va_queue_first;
-                bcp->va_queue_last      = bau_tablesp->va_queue_last;
-                bcp->msg_statuses       = bau_tablesp->msg_statuses;
-                bcp->descriptor_base    = adp;
-        }
-}
-
-/*
  * initialize the sending side's sending buffers
  */
-static struct bau_desc * __init
+static void
 uv_activation_descriptor_init(int node, int pnode)
 {
         int i;
+        int cpu;
         unsigned long pa;
         unsigned long m;
         unsigned long n;
-        struct bau_desc *adp;
-        struct bau_desc *ad2;
+        struct bau_desc *bau_desc;
+        struct bau_desc *bd2;
+        struct bau_control *bcp;
 
         /*
          * each bau_desc is 64 bytes; there are 8 (UV_ITEMS_PER_DESCRIPTOR)
-         * per cpu; and up to 32 (UV_ADP_SIZE) cpu's per blade
+         * per cpu; and up to 32 (UV_ADP_SIZE) cpu's per uvhub
          */
-        adp = (struct bau_desc *)kmalloc_node(sizeof(struct bau_desc)*
+        bau_desc = (struct bau_desc *)kmalloc_node(sizeof(struct bau_desc)*
                 UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR, GFP_KERNEL, node);
-        BUG_ON(!adp);
+        BUG_ON(!bau_desc);
 
-        pa = uv_gpa(adp); /* need the real nasid*/
-        n = uv_gpa_to_pnode(pa);
+        pa = uv_gpa(bau_desc); /* need the real nasid*/
+        n = pa >> uv_nshift;
         m = pa & uv_mmask;
 
         uv_write_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE,
@@ -731,96 +1155,188 @@ uv_activation_descriptor_init(int node, int pnode)
         /*
          * initializing all 8 (UV_ITEMS_PER_DESCRIPTOR) descriptors for each
          * cpu even though we only use the first one; one descriptor can
-         * describe a broadcast to 256 nodes.
+         * describe a broadcast to 256 uv hubs.
          */
-        for (i = 0, ad2 = adp; i < (UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR);
-                i++, ad2++) {
-                memset(ad2, 0, sizeof(struct bau_desc));
-                ad2->header.sw_ack_flag = 1;
+        for (i = 0, bd2 = bau_desc; i < (UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR);
+                i++, bd2++) {
+                memset(bd2, 0, sizeof(struct bau_desc));
+                bd2->header.sw_ack_flag = 1;
                 /*
-                 * base_dest_nodeid is the first node in the partition, so
-                 * the bit map will indicate partition-relative node numbers.
-                 * note that base_dest_nodeid is actually a nasid.
+                 * base_dest_nodeid is the nasid (pnode<<1) of the first uvhub
+                 * in the partition. The bit map will indicate uvhub numbers,
+                 * which are 0-N in a partition. Pnodes are unique system-wide.
                  */
-                ad2->header.base_dest_nodeid = uv_partition_base_pnode << 1;
-                ad2->header.dest_subnodeid = 0x10; /* the LB */
-                ad2->header.command = UV_NET_ENDPOINT_INTD;
-                ad2->header.int_both = 1;
+                bd2->header.base_dest_nodeid = uv_partition_base_pnode << 1;
+                bd2->header.dest_subnodeid = 0x10; /* the LB */
+                bd2->header.command = UV_NET_ENDPOINT_INTD;
+                bd2->header.int_both = 1;
                 /*
                  * all others need to be set to zero:
                  *   fairness chaining multilevel count replied_to
                  */
         }
-        return adp;
+        for_each_present_cpu(cpu) {
+                if (pnode != uv_blade_to_pnode(uv_cpu_to_blade_id(cpu)))
+                        continue;
+                bcp = &per_cpu(bau_control, cpu);
+                bcp->descriptor_base = bau_desc;
+        }
 }
 
 /*
  * initialize the destination side's receiving buffers
+ * entered for each uvhub in the partition
+ * - node is first node (kernel memory notion) on the uvhub
+ * - pnode is the uvhub's physical identifier
  */
-static struct bau_payload_queue_entry * __init
-uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp)
+static void
+uv_payload_queue_init(int node, int pnode)
 {
-        struct bau_payload_queue_entry *pqp;
-        unsigned long pa;
         int pn;
+        int cpu;
         char *cp;
+        unsigned long pa;
+        struct bau_payload_queue_entry *pqp;
+        struct bau_payload_queue_entry *pqp_malloc;
+        struct bau_control *bcp;
 
         pqp = (struct bau_payload_queue_entry *) kmalloc_node(
                 (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),
                 GFP_KERNEL, node);
         BUG_ON(!pqp);
+        pqp_malloc = pqp;
 
         cp = (char *)pqp + 31;
         pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
-        bau_tablesp->va_queue_first = pqp;
+
+        for_each_present_cpu(cpu) {
+                if (pnode != uv_cpu_to_pnode(cpu))
+                        continue;
+                /* for every cpu on this pnode: */
+                bcp = &per_cpu(bau_control, cpu);
+                bcp->va_queue_first = pqp;
+                bcp->bau_msg_head = pqp;
+                bcp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
+        }
         /*
          * need the pnode of where the memory was really allocated
          */
         pa = uv_gpa(pqp);
-        pn = uv_gpa_to_pnode(pa);
+        pn = pa >> uv_nshift;
         uv_write_global_mmr64(pnode,
                               UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST,
                               ((unsigned long)pn << UV_PAYLOADQ_PNODE_SHIFT) |
                               uv_physnodeaddr(pqp));
         uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
                               uv_physnodeaddr(pqp));
-        bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
         uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
                               (unsigned long)
-                              uv_physnodeaddr(bau_tablesp->va_queue_last));
+                              uv_physnodeaddr(pqp + (DEST_Q_SIZE - 1)));
+        /* in effect, all msg_type's are set to MSG_NOOP */
         memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);
-
-        return pqp;
 }
 
 /*
- * Initialization of each UV blade's structures
+ * Initialization of each UV hub's structures
  */
-static int __init uv_init_blade(int blade)
+static void __init uv_init_uvhub(int uvhub, int vector)
 {
         int node;
         int pnode;
-        unsigned long pa;
         unsigned long apicid;
-        struct bau_desc *adp;
-        struct bau_payload_queue_entry *pqp;
-        struct bau_control *bau_tablesp;
-
-        node = blade_to_first_node(blade);
-        bau_tablesp = uv_table_bases_init(blade, node);
-        pnode = uv_blade_to_pnode(blade);
-        adp = uv_activation_descriptor_init(node, pnode);
-        pqp = uv_payload_queue_init(node, pnode, bau_tablesp);
-        uv_table_bases_finish(blade, bau_tablesp, adp);
+
+        node = uvhub_to_first_node(uvhub);
+        pnode = uv_blade_to_pnode(uvhub);
+        uv_activation_descriptor_init(node, pnode);
+        uv_payload_queue_init(node, pnode);
         /*
          * the below initialization can't be in firmware because the
          * messaging IRQ will be determined by the OS
          */
-        apicid = blade_to_first_apicid(blade);
-        pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG);
+        apicid = uvhub_to_first_apicid(uvhub);
         uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG,
-                                      ((apicid << 32) | UV_BAU_MESSAGE));
-        return 0;
+                                      ((apicid << 32) | vector));
+}
+
+/*
+ * initialize the bau_control structure for each cpu
+ */
+static void uv_init_per_cpu(int nuvhubs)
+{
+        int i, j, k;
+        int cpu;
+        int pnode;
+        int uvhub;
+        short socket = 0;
+        struct bau_control *bcp;
+        struct uvhub_desc *bdp;
+        struct socket_desc *sdp;
+        struct bau_control *hmaster = NULL;
+        struct bau_control *smaster = NULL;
+        struct socket_desc {
+                short num_cpus;
+                short cpu_number[16];
+        };
+        struct uvhub_desc {
+                short num_sockets;
+                short num_cpus;
+                short uvhub;
+                short pnode;
+                struct socket_desc socket[2];
+        };
+        struct uvhub_desc *uvhub_descs;
+
+        uvhub_descs = (struct uvhub_desc *)
+                kmalloc(nuvhubs * sizeof(struct uvhub_desc), GFP_KERNEL);
+        memset(uvhub_descs, 0, nuvhubs * sizeof(struct uvhub_desc));
+        for_each_present_cpu(cpu) {
+                bcp = &per_cpu(bau_control, cpu);
+                memset(bcp, 0, sizeof(struct bau_control));
+                spin_lock_init(&bcp->masks_lock);
+                bcp->max_concurrent = uv_bau_max_concurrent;
+                pnode = uv_cpu_hub_info(cpu)->pnode;
+                uvhub = uv_cpu_hub_info(cpu)->numa_blade_id;
+                bdp = &uvhub_descs[uvhub];
+                bdp->num_cpus++;
+                bdp->uvhub = uvhub;
+                bdp->pnode = pnode;
+                /* time interval to catch a hardware stay-busy bug */
+                bcp->timeout_interval = millisec_2_cycles(3);
+                /* kludge: assume uv_hub.h is constant */
+                socket = (cpu_physical_id(cpu)>>5)&1;
+                if (socket >= bdp->num_sockets)
+                        bdp->num_sockets = socket+1;
+                sdp = &bdp->socket[socket];
+                sdp->cpu_number[sdp->num_cpus] = cpu;
+                sdp->num_cpus++;
+        }
+        socket = 0;
+        for_each_possible_blade(uvhub) {
+                bdp = &uvhub_descs[uvhub];
+                for (i = 0; i < bdp->num_sockets; i++) {
+                        sdp = &bdp->socket[i];
+                        for (j = 0; j < sdp->num_cpus; j++) {
+                                cpu = sdp->cpu_number[j];
+                                bcp = &per_cpu(bau_control, cpu);
+                                bcp->cpu = cpu;
+                                if (j == 0) {
+                                        smaster = bcp;
+                                        if (i == 0)
+                                                hmaster = bcp;
+                                }
+                                bcp->cpus_in_uvhub = bdp->num_cpus;
+                                bcp->cpus_in_socket = sdp->num_cpus;
+                                bcp->socket_master = smaster;
+                                bcp->uvhub_master = hmaster;
+                                for (k = 0; k < DEST_Q_SIZE; k++)
+                                        bcp->socket_acknowledge_count[k] = 0;
+                                bcp->uvhub_cpu =
+                                  uv_cpu_hub_info(cpu)->blade_processor_id;
+                        }
+                        socket++;
+                }
+        }
+        kfree(uvhub_descs);
 }
 
 /*
@@ -828,38 +1344,54 @@ static int __init uv_init_blade(int blade)
  */
 static int __init uv_bau_init(void)
 {
-        int blade;
-        int nblades;
+        int uvhub;
+        int pnode;
+        int nuvhubs;
         int cur_cpu;
+        int vector;
+        unsigned long mmr;
 
         if (!is_uv_system())
                 return 0;
 
+        if (nobau)
+                return 0;
+
         for_each_possible_cpu(cur_cpu)
                 zalloc_cpumask_var_node(&per_cpu(uv_flush_tlb_mask, cur_cpu),
                                        GFP_KERNEL, cpu_to_node(cur_cpu));
 
-        uv_bau_retry_limit = 1;
+        uv_bau_max_concurrent = MAX_BAU_CONCURRENT;
+        uv_nshift = uv_hub_info->m_val;
         uv_mmask = (1UL << uv_hub_info->m_val) - 1;
-        nblades = uv_num_possible_blades();
+        nuvhubs = uv_num_possible_blades();
 
-        uv_bau_table_bases = (struct bau_control **)
-            kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
-        BUG_ON(!uv_bau_table_bases);
+        uv_init_per_cpu(nuvhubs);
 
         uv_partition_base_pnode = 0x7fffffff;
-        for (blade = 0; blade < nblades; blade++)
-                if (uv_blade_nr_possible_cpus(blade) &&
-                        (uv_blade_to_pnode(blade) < uv_partition_base_pnode))
-                        uv_partition_base_pnode = uv_blade_to_pnode(blade);
-        for (blade = 0; blade < nblades; blade++)
-                if (uv_blade_nr_possible_cpus(blade))
-                        uv_init_blade(blade);
-
-        alloc_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1);
+        for (uvhub = 0; uvhub < nuvhubs; uvhub++)
+                if (uv_blade_nr_possible_cpus(uvhub) &&
+                        (uv_blade_to_pnode(uvhub) < uv_partition_base_pnode))
+                        uv_partition_base_pnode = uv_blade_to_pnode(uvhub);
+
+        vector = UV_BAU_MESSAGE;
+        for_each_possible_blade(uvhub)
+                if (uv_blade_nr_possible_cpus(uvhub))
+                        uv_init_uvhub(uvhub, vector);
+
         uv_enable_timeouts();
+        alloc_intr_gate(vector, uv_bau_message_intr1);
+
+        for_each_possible_blade(uvhub) {
+                pnode = uv_blade_to_pnode(uvhub);
+                /* INIT the bau */
+                uv_write_global_mmr64(pnode, UVH_LB_BAU_SB_ACTIVATION_CONTROL,
+                                      ((unsigned long)1 << 63));
+                mmr = 1; /* should be 1 to broadcast to both sockets */
+                uv_write_global_mmr64(pnode, UVH_BAU_DATA_BROADCAST, mmr);
+        }
 
         return 0;
 }
-__initcall(uv_bau_init);
-__initcall(uv_ptc_init);
+core_initcall(uv_bau_init);
+core_initcall(uv_ptc_init);
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 36f1bd9f8e76..02cfb9b8f5b1 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -108,15 +108,6 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
         dec_preempt_count();
 }
 
-#ifdef CONFIG_X86_32
-static inline void
-die_if_kernel(const char *str, struct pt_regs *regs, long err)
-{
-        if (!user_mode_vm(regs))
-                die(str, regs, err);
-}
-#endif
-
 static void __kprobes
 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
         long error_code, siginfo_t *info)
@@ -585,55 +576,67 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
         return;
 }
 
-#ifdef CONFIG_X86_64
-static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr)
-{
-        if (fixup_exception(regs))
-                return 1;
-
-        notify_die(DIE_GPF, str, regs, 0, trapnr, SIGFPE);
-        /* Illegal floating point operation in the kernel */
-        current->thread.trap_no = trapnr;
-        die(str, regs, 0);
-        return 0;
-}
-#endif
-
 /*
  * Note that we play around with the 'TS' bit in an attempt to get
  * the correct behaviour even in the presence of the asynchronous
  * IRQ13 behaviour
  */
-void math_error(void __user *ip)
+void math_error(struct pt_regs *regs, int error_code, int trapnr)
 {
-        struct task_struct *task;
+        struct task_struct *task = current;
         siginfo_t info;
-        unsigned short cwd, swd, err;
+        unsigned short err;
+        char *str = (trapnr == 16) ? "fpu exception" : "simd exception";
+
+        if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
+                return;
+        conditional_sti(regs);
+
+        if (!user_mode_vm(regs))
+        {
+                if (!fixup_exception(regs)) {
+                        task->thread.error_code = error_code;
+                        task->thread.trap_no = trapnr;
+                        die(str, regs, error_code);
+                }
+                return;
+        }
 
         /*
          * Save the info for the exception handler and clear the error.
          */
-        task = current;
         save_init_fpu(task);
-        task->thread.trap_no = 16;
-        task->thread.error_code = 0;
+        task->thread.trap_no = trapnr;
+        task->thread.error_code = error_code;
         info.si_signo = SIGFPE;
         info.si_errno = 0;
-        info.si_addr = ip;
-        /*
-         * (~cwd & swd) will mask out exceptions that are not set to unmasked
-         * status.  0x3f is the exception bits in these regs, 0x200 is the
-         * C1 reg you need in case of a stack fault, 0x040 is the stack
-         * fault bit.  We should only be taking one exception at a time,
-         * so if this combination doesn't produce any single exception,
-         * then we have a bad program that isn't synchronizing its FPU usage
-         * and it will suffer the consequences since we won't be able to
-         * fully reproduce the context of the exception
-         */
-        cwd = get_fpu_cwd(task);
-        swd = get_fpu_swd(task);
+        info.si_addr = (void __user *)regs->ip;
+        if (trapnr == 16) {
+                unsigned short cwd, swd;
+                /*
+                 * (~cwd & swd) will mask out exceptions that are not set to unmasked
+                 * status.  0x3f is the exception bits in these regs, 0x200 is the
+                 * C1 reg you need in case of a stack fault, 0x040 is the stack
+                 * fault bit.  We should only be taking one exception at a time,
+                 * so if this combination doesn't produce any single exception,
+                 * then we have a bad program that isn't synchronizing its FPU usage
+                 * and it will suffer the consequences since we won't be able to
+                 * fully reproduce the context of the exception
+                 */
+                cwd = get_fpu_cwd(task);
+                swd = get_fpu_swd(task);
 
-        err = swd & ~cwd;
+                err = swd & ~cwd;
+        } else {
+                /*
+                 * The SIMD FPU exceptions are handled a little differently, as there
+                 * is only a single status/control register.  Thus, to determine which
+                 * unmasked exception was caught we must mask the exception mask bits
+                 * at 0x1f80, and then use these to mask the exception bits at 0x3f.
+                 */
+                unsigned short mxcsr = get_fpu_mxcsr(task);
+                err = ~(mxcsr >> 7) & mxcsr;
+        }
 
         if (err & 0x001) {      /* Invalid op */
                 /*
@@ -662,97 +665,17 @@ void math_error(void __user *ip)
 
 dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
 {
-        conditional_sti(regs);
-
 #ifdef CONFIG_X86_32
         ignore_fpu_irq = 1;
-#else
-        if (!user_mode(regs) &&
-            kernel_math_error(regs, "kernel x87 math error", 16))
-                return;
 #endif
 
-        math_error((void __user *)regs->ip);
-}
-
-static void simd_math_error(void __user *ip)
-{
-        struct task_struct *task;
-        siginfo_t info;
-        unsigned short mxcsr;
-
-        /*
-         * Save the info for the exception handler and clear the error.
-         */
-        task = current;
-        save_init_fpu(task);
-        task->thread.trap_no = 19;
-        task->thread.error_code = 0;
-        info.si_signo = SIGFPE;
-        info.si_errno = 0;
-        info.si_code = __SI_FAULT;
-        info.si_addr = ip;
-        /*
-         * The SIMD FPU exceptions are handled a little differently, as there
-         * is only a single status/control register.  Thus, to determine which
-         * unmasked exception was caught we must mask the exception mask bits
-         * at 0x1f80, and then use these to mask the exception bits at 0x3f.
-         */
-        mxcsr = get_fpu_mxcsr(task);
-        switch (~((mxcsr & 0x1f80) >> 7) & (mxcsr & 0x3f)) {
-        case 0x000:
-        default:
-                break;
-        case 0x001: /* Invalid Op */
-                info.si_code = FPE_FLTINV;
-                break;
-        case 0x002: /* Denormalize */
-        case 0x010: /* Underflow */
-                info.si_code = FPE_FLTUND;
-                break;
-        case 0x004: /* Zero Divide */
-                info.si_code = FPE_FLTDIV;
-                break;
-        case 0x008: /* Overflow */
-                info.si_code = FPE_FLTOVF;
-                break;
-        case 0x020: /* Precision */
-                info.si_code = FPE_FLTRES;
-                break;
-        }
-        force_sig_info(SIGFPE, &info, task);
+        math_error(regs, error_code, 16);
 }
 
 dotraplinkage void
 do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
 {
-        conditional_sti(regs);
-
-#ifdef CONFIG_X86_32
-        if (cpu_has_xmm) {
-                /* Handle SIMD FPU exceptions on PIII+ processors. */
-                ignore_fpu_irq = 1;
-                simd_math_error((void __user *)regs->ip);
-                return;
-        }
-        /*
-         * Handle strange cache flush from user space exception
-         * in all other cases.  This is undocumented behaviour.
-         */
-        if (regs->flags & X86_VM_MASK) {
-                handle_vm86_fault((struct kernel_vm86_regs *)regs, error_code);
-                return;
-        }
-        current->thread.trap_no = 19;
-        current->thread.error_code = error_code;
-        die_if_kernel("cache flush denied", regs, error_code);
-        force_sig(SIGSEGV, current);
-#else
-        if (!user_mode(regs) &&
-                        kernel_math_error(regs, "kernel simd math error", 19))
-                return;
-        simd_math_error((void __user *)regs->ip);
-#endif
+        math_error(regs, error_code, 19);
 }
 
 dotraplinkage void
diff --git a/arch/x86/kernel/uv_irq.c b/arch/x86/kernel/uv_irq.c
index 1d40336b030a..1132129db792 100644
--- a/arch/x86/kernel/uv_irq.c
+++ b/arch/x86/kernel/uv_irq.c
@@ -44,7 +44,7 @@ static void uv_ack_apic(unsigned int irq)
         ack_APIC_irq();
 }
 
-struct irq_chip uv_irq_chip = {
+static struct irq_chip uv_irq_chip = {
         .name           = "UV-CORE",
         .startup        = uv_noop_ret,
         .shutdown       = uv_noop,
@@ -141,7 +141,7 @@ int uv_irq_2_mmr_info(int irq, unsigned long *offset, int *pnode)
  */
 static int
 arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
-                       unsigned long mmr_offset, int restrict)
+                       unsigned long mmr_offset, int limit)
 {
         const struct cpumask *eligible_cpu = cpumask_of(cpu);
         struct irq_desc *desc = irq_to_desc(irq);
@@ -160,7 +160,7 @@ arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
         if (err != 0)
                 return err;
 
-        if (restrict == UV_AFFINITY_CPU)
+        if (limit == UV_AFFINITY_CPU)
                 desc->status |= IRQ_NO_BALANCING;
         else
                 desc->status |= IRQ_MOVE_PCNTXT;
@@ -214,7 +214,7 @@ static int uv_set_irq_affinity(unsigned int irq, const struct cpumask *mask)
         unsigned long mmr_value;
         struct uv_IO_APIC_route_entry *entry;
         unsigned long mmr_offset;
-        unsigned mmr_pnode;
+        int mmr_pnode;
 
         if (set_desc_affinity(desc, mask, &dest))
                 return -1;
@@ -248,7 +248,7 @@ static int uv_set_irq_affinity(unsigned int irq, const struct cpumask *mask)
  * interrupt is raised.
  */
 int uv_setup_irq(char *irq_name, int cpu, int mmr_blade,
-                 unsigned long mmr_offset, int restrict)
+                 unsigned long mmr_offset, int limit)
 {
         int irq, ret;
 
@@ -258,7 +258,7 @@ int uv_setup_irq(char *irq_name, int cpu, int mmr_blade,
                 return -EBUSY;
 
         ret = arch_enable_uv_irq(irq_name, irq, cpu, mmr_blade, mmr_offset,
-                restrict);
+                limit);
         if (ret == irq)
                 uv_set_irq_2_mmr_info(irq, mmr_offset, mmr_blade);
         else
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 693920b22496..1b950d151e58 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -54,7 +54,6 @@ EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(__memcpy);
 
 EXPORT_SYMBOL(empty_zero_page);
-EXPORT_SYMBOL(init_level4_pgt);
 #ifndef CONFIG_PARAVIRT
 EXPORT_SYMBOL(native_load_gs_index);
 #endif
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 782c3a362ec6..37e68fc5e24a 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -99,7 +99,7 @@ int save_i387_xstate(void __user *buf)
                 if (err)
                         return err;
 
-                if (task_thread_info(tsk)->status & TS_XSAVE)
+                if (use_xsave())
                         err = xsave_user(buf);
                 else
                         err = fxsave_user(buf);
@@ -109,14 +109,14 @@ int save_i387_xstate(void __user *buf)
                 task_thread_info(tsk)->status &= ~TS_USEDFPU;
                 stts();
         } else {
-                if (__copy_to_user(buf, &tsk->thread.xstate->fxsave,
+                if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
                                    xstate_size))
                         return -1;
         }
 
         clear_used_math(); /* trigger finit */
 
-        if (task_thread_info(tsk)->status & TS_XSAVE) {
+        if (use_xsave()) {
                 struct _fpstate __user *fx = buf;
                 struct _xstate __user *x = buf;
                 u64 xstate_bv;
@@ -225,7 +225,7 @@ int restore_i387_xstate(void __user *buf)
                 clts();
                 task_thread_info(current)->status |= TS_USEDFPU;
         }
-        if (task_thread_info(tsk)->status & TS_XSAVE)
+        if (use_xsave())
                 err = restore_user_xstate(buf);
         else
                 err = fxrstor_checking((__force struct i387_fxsave_struct *)