16 files changed, 319 insertions, 101 deletions

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index d6192bcf9f09..dc5dddafe5c2 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/pci.h>
+#include <linux/pci-ats.h>
 #include <linux/bitmap.h>
 #include <linux/slab.h>
 #include <linux/debugfs.h>
@@ -463,6 +464,37 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
                 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 }
 
+static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
+                                  u64 address, size_t size)
+{
+        u64 pages;
+        int s;
+
+        pages = iommu_num_pages(address, size, PAGE_SIZE);
+        s     = 0;
+
+        if (pages > 1) {
+                /*
+                 * If we have to flush more than one page, flush all
+                 * TLB entries for this domain
+                 */
+                address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
+                s = 1;
+        }
+
+        address &= PAGE_MASK;
+
+        memset(cmd, 0, sizeof(*cmd));
+        cmd->data[0]  = devid;
+        cmd->data[0] |= (qdep & 0xff) << 24;
+        cmd->data[1]  = devid;
+        cmd->data[2]  = lower_32_bits(address);
+        cmd->data[3]  = upper_32_bits(address);
+        CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
+        if (s)
+                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
+}
+
 static void build_inv_all(struct iommu_cmd *cmd)
 {
         memset(cmd, 0, sizeof(*cmd));
@@ -594,17 +626,47 @@ void iommu_flush_all_caches(struct amd_iommu *iommu)
 }
 
 /*
+ * Command send function for flushing on-device TLB
+ */
+static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct amd_iommu *iommu;
+        struct iommu_cmd cmd;
+        u16 devid;
+        int qdep;
+
+        qdep  = pci_ats_queue_depth(pdev);
+        devid = get_device_id(dev);
+        iommu = amd_iommu_rlookup_table[devid];
+
+        build_inv_iotlb_pages(&cmd, devid, qdep, address, size);
+
+        return iommu_queue_command(iommu, &cmd);
+}
+
+/*
  * Command send function for invalidating a device table entry
  */
 static int device_flush_dte(struct device *dev)
 {
         struct amd_iommu *iommu;
+        struct pci_dev *pdev;
         u16 devid;
+        int ret;
 
+        pdev  = to_pci_dev(dev);
         devid = get_device_id(dev);
         iommu = amd_iommu_rlookup_table[devid];
 
-        return iommu_flush_dte(iommu, devid);
+        ret = iommu_flush_dte(iommu, devid);
+        if (ret)
+                return ret;
+
+        if (pci_ats_enabled(pdev))
+                ret = device_flush_iotlb(dev, 0, ~0UL);
+
+        return ret;
 }
 
 /*
@@ -615,6 +677,7 @@ static int device_flush_dte(struct device *dev)
 static void __domain_flush_pages(struct protection_domain *domain,
                                  u64 address, size_t size, int pde)
 {
+        struct iommu_dev_data *dev_data;
         struct iommu_cmd cmd;
         int ret = 0, i;
 
@@ -631,6 +694,15 @@ static void __domain_flush_pages(struct protection_domain *domain,
                 ret |= iommu_queue_command(amd_iommus[i], &cmd);
         }
 
+        list_for_each_entry(dev_data, &domain->dev_list, list) {
+                struct pci_dev *pdev = to_pci_dev(dev_data->dev);
+
+                if (!pci_ats_enabled(pdev))
+                        continue;
+
+                ret |= device_flush_iotlb(dev_data->dev, address, size);
+        }
+
         WARN_ON(ret);
 }
 
@@ -1400,17 +1472,22 @@ static bool dma_ops_domain(struct protection_domain *domain)
         return domain->flags & PD_DMA_OPS_MASK;
 }
 
-static void set_dte_entry(u16 devid, struct protection_domain *domain)
+static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 {
         u64 pte_root = virt_to_phys(domain->pt_root);
+        u32 flags = 0;
 
         pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
                     << DEV_ENTRY_MODE_SHIFT;
         pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
 
-        amd_iommu_dev_table[devid].data[2] = domain->id;
-        amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
-        amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
+        if (ats)
+                flags |= DTE_FLAG_IOTLB;
+
+        amd_iommu_dev_table[devid].data[3] |= flags;
+        amd_iommu_dev_table[devid].data[2]  = domain->id;
+        amd_iommu_dev_table[devid].data[1]  = upper_32_bits(pte_root);
+        amd_iommu_dev_table[devid].data[0]  = lower_32_bits(pte_root);
 }
 
 static void clear_dte_entry(u16 devid)
@@ -1427,16 +1504,22 @@ static void do_attach(struct device *dev, struct protection_domain *domain)
 {
         struct iommu_dev_data *dev_data;
         struct amd_iommu *iommu;
+        struct pci_dev *pdev;
+        bool ats = false;
         u16 devid;
 
         devid    = get_device_id(dev);
         iommu    = amd_iommu_rlookup_table[devid];
         dev_data = get_dev_data(dev);
+        pdev     = to_pci_dev(dev);
+
+        if (amd_iommu_iotlb_sup)
+                ats = pci_ats_enabled(pdev);
 
         /* Update data structures */
         dev_data->domain = domain;
         list_add(&dev_data->list, &domain->dev_list);
-        set_dte_entry(devid, domain);
+        set_dte_entry(devid, domain, ats);
 
         /* Do reference counting */
         domain->dev_iommu[iommu->index] += 1;
@@ -1450,11 +1533,13 @@ static void do_detach(struct device *dev)
 {
         struct iommu_dev_data *dev_data;
         struct amd_iommu *iommu;
+        struct pci_dev *pdev;
         u16 devid;
 
         devid    = get_device_id(dev);
         iommu    = amd_iommu_rlookup_table[devid];
         dev_data = get_dev_data(dev);
+        pdev     = to_pci_dev(dev);
 
         /* decrease reference counters */
         dev_data->domain->dev_iommu[iommu->index] -= 1;
@@ -1529,9 +1614,13 @@ out_unlock:
 static int attach_device(struct device *dev,
                          struct protection_domain *domain)
 {
+        struct pci_dev *pdev = to_pci_dev(dev);
         unsigned long flags;
         int ret;
 
+        if (amd_iommu_iotlb_sup)
+                pci_enable_ats(pdev, PAGE_SHIFT);
+
         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
         ret = __attach_device(dev, domain);
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
@@ -1588,12 +1677,16 @@ static void __detach_device(struct device *dev)
  */
 static void detach_device(struct device *dev)
 {
+        struct pci_dev *pdev = to_pci_dev(dev);
         unsigned long flags;
 
         /* lock device table */
         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
         __detach_device(dev);
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+
+        if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev))
+                pci_disable_ats(pdev);
 }
 
 /*
@@ -1743,8 +1836,9 @@ static void update_device_table(struct protection_domain *domain)
         struct iommu_dev_data *dev_data;
 
         list_for_each_entry(dev_data, &domain->dev_list, list) {
+                struct pci_dev *pdev = to_pci_dev(dev_data->dev);
                 u16 devid = get_device_id(dev_data->dev);
-                set_dte_entry(devid, domain);
+                set_dte_entry(devid, domain, pci_ats_enabled(pdev));
         }
 }
 
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index 047905dc3e14..28b078133688 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -137,6 +137,7 @@ int amd_iommus_present;
 
 /* IOMMUs have a non-present cache? */
 bool amd_iommu_np_cache __read_mostly;
+bool amd_iommu_iotlb_sup __read_mostly = true;
 
 /*
  * The ACPI table parsing functions set this variable on an error
@@ -687,6 +688,9 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
                                         MMIO_GET_LD(range));
         iommu->evt_msi_num = MMIO_MSI_NUM(misc);
 
+        if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
+                amd_iommu_iotlb_sup = false;
+
         /* read extended feature bits */
         low  = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
         high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 86d1ad4962a7..73fb469908c6 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -499,7 +499,7 @@ out:
                  * Don't enable translation yet but enable GART IO and CPU
                  * accesses and set DISTLBWALKPRB since GART table memory is UC.
                  */
-                u32 ctl = DISTLBWALKPRB | aper_order << 1;
+                u32 ctl = aper_order << 1;
 
                 bus = amd_nb_bus_dev_ranges[i].bus;
                 dev_base = amd_nb_bus_dev_ranges[i].dev_base;
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 68df09bba92e..45fd33d1fd3a 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -128,8 +128,8 @@ static int __init parse_noapic(char *str)
 }
 early_param("noapic", parse_noapic);
 
-static int io_apic_setup_irq_pin_once(unsigned int irq, int node,
-                                      struct io_apic_irq_attr *attr);
+static int io_apic_setup_irq_pin(unsigned int irq, int node,
+                                 struct io_apic_irq_attr *attr);
 
 /* Will be called in mpparse/acpi/sfi codes for saving IRQ info */
 void mp_save_irq(struct mpc_intsrc *m)
@@ -3570,7 +3570,7 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
 }
 #endif /* CONFIG_HT_IRQ */
 
-int
+static int
 io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
 {
         struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
@@ -3585,8 +3585,8 @@ io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
         return ret;
 }
 
-static int io_apic_setup_irq_pin_once(unsigned int irq, int node,
-                                      struct io_apic_irq_attr *attr)
+int io_apic_setup_irq_pin_once(unsigned int irq, int node,
+                               struct io_apic_irq_attr *attr)
 {
         unsigned int id = attr->ioapic, pin = attr->ioapic_pin;
         int ret;
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index 0b4be431c620..adee12e0da1f 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -228,6 +228,7 @@
 #include <linux/kthread.h>
 #include <linux/jiffies.h>
 #include <linux/acpi.h>
+#include <linux/syscore_ops.h>
 
 #include <asm/system.h>
 #include <asm/uaccess.h>
@@ -1238,6 +1239,7 @@ static int suspend(int vetoable)
 
         local_irq_disable();
         sysdev_suspend(PMSG_SUSPEND);
+        syscore_suspend();
 
         local_irq_enable();
 
@@ -1255,6 +1257,7 @@ static int suspend(int vetoable)
                 apm_error("suspend", err);
         err = (err == APM_SUCCESS) ? 0 : -EIO;
 
+        syscore_resume();
         sysdev_resume();
         local_irq_enable();
 
@@ -1280,6 +1283,7 @@ static void standby(void)
 
         local_irq_disable();
         sysdev_suspend(PMSG_SUSPEND);
+        syscore_suspend();
         local_irq_enable();
 
         err = set_system_power_state(APM_STATE_STANDBY);
@@ -1287,6 +1291,7 @@ static void standby(void)
                 apm_error("standby", err);
 
         local_irq_disable();
+        syscore_resume();
         sysdev_resume();
         local_irq_enable();
 
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 3ecece0217ef..bb9eb29a52dd 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -615,6 +615,25 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
         /* As a rule processors have APIC timer running in deep C states */
         if (c->x86 >= 0xf && !cpu_has_amd_erratum(amd_erratum_400))
                 set_cpu_cap(c, X86_FEATURE_ARAT);
+
+        /*
+         * Disable GART TLB Walk Errors on Fam10h. We do this here
+         * because this is always needed when GART is enabled, even in a
+         * kernel which has no MCE support built in.
+         */
+        if (c->x86 == 0x10) {
+                /*
+                 * BIOS should disable GartTlbWlk Errors themself. If
+                 * it doesn't do it here as suggested by the BKDG.
+                 *
+                 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
+                 */
+                u64 mask;
+
+                rdmsrl(MSR_AMD64_MCx_MASK(4), mask);
+                mask |= (1 << 10);
+                wrmsrl(MSR_AMD64_MCx_MASK(4), mask);
+        }
 }
 
 #ifdef CONFIG_X86_32
@@ -679,7 +698,7 @@ cpu_dev_register(amd_cpu_dev);
  */
 
 const int amd_erratum_400[] =
-        AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
+        AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0x0f, 0x4, 0x2, 0xff, 0xf),
                             AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));
 EXPORT_SYMBOL_GPL(amd_erratum_400);
 
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index eed3673a8656..e638689279d3 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -586,8 +586,12 @@ static int x86_setup_perfctr(struct perf_event *event)
                         return -EOPNOTSUPP;
         }
 
+        /*
+         * Do not allow config1 (extended registers) to propagate,
+         * there's no sane user-space generalization yet:
+         */
         if (attr->type == PERF_TYPE_RAW)
-                return x86_pmu_extra_regs(event->attr.config, event);
+                return 0;
 
         if (attr->type == PERF_TYPE_HW_CACHE)
                 return set_ext_hw_attr(hwc, event);
@@ -609,8 +613,8 @@ static int x86_setup_perfctr(struct perf_event *event)
         /*
          * Branch tracing:
          */
-        if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
-            (hwc->sample_period == 1)) {
+        if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
+            !attr->freq && hwc->sample_period == 1) {
                 /* BTS is not supported by this architecture. */
                 if (!x86_pmu.bts_active)
                         return -EOPNOTSUPP;
@@ -1284,6 +1288,16 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
 
         cpuc = &__get_cpu_var(cpu_hw_events);
 
+        /*
+         * Some chipsets need to unmask the LVTPC in a particular spot
+         * inside the nmi handler.  As a result, the unmasking was pushed
+         * into all the nmi handlers.
+         *
+         * This generic handler doesn't seem to have any issues where the
+         * unmasking occurs so it was left at the top.
+         */
+        apic_write(APIC_LVTPC, APIC_DM_NMI);
+
         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
                 if (!test_bit(idx, cpuc->active_mask)) {
                         /*
@@ -1370,8 +1384,6 @@ perf_event_nmi_handler(struct notifier_block *self,
                 return NOTIFY_DONE;
         }
 
-        apic_write(APIC_LVTPC, APIC_DM_NMI);
-
         handled = x86_pmu.handle_irq(args->regs);
         if (!handled)
                 return NOTIFY_DONE;
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index 461f62bbd774..cf4e369cea67 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -8,7 +8,7 @@ static __initconst const u64 amd_hw_cache_event_ids
  [ C(L1D) ] = {
         [ C(OP_READ) ] = {
                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
-                [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
+                [ C(RESULT_MISS)   ] = 0x0141, /* Data Cache Misses          */
         },
         [ C(OP_WRITE) ] = {
                 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
@@ -427,7 +427,9 @@ static __initconst const struct x86_pmu amd_pmu = {
  *
  * Exceptions:
  *
+ * 0x000        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
  * 0x003        FP      PERF_CTL[3]
+ * 0x004        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
  * 0x00B        FP      PERF_CTL[3]
  * 0x00D        FP      PERF_CTL[3]
  * 0x023        DE      PERF_CTL[2:0]
@@ -448,6 +450,8 @@ static __initconst const struct x86_pmu amd_pmu = {
  * 0x0DF        LS      PERF_CTL[5:0]
  * 0x1D6        EX      PERF_CTL[5:0]
  * 0x1D8        EX      PERF_CTL[5:0]
+ *
+ * (*) depending on the umask all FPU counters may be used
  */
 
 static struct event_constraint amd_f15_PMC0  = EVENT_CONSTRAINT(0, 0x01, 0);
@@ -460,18 +464,28 @@ static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
 static struct event_constraint *
 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
 {
-        unsigned int event_code = amd_get_event_code(&event->hw);
+        struct hw_perf_event *hwc = &event->hw;
+        unsigned int event_code = amd_get_event_code(hwc);
 
         switch (event_code & AMD_EVENT_TYPE_MASK) {
         case AMD_EVENT_FP:
                 switch (event_code) {
+                case 0x000:
+                        if (!(hwc->config & 0x0000F000ULL))
+                                break;
+                        if (!(hwc->config & 0x00000F00ULL))
+                                break;
+                        return &amd_f15_PMC3;
+                case 0x004:
+                        if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
+                                break;
+                        return &amd_f15_PMC3;
                 case 0x003:
                 case 0x00B:
                 case 0x00D:
                         return &amd_f15_PMC3;
-                default:
-                        return &amd_f15_PMC53;
                 }
+                return &amd_f15_PMC53;
         case AMD_EVENT_LS:
         case AMD_EVENT_DC:
         case AMD_EVENT_EX_LS:
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 8fc2b2cee1da..447a28de6f09 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -25,7 +25,7 @@ struct intel_percore {
 /*
  * Intel PerfMon, used on Core and later.
  */
-static const u64 intel_perfmon_event_map[] =
+static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
 {
   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
@@ -184,26 +184,23 @@ static __initconst const u64 snb_hw_cache_event_ids
         },
  },
  [ C(LL  ) ] = {
-        /*
-         * TBD: Need Off-core Response Performance Monitoring support
-         */
         [ C(OP_READ) ] = {
-                /* OFFCORE_RESPONSE_0.ANY_DATA.LOCAL_CACHE */
+                /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
                 [ C(RESULT_ACCESS) ] = 0x01b7,
-                /* OFFCORE_RESPONSE_1.ANY_DATA.ANY_LLC_MISS */
-                [ C(RESULT_MISS)   ] = 0x01bb,
+                /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
         [ C(OP_WRITE) ] = {
-                /* OFFCORE_RESPONSE_0.ANY_RFO.LOCAL_CACHE */
+                /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
                 [ C(RESULT_ACCESS) ] = 0x01b7,
-                /* OFFCORE_RESPONSE_1.ANY_RFO.ANY_LLC_MISS */
-                [ C(RESULT_MISS)   ] = 0x01bb,
+                /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
         [ C(OP_PREFETCH) ] = {
-                /* OFFCORE_RESPONSE_0.PREFETCH.LOCAL_CACHE */
+                /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
                 [ C(RESULT_ACCESS) ] = 0x01b7,
-                /* OFFCORE_RESPONSE_1.PREFETCH.ANY_LLC_MISS */
-                [ C(RESULT_MISS)   ] = 0x01bb,
+                /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
  },
  [ C(DTLB) ] = {
@@ -285,26 +282,26 @@ static __initconst const u64 westmere_hw_cache_event_ids
  },
  [ C(LL  ) ] = {
         [ C(OP_READ) ] = {
-                /* OFFCORE_RESPONSE_0.ANY_DATA.LOCAL_CACHE */
+                /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
                 [ C(RESULT_ACCESS) ] = 0x01b7,
-                /* OFFCORE_RESPONSE_1.ANY_DATA.ANY_LLC_MISS */
-                [ C(RESULT_MISS)   ] = 0x01bb,
+                /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
         /*
          * Use RFO, not WRITEBACK, because a write miss would typically occur
          * on RFO.
          */
         [ C(OP_WRITE) ] = {
-                /* OFFCORE_RESPONSE_1.ANY_RFO.LOCAL_CACHE */
-                [ C(RESULT_ACCESS) ] = 0x01bb,
-                /* OFFCORE_RESPONSE_0.ANY_RFO.ANY_LLC_MISS */
+                /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
                 [ C(RESULT_MISS)   ] = 0x01b7,
         },
         [ C(OP_PREFETCH) ] = {
-                /* OFFCORE_RESPONSE_0.PREFETCH.LOCAL_CACHE */
+                /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
                 [ C(RESULT_ACCESS) ] = 0x01b7,
-                /* OFFCORE_RESPONSE_1.PREFETCH.ANY_LLC_MISS */
-                [ C(RESULT_MISS)   ] = 0x01bb,
+                /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
  },
  [ C(DTLB) ] = {
@@ -352,16 +349,36 @@ static __initconst const u64 westmere_hw_cache_event_ids
 };
 
 /*
- * OFFCORE_RESPONSE MSR bits (subset), See IA32 SDM Vol 3 30.6.1.3
+ * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits;
+ * See IA32 SDM Vol 3B 30.6.1.3
  */
 
-#define DMND_DATA_RD     (1 << 0)
-#define DMND_RFO         (1 << 1)
-#define DMND_WB          (1 << 3)
-#define PF_DATA_RD       (1 << 4)
-#define PF_DATA_RFO      (1 << 5)
-#define RESP_UNCORE_HIT  (1 << 8)
-#define RESP_MISS        (0xf600) /* non uncore hit */
+#define NHM_DMND_DATA_RD        (1 << 0)
+#define NHM_DMND_RFO            (1 << 1)
+#define NHM_DMND_IFETCH         (1 << 2)
+#define NHM_DMND_WB             (1 << 3)
+#define NHM_PF_DATA_RD          (1 << 4)
+#define NHM_PF_DATA_RFO         (1 << 5)
+#define NHM_PF_IFETCH           (1 << 6)
+#define NHM_OFFCORE_OTHER       (1 << 7)
+#define NHM_UNCORE_HIT          (1 << 8)
+#define NHM_OTHER_CORE_HIT_SNP  (1 << 9)
+#define NHM_OTHER_CORE_HITM     (1 << 10)
+                                /* reserved */
+#define NHM_REMOTE_CACHE_FWD    (1 << 12)
+#define NHM_REMOTE_DRAM         (1 << 13)
+#define NHM_LOCAL_DRAM          (1 << 14)
+#define NHM_NON_DRAM            (1 << 15)
+
+#define NHM_ALL_DRAM            (NHM_REMOTE_DRAM|NHM_LOCAL_DRAM)
+
+#define NHM_DMND_READ           (NHM_DMND_DATA_RD)
+#define NHM_DMND_WRITE          (NHM_DMND_RFO|NHM_DMND_WB)
+#define NHM_DMND_PREFETCH       (NHM_PF_DATA_RD|NHM_PF_DATA_RFO)
+
+#define NHM_L3_HIT      (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)
+#define NHM_L3_MISS     (NHM_NON_DRAM|NHM_ALL_DRAM|NHM_REMOTE_CACHE_FWD)
+#define NHM_L3_ACCESS   (NHM_L3_HIT|NHM_L3_MISS)
 
 static __initconst const u64 nehalem_hw_cache_extra_regs
                                 [PERF_COUNT_HW_CACHE_MAX]
@@ -370,16 +387,16 @@ static __initconst const u64 nehalem_hw_cache_extra_regs
 {
  [ C(LL  ) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = DMND_DATA_RD|RESP_UNCORE_HIT,
-                [ C(RESULT_MISS)   ] = DMND_DATA_RD|RESP_MISS,
+                [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS,
+                [ C(RESULT_MISS)   ] = NHM_DMND_READ|NHM_L3_MISS,
         },
         [ C(OP_WRITE) ] = {
-                [ C(RESULT_ACCESS) ] = DMND_RFO|DMND_WB|RESP_UNCORE_HIT,
-                [ C(RESULT_MISS)   ] = DMND_RFO|DMND_WB|RESP_MISS,
+                [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS,
+                [ C(RESULT_MISS)   ] = NHM_DMND_WRITE|NHM_L3_MISS,
         },
         [ C(OP_PREFETCH) ] = {
-                [ C(RESULT_ACCESS) ] = PF_DATA_RD|PF_DATA_RFO|RESP_UNCORE_HIT,
-                [ C(RESULT_MISS)   ] = PF_DATA_RD|PF_DATA_RFO|RESP_MISS,
+                [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS,
+                [ C(RESULT_MISS)   ] = NHM_DMND_PREFETCH|NHM_L3_MISS,
         },
  }
 };
@@ -391,12 +408,12 @@ static __initconst const u64 nehalem_hw_cache_event_ids
 {
  [ C(L1D) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
-                [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
+                [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
+                [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
         },
         [ C(OP_WRITE) ] = {
-                [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
-                [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
+                [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
+                [ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
         },
         [ C(OP_PREFETCH) ] = {
                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
@@ -933,6 +950,16 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
 
         cpuc = &__get_cpu_var(cpu_hw_events);
 
+        /*
+         * Some chipsets need to unmask the LVTPC in a particular spot
+         * inside the nmi handler.  As a result, the unmasking was pushed
+         * into all the nmi handlers.
+         *
+         * This handler doesn't seem to have any issues with the unmasking
+         * so it was left at the top.
+         */
+        apic_write(APIC_LVTPC, APIC_DM_NMI);
+
         intel_pmu_disable_all();
         handled = intel_pmu_drain_bts_buffer();
         status = intel_pmu_get_status();
@@ -998,6 +1025,9 @@ intel_bts_constraints(struct perf_event *event)
         struct hw_perf_event *hwc = &event->hw;
         unsigned int hw_event, bts_event;
 
+        if (event->attr.freq)
+                return NULL;
+
         hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
         bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
 
@@ -1305,7 +1335,7 @@ static void intel_clovertown_quirks(void)
          * AJ106 could possibly be worked around by not allowing LBR
          *       usage from PEBS, including the fixup.
          * AJ68  could possibly be worked around by always programming
-         *       a pebs_event_reset[0] value and coping with the lost events.
+         *       a pebs_event_reset[0] value and coping with the lost events.
          *
          * But taken together it might just make sense to not enable PEBS on
          * these chips.
@@ -1409,6 +1439,18 @@ static __init int intel_pmu_init(void)
                 x86_pmu.percore_constraints = intel_nehalem_percore_constraints;
                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
                 x86_pmu.extra_regs = intel_nehalem_extra_regs;
+
+                if (ebx & 0x40) {
+                        /*
+                         * Erratum AAJ80 detected, we work it around by using
+                         * the BR_MISP_EXEC.ANY event. This will over-count
+                         * branch-misses, but it's still much better than the
+                         * architectural event which is often completely bogus:
+                         */
+                        intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89;
+
+                        pr_cont("erratum AAJ80 worked around, ");
+                }
                 pr_cont("Nehalem events, ");
                 break;
 
@@ -1425,6 +1467,7 @@ static __init int intel_pmu_init(void)
 
         case 37: /* 32 nm nehalem, "Clarkdale" */
         case 44: /* 32 nm nehalem, "Gulftown" */
+        case 47: /* 32 nm Xeon E7 */
                 memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
                        sizeof(hw_cache_event_ids));
                 memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c
index c2520e178d32..e93fcd55fae1 100644
--- a/arch/x86/kernel/cpu/perf_event_p4.c
+++ b/arch/x86/kernel/cpu/perf_event_p4.c
@@ -947,14 +947,23 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
                 if (!x86_perf_event_set_period(event))
                         continue;
                 if (perf_event_overflow(event, 1, &data, regs))
-                        p4_pmu_disable_event(event);
+                        x86_pmu_stop(event, 0);
         }
 
-        if (handled) {
-                /* p4 quirk: unmask it again */
-                apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);
+        if (handled)
                 inc_irq_stat(apic_perf_irqs);
-        }
+
+        /*
+         * When dealing with the unmasking of the LVTPC on P4 perf hw, it has
+         * been observed that the OVF bit flag has to be cleared first _before_
+         * the LVTPC can be unmasked.
+         *
+         * The reason is the NMI line will continue to be asserted while the OVF
+         * bit is set.  This causes a second NMI to generate if the LVTPC is
+         * unmasked before the OVF bit is cleared, leading to unknown NMI
+         * messages.
+         */
+        apic_write(APIC_LVTPC, APIC_DM_NMI);
 
         return handled;
 }
diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c
index 706a9fb46a58..e90f08458e6b 100644
--- a/arch/x86/kernel/devicetree.c
+++ b/arch/x86/kernel/devicetree.c
@@ -391,7 +391,7 @@ static int ioapic_xlate(struct irq_domain *id, const u32 *intspec, u32 intsize,
 
         set_io_apic_irq_attr(&attr, idx, line, it->trigger, it->polarity);
 
-        return io_apic_setup_irq_pin(*out_hwirq, cpu_to_node(0), &attr);
+        return io_apic_setup_irq_pin_once(*out_hwirq, cpu_to_node(0), &attr);
 }
 
 static void __init ioapic_add_ofnode(struct device_node *np)
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index 82ada01625b9..b117efd24f71 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -81,6 +81,9 @@ static u32 gart_unmapped_entry;
 #define AGPEXTERN
 #endif
 
+/* GART can only remap to physical addresses < 1TB */
+#define GART_MAX_PHYS_ADDR      (1ULL << 40)
+
 /* backdoor interface to AGP driver */
 AGPEXTERN int agp_memory_reserved;
 AGPEXTERN __u32 *agp_gatt_table;
@@ -212,9 +215,13 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
                                 size_t size, int dir, unsigned long align_mask)
 {
         unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
-        unsigned long iommu_page = alloc_iommu(dev, npages, align_mask);
+        unsigned long iommu_page;
         int i;
 
+        if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR))
+                return bad_dma_addr;
+
+        iommu_page = alloc_iommu(dev, npages, align_mask);
         if (iommu_page == -1) {
                 if (!nonforced_iommu(dev, phys_mem, size))
                         return phys_mem;
diff --git a/arch/x86/kernel/pci-iommu_table.c b/arch/x86/kernel/pci-iommu_table.c
index 55d745ec1181..35ccf75696eb 100644
--- a/arch/x86/kernel/pci-iommu_table.c
+++ b/arch/x86/kernel/pci-iommu_table.c
@@ -50,20 +50,14 @@ void __init check_iommu_entries(struct iommu_table_entry *start,
                                 struct iommu_table_entry *finish)
 {
         struct iommu_table_entry *p, *q, *x;
-        char sym_p[KSYM_SYMBOL_LEN];
-        char sym_q[KSYM_SYMBOL_LEN];
 
         /* Simple cyclic dependency checker. */
         for (p = start; p < finish; p++) {
                 q = find_dependents_of(start, finish, p);
                 x = find_dependents_of(start, finish, q);
                 if (p == x) {
-                        sprint_symbol(sym_p, (unsigned long)p->detect);
-                        sprint_symbol(sym_q, (unsigned long)q->detect);
-
-                        printk(KERN_ERR "CYCLIC DEPENDENCY FOUND! %s depends" \
-                                        " on %s and vice-versa. BREAKING IT.\n",
-                                        sym_p, sym_q);
+                        printk(KERN_ERR "CYCLIC DEPENDENCY FOUND! %pS depends on %pS and vice-versa. BREAKING IT.\n",
+                               p->detect, q->detect);
                         /* Heavy handed way..*/
                         x->depend = 0;
                 }
@@ -72,12 +66,8 @@ void __init check_iommu_entries(struct iommu_table_entry *start,
         for (p = start; p < finish; p++) {
                 q = find_dependents_of(p, finish, p);
                 if (q && q > p) {
-                        sprint_symbol(sym_p, (unsigned long)p->detect);
-                        sprint_symbol(sym_q, (unsigned long)q->detect);
-
-                        printk(KERN_ERR "EXECUTION ORDER INVALID! %s "\
-                                        "should be called before %s!\n",
-                                        sym_p, sym_q);
+                        printk(KERN_ERR "EXECUTION ORDER INVALID! %pS should be called before %pS!\n",
+                               p->detect, q->detect);
                 }
         }
 }
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 45892dc4b72a..f65e5b521dbd 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -608,6 +608,9 @@ static int ptrace_write_dr7(struct task_struct *tsk, unsigned long data)
         unsigned len, type;
         struct perf_event *bp;
 
+        if (ptrace_get_breakpoints(tsk) < 0)
+                return -ESRCH;
+
         data &= ~DR_CONTROL_RESERVED;
         old_dr7 = ptrace_get_dr7(thread->ptrace_bps);
 restore:
@@ -655,6 +658,9 @@ restore:
                 }
                 goto restore;
         }
+
+        ptrace_put_breakpoints(tsk);
+
         return ((orig_ret < 0) ? orig_ret : rc);
 }
 
@@ -668,10 +674,17 @@ static unsigned long ptrace_get_debugreg(struct task_struct *tsk, int n)
 
         if (n < HBP_NUM) {
                 struct perf_event *bp;
+
+                if (ptrace_get_breakpoints(tsk) < 0)
+                        return -ESRCH;
+
                 bp = thread->ptrace_bps[n];
                 if (!bp)
-                        return 0;
-                val = bp->hw.info.address;
+                        val = 0;
+                else
+                        val = bp->hw.info.address;
+
+                ptrace_put_breakpoints(tsk);
         } else if (n == 6) {
                 val = thread->debugreg6;
          } else if (n == 7) {
@@ -686,6 +699,10 @@ static int ptrace_set_breakpoint_addr(struct task_struct *tsk, int nr,
         struct perf_event *bp;
         struct thread_struct *t = &tsk->thread;
         struct perf_event_attr attr;
+        int err = 0;
+
+        if (ptrace_get_breakpoints(tsk) < 0)
+                return -ESRCH;
 
         if (!t->ptrace_bps[nr]) {
                 ptrace_breakpoint_init(&attr);
@@ -709,24 +726,23 @@ static int ptrace_set_breakpoint_addr(struct task_struct *tsk, int nr,
                  * writing for the user. And anyway this is the previous
                  * behaviour.
                  */
-                if (IS_ERR(bp))
-                        return PTR_ERR(bp);
+                if (IS_ERR(bp)) {
+                        err = PTR_ERR(bp);
+                        goto put;
+                }
 
                 t->ptrace_bps[nr] = bp;
         } else {
-                int err;
-
                 bp = t->ptrace_bps[nr];
 
                 attr = bp->attr;
                 attr.bp_addr = addr;
                 err = modify_user_hw_breakpoint(bp, &attr);
-                if (err)
-                        return err;
         }
 
-
-        return 0;
+put:
+        ptrace_put_breakpoints(tsk);
+        return err;
 }
 
 /*
diff --git a/arch/x86/kernel/reboot_32.S b/arch/x86/kernel/reboot_32.S
index 29092b38d816..1d5c46df0d78 100644
--- a/arch/x86/kernel/reboot_32.S
+++ b/arch/x86/kernel/reboot_32.S
@@ -21,26 +21,26 @@ r_base = .
         /* Get our own relocated address */
         call    1f
 1:      popl    %ebx
-        subl    $1b, %ebx
+        subl    $(1b - r_base), %ebx
 
         /* Compute the equivalent real-mode segment */
         movl    %ebx, %ecx
         shrl    $4, %ecx
         
         /* Patch post-real-mode segment jump */
-        movw    dispatch_table(%ebx,%eax,2),%ax
-        movw    %ax, 101f(%ebx)
-        movw    %cx, 102f(%ebx)
+        movw    (dispatch_table - r_base)(%ebx,%eax,2),%ax
+        movw    %ax, (101f - r_base)(%ebx)
+        movw    %cx, (102f - r_base)(%ebx)
 
         /* Set up the IDT for real mode. */
-        lidtl   machine_real_restart_idt(%ebx)
+        lidtl   (machine_real_restart_idt - r_base)(%ebx)
 
         /*
          * Set up a GDT from which we can load segment descriptors for real
          * mode.  The GDT is not used in real mode; it is just needed here to
          * prepare the descriptors.
          */
-        lgdtl   machine_real_restart_gdt(%ebx)
+        lgdtl   (machine_real_restart_gdt - r_base)(%ebx)
 
         /*
          * Load the data segment registers with 16-bit compatible values
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 5a0484a95ad6..4be9b398470e 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -976,6 +976,11 @@ void __init setup_arch(char **cmdline_p)
         paging_init();
         x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
+        if (boot_cpu_data.cpuid_level >= 0) {
+                /* A CPU has %cr4 if and only if it has CPUID */
+                mmu_cr4_features = read_cr4();
+        }
+
 #ifdef CONFIG_X86_32
         /* sync back kernel address range */
         clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,