Merge commit 'v2.6.31-rc7' into x86/cpu

23 files changed, 1122 insertions, 847 deletions

diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index e5b27d8f1b47..63fddcd082cd 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -258,13 +258,15 @@ static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_X86_HT
         unsigned bits;
+        int cpu = smp_processor_id();
 
         bits = c->x86_coreid_bits;
-
         /* Low order bits define the core id (index of core in socket) */
         c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
         /* Convert the initial APIC ID into the socket ID */
         c->phys_proc_id = c->initial_apicid >> bits;
+        /* use socket ID also for last level cache */
+        per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
 #endif
 }
 
@@ -354,7 +356,7 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
 #endif
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
         /* check CPU config space for extended APIC ID */
-        if (c->x86 >= 0xf) {
+        if (cpu_has_apic && c->x86 >= 0xf) {
                 unsigned int val;
                 val = read_pci_config(0, 24, 0, 0x68);
                 if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
@@ -398,6 +400,13 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
                 level = cpuid_eax(1);
                 if((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
                         set_cpu_cap(c, X86_FEATURE_REP_GOOD);
+
+                /*
+                 * Some BIOSes incorrectly force this feature, but only K8
+                 * revision D (model = 0x14) and later actually support it.
+                 */
+                if (c->x86_model < 0x14)
+                        clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
         }
         if (c->x86 == 0x10 || c->x86 == 0x11)
                 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 3ffdcfa9abdf..5ce60a88027b 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -59,7 +59,30 @@ void __init setup_cpu_local_masks(void)
         alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
 }
 
-static const struct cpu_dev *this_cpu __cpuinitdata;
+static void __cpuinit default_init(struct cpuinfo_x86 *c)
+{
+#ifdef CONFIG_X86_64
+        display_cacheinfo(c);
+#else
+        /* Not much we can do here... */
+        /* Check if at least it has cpuid */
+        if (c->cpuid_level == -1) {
+                /* No cpuid. It must be an ancient CPU */
+                if (c->x86 == 4)
+                        strcpy(c->x86_model_id, "486");
+                else if (c->x86 == 3)
+                        strcpy(c->x86_model_id, "386");
+        }
+#endif
+}
+
+static const struct cpu_dev __cpuinitconst default_cpu = {
+        .c_init         = default_init,
+        .c_vendor       = "Unknown",
+        .c_x86_vendor   = X86_VENDOR_UNKNOWN,
+};
+
+static const struct cpu_dev *this_cpu __cpuinitdata = &default_cpu;
 
 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 #ifdef CONFIG_X86_64
@@ -108,7 +131,7 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
         /* data */
         [GDT_ENTRY_APMBIOS_BASE+2]      = { { { 0x0000ffff, 0x00409200 } } },
 
-        [GDT_ENTRY_ESPFIX_SS]           = { { { 0x00000000, 0x00c09200 } } },
+        [GDT_ENTRY_ESPFIX_SS]           = { { { 0x0000ffff, 0x00cf9200 } } },
         [GDT_ENTRY_PERCPU]              = { { { 0x0000ffff, 0x00cf9200 } } },
         GDT_STACK_CANARY_INIT
 #endif
@@ -332,29 +355,6 @@ void switch_to_new_gdt(int cpu)
 
 static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {};
 
-static void __cpuinit default_init(struct cpuinfo_x86 *c)
-{
-#ifdef CONFIG_X86_64
-        display_cacheinfo(c);
-#else
-        /* Not much we can do here... */
-        /* Check if at least it has cpuid */
-        if (c->cpuid_level == -1) {
-                /* No cpuid. It must be an ancient CPU */
-                if (c->x86 == 4)
-                        strcpy(c->x86_model_id, "486");
-                else if (c->x86 == 3)
-                        strcpy(c->x86_model_id, "386");
-        }
-#endif
-}
-
-static const struct cpu_dev __cpuinitconst default_cpu = {
-        .c_init = default_init,
-        .c_vendor = "Unknown",
-        .c_x86_vendor = X86_VENDOR_UNKNOWN,
-};
-
 static void __cpuinit get_model_name(struct cpuinfo_x86 *c)
 {
         unsigned int *v;
@@ -487,7 +487,6 @@ out:
 static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
 {
         char *v = c->x86_vendor_id;
-        static int printed;
         int i;
 
         for (i = 0; i < X86_VENDOR_NUM; i++) {
@@ -504,13 +503,9 @@ static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
                 }
         }
 
-        if (!printed) {
-                printed++;
-                printk(KERN_ERR
-                    "CPU: vendor_id '%s' unknown, using generic init.\n", v);
-
-                printk(KERN_ERR "CPU: Your system may be unstable.\n");
-        }
+        printk_once(KERN_ERR
+                        "CPU: vendor_id '%s' unknown, using generic init.\n" \
+                        "CPU: Your system may be unstable.\n", v);
 
         c->x86_vendor = X86_VENDOR_UNKNOWN;
         this_cpu = &default_cpu;
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
index cf52215d9eb1..2a50ef891000 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
@@ -1,3 +1,4 @@
+
 /*
  *   (c) 2003-2006 Advanced Micro Devices, Inc.
  *  Your use of this code is subject to the terms and conditions of the
@@ -117,20 +118,17 @@ static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
         u32 i = 0;
 
         if (cpu_family == CPU_HW_PSTATE) {
-                if (data->currpstate == HW_PSTATE_INVALID) {
-                        /* read (initial) hw pstate if not yet set */
-                        rdmsr(MSR_PSTATE_STATUS, lo, hi);
-                        i = lo & HW_PSTATE_MASK;
-
-                        /*
-                         * a workaround for family 11h erratum 311 might cause
-                         * an "out-of-range Pstate if the core is in Pstate-0
-                         */
-                        if (i >= data->numps)
-                                data->currpstate = HW_PSTATE_0;
-                        else
-                                data->currpstate = i;
-                }
+                rdmsr(MSR_PSTATE_STATUS, lo, hi);
+                i = lo & HW_PSTATE_MASK;
+                data->currpstate = i;
+
+                /*
+                 * a workaround for family 11h erratum 311 might cause
+                 * an "out-of-range Pstate if the core is in Pstate-0
+                 */
+                if ((boot_cpu_data.x86 == 0x11) && (i >= data->numps))
+                        data->currpstate = HW_PSTATE_0;
+
                 return 0;
         }
         do {
@@ -301,7 +299,7 @@ static int transition_pstate(struct powernow_k8_data *data, u32 pstate)
 static int transition_fid_vid(struct powernow_k8_data *data,
                 u32 reqfid, u32 reqvid)
 {
-        if (core_voltage_pre_transition(data, reqvid))
+        if (core_voltage_pre_transition(data, reqvid, reqfid))
                 return 1;
 
         if (core_frequency_transition(data, reqfid))
@@ -329,17 +327,20 @@ static int transition_fid_vid(struct powernow_k8_data *data,
 
 /* Phase 1 - core voltage transition ... setup voltage */
 static int core_voltage_pre_transition(struct powernow_k8_data *data,
-                u32 reqvid)
+                u32 reqvid, u32 reqfid)
 {
         u32 rvosteps = data->rvo;
         u32 savefid = data->currfid;
-        u32 maxvid, lo;
+        u32 maxvid, lo, rvomult = 1;
 
         dprintk("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, "
                 "reqvid 0x%x, rvo 0x%x\n",
                 smp_processor_id(),
                 data->currfid, data->currvid, reqvid, data->rvo);
 
+        if ((savefid < LO_FID_TABLE_TOP) && (reqfid < LO_FID_TABLE_TOP))
+                rvomult = 2;
+        rvosteps *= rvomult;
         rdmsr(MSR_FIDVID_STATUS, lo, maxvid);
         maxvid = 0x1f & (maxvid >> 16);
         dprintk("ph1 maxvid=0x%x\n", maxvid);
@@ -353,7 +354,8 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data,
                         return 1;
         }
 
-        while ((rvosteps > 0) && ((data->rvo + data->currvid) > reqvid)) {
+        while ((rvosteps > 0) &&
+                        ((rvomult * data->rvo + data->currvid) > reqvid)) {
                 if (data->currvid == maxvid) {
                         rvosteps = 0;
                 } else {
@@ -386,13 +388,6 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid)
         u32 vcoreqfid, vcocurrfid, vcofiddiff;
         u32 fid_interval, savevid = data->currvid;
 
-        if ((reqfid < HI_FID_TABLE_BOTTOM) &&
-            (data->currfid < HI_FID_TABLE_BOTTOM)) {
-                printk(KERN_ERR PFX "ph2: illegal lo-lo transition "
-                                "0x%x 0x%x\n", reqfid, data->currfid);
-                return 1;
-        }
-
         if (data->currfid == reqfid) {
                 printk(KERN_ERR PFX "ph2 null fid transition 0x%x\n",
                                 data->currfid);
@@ -409,6 +404,9 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid)
         vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
             : vcoreqfid - vcocurrfid;
 
+        if ((reqfid <= LO_FID_TABLE_TOP) && (data->currfid <= LO_FID_TABLE_TOP))
+                vcofiddiff = 0;
+
         while (vcofiddiff > 2) {
                 (data->currfid & 1) ? (fid_interval = 1) : (fid_interval = 2);
 
@@ -510,41 +508,34 @@ static int core_voltage_post_transition(struct powernow_k8_data *data,
         return 0;
 }
 
-static int check_supported_cpu(unsigned int cpu)
+static void check_supported_cpu(void *_rc)
 {
-        cpumask_t oldmask;
         u32 eax, ebx, ecx, edx;
-        unsigned int rc = 0;
+        int *rc = _rc;
 
-        oldmask = current->cpus_allowed;
-        set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
-
-        if (smp_processor_id() != cpu) {
-                printk(KERN_ERR PFX "limiting to cpu %u failed\n", cpu);
-                goto out;
-        }
+        *rc = -ENODEV;
 
         if (current_cpu_data.x86_vendor != X86_VENDOR_AMD)
-                goto out;
+                return;
 
         eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
         if (((eax & CPUID_XFAM) != CPUID_XFAM_K8) &&
             ((eax & CPUID_XFAM) < CPUID_XFAM_10H))
-                goto out;
+                return;
 
         if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) {
                 if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) ||
                     ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) {
                         printk(KERN_INFO PFX
                                 "Processor cpuid %x not supported\n", eax);
-                        goto out;
+                        return;
                 }
 
                 eax = cpuid_eax(CPUID_GET_MAX_CAPABILITIES);
                 if (eax < CPUID_FREQ_VOLT_CAPABILITIES) {
                         printk(KERN_INFO PFX
                                "No frequency change capabilities detected\n");
-                        goto out;
+                        return;
                 }
 
                 cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
@@ -552,21 +543,17 @@ static int check_supported_cpu(unsigned int cpu)
                         != P_STATE_TRANSITION_CAPABLE) {
                         printk(KERN_INFO PFX
                                 "Power state transitions not supported\n");
-                        goto out;
+                        return;
                 }
         } else { /* must be a HW Pstate capable processor */
                 cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
                 if ((edx & USE_HW_PSTATE) == USE_HW_PSTATE)
                         cpu_family = CPU_HW_PSTATE;
                 else
-                        goto out;
+                        return;
         }
 
-        rc = 1;
-
-out:
-        set_cpus_allowed_ptr(current, &oldmask);
-        return rc;
+        *rc = 0;
 }
 
 static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst,
@@ -823,13 +810,14 @@ static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data,
         if (!data->acpi_data.state_count || (cpu_family == CPU_HW_PSTATE))
                 return;
 
-        control = data->acpi_data.states[index].control; data->irt = (control
-                        >> IRT_SHIFT) & IRT_MASK; data->rvo = (control >>
-                                RVO_SHIFT) & RVO_MASK; data->exttype = (control
-                                        >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK;
-        data->plllock = (control >> PLL_L_SHIFT) & PLL_L_MASK; data->vidmvs = 1
-                << ((control >> MVS_SHIFT) & MVS_MASK); data->vstable =
-                (control >> VST_SHIFT) & VST_MASK; }
+        control = data->acpi_data.states[index].control;
+        data->irt = (control >> IRT_SHIFT) & IRT_MASK;
+        data->rvo = (control >> RVO_SHIFT) & RVO_MASK;
+        data->exttype = (control >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK;
+        data->plllock = (control >> PLL_L_SHIFT) & PLL_L_MASK;
+        data->vidmvs = 1 << ((control >> MVS_SHIFT) & MVS_MASK);
+        data->vstable = (control >> VST_SHIFT) & VST_MASK;
+}
 
 static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
 {
@@ -1046,6 +1034,19 @@ static int get_transition_latency(struct powernow_k8_data *data)
                 if (cur_latency > max_latency)
                         max_latency = cur_latency;
         }
+        if (max_latency == 0) {
+                /*
+                 * Fam 11h always returns 0 as transition latency.
+                 * This is intended and means "very fast". While cpufreq core
+                 * and governors currently can handle that gracefully, better
+                 * set it to 1 to avoid problems in the future.
+                 * For all others it's a BIOS bug.
+                 */
+                if (!boot_cpu_data.x86 == 0x11)
+                        printk(KERN_ERR FW_WARN PFX "Invalid zero transition "
+                                "latency\n");
+                max_latency = 1;
+        }
         /* value in usecs, needs to be in nanoseconds */
         return 1000 * max_latency;
 }
@@ -1080,20 +1081,12 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data,
                 return 0;
         }
 
-        if ((fid < HI_FID_TABLE_BOTTOM) &&
-            (data->currfid < HI_FID_TABLE_BOTTOM)) {
-                printk(KERN_ERR PFX
-                       "ignoring illegal change in lo freq table-%x to 0x%x\n",
-                       data->currfid, fid);
-                return 1;
-        }
-
         dprintk("cpu %d, changing to fid 0x%x, vid 0x%x\n",
                 smp_processor_id(), fid, vid);
         freqs.old = find_khz_freq_from_fid(data->currfid);
         freqs.new = find_khz_freq_from_fid(fid);
 
-        for_each_cpu_mask_nr(i, *(data->available_cores)) {
+        for_each_cpu(i, data->available_cores) {
                 freqs.cpu = i;
                 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
         }
@@ -1101,7 +1094,7 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data,
         res = transition_fid_vid(data, fid, vid);
         freqs.new = find_khz_freq_from_fid(data->currfid);
 
-        for_each_cpu_mask_nr(i, *(data->available_cores)) {
+        for_each_cpu(i, data->available_cores) {
                 freqs.cpu = i;
                 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
         }
@@ -1126,7 +1119,7 @@ static int transition_frequency_pstate(struct powernow_k8_data *data,
                         data->currpstate);
         freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate);
 
-        for_each_cpu_mask_nr(i, *(data->available_cores)) {
+        for_each_cpu(i, data->available_cores) {
                 freqs.cpu = i;
                 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
         }
@@ -1134,7 +1127,7 @@ static int transition_frequency_pstate(struct powernow_k8_data *data,
         res = transition_pstate(data, pstate);
         freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate);
 
-        for_each_cpu_mask_nr(i, *(data->available_cores)) {
+        for_each_cpu(i, data->available_cores) {
                 freqs.cpu = i;
                 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
         }
@@ -1235,21 +1228,47 @@ static int powernowk8_verify(struct cpufreq_policy *pol)
         return cpufreq_frequency_table_verify(pol, data->powernow_table);
 }
 
-static const char ACPI_PSS_BIOS_BUG_MSG[] =
-        KERN_ERR FW_BUG PFX "No compatible ACPI _PSS objects found.\n"
-        KERN_ERR FW_BUG PFX "Try again with latest BIOS.\n";
+struct init_on_cpu {
+        struct powernow_k8_data *data;
+        int rc;
+};
+
+static void __cpuinit powernowk8_cpu_init_on_cpu(void *_init_on_cpu)
+{
+        struct init_on_cpu *init_on_cpu = _init_on_cpu;
+
+        if (pending_bit_stuck()) {
+                printk(KERN_ERR PFX "failing init, change pending bit set\n");
+                init_on_cpu->rc = -ENODEV;
+                return;
+        }
+
+        if (query_current_values_with_pending_wait(init_on_cpu->data)) {
+                init_on_cpu->rc = -ENODEV;
+                return;
+        }
+
+        if (cpu_family == CPU_OPTERON)
+                fidvid_msr_init();
+
+        init_on_cpu->rc = 0;
+}
 
 /* per CPU init entry point to the driver */
 static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 {
+        static const char ACPI_PSS_BIOS_BUG_MSG[] =
+                KERN_ERR FW_BUG PFX "No compatible ACPI _PSS objects found.\n"
+                FW_BUG PFX "Try again with latest BIOS.\n";
         struct powernow_k8_data *data;
-        cpumask_t oldmask;
+        struct init_on_cpu init_on_cpu;
         int rc;
 
         if (!cpu_online(pol->cpu))
                 return -ENODEV;
 
-        if (!check_supported_cpu(pol->cpu))
+        smp_call_function_single(pol->cpu, check_supported_cpu, &rc, 1);
+        if (rc)
                 return -ENODEV;
 
         data = kzalloc(sizeof(struct powernow_k8_data), GFP_KERNEL);
@@ -1289,27 +1308,12 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
                 pol->cpuinfo.transition_latency = get_transition_latency(data);
 
         /* only run on specific CPU from here on */
-        oldmask = current->cpus_allowed;
-        set_cpus_allowed_ptr(current, &cpumask_of_cpu(pol->cpu));
-
-        if (smp_processor_id() != pol->cpu) {
-                printk(KERN_ERR PFX "limiting to cpu %u failed\n", pol->cpu);
-                goto err_out_unmask;
-        }
-
-        if (pending_bit_stuck()) {
-                printk(KERN_ERR PFX "failing init, change pending bit set\n");
-                goto err_out_unmask;
-        }
-
-        if (query_current_values_with_pending_wait(data))
-                goto err_out_unmask;
-
-        if (cpu_family == CPU_OPTERON)
-                fidvid_msr_init();
-
-        /* run on any CPU again */
-        set_cpus_allowed_ptr(current, &oldmask);
+        init_on_cpu.data = data;
+        smp_call_function_single(data->cpu, powernowk8_cpu_init_on_cpu,
+                                 &init_on_cpu, 1);
+        rc = init_on_cpu.rc;
+        if (rc != 0)
+                goto err_out_exit_acpi;
 
         if (cpu_family == CPU_HW_PSTATE)
                 cpumask_copy(pol->cpus, cpumask_of(pol->cpu));
@@ -1346,8 +1350,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 
         return 0;
 
-err_out_unmask:
-        set_cpus_allowed_ptr(current, &oldmask);
+err_out_exit_acpi:
         powernow_k8_cpu_exit_acpi(data);
 
 err_out:
@@ -1372,28 +1375,25 @@ static int __devexit powernowk8_cpu_exit(struct cpufreq_policy *pol)
         return 0;
 }
 
+static void query_values_on_cpu(void *_err)
+{
+        int *err = _err;
+        struct powernow_k8_data *data = __get_cpu_var(powernow_data);
+
+        *err = query_current_values_with_pending_wait(data);
+}
+
 static unsigned int powernowk8_get(unsigned int cpu)
 {
-        struct powernow_k8_data *data;
-        cpumask_t oldmask = current->cpus_allowed;
+        struct powernow_k8_data *data = per_cpu(powernow_data, cpu);
         unsigned int khz = 0;
-        unsigned int first;
-
-        first = cpumask_first(cpu_core_mask(cpu));
-        data = per_cpu(powernow_data, first);
+        int err;
 
         if (!data)
                 return -EINVAL;
 
-        set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
-        if (smp_processor_id() != cpu) {
-                printk(KERN_ERR PFX
-                        "limiting to CPU %d failed in powernowk8_get\n", cpu);
-                set_cpus_allowed_ptr(current, &oldmask);
-                return 0;
-        }
-
-        if (query_current_values_with_pending_wait(data))
+        smp_call_function_single(cpu, query_values_on_cpu, &err, true);
+        if (err)
                 goto out;
 
         if (cpu_family == CPU_HW_PSTATE)
@@ -1404,7 +1404,6 @@ static unsigned int powernowk8_get(unsigned int cpu)
 
 
 out:
-        set_cpus_allowed_ptr(current, &oldmask);
         return khz;
 }
 
@@ -1430,7 +1429,9 @@ static int __cpuinit powernowk8_init(void)
         unsigned int i, supported_cpus = 0;
 
         for_each_online_cpu(i) {
-                if (check_supported_cpu(i))
+                int rc;
+                smp_call_function_single(i, check_supported_cpu, &rc, 1);
+                if (rc == 0)
                         supported_cpus++;
         }
 
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.h b/arch/x86/kernel/cpu/cpufreq/powernow-k8.h
index 6c6698feade1..02ce824073cb 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.h
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.h
@@ -215,7 +215,8 @@ struct pst_s {
 
 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "powernow-k8", msg)
 
-static int core_voltage_pre_transition(struct powernow_k8_data *data, u32 reqvid);
+static int core_voltage_pre_transition(struct powernow_k8_data *data,
+        u32 reqvid, u32 regfid);
 static int core_voltage_post_transition(struct powernow_k8_data *data, u32 reqvid);
 static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid);
 
@@ -223,14 +224,3 @@ static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned
 
 static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table);
 static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table);
-
-#ifdef CONFIG_SMP
-static inline void define_siblings(int cpu, cpumask_t cpu_sharedcore_mask[])
-{
-}
-#else
-static inline void define_siblings(int cpu, cpumask_t cpu_sharedcore_mask[])
-{
-        cpu_set(0, cpu_sharedcore_mask[0]);
-}
-#endif
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
index 55c831ed71ce..8d672ef162ce 100644
--- a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
+++ b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
@@ -323,14 +323,8 @@ static unsigned int get_cur_freq(unsigned int cpu)
 {
         unsigned l, h;
         unsigned clock_freq;
-        cpumask_t saved_mask;
 
-        saved_mask = current->cpus_allowed;
-        set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
-        if (smp_processor_id() != cpu)
-                return 0;
-
-        rdmsr(MSR_IA32_PERF_STATUS, l, h);
+        rdmsr_on_cpu(cpu, MSR_IA32_PERF_STATUS, &l, &h);
         clock_freq = extract_clock(l, cpu, 0);
 
         if (unlikely(clock_freq == 0)) {
@@ -340,11 +334,9 @@ static unsigned int get_cur_freq(unsigned int cpu)
                  * P-state transition (like TM2). Get the last freq set 
                  * in PERF_CTL.
                  */
-                rdmsr(MSR_IA32_PERF_CTL, l, h);
+                rdmsr_on_cpu(cpu, MSR_IA32_PERF_CTL, &l, &h);
                 clock_freq = extract_clock(l, cpu, 1);
         }
-
-        set_cpus_allowed_ptr(current, &saved_mask);
         return clock_freq;
 }
 
@@ -467,15 +459,10 @@ static int centrino_target (struct cpufreq_policy *policy,
         struct cpufreq_freqs    freqs;
         int                     retval = 0;
         unsigned int            j, k, first_cpu, tmp;
-        cpumask_var_t saved_mask, covered_cpus;
+        cpumask_var_t covered_cpus;
 
-        if (unlikely(!alloc_cpumask_var(&saved_mask, GFP_KERNEL)))
-                return -ENOMEM;
-        if (unlikely(!zalloc_cpumask_var(&covered_cpus, GFP_KERNEL))) {
-                free_cpumask_var(saved_mask);
+        if (unlikely(!zalloc_cpumask_var(&covered_cpus, GFP_KERNEL)))
                 return -ENOMEM;
-        }
-        cpumask_copy(saved_mask, &current->cpus_allowed);
 
         if (unlikely(per_cpu(centrino_model, cpu) == NULL)) {
                 retval = -ENODEV;
@@ -493,7 +480,7 @@ static int centrino_target (struct cpufreq_policy *policy,
 
         first_cpu = 1;
         for_each_cpu(j, policy->cpus) {
-                const struct cpumask *mask;
+                int good_cpu;
 
                 /* cpufreq holds the hotplug lock, so we are safe here */
                 if (!cpu_online(j))
@@ -504,32 +491,30 @@ static int centrino_target (struct cpufreq_policy *policy,
                  * Make sure we are running on CPU that wants to change freq
                  */
                 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY)
-                        mask = policy->cpus;
+                        good_cpu = cpumask_any_and(policy->cpus,
+                                                   cpu_online_mask);
                 else
-                        mask = cpumask_of(j);
+                        good_cpu = j;
 
-                set_cpus_allowed_ptr(current, mask);
-                preempt_disable();
-                if (unlikely(!cpu_isset(smp_processor_id(), *mask))) {
+                if (good_cpu >= nr_cpu_ids) {
                         dprintk("couldn't limit to CPUs in this domain\n");
                         retval = -EAGAIN;
                         if (first_cpu) {
                                 /* We haven't started the transition yet. */
-                                goto migrate_end;
+                                goto out;
                         }
-                        preempt_enable();
                         break;
                 }
 
                 msr = per_cpu(centrino_model, cpu)->op_points[newstate].index;
 
                 if (first_cpu) {
-                        rdmsr(MSR_IA32_PERF_CTL, oldmsr, h);
+                        rdmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, &oldmsr, &h);
                         if (msr == (oldmsr & 0xffff)) {
                                 dprintk("no change needed - msr was and needs "
                                         "to be %x\n", oldmsr);
                                 retval = 0;
-                                goto migrate_end;
+                                goto out;
                         }
 
                         freqs.old = extract_clock(oldmsr, cpu, 0);
@@ -553,14 +538,11 @@ static int centrino_target (struct cpufreq_policy *policy,
                         oldmsr |= msr;
                 }
 
-                wrmsr(MSR_IA32_PERF_CTL, oldmsr, h);
-                if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
-                        preempt_enable();
+                wrmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, oldmsr, h);
+                if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY)
                         break;
-                }
 
-                cpu_set(j, *covered_cpus);
-                preempt_enable();
+                cpumask_set_cpu(j, covered_cpus);
         }
 
         for_each_cpu(k, policy->cpus) {
@@ -578,10 +560,8 @@ static int centrino_target (struct cpufreq_policy *policy,
                  * Best effort undo..
                  */
 
-                for_each_cpu_mask_nr(j, *covered_cpus) {
-                        set_cpus_allowed_ptr(current, &cpumask_of_cpu(j));
-                        wrmsr(MSR_IA32_PERF_CTL, oldmsr, h);
-                }
+                for_each_cpu(j, covered_cpus)
+                        wrmsr_on_cpu(j, MSR_IA32_PERF_CTL, oldmsr, h);
 
                 tmp = freqs.new;
                 freqs.new = freqs.old;
@@ -593,15 +573,9 @@ static int centrino_target (struct cpufreq_policy *policy,
                         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
                 }
         }
-        set_cpus_allowed_ptr(current, saved_mask);
         retval = 0;
-        goto out;
 
-migrate_end:
-        preempt_enable();
-        set_cpus_allowed_ptr(current, saved_mask);
 out:
-        free_cpumask_var(saved_mask);
         free_cpumask_var(covered_cpus);
         return retval;
 }
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c
index 016c1a4fa3fc..6911e91fb4f6 100644
--- a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c
+++ b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c
@@ -89,7 +89,8 @@ static int speedstep_find_register(void)
  * speedstep_set_state - set the SpeedStep state
  * @state: new processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH)
  *
- *   Tries to change the SpeedStep state.
+ *   Tries to change the SpeedStep state.  Can be called from
+ *   smp_call_function_single.
  */
 static void speedstep_set_state(unsigned int state)
 {
@@ -143,6 +144,11 @@ static void speedstep_set_state(unsigned int state)
         return;
 }
 
+/* Wrapper for smp_call_function_single. */
+static void _speedstep_set_state(void *_state)
+{
+        speedstep_set_state(*(unsigned int *)_state);
+}
 
 /**
  * speedstep_activate - activate SpeedStep control in the chipset
@@ -226,22 +232,28 @@ static unsigned int speedstep_detect_chipset(void)
         return 0;
 }
 
-static unsigned int _speedstep_get(const struct cpumask *cpus)
-{
+struct get_freq_data {
         unsigned int speed;
-        cpumask_t cpus_allowed;
-
-        cpus_allowed = current->cpus_allowed;
-        set_cpus_allowed_ptr(current, cpus);
-        speed = speedstep_get_frequency(speedstep_processor);
-        set_cpus_allowed_ptr(current, &cpus_allowed);
-        dprintk("detected %u kHz as current frequency\n", speed);
-        return speed;
+        unsigned int processor;
+};
+
+static void get_freq_data(void *_data)
+{
+        struct get_freq_data *data = _data;
+
+        data->speed = speedstep_get_frequency(data->processor);
 }
 
 static unsigned int speedstep_get(unsigned int cpu)
 {
-        return _speedstep_get(cpumask_of(cpu));
+        struct get_freq_data data = { .processor = cpu };
+
+        /* You're supposed to ensure CPU is online. */
+        if (smp_call_function_single(cpu, get_freq_data, &data, 1) != 0)
+                BUG();
+
+        dprintk("detected %u kHz as current frequency\n", data.speed);
+        return data.speed;
 }
 
 /**
@@ -257,16 +269,16 @@ static int speedstep_target(struct cpufreq_policy *policy,
                              unsigned int target_freq,
                              unsigned int relation)
 {
-        unsigned int newstate = 0;
+        unsigned int newstate = 0, policy_cpu;
         struct cpufreq_freqs freqs;
-        cpumask_t cpus_allowed;
         int i;
 
         if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0],
                                 target_freq, relation, &newstate))
                 return -EINVAL;
 
-        freqs.old = _speedstep_get(policy->cpus);
+        policy_cpu = cpumask_any_and(policy->cpus, cpu_online_mask);
+        freqs.old = speedstep_get(policy_cpu);
         freqs.new = speedstep_freqs[newstate].frequency;
         freqs.cpu = policy->cpu;
 
@@ -276,20 +288,13 @@ static int speedstep_target(struct cpufreq_policy *policy,
         if (freqs.old == freqs.new)
                 return 0;
 
-        cpus_allowed = current->cpus_allowed;
-
         for_each_cpu(i, policy->cpus) {
                 freqs.cpu = i;
                 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
         }
 
-        /* switch to physical CPU where state is to be changed */
-        set_cpus_allowed_ptr(current, policy->cpus);
-
-        speedstep_set_state(newstate);
-
-        /* allow to be run on all CPUs */
-        set_cpus_allowed_ptr(current, &cpus_allowed);
+        smp_call_function_single(policy_cpu, _speedstep_set_state, &newstate,
+                                 true);
 
         for_each_cpu(i, policy->cpus) {
                 freqs.cpu = i;
@@ -312,33 +317,43 @@ static int speedstep_verify(struct cpufreq_policy *policy)
         return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]);
 }
 
+struct get_freqs {
+        struct cpufreq_policy *policy;
+        int ret;
+};
+
+static void get_freqs_on_cpu(void *_get_freqs)
+{
+        struct get_freqs *get_freqs = _get_freqs;
+
+        get_freqs->ret =
+                speedstep_get_freqs(speedstep_processor,
+                            &speedstep_freqs[SPEEDSTEP_LOW].frequency,
+                            &speedstep_freqs[SPEEDSTEP_HIGH].frequency,
+                            &get_freqs->policy->cpuinfo.transition_latency,
+                            &speedstep_set_state);
+}
 
 static int speedstep_cpu_init(struct cpufreq_policy *policy)
 {
-        int result = 0;
-        unsigned int speed;
-        cpumask_t cpus_allowed;
+        int result;
+        unsigned int policy_cpu, speed;
+        struct get_freqs gf;
 
         /* only run on CPU to be set, or on its sibling */
 #ifdef CONFIG_SMP
         cpumask_copy(policy->cpus, cpu_sibling_mask(policy->cpu));
 #endif
-
-        cpus_allowed = current->cpus_allowed;
-        set_cpus_allowed_ptr(current, policy->cpus);
+        policy_cpu = cpumask_any_and(policy->cpus, cpu_online_mask);
 
         /* detect low and high frequency and transition latency */
-        result = speedstep_get_freqs(speedstep_processor,
-                                     &speedstep_freqs[SPEEDSTEP_LOW].frequency,
-                                     &speedstep_freqs[SPEEDSTEP_HIGH].frequency,
-                                     &policy->cpuinfo.transition_latency,
-                                     &speedstep_set_state);
-        set_cpus_allowed_ptr(current, &cpus_allowed);
-        if (result)
-                return result;
+        gf.policy = policy;
+        smp_call_function_single(policy_cpu, get_freqs_on_cpu, &gf, 1);
+        if (gf.ret)
+                return gf.ret;
 
         /* get current speed setting */
-        speed = _speedstep_get(policy->cpus);
+        speed = speedstep_get(policy_cpu);
         if (!speed)
                 return -EIO;
 
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c
index 2e3c6862657b..f4c290b8482f 100644
--- a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c
+++ b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c
@@ -226,6 +226,7 @@ static unsigned int pentium4_get_frequency(void)
 }
 
 
+/* Warning: may get called from smp_call_function_single. */
 unsigned int speedstep_get_frequency(unsigned int processor)
 {
         switch (processor) {
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index daed39ba2614..3260ab044996 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -86,6 +86,29 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
          */
         if (c->x86 == 6 && c->x86_model < 15)
                 clear_cpu_cap(c, X86_FEATURE_PAT);
+
+#ifdef CONFIG_KMEMCHECK
+        /*
+         * P4s have a "fast strings" feature which causes single-
+         * stepping REP instructions to only generate a #DB on
+         * cache-line boundaries.
+         *
+         * Ingo Molnar reported a Pentium D (model 6) and a Xeon
+         * (model 2) with the same problem.
+         */
+        if (c->x86 == 15) {
+                u64 misc_enable;
+
+                rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
+
+                if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) {
+                        printk(KERN_INFO "kmemcheck: Disabling fast string operations\n");
+
+                        misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING;
+                        wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
+                }
+        }
+#endif
 }
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/cpu/mcheck/Makefile b/arch/x86/kernel/cpu/mcheck/Makefile
index 45004faf67ea..188a1ca5ad2b 100644
--- a/arch/x86/kernel/cpu/mcheck/Makefile
+++ b/arch/x86/kernel/cpu/mcheck/Makefile
@@ -1,11 +1,12 @@
-obj-y                           =  mce.o therm_throt.o
+obj-y                           =  mce.o
 
 obj-$(CONFIG_X86_NEW_MCE)       += mce-severity.o
 obj-$(CONFIG_X86_OLD_MCE)       += k7.o p4.o p6.o
 obj-$(CONFIG_X86_ANCIENT_MCE)   += winchip.o p5.o
-obj-$(CONFIG_X86_MCE_P4THERMAL) += mce_intel.o
-obj-$(CONFIG_X86_MCE_INTEL)     += mce_intel_64.o mce_intel.o
-obj-$(CONFIG_X86_MCE_AMD)       += mce_amd_64.o
+obj-$(CONFIG_X86_MCE_INTEL)     += mce_intel.o
+obj-$(CONFIG_X86_MCE_AMD)       += mce_amd.o
 obj-$(CONFIG_X86_MCE_NONFATAL)  += non-fatal.o
 obj-$(CONFIG_X86_MCE_THRESHOLD) += threshold.o
 obj-$(CONFIG_X86_MCE_INJECT)    += mce-inject.o
+
+obj-$(CONFIG_X86_THERMAL_VECTOR) += therm_throt.o
diff --git a/arch/x86/kernel/cpu/mcheck/k7.c b/arch/x86/kernel/cpu/mcheck/k7.c
index 89e510424152..b945d5dbc609 100644
--- a/arch/x86/kernel/cpu/mcheck/k7.c
+++ b/arch/x86/kernel/cpu/mcheck/k7.c
@@ -10,10 +10,9 @@
 
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/mce.h>
 #include <asm/msr.h>
 
-#include "mce.h"
-
 /* Machine Check Handler For AMD Athlon/Duron: */
 static void k7_machine_check(struct pt_regs *regs, long error_code)
 {
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index fabba15e4558..01213048f62f 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -44,7 +44,6 @@
 #include <asm/msr.h>
 
 #include "mce-internal.h"
-#include "mce.h"
 
 /* Handle unconfigured int18 (should never happen) */
 static void unexpected_machine_check(struct pt_regs *regs, long error_code)
@@ -57,7 +56,7 @@ static void unexpected_machine_check(struct pt_regs *regs, long error_code)
 void (*machine_check_vector)(struct pt_regs *, long error_code) =
                                                 unexpected_machine_check;
 
-int                             mce_disabled;
+int mce_disabled __read_mostly;
 
 #ifdef CONFIG_X86_NEW_MCE
 
@@ -76,21 +75,22 @@ DEFINE_PER_CPU(unsigned, mce_exception_count);
  *   2: SIGBUS or log uncorrected errors (if possible), log corrected errors
  *   3: never panic or SIGBUS, log all errors (for testing only)
  */
-static int                      tolerant = 1;
-static int                      banks;
-static u64                      *bank;
-static unsigned long            notify_user;
-static int                      rip_msr;
-static int                      mce_bootlog = -1;
-static int                      monarch_timeout = -1;
-static int                      mce_panic_timeout;
-static int                      mce_dont_log_ce;
-int                             mce_cmci_disabled;
-int                             mce_ignore_ce;
-int                             mce_ser;
-
-static char                     trigger[128];
-static char                     *trigger_argv[2] = { trigger, NULL };
+static int                      tolerant                __read_mostly = 1;
+static int                      banks                   __read_mostly;
+static u64                      *bank                   __read_mostly;
+static int                      rip_msr                 __read_mostly;
+static int                      mce_bootlog             __read_mostly = -1;
+static int                      monarch_timeout         __read_mostly = -1;
+static int                      mce_panic_timeout       __read_mostly;
+static int                      mce_dont_log_ce         __read_mostly;
+int                             mce_cmci_disabled       __read_mostly;
+int                             mce_ignore_ce           __read_mostly;
+int                             mce_ser                 __read_mostly;
+
+/* User mode helper program triggered by machine check event */
+static unsigned long            mce_need_notify;
+static char                     mce_helper[128];
+static char                     *mce_helper_argv[2] = { mce_helper, NULL };
 
 static unsigned long            dont_init_banks;
 
@@ -180,7 +180,7 @@ void mce_log(struct mce *mce)
         wmb();
 
         mce->finished = 1;
-        set_bit(0, &notify_user);
+        set_bit(0, &mce_need_notify);
 }
 
 static void print_mce(struct mce *m)
@@ -194,14 +194,14 @@ static void print_mce(struct mce *m)
                        m->cs, m->ip);
                 if (m->cs == __KERNEL_CS)
                         print_symbol("{%s}", m->ip);
-                printk("\n");
+                printk(KERN_CONT "\n");
         }
         printk(KERN_EMERG "TSC %llx ", m->tsc);
         if (m->addr)
-                printk("ADDR %llx ", m->addr);
+                printk(KERN_CONT "ADDR %llx ", m->addr);
         if (m->misc)
-                printk("MISC %llx ", m->misc);
-        printk("\n");
+                printk(KERN_CONT "MISC %llx ", m->misc);
+        printk(KERN_CONT "\n");
         printk(KERN_EMERG "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
                         m->cpuvendor, m->cpuid, m->time, m->socketid,
                         m->apicid);
@@ -209,13 +209,13 @@ static void print_mce(struct mce *m)
 
 static void print_mce_head(void)
 {
-        printk(KERN_EMERG "\n" KERN_EMERG "HARDWARE ERROR\n");
+        printk(KERN_EMERG "\nHARDWARE ERROR\n");
 }
 
 static void print_mce_tail(void)
 {
         printk(KERN_EMERG "This is not a software problem!\n"
-               KERN_EMERG "Run through mcelog --ascii to decode and contact your hardware vendor\n");
+               "Run through mcelog --ascii to decode and contact your hardware vendor\n");
 }
 
 #define PANIC_TIMEOUT 5 /* 5 seconds */
@@ -691,18 +691,21 @@ static atomic_t global_nwo;
  * in the entry order.
  * TBD double check parallel CPU hotunplug
  */
-static int mce_start(int no_way_out, int *order)
+static int mce_start(int *no_way_out)
 {
-        int nwo;
+        int order;
         int cpus = num_online_cpus();
         u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;
 
-        if (!timeout) {
-                *order = -1;
-                return no_way_out;
-        }
+        if (!timeout)
+                return -1;
 
-        atomic_add(no_way_out, &global_nwo);
+        atomic_add(*no_way_out, &global_nwo);
+        /*
+         * global_nwo should be updated before mce_callin
+         */
+        smp_wmb();
+        order = atomic_add_return(1, &mce_callin);
 
         /*
          * Wait for everyone.
@@ -710,40 +713,43 @@ static int mce_start(int no_way_out, int *order)
         while (atomic_read(&mce_callin) != cpus) {
                 if (mce_timed_out(&timeout)) {
                         atomic_set(&global_nwo, 0);
-                        *order = -1;
-                        return no_way_out;
+                        return -1;
                 }
                 ndelay(SPINUNIT);
         }
 
         /*
-         * Cache the global no_way_out state.
+         * mce_callin should be read before global_nwo
          */
-        nwo = atomic_read(&global_nwo);
+        smp_rmb();
 
-        /*
-         * Monarch starts executing now, the others wait.
-         */
-        if (*order == 1) {
+        if (order == 1) {
+                /*
+                 * Monarch: Starts executing now, the others wait.
+                 */
                 atomic_set(&mce_executing, 1);
-                return nwo;
+        } else {
+                /*
+                 * Subject: Now start the scanning loop one by one in
+                 * the original callin order.
+                 * This way when there are any shared banks it will be
+                 * only seen by one CPU before cleared, avoiding duplicates.
+                 */
+                while (atomic_read(&mce_executing) < order) {
+                        if (mce_timed_out(&timeout)) {
+                                atomic_set(&global_nwo, 0);
+                                return -1;
+                        }
+                        ndelay(SPINUNIT);
+                }
         }
 
         /*
-         * Now start the scanning loop one by one
-         * in the original callin order.
-         * This way when there are any shared banks it will
-         * be only seen by one CPU before cleared, avoiding duplicates.
+         * Cache the global no_way_out state.
          */
-        while (atomic_read(&mce_executing) < *order) {
-                if (mce_timed_out(&timeout)) {
-                        atomic_set(&global_nwo, 0);
-                        *order = -1;
-                        return no_way_out;
-                }
-                ndelay(SPINUNIT);
-        }
-        return nwo;
+        *no_way_out = atomic_read(&global_nwo);
+
+        return order;
 }
 
 /*
@@ -863,7 +869,6 @@ void do_machine_check(struct pt_regs *regs, long error_code)
          * check handler.
          */
         int order;
-
         /*
          * If no_way_out gets set, there is no safe way to recover from this
          * MCE.  If tolerant is cranked up, we'll try anyway.
@@ -887,7 +892,6 @@ void do_machine_check(struct pt_regs *regs, long error_code)
         if (!banks)
                 goto out;
 
-        order = atomic_add_return(1, &mce_callin);
         mce_setup(&m);
 
         m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
@@ -909,7 +913,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
          * This way we don't report duplicated events on shared banks
          * because the first one to see it will clear it.
          */
-        no_way_out = mce_start(no_way_out, &order);
+        order = mce_start(&no_way_out);
         for (i = 0; i < banks; i++) {
                 __clear_bit(i, toclear);
                 if (!bank[i])
@@ -1113,12 +1117,12 @@ static void mcheck_timer(unsigned long data)
                 *n = min(*n*2, (int)round_jiffies_relative(check_interval*HZ));
 
         t->expires = jiffies + *n;
-        add_timer(t);
+        add_timer_on(t, smp_processor_id());
 }
 
 static void mce_do_trigger(struct work_struct *work)
 {
-        call_usermodehelper(trigger, trigger_argv, NULL, UMH_NO_WAIT);
+        call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
 }
 
 static DECLARE_WORK(mce_trigger_work, mce_do_trigger);
@@ -1135,7 +1139,7 @@ int mce_notify_irq(void)
 
         clear_thread_flag(TIF_MCE_NOTIFY);
 
-        if (test_and_clear_bit(0, &notify_user)) {
+        if (test_and_clear_bit(0, &mce_need_notify)) {
                 wake_up_interruptible(&mce_wait);
 
                 /*
@@ -1143,7 +1147,7 @@ int mce_notify_irq(void)
                  * work_pending is always cleared before the function is
                  * executed.
                  */
-                if (trigger[0] && !work_pending(&mce_trigger_work))
+                if (mce_helper[0] && !work_pending(&mce_trigger_work))
                         schedule_work(&mce_trigger_work);
 
                 if (__ratelimit(&ratelimit))
@@ -1222,8 +1226,13 @@ static void mce_init(void)
 }
 
 /* Add per CPU specific workarounds here */
-static void mce_cpu_quirks(struct cpuinfo_x86 *c)
+static int mce_cpu_quirks(struct cpuinfo_x86 *c)
 {
+        if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
+                pr_info("MCE: unknown CPU type - not enabling MCE support.\n");
+                return -EOPNOTSUPP;
+        }
+
         /* This should be disabled by the BIOS, but isn't always */
         if (c->x86_vendor == X86_VENDOR_AMD) {
                 if (c->x86 == 15 && banks > 4) {
@@ -1245,7 +1254,7 @@ static void mce_cpu_quirks(struct cpuinfo_x86 *c)
                  * Various K7s with broken bank 0 around. Always disable
                  * by default.
                  */
-                 if (c->x86 == 6)
+                 if (c->x86 == 6 && banks > 0)
                         bank[0] = 0;
         }
 
@@ -1269,11 +1278,20 @@ static void mce_cpu_quirks(struct cpuinfo_x86 *c)
                 if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
                         monarch_timeout < 0)
                         monarch_timeout = USEC_PER_SEC;
+
+                /*
+                 * There are also broken BIOSes on some Pentium M and
+                 * earlier systems:
+                 */
+                if (c->x86 == 6 && c->x86_model <= 13 && mce_bootlog < 0)
+                        mce_bootlog = 0;
         }
         if (monarch_timeout < 0)
                 monarch_timeout = 0;
         if (mce_bootlog != 0)
                 mce_panic_timeout = 30;
+
+        return 0;
 }
 
 static void __cpuinit mce_ancient_init(struct cpuinfo_x86 *c)
@@ -1282,8 +1300,7 @@ static void __cpuinit mce_ancient_init(struct cpuinfo_x86 *c)
                 return;
         switch (c->x86_vendor) {
         case X86_VENDOR_INTEL:
-                if (mce_p5_enabled())
-                        intel_p5_mcheck_init(c);
+                intel_p5_mcheck_init(c);
                 break;
         case X86_VENDOR_CENTAUR:
                 winchip_mcheck_init(c);
@@ -1318,7 +1335,7 @@ static void mce_init_timer(void)
                 return;
         setup_timer(t, mcheck_timer, smp_processor_id());
         t->expires = round_jiffies(jiffies + *n);
-        add_timer(t);
+        add_timer_on(t, smp_processor_id());
 }
 
 /*
@@ -1335,11 +1352,10 @@ void __cpuinit mcheck_init(struct cpuinfo_x86 *c)
         if (!mce_available(c))
                 return;
 
-        if (mce_cap_init() < 0) {
+        if (mce_cap_init() < 0 || mce_cpu_quirks(c) < 0) {
                 mce_disabled = 1;
                 return;
         }
-        mce_cpu_quirks(c);
 
         machine_check_vector = do_machine_check;
 
@@ -1609,8 +1625,9 @@ static int mce_resume(struct sys_device *dev)
 static void mce_cpu_restart(void *data)
 {
         del_timer_sync(&__get_cpu_var(mce_timer));
-        if (mce_available(&current_cpu_data))
-                mce_init();
+        if (!mce_available(&current_cpu_data))
+                return;
+        mce_init();
         mce_init_timer();
 }
 
@@ -1620,6 +1637,26 @@ static void mce_restart(void)
         on_each_cpu(mce_cpu_restart, NULL, 1);
 }
 
+/* Toggle features for corrected errors */
+static void mce_disable_ce(void *all)
+{
+        if (!mce_available(&current_cpu_data))
+                return;
+        if (all)
+                del_timer_sync(&__get_cpu_var(mce_timer));
+        cmci_clear();
+}
+
+static void mce_enable_ce(void *all)
+{
+        if (!mce_available(&current_cpu_data))
+                return;
+        cmci_reenable();
+        cmci_recheck();
+        if (all)
+                mce_init_timer();
+}
+
 static struct sysdev_class mce_sysclass = {
         .suspend        = mce_suspend,
         .shutdown       = mce_shutdown,
@@ -1659,26 +1696,70 @@ static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
 static ssize_t
 show_trigger(struct sys_device *s, struct sysdev_attribute *attr, char *buf)
 {
-        strcpy(buf, trigger);
+        strcpy(buf, mce_helper);
         strcat(buf, "\n");
-        return strlen(trigger) + 1;
+        return strlen(mce_helper) + 1;
 }
 
 static ssize_t set_trigger(struct sys_device *s, struct sysdev_attribute *attr,
                                 const char *buf, size_t siz)
 {
         char *p;
-        int len;
 
-        strncpy(trigger, buf, sizeof(trigger));
-        trigger[sizeof(trigger)-1] = 0;
-        len = strlen(trigger);
-        p = strchr(trigger, '\n');
+        strncpy(mce_helper, buf, sizeof(mce_helper));
+        mce_helper[sizeof(mce_helper)-1] = 0;
+        p = strchr(mce_helper, '\n');
 
-        if (*p)
+        if (p)
                 *p = 0;
 
-        return len;
+        return strlen(mce_helper) + !!p;
+}
+
+static ssize_t set_ignore_ce(struct sys_device *s,
+                             struct sysdev_attribute *attr,
+                             const char *buf, size_t size)
+{
+        u64 new;
+
+        if (strict_strtoull(buf, 0, &new) < 0)
+                return -EINVAL;
+
+        if (mce_ignore_ce ^ !!new) {
+                if (new) {
+                        /* disable ce features */
+                        on_each_cpu(mce_disable_ce, (void *)1, 1);
+                        mce_ignore_ce = 1;
+                } else {
+                        /* enable ce features */
+                        mce_ignore_ce = 0;
+                        on_each_cpu(mce_enable_ce, (void *)1, 1);
+                }
+        }
+        return size;
+}
+
+static ssize_t set_cmci_disabled(struct sys_device *s,
+                                 struct sysdev_attribute *attr,
+                                 const char *buf, size_t size)
+{
+        u64 new;
+
+        if (strict_strtoull(buf, 0, &new) < 0)
+                return -EINVAL;
+
+        if (mce_cmci_disabled ^ !!new) {
+                if (new) {
+                        /* disable cmci */
+                        on_each_cpu(mce_disable_ce, NULL, 1);
+                        mce_cmci_disabled = 1;
+                } else {
+                        /* enable cmci */
+                        mce_cmci_disabled = 0;
+                        on_each_cpu(mce_enable_ce, NULL, 1);
+                }
+        }
+        return size;
 }
 
 static ssize_t store_int_with_restart(struct sys_device *s,
@@ -1693,6 +1774,7 @@ static ssize_t store_int_with_restart(struct sys_device *s,
 static SYSDEV_ATTR(trigger, 0644, show_trigger, set_trigger);
 static SYSDEV_INT_ATTR(tolerant, 0644, tolerant);
 static SYSDEV_INT_ATTR(monarch_timeout, 0644, monarch_timeout);
+static SYSDEV_INT_ATTR(dont_log_ce, 0644, mce_dont_log_ce);
 
 static struct sysdev_ext_attribute attr_check_interval = {
         _SYSDEV_ATTR(check_interval, 0644, sysdev_show_int,
@@ -1700,9 +1782,24 @@ static struct sysdev_ext_attribute attr_check_interval = {
         &check_interval
 };
 
+static struct sysdev_ext_attribute attr_ignore_ce = {
+        _SYSDEV_ATTR(ignore_ce, 0644, sysdev_show_int, set_ignore_ce),
+        &mce_ignore_ce
+};
+
+static struct sysdev_ext_attribute attr_cmci_disabled = {
+        _SYSDEV_ATTR(cmci_disabled, 0644, sysdev_show_int, set_cmci_disabled),
+        &mce_cmci_disabled
+};
+
 static struct sysdev_attribute *mce_attrs[] = {
-        &attr_tolerant.attr, &attr_check_interval.attr, &attr_trigger,
+        &attr_tolerant.attr,
+        &attr_check_interval.attr,
+        &attr_trigger,
         &attr_monarch_timeout.attr,
+        &attr_dont_log_ce.attr,
+        &attr_ignore_ce.attr,
+        &attr_cmci_disabled.attr,
         NULL
 };
 
@@ -1712,7 +1809,7 @@ static cpumask_var_t mce_dev_initialized;
 static __cpuinit int mce_create_device(unsigned int cpu)
 {
         int err;
-        int i;
+        int i, j;
 
         if (!mce_available(&boot_cpu_data))
                 return -EIO;
@@ -1730,9 +1827,9 @@ static __cpuinit int mce_create_device(unsigned int cpu)
                 if (err)
                         goto error;
         }
-        for (i = 0; i < banks; i++) {
+        for (j = 0; j < banks; j++) {
                 err = sysdev_create_file(&per_cpu(mce_dev, cpu),
-                                        &bank_attrs[i]);
+                                        &bank_attrs[j]);
                 if (err)
                         goto error2;
         }
@@ -1740,8 +1837,8 @@ static __cpuinit int mce_create_device(unsigned int cpu)
 
         return 0;
 error2:
-        while (--i >= 0)
-                sysdev_remove_file(&per_cpu(mce_dev, cpu), &bank_attrs[i]);
+        while (--j >= 0)
+                sysdev_remove_file(&per_cpu(mce_dev, cpu), &bank_attrs[j]);
 error:
         while (--i >= 0)
                 sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
@@ -1883,7 +1980,7 @@ static __init int mce_init_device(void)
         if (!mce_available(&boot_cpu_data))
                 return -EIO;
 
-        alloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL);
+        zalloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL);
 
         err = mce_init_banks();
         if (err)
@@ -1915,7 +2012,7 @@ EXPORT_SYMBOL_GPL(nr_mce_banks);	/* non-fatal.o */
 /* This has to be run for each processor */
 void mcheck_init(struct cpuinfo_x86 *c)
 {
-        if (mce_disabled == 1)
+        if (mce_disabled)
                 return;
 
         switch (c->x86_vendor) {
@@ -1945,10 +2042,9 @@ void mcheck_init(struct cpuinfo_x86 *c)
 
 static int __init mcheck_enable(char *str)
 {
-        mce_disabled = -1;
+        mce_p5_enabled = 1;
         return 1;
 }
-
 __setup("mce", mcheck_enable);
 
 #endif /* CONFIG_X86_OLD_MCE */
diff --git a/arch/x86/kernel/cpu/mcheck/mce.h b/arch/x86/kernel/cpu/mcheck/mce.h
deleted file mode 100644
index 84a552b458c8..000000000000
--- a/arch/x86/kernel/cpu/mcheck/mce.h
+++ /dev/null
@@ -1,38 +0,0 @@
-#include <linux/init.h>
-#include <asm/mce.h>
-
-#ifdef CONFIG_X86_OLD_MCE
-void amd_mcheck_init(struct cpuinfo_x86 *c);
-void intel_p4_mcheck_init(struct cpuinfo_x86 *c);
-void intel_p6_mcheck_init(struct cpuinfo_x86 *c);
-#endif
-
-#ifdef CONFIG_X86_ANCIENT_MCE
-void intel_p5_mcheck_init(struct cpuinfo_x86 *c);
-void winchip_mcheck_init(struct cpuinfo_x86 *c);
-extern int mce_p5_enable;
-static inline int mce_p5_enabled(void) { return mce_p5_enable; }
-static inline void enable_p5_mce(void) { mce_p5_enable = 1; }
-#else
-static inline void intel_p5_mcheck_init(struct cpuinfo_x86 *c) {}
-static inline void winchip_mcheck_init(struct cpuinfo_x86 *c) {}
-static inline int mce_p5_enabled(void) { return 0; }
-static inline void enable_p5_mce(void) { }
-#endif
-
-/* Call the installed machine check handler for this CPU setup. */
-extern void (*machine_check_vector)(struct pt_regs *, long error_code);
-
-#ifdef CONFIG_X86_OLD_MCE
-
-extern int nr_mce_banks;
-
-void intel_set_thermal_handler(void);
-
-#else
-
-static inline void intel_set_thermal_handler(void) { }
-
-#endif
-
-void intel_init_thermal(struct cpuinfo_x86 *c);
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index ddae21620bda..ddae21620bda 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
index 2b011d2d8579..e1acec0f7a32 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
@@ -1,74 +1,226 @@
 /*
- * Common code for Intel machine checks
+ * Intel specific MCE features.
+ * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
+ * Copyright (C) 2008, 2009 Intel Corporation
+ * Author: Andi Kleen
  */
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/smp.h>
 
-#include <asm/therm_throt.h>
-#include <asm/processor.h>
-#include <asm/system.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/percpu.h>
 #include <asm/apic.h>
+#include <asm/processor.h>
 #include <asm/msr.h>
+#include <asm/mce.h>
+
+/*
+ * Support for Intel Correct Machine Check Interrupts. This allows
+ * the CPU to raise an interrupt when a corrected machine check happened.
+ * Normally we pick those up using a regular polling timer.
+ * Also supports reliable discovery of shared banks.
+ */
 
-#include "mce.h"
+static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
 
-void intel_init_thermal(struct cpuinfo_x86 *c)
+/*
+ * cmci_discover_lock protects against parallel discovery attempts
+ * which could race against each other.
+ */
+static DEFINE_SPINLOCK(cmci_discover_lock);
+
+#define CMCI_THRESHOLD 1
+
+static int cmci_supported(int *banks)
 {
-        unsigned int cpu = smp_processor_id();
-        int tm2 = 0;
-        u32 l, h;
+        u64 cap;
 
-        /* Thermal monitoring depends on ACPI and clock modulation*/
-        if (!cpu_has(c, X86_FEATURE_ACPI) || !cpu_has(c, X86_FEATURE_ACC))
-                return;
+        if (mce_cmci_disabled || mce_ignore_ce)
+                return 0;
 
         /*
-         * First check if its enabled already, in which case there might
-         * be some SMM goo which handles it, so we can't even put a handler
-         * since it might be delivered via SMI already:
+         * Vendor check is not strictly needed, but the initial
+         * initialization is vendor keyed and this
+         * makes sure none of the backdoors are entered otherwise.
          */
-        rdmsr(MSR_IA32_MISC_ENABLE, l, h);
-        h = apic_read(APIC_LVTTHMR);
-        if ((l & MSR_IA32_MISC_ENABLE_TM1) && (h & APIC_DM_SMI)) {
-                printk(KERN_DEBUG
-                       "CPU%d: Thermal monitoring handled by SMI\n", cpu);
-                return;
+        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+                return 0;
+        if (!cpu_has_apic || lapic_get_maxlvt() < 6)
+                return 0;
+        rdmsrl(MSR_IA32_MCG_CAP, cap);
+        *banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
+        return !!(cap & MCG_CMCI_P);
+}
+
+/*
+ * The interrupt handler. This is called on every event.
+ * Just call the poller directly to log any events.
+ * This could in theory increase the threshold under high load,
+ * but doesn't for now.
+ */
+static void intel_threshold_interrupt(void)
+{
+        machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
+        mce_notify_irq();
+}
+
+static void print_update(char *type, int *hdr, int num)
+{
+        if (*hdr == 0)
+                printk(KERN_INFO "CPU %d MCA banks", smp_processor_id());
+        *hdr = 1;
+        printk(KERN_CONT " %s:%d", type, num);
+}
+
+/*
+ * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
+ * on this CPU. Use the algorithm recommended in the SDM to discover shared
+ * banks.
+ */
+static void cmci_discover(int banks, int boot)
+{
+        unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
+        unsigned long flags;
+        int hdr = 0;
+        int i;
+
+        spin_lock_irqsave(&cmci_discover_lock, flags);
+        for (i = 0; i < banks; i++) {
+                u64 val;
+
+                if (test_bit(i, owned))
+                        continue;
+
+                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
+
+                /* Already owned by someone else? */
+                if (val & CMCI_EN) {
+                        if (test_and_clear_bit(i, owned) || boot)
+                                print_update("SHD", &hdr, i);
+                        __clear_bit(i, __get_cpu_var(mce_poll_banks));
+                        continue;
+                }
+
+                val |= CMCI_EN | CMCI_THRESHOLD;
+                wrmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
+
+                /* Did the enable bit stick? -- the bank supports CMCI */
+                if (val & CMCI_EN) {
+                        if (!test_and_set_bit(i, owned) || boot)
+                                print_update("CMCI", &hdr, i);
+                        __clear_bit(i, __get_cpu_var(mce_poll_banks));
+                } else {
+                        WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
+                }
         }
+        spin_unlock_irqrestore(&cmci_discover_lock, flags);
+        if (hdr)
+                printk(KERN_CONT "\n");
+}
+
+/*
+ * Just in case we missed an event during initialization check
+ * all the CMCI owned banks.
+ */
+void cmci_recheck(void)
+{
+        unsigned long flags;
+        int banks;
+
+        if (!mce_available(&current_cpu_data) || !cmci_supported(&banks))
+                return;
+        local_irq_save(flags);
+        machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
+        local_irq_restore(flags);
+}
 
-        if (cpu_has(c, X86_FEATURE_TM2) && (l & MSR_IA32_MISC_ENABLE_TM2))
-                tm2 = 1;
+/*
+ * Disable CMCI on this CPU for all banks it owns when it goes down.
+ * This allows other CPUs to claim the banks on rediscovery.
+ */
+void cmci_clear(void)
+{
+        unsigned long flags;
+        int i;
+        int banks;
+        u64 val;
 
-        /* Check whether a vector already exists */
-        if (h & APIC_VECTOR_MASK) {
-                printk(KERN_DEBUG
-                       "CPU%d: Thermal LVT vector (%#x) already installed\n",
-                       cpu, (h & APIC_VECTOR_MASK));
+        if (!cmci_supported(&banks))
                 return;
+        spin_lock_irqsave(&cmci_discover_lock, flags);
+        for (i = 0; i < banks; i++) {
+                if (!test_bit(i, __get_cpu_var(mce_banks_owned)))
+                        continue;
+                /* Disable CMCI */
+                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                val &= ~(CMCI_EN|CMCI_THRESHOLD_MASK);
+                wrmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                __clear_bit(i, __get_cpu_var(mce_banks_owned));
         }
+        spin_unlock_irqrestore(&cmci_discover_lock, flags);
+}
+
+/*
+ * After a CPU went down cycle through all the others and rediscover
+ * Must run in process context.
+ */
+void cmci_rediscover(int dying)
+{
+        int banks;
+        int cpu;
+        cpumask_var_t old;
+
+        if (!cmci_supported(&banks))
+                return;
+        if (!alloc_cpumask_var(&old, GFP_KERNEL))
+                return;
+        cpumask_copy(old, &current->cpus_allowed);
 
-        /* We'll mask the thermal vector in the lapic till we're ready: */
-        h = THERMAL_APIC_VECTOR | APIC_DM_FIXED | APIC_LVT_MASKED;
-        apic_write(APIC_LVTTHMR, h);
+        for_each_online_cpu(cpu) {
+                if (cpu == dying)
+                        continue;
+                if (set_cpus_allowed_ptr(current, cpumask_of(cpu)))
+                        continue;
+                /* Recheck banks in case CPUs don't all have the same */
+                if (cmci_supported(&banks))
+                        cmci_discover(banks, 0);
+        }
 
-        rdmsr(MSR_IA32_THERM_INTERRUPT, l, h);
-        wrmsr(MSR_IA32_THERM_INTERRUPT,
-                l | (THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE), h);
+        set_cpus_allowed_ptr(current, old);
+        free_cpumask_var(old);
+}
 
-        intel_set_thermal_handler();
+/*
+ * Reenable CMCI on this CPU in case a CPU down failed.
+ */
+void cmci_reenable(void)
+{
+        int banks;
+        if (cmci_supported(&banks))
+                cmci_discover(banks, 0);
+}
 
-        rdmsr(MSR_IA32_MISC_ENABLE, l, h);
-        wrmsr(MSR_IA32_MISC_ENABLE, l | MSR_IA32_MISC_ENABLE_TM1, h);
+static void intel_init_cmci(void)
+{
+        int banks;
 
-        /* Unmask the thermal vector: */
-        l = apic_read(APIC_LVTTHMR);
-        apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
+        if (!cmci_supported(&banks))
+                return;
 
-        printk(KERN_INFO "CPU%d: Thermal monitoring enabled (%s)\n",
-               cpu, tm2 ? "TM2" : "TM1");
+        mce_threshold_vector = intel_threshold_interrupt;
+        cmci_discover(banks, 1);
+        /*
+         * For CPU #0 this runs with still disabled APIC, but that's
+         * ok because only the vector is set up. We still do another
+         * check for the banks later for CPU #0 just to make sure
+         * to not miss any events.
+         */
+        apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED);
+        cmci_recheck();
+}
 
-        /* enable thermal throttle processing */
-        atomic_set(&therm_throt_en, 1);
+void mce_intel_feature_init(struct cpuinfo_x86 *c)
+{
+        intel_init_thermal(c);
+        intel_init_cmci();
 }
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
deleted file mode 100644
index f2ef6952c400..000000000000
--- a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Intel specific MCE features.
- * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
- * Copyright (C) 2008, 2009 Intel Corporation
- * Author: Andi Kleen
- */
-
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/percpu.h>
-#include <asm/processor.h>
-#include <asm/apic.h>
-#include <asm/msr.h>
-#include <asm/mce.h>
-#include <asm/hw_irq.h>
-#include <asm/idle.h>
-#include <asm/therm_throt.h>
-
-#include "mce.h"
-
-asmlinkage void smp_thermal_interrupt(void)
-{
-        __u64 msr_val;
-
-        ack_APIC_irq();
-
-        exit_idle();
-        irq_enter();
-
-        rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
-        if (therm_throt_process(msr_val & THERM_STATUS_PROCHOT))
-                mce_log_therm_throt_event(msr_val);
-
-        inc_irq_stat(irq_thermal_count);
-        irq_exit();
-}
-
-/*
- * Support for Intel Correct Machine Check Interrupts. This allows
- * the CPU to raise an interrupt when a corrected machine check happened.
- * Normally we pick those up using a regular polling timer.
- * Also supports reliable discovery of shared banks.
- */
-
-static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
-
-/*
- * cmci_discover_lock protects against parallel discovery attempts
- * which could race against each other.
- */
-static DEFINE_SPINLOCK(cmci_discover_lock);
-
-#define CMCI_THRESHOLD 1
-
-static int cmci_supported(int *banks)
-{
-        u64 cap;
-
-        if (mce_cmci_disabled || mce_ignore_ce)
-                return 0;
-
-        /*
-         * Vendor check is not strictly needed, but the initial
-         * initialization is vendor keyed and this
-         * makes sure none of the backdoors are entered otherwise.
-         */
-        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
-                return 0;
-        if (!cpu_has_apic || lapic_get_maxlvt() < 6)
-                return 0;
-        rdmsrl(MSR_IA32_MCG_CAP, cap);
-        *banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
-        return !!(cap & MCG_CMCI_P);
-}
-
-/*
- * The interrupt handler. This is called on every event.
- * Just call the poller directly to log any events.
- * This could in theory increase the threshold under high load,
- * but doesn't for now.
- */
-static void intel_threshold_interrupt(void)
-{
-        machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
-        mce_notify_irq();
-}
-
-static void print_update(char *type, int *hdr, int num)
-{
-        if (*hdr == 0)
-                printk(KERN_INFO "CPU %d MCA banks", smp_processor_id());
-        *hdr = 1;
-        printk(KERN_CONT " %s:%d", type, num);
-}
-
-/*
- * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
- * on this CPU. Use the algorithm recommended in the SDM to discover shared
- * banks.
- */
-static void cmci_discover(int banks, int boot)
-{
-        unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
-        unsigned long flags;
-        int hdr = 0;
-        int i;
-
-        spin_lock_irqsave(&cmci_discover_lock, flags);
-        for (i = 0; i < banks; i++) {
-                u64 val;
-
-                if (test_bit(i, owned))
-                        continue;
-
-                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
-
-                /* Already owned by someone else? */
-                if (val & CMCI_EN) {
-                        if (test_and_clear_bit(i, owned) || boot)
-                                print_update("SHD", &hdr, i);
-                        __clear_bit(i, __get_cpu_var(mce_poll_banks));
-                        continue;
-                }
-
-                val |= CMCI_EN | CMCI_THRESHOLD;
-                wrmsrl(MSR_IA32_MC0_CTL2 + i, val);
-                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
-
-                /* Did the enable bit stick? -- the bank supports CMCI */
-                if (val & CMCI_EN) {
-                        if (!test_and_set_bit(i, owned) || boot)
-                                print_update("CMCI", &hdr, i);
-                        __clear_bit(i, __get_cpu_var(mce_poll_banks));
-                } else {
-                        WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
-                }
-        }
-        spin_unlock_irqrestore(&cmci_discover_lock, flags);
-        if (hdr)
-                printk(KERN_CONT "\n");
-}
-
-/*
- * Just in case we missed an event during initialization check
- * all the CMCI owned banks.
- */
-void cmci_recheck(void)
-{
-        unsigned long flags;
-        int banks;
-
-        if (!mce_available(&current_cpu_data) || !cmci_supported(&banks))
-                return;
-        local_irq_save(flags);
-        machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
-        local_irq_restore(flags);
-}
-
-/*
- * Disable CMCI on this CPU for all banks it owns when it goes down.
- * This allows other CPUs to claim the banks on rediscovery.
- */
-void cmci_clear(void)
-{
-        unsigned long flags;
-        int i;
-        int banks;
-        u64 val;
-
-        if (!cmci_supported(&banks))
-                return;
-        spin_lock_irqsave(&cmci_discover_lock, flags);
-        for (i = 0; i < banks; i++) {
-                if (!test_bit(i, __get_cpu_var(mce_banks_owned)))
-                        continue;
-                /* Disable CMCI */
-                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
-                val &= ~(CMCI_EN|CMCI_THRESHOLD_MASK);
-                wrmsrl(MSR_IA32_MC0_CTL2 + i, val);
-                __clear_bit(i, __get_cpu_var(mce_banks_owned));
-        }
-        spin_unlock_irqrestore(&cmci_discover_lock, flags);
-}
-
-/*
- * After a CPU went down cycle through all the others and rediscover
- * Must run in process context.
- */
-void cmci_rediscover(int dying)
-{
-        int banks;
-        int cpu;
-        cpumask_var_t old;
-
-        if (!cmci_supported(&banks))
-                return;
-        if (!alloc_cpumask_var(&old, GFP_KERNEL))
-                return;
-        cpumask_copy(old, &current->cpus_allowed);
-
-        for_each_online_cpu(cpu) {
-                if (cpu == dying)
-                        continue;
-                if (set_cpus_allowed_ptr(current, cpumask_of(cpu)))
-                        continue;
-                /* Recheck banks in case CPUs don't all have the same */
-                if (cmci_supported(&banks))
-                        cmci_discover(banks, 0);
-        }
-
-        set_cpus_allowed_ptr(current, old);
-        free_cpumask_var(old);
-}
-
-/*
- * Reenable CMCI on this CPU in case a CPU down failed.
- */
-void cmci_reenable(void)
-{
-        int banks;
-        if (cmci_supported(&banks))
-                cmci_discover(banks, 0);
-}
-
-static void intel_init_cmci(void)
-{
-        int banks;
-
-        if (!cmci_supported(&banks))
-                return;
-
-        mce_threshold_vector = intel_threshold_interrupt;
-        cmci_discover(banks, 1);
-        /*
-         * For CPU #0 this runs with still disabled APIC, but that's
-         * ok because only the vector is set up. We still do another
-         * check for the banks later for CPU #0 just to make sure
-         * to not miss any events.
-         */
-        apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED);
-        cmci_recheck();
-}
-
-void mce_intel_feature_init(struct cpuinfo_x86 *c)
-{
-        intel_init_thermal(c);
-        intel_init_cmci();
-}
diff --git a/arch/x86/kernel/cpu/mcheck/non-fatal.c b/arch/x86/kernel/cpu/mcheck/non-fatal.c
index 70b710420f74..f5f2d6f71fb6 100644
--- a/arch/x86/kernel/cpu/mcheck/non-fatal.c
+++ b/arch/x86/kernel/cpu/mcheck/non-fatal.c
@@ -17,10 +17,9 @@
 
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/mce.h>
 #include <asm/msr.h>
 
-#include "mce.h"
-
 static int              firstbank;
 
 #define MCE_RATE        (15*HZ) /* timer rate is 15s */
diff --git a/arch/x86/kernel/cpu/mcheck/p4.c b/arch/x86/kernel/cpu/mcheck/p4.c
index 82cee108a2d3..4482aea9aa2e 100644
--- a/arch/x86/kernel/cpu/mcheck/p4.c
+++ b/arch/x86/kernel/cpu/mcheck/p4.c
@@ -1,21 +1,15 @@
 /*
  * P4 specific Machine Check Exception Reporting
  */
-
-#include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/smp.h>
 
-#include <asm/therm_throt.h>
 #include <asm/processor.h>
-#include <asm/system.h>
-#include <asm/apic.h>
+#include <asm/mce.h>
 #include <asm/msr.h>
 
-#include "mce.h"
-
 /* as supported by the P4/Xeon family */
 struct intel_mce_extended_msrs {
         u32 eax;
@@ -33,46 +27,6 @@ struct intel_mce_extended_msrs {
 
 static int mce_num_extended_msrs;
 
-
-#ifdef CONFIG_X86_MCE_P4THERMAL
-
-static void unexpected_thermal_interrupt(struct pt_regs *regs)
-{
-        printk(KERN_ERR "CPU%d: Unexpected LVT TMR interrupt!\n",
-                        smp_processor_id());
-        add_taint(TAINT_MACHINE_CHECK);
-}
-
-/* P4/Xeon Thermal transition interrupt handler: */
-static void intel_thermal_interrupt(struct pt_regs *regs)
-{
-        __u64 msr_val;
-
-        ack_APIC_irq();
-
-        rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
-        therm_throt_process(msr_val & THERM_STATUS_PROCHOT);
-}
-
-/* Thermal interrupt handler for this CPU setup: */
-static void (*vendor_thermal_interrupt)(struct pt_regs *regs) =
-                                                unexpected_thermal_interrupt;
-
-void smp_thermal_interrupt(struct pt_regs *regs)
-{
-        irq_enter();
-        vendor_thermal_interrupt(regs);
-        __get_cpu_var(irq_stat).irq_thermal_count++;
-        irq_exit();
-}
-
-void intel_set_thermal_handler(void)
-{
-        vendor_thermal_interrupt = intel_thermal_interrupt;
-}
-
-#endif /* CONFIG_X86_MCE_P4THERMAL */
-
 /* P4/Xeon Extended MCE MSR retrieval, return 0 if unsupported */
 static void intel_get_extended_msrs(struct intel_mce_extended_msrs *r)
 {
diff --git a/arch/x86/kernel/cpu/mcheck/p5.c b/arch/x86/kernel/cpu/mcheck/p5.c
index 015f481ab1b0..5c0e6533d9bc 100644
--- a/arch/x86/kernel/cpu/mcheck/p5.c
+++ b/arch/x86/kernel/cpu/mcheck/p5.c
@@ -10,12 +10,11 @@
 
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/mce.h>
 #include <asm/msr.h>
 
-#include "mce.h"
-
 /* By default disabled */
-int             mce_p5_enable;
+int mce_p5_enabled __read_mostly;
 
 /* Machine check handler for Pentium class Intel CPUs: */
 static void pentium_machine_check(struct pt_regs *regs, long error_code)
@@ -43,15 +42,13 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
 {
         u32 l, h;
 
-        /* Check for MCE support: */
-        if (!cpu_has(c, X86_FEATURE_MCE))
+        /* Default P5 to off as its often misconnected: */
+        if (!mce_p5_enabled)
                 return;
 
-#ifdef CONFIG_X86_OLD_MCE
-        /* Default P5 to off as its often misconnected: */
-        if (mce_disabled != -1)
+        /* Check for MCE support: */
+        if (!cpu_has(c, X86_FEATURE_MCE))
                 return;
-#endif
 
         machine_check_vector = pentium_machine_check;
         /* Make sure the vector pointer is visible before we enable MCEs: */
diff --git a/arch/x86/kernel/cpu/mcheck/p6.c b/arch/x86/kernel/cpu/mcheck/p6.c
index 43c24e667457..01e4f8178183 100644
--- a/arch/x86/kernel/cpu/mcheck/p6.c
+++ b/arch/x86/kernel/cpu/mcheck/p6.c
@@ -10,10 +10,9 @@
 
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/mce.h>
 #include <asm/msr.h>
 
-#include "mce.h"
-
 /* Machine Check Handler For PII/PIII */
 static void intel_machine_check(struct pt_regs *regs, long error_code)
 {
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 7b1ae2e20ba5..5957a93e5173 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -13,21 +13,32 @@
  * Credits: Adapted from Zwane Mwaikambo's original code in mce_intel.c.
  *          Inspired by Ross Biro's and Al Borchers' counter code.
  */
+#include <linux/interrupt.h>
 #include <linux/notifier.h>
 #include <linux/jiffies.h>
+#include <linux/kernel.h>
 #include <linux/percpu.h>
 #include <linux/sysdev.h>
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/smp.h>
 #include <linux/cpu.h>
 
-#include <asm/therm_throt.h>
+#include <asm/processor.h>
+#include <asm/system.h>
+#include <asm/apic.h>
+#include <asm/idle.h>
+#include <asm/mce.h>
+#include <asm/msr.h>
 
 /* How long to wait between reporting thermal events */
 #define CHECK_INTERVAL          (300 * HZ)
 
 static DEFINE_PER_CPU(__u64, next_check) = INITIAL_JIFFIES;
 static DEFINE_PER_CPU(unsigned long, thermal_throttle_count);
+static DEFINE_PER_CPU(bool, thermal_throttle_active);
 
-atomic_t therm_throt_en         = ATOMIC_INIT(0);
+static atomic_t therm_throt_en          = ATOMIC_INIT(0);
 
 #ifdef CONFIG_SYSFS
 #define define_therm_throt_sysdev_one_ro(_name)                         \
@@ -82,31 +93,37 @@ static struct attribute_group thermal_throttle_attr_group = {
  *          1 : Event should be logged further, and a message has been
  *              printed to the syslog.
  */
-int therm_throt_process(int curr)
+static int therm_throt_process(int curr)
 {
         unsigned int cpu = smp_processor_id();
         __u64 tmp_jiffs = get_jiffies_64();
+        bool was_throttled = __get_cpu_var(thermal_throttle_active);
+        bool is_throttled = __get_cpu_var(thermal_throttle_active) = curr;
 
-        if (curr)
+        if (is_throttled)
                 __get_cpu_var(thermal_throttle_count)++;
 
-        if (time_before64(tmp_jiffs, __get_cpu_var(next_check)))
+        if (!(was_throttled ^ is_throttled) &&
+            time_before64(tmp_jiffs, __get_cpu_var(next_check)))
                 return 0;
 
         __get_cpu_var(next_check) = tmp_jiffs + CHECK_INTERVAL;
 
         /* if we just entered the thermal event */
-        if (curr) {
+        if (is_throttled) {
                 printk(KERN_CRIT "CPU%d: Temperature above threshold, "
-                       "cpu clock throttled (total events = %lu)\n", cpu,
-                       __get_cpu_var(thermal_throttle_count));
+                       "cpu clock throttled (total events = %lu)\n",
+                       cpu, __get_cpu_var(thermal_throttle_count));
 
                 add_taint(TAINT_MACHINE_CHECK);
-        } else {
-                printk(KERN_CRIT "CPU%d: Temperature/speed normal\n", cpu);
+                return 1;
+        }
+        if (was_throttled) {
+                printk(KERN_INFO "CPU%d: Temperature/speed normal\n", cpu);
+                return 1;
         }
 
-        return 1;
+        return 0;
 }
 
 #ifdef CONFIG_SYSFS
@@ -186,6 +203,94 @@ static __init int thermal_throttle_init_device(void)
 
         return 0;
 }
-
 device_initcall(thermal_throttle_init_device);
+
 #endif /* CONFIG_SYSFS */
+
+/* Thermal transition interrupt handler */
+static void intel_thermal_interrupt(void)
+{
+        __u64 msr_val;
+
+        rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
+        if (therm_throt_process(msr_val & THERM_STATUS_PROCHOT))
+                mce_log_therm_throt_event(msr_val);
+}
+
+static void unexpected_thermal_interrupt(void)
+{
+        printk(KERN_ERR "CPU%d: Unexpected LVT TMR interrupt!\n",
+                        smp_processor_id());
+        add_taint(TAINT_MACHINE_CHECK);
+}
+
+static void (*smp_thermal_vector)(void) = unexpected_thermal_interrupt;
+
+asmlinkage void smp_thermal_interrupt(struct pt_regs *regs)
+{
+        exit_idle();
+        irq_enter();
+        inc_irq_stat(irq_thermal_count);
+        smp_thermal_vector();
+        irq_exit();
+        /* Ack only at the end to avoid potential reentry */
+        ack_APIC_irq();
+}
+
+void intel_init_thermal(struct cpuinfo_x86 *c)
+{
+        unsigned int cpu = smp_processor_id();
+        int tm2 = 0;
+        u32 l, h;
+
+        /* Thermal monitoring depends on ACPI and clock modulation*/
+        if (!cpu_has(c, X86_FEATURE_ACPI) || !cpu_has(c, X86_FEATURE_ACC))
+                return;
+
+        /*
+         * First check if its enabled already, in which case there might
+         * be some SMM goo which handles it, so we can't even put a handler
+         * since it might be delivered via SMI already:
+         */
+        rdmsr(MSR_IA32_MISC_ENABLE, l, h);
+        h = apic_read(APIC_LVTTHMR);
+        if ((l & MSR_IA32_MISC_ENABLE_TM1) && (h & APIC_DM_SMI)) {
+                printk(KERN_DEBUG
+                       "CPU%d: Thermal monitoring handled by SMI\n", cpu);
+                return;
+        }
+
+        if (cpu_has(c, X86_FEATURE_TM2) && (l & MSR_IA32_MISC_ENABLE_TM2))
+                tm2 = 1;
+
+        /* Check whether a vector already exists */
+        if (h & APIC_VECTOR_MASK) {
+                printk(KERN_DEBUG
+                       "CPU%d: Thermal LVT vector (%#x) already installed\n",
+                       cpu, (h & APIC_VECTOR_MASK));
+                return;
+        }
+
+        /* We'll mask the thermal vector in the lapic till we're ready: */
+        h = THERMAL_APIC_VECTOR | APIC_DM_FIXED | APIC_LVT_MASKED;
+        apic_write(APIC_LVTTHMR, h);
+
+        rdmsr(MSR_IA32_THERM_INTERRUPT, l, h);
+        wrmsr(MSR_IA32_THERM_INTERRUPT,
+                l | (THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE), h);
+
+        smp_thermal_vector = intel_thermal_interrupt;
+
+        rdmsr(MSR_IA32_MISC_ENABLE, l, h);
+        wrmsr(MSR_IA32_MISC_ENABLE, l | MSR_IA32_MISC_ENABLE_TM1, h);
+
+        /* Unmask the thermal vector: */
+        l = apic_read(APIC_LVTTHMR);
+        apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
+
+        printk(KERN_INFO "CPU%d: Thermal monitoring enabled (%s)\n",
+               cpu, tm2 ? "TM2" : "TM1");
+
+        /* enable thermal throttle processing */
+        atomic_set(&therm_throt_en, 1);
+}
diff --git a/arch/x86/kernel/cpu/mcheck/winchip.c b/arch/x86/kernel/cpu/mcheck/winchip.c
index 81b02487090b..54060f565974 100644
--- a/arch/x86/kernel/cpu/mcheck/winchip.c
+++ b/arch/x86/kernel/cpu/mcheck/winchip.c
@@ -9,10 +9,9 @@
 
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/mce.h>
 #include <asm/msr.h>
 
-#include "mce.h"
-
 /* Machine check handler for WinChip C6: */
 static void winchip_machine_check(struct pt_regs *regs, long error_code)
 {
diff --git a/arch/x86/kernel/cpu/perf_counter.c b/arch/x86/kernel/cpu/perf_counter.c
index 275bc142cd5d..900332b800f8 100644
--- a/arch/x86/kernel/cpu/perf_counter.c
+++ b/arch/x86/kernel/cpu/perf_counter.c
@@ -19,6 +19,7 @@
 #include <linux/kdebug.h>
 #include <linux/sched.h>
 #include <linux/uaccess.h>
+#include <linux/highmem.h>
 
 #include <asm/apic.h>
 #include <asm/stacktrace.h>
@@ -54,6 +55,7 @@ struct x86_pmu {
         int             num_counters_fixed;
         int             counter_bits;
         u64             counter_mask;
+        int             apic;
         u64             max_period;
         u64             intel_ctrl;
 };
@@ -65,6 +67,52 @@ static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
 };
 
 /*
+ * Not sure about some of these
+ */
+static const u64 p6_perfmon_event_map[] =
+{
+  [PERF_COUNT_HW_CPU_CYCLES]            = 0x0079,
+  [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
+  [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0f2e,
+  [PERF_COUNT_HW_CACHE_MISSES]          = 0x012e,
+  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
+  [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
+  [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
+};
+
+static u64 p6_pmu_event_map(int event)
+{
+        return p6_perfmon_event_map[event];
+}
+
+/*
+ * Counter setting that is specified not to count anything.
+ * We use this to effectively disable a counter.
+ *
+ * L2_RQSTS with 0 MESI unit mask.
+ */
+#define P6_NOP_COUNTER                  0x0000002EULL
+
+static u64 p6_pmu_raw_event(u64 event)
+{
+#define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
+#define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
+#define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
+#define P6_EVNTSEL_INV_MASK             0x00800000ULL
+#define P6_EVNTSEL_COUNTER_MASK         0xFF000000ULL
+
+#define P6_EVNTSEL_MASK                 \
+        (P6_EVNTSEL_EVENT_MASK |        \
+         P6_EVNTSEL_UNIT_MASK  |        \
+         P6_EVNTSEL_EDGE_MASK  |        \
+         P6_EVNTSEL_INV_MASK   |        \
+         P6_EVNTSEL_COUNTER_MASK)
+
+        return event & P6_EVNTSEL_MASK;
+}
+
+
+/*
  * Intel PerfMon v3. Used on Core2 and later.
  */
 static const u64 intel_perfmon_event_map[] =
@@ -389,23 +437,23 @@ static u64 intel_pmu_raw_event(u64 event)
         return event & CORE_EVNTSEL_MASK;
 }
 
-static const u64 amd_0f_hw_cache_event_ids
+static const u64 amd_hw_cache_event_ids
                                 [PERF_COUNT_HW_CACHE_MAX]
                                 [PERF_COUNT_HW_CACHE_OP_MAX]
                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 {
  [ C(L1D) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
-                [ C(RESULT_MISS)   ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
+                [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
         },
         [ C(OP_WRITE) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
                 [ C(RESULT_MISS)   ] = 0,
         },
         [ C(OP_PREFETCH) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
-                [ C(RESULT_MISS)   ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
+                [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
         },
  },
  [ C(L1I ) ] = {
@@ -418,17 +466,17 @@ static const u64 amd_0f_hw_cache_event_ids
                 [ C(RESULT_MISS)   ] = -1,
         },
         [ C(OP_PREFETCH) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
                 [ C(RESULT_MISS)   ] = 0,
         },
  },
  [ C(LL  ) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
-                [ C(RESULT_MISS)   ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
+                [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
         },
         [ C(OP_WRITE) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
                 [ C(RESULT_MISS)   ] = 0,
         },
         [ C(OP_PREFETCH) ] = {
@@ -438,8 +486,8 @@ static const u64 amd_0f_hw_cache_event_ids
  },
  [ C(DTLB) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = 0,
-                [ C(RESULT_MISS)   ] = 0,
+                [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
+                [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
         },
         [ C(OP_WRITE) ] = {
                 [ C(RESULT_ACCESS) ] = 0,
@@ -566,6 +614,7 @@ static DEFINE_MUTEX(pmc_reserve_mutex);
 
 static bool reserve_pmc_hardware(void)
 {
+#ifdef CONFIG_X86_LOCAL_APIC
         int i;
 
         if (nmi_watchdog == NMI_LOCAL_APIC)
@@ -580,9 +629,11 @@ static bool reserve_pmc_hardware(void)
                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
                         goto eventsel_fail;
         }
+#endif
 
         return true;
 
+#ifdef CONFIG_X86_LOCAL_APIC
 eventsel_fail:
         for (i--; i >= 0; i--)
                 release_evntsel_nmi(x86_pmu.eventsel + i);
@@ -597,10 +648,12 @@ perfctr_fail:
                 enable_lapic_nmi_watchdog();
 
         return false;
+#endif
 }
 
 static void release_pmc_hardware(void)
 {
+#ifdef CONFIG_X86_LOCAL_APIC
         int i;
 
         for (i = 0; i < x86_pmu.num_counters; i++) {
@@ -610,6 +663,7 @@ static void release_pmc_hardware(void)
 
         if (nmi_watchdog == NMI_LOCAL_APIC)
                 enable_lapic_nmi_watchdog();
+#endif
 }
 
 static void hw_perf_counter_destroy(struct perf_counter *counter)
@@ -665,6 +719,7 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
 {
         struct perf_counter_attr *attr = &counter->attr;
         struct hw_perf_counter *hwc = &counter->hw;
+        u64 config;
         int err;
 
         if (!x86_pmu_initialized())
@@ -700,6 +755,15 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
                 hwc->sample_period = x86_pmu.max_period;
                 hwc->last_period = hwc->sample_period;
                 atomic64_set(&hwc->period_left, hwc->sample_period);
+        } else {
+                /*
+                 * If we have a PMU initialized but no APIC
+                 * interrupts, we cannot sample hardware
+                 * counters (user-space has to fall back and
+                 * sample via a hrtimer based software counter):
+                 */
+                if (!x86_pmu.apic)
+                        return -EOPNOTSUPP;
         }
 
         counter->destroy = hw_perf_counter_destroy;
@@ -717,14 +781,40 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
 
         if (attr->config >= x86_pmu.max_events)
                 return -EINVAL;
+
         /*
          * The generic map:
          */
-        hwc->config |= x86_pmu.event_map(attr->config);
+        config = x86_pmu.event_map(attr->config);
+
+        if (config == 0)
+                return -ENOENT;
+
+        if (config == -1LL)
+                return -EINVAL;
+
+        hwc->config |= config;
 
         return 0;
 }
 
+static void p6_pmu_disable_all(void)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        u64 val;
+
+        if (!cpuc->enabled)
+                return;
+
+        cpuc->enabled = 0;
+        barrier();
+
+        /* p6 only has one enable register */
+        rdmsrl(MSR_P6_EVNTSEL0, val);
+        val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
+        wrmsrl(MSR_P6_EVNTSEL0, val);
+}
+
 static void intel_pmu_disable_all(void)
 {
         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
@@ -766,6 +856,23 @@ void hw_perf_disable(void)
         return x86_pmu.disable_all();
 }
 
+static void p6_pmu_enable_all(void)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        unsigned long val;
+
+        if (cpuc->enabled)
+                return;
+
+        cpuc->enabled = 1;
+        barrier();
+
+        /* p6 only has one enable register */
+        rdmsrl(MSR_P6_EVNTSEL0, val);
+        val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+        wrmsrl(MSR_P6_EVNTSEL0, val);
+}
+
 static void intel_pmu_enable_all(void)
 {
         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
@@ -783,13 +890,13 @@ static void amd_pmu_enable_all(void)
         barrier();
 
         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                struct perf_counter *counter = cpuc->counters[idx];
                 u64 val;
 
                 if (!test_bit(idx, cpuc->active_mask))
                         continue;
-                rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
-                if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
-                        continue;
+
+                val = counter->hw.config;
                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
         }
@@ -818,16 +925,13 @@ static inline void intel_pmu_ack_status(u64 ack)
 
 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 {
-        int err;
-        err = checking_wrmsrl(hwc->config_base + idx,
+        (void)checking_wrmsrl(hwc->config_base + idx,
                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 }
 
 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 {
-        int err;
-        err = checking_wrmsrl(hwc->config_base + idx,
-                              hwc->config);
+        (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
 }
 
 static inline void
@@ -835,13 +939,24 @@ intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 {
         int idx = __idx - X86_PMC_IDX_FIXED;
         u64 ctrl_val, mask;
-        int err;
 
         mask = 0xfULL << (idx * 4);
 
         rdmsrl(hwc->config_base, ctrl_val);
         ctrl_val &= ~mask;
-        err = checking_wrmsrl(hwc->config_base, ctrl_val);
+        (void)checking_wrmsrl(hwc->config_base, ctrl_val);
+}
+
+static inline void
+p6_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        u64 val = P6_NOP_COUNTER;
+
+        if (cpuc->enabled)
+                val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+
+        (void)checking_wrmsrl(hwc->config_base + idx, val);
 }
 
 static inline void
@@ -911,6 +1026,8 @@ x86_perf_counter_set_period(struct perf_counter *counter,
         err = checking_wrmsrl(hwc->counter_base + idx,
                              (u64)(-left) & x86_pmu.counter_mask);
 
+        perf_counter_update_userpage(counter);
+
         return ret;
 }
 
@@ -940,6 +1057,19 @@ intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 }
 
+static void p6_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        u64 val;
+
+        val = hwc->config;
+        if (cpuc->enabled)
+                val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+
+        (void)checking_wrmsrl(hwc->config_base + idx, val);
+}
+
+
 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 {
         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
@@ -956,8 +1086,6 @@ static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 
         if (cpuc->enabled)
                 x86_pmu_enable_counter(hwc, idx);
-        else
-                x86_pmu_disable_counter(hwc, idx);
 }
 
 static int
@@ -968,13 +1096,6 @@ fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
         if (!x86_pmu.num_counters_fixed)
                 return -1;
 
-        /*
-         * Quirk, IA32_FIXED_CTRs do not work on current Atom processors:
-         */
-        if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
-                                        boot_cpu_data.x86_model == 28)
-                return -1;
-
         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 
         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
@@ -1040,6 +1161,8 @@ try_generic:
         x86_perf_counter_set_period(counter, hwc, idx);
         x86_pmu.enable(hwc, idx);
 
+        perf_counter_update_userpage(counter);
+
         return 0;
 }
 
@@ -1132,6 +1255,8 @@ static void x86_pmu_disable(struct perf_counter *counter)
         x86_perf_counter_update(counter, hwc, idx);
         cpuc->counters[idx] = NULL;
         clear_bit(idx, cpuc->used_mask);
+
+        perf_counter_update_userpage(counter);
 }
 
 /*
@@ -1176,6 +1301,49 @@ static void intel_pmu_reset(void)
         local_irq_restore(flags);
 }
 
+static int p6_pmu_handle_irq(struct pt_regs *regs)
+{
+        struct perf_sample_data data;
+        struct cpu_hw_counters *cpuc;
+        struct perf_counter *counter;
+        struct hw_perf_counter *hwc;
+        int idx, handled = 0;
+        u64 val;
+
+        data.regs = regs;
+        data.addr = 0;
+
+        cpuc = &__get_cpu_var(cpu_hw_counters);
+
+        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+
+                counter = cpuc->counters[idx];
+                hwc = &counter->hw;
+
+                val = x86_perf_counter_update(counter, hwc, idx);
+                if (val & (1ULL << (x86_pmu.counter_bits - 1)))
+                        continue;
+
+                /*
+                 * counter overflow
+                 */
+                handled         = 1;
+                data.period     = counter->hw.last_period;
+
+                if (!x86_perf_counter_set_period(counter, hwc, idx))
+                        continue;
+
+                if (perf_counter_overflow(counter, 1, &data))
+                        p6_pmu_disable_counter(hwc, idx);
+        }
+
+        if (handled)
+                inc_irq_stat(apic_perf_irqs);
+
+        return handled;
+}
 
 /*
  * This handler is triggered by the local APIC, so the APIC IRQ handling
@@ -1185,14 +1353,13 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
 {
         struct perf_sample_data data;
         struct cpu_hw_counters *cpuc;
-        int bit, cpu, loops;
+        int bit, loops;
         u64 ack, status;
 
         data.regs = regs;
         data.addr = 0;
 
-        cpu = smp_processor_id();
-        cpuc = &per_cpu(cpu_hw_counters, cpu);
+        cpuc = &__get_cpu_var(cpu_hw_counters);
 
         perf_disable();
         status = intel_pmu_get_status();
@@ -1223,6 +1390,8 @@ again:
                 if (!intel_pmu_save_and_restart(counter))
                         continue;
 
+                data.period = counter->hw.last_period;
+
                 if (perf_counter_overflow(counter, 1, &data))
                         intel_pmu_disable_counter(&counter->hw, bit);
         }
@@ -1247,14 +1416,13 @@ static int amd_pmu_handle_irq(struct pt_regs *regs)
         struct cpu_hw_counters *cpuc;
         struct perf_counter *counter;
         struct hw_perf_counter *hwc;
-        int cpu, idx, handled = 0;
+        int idx, handled = 0;
         u64 val;
 
         data.regs = regs;
         data.addr = 0;
 
-        cpu = smp_processor_id();
-        cpuc = &per_cpu(cpu_hw_counters, cpu);
+        cpuc = &__get_cpu_var(cpu_hw_counters);
 
         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
                 if (!test_bit(idx, cpuc->active_mask))
@@ -1297,18 +1465,22 @@ void smp_perf_pending_interrupt(struct pt_regs *regs)
 
 void set_perf_counter_pending(void)
 {
+#ifdef CONFIG_X86_LOCAL_APIC
         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
+#endif
 }
 
 void perf_counters_lapic_init(void)
 {
-        if (!x86_pmu_initialized())
+#ifdef CONFIG_X86_LOCAL_APIC
+        if (!x86_pmu.apic || !x86_pmu_initialized())
                 return;
 
         /*
          * Always use NMI for PMU
          */
         apic_write(APIC_LVTPC, APIC_DM_NMI);
+#endif
 }
 
 static int __kprobes
@@ -1332,7 +1504,9 @@ perf_counter_nmi_handler(struct notifier_block *self,
 
         regs = args->regs;
 
+#ifdef CONFIG_X86_LOCAL_APIC
         apic_write(APIC_LVTPC, APIC_DM_NMI);
+#endif
         /*
          * Can't rely on the handled return value to say it was our NMI, two
          * counters could trigger 'simultaneously' raising two back-to-back NMIs.
@@ -1351,6 +1525,33 @@ static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
         .priority               = 1
 };
 
+static struct x86_pmu p6_pmu = {
+        .name                   = "p6",
+        .handle_irq             = p6_pmu_handle_irq,
+        .disable_all            = p6_pmu_disable_all,
+        .enable_all             = p6_pmu_enable_all,
+        .enable                 = p6_pmu_enable_counter,
+        .disable                = p6_pmu_disable_counter,
+        .eventsel               = MSR_P6_EVNTSEL0,
+        .perfctr                = MSR_P6_PERFCTR0,
+        .event_map              = p6_pmu_event_map,
+        .raw_event              = p6_pmu_raw_event,
+        .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
+        .apic                   = 1,
+        .max_period             = (1ULL << 31) - 1,
+        .version                = 0,
+        .num_counters           = 2,
+        /*
+         * Counters have 40 bits implemented. However they are designed such
+         * that bits [32-39] are sign extensions of bit 31. As such the
+         * effective width of a counter for P6-like PMU is 32 bits only.
+         *
+         * See IA-32 Intel Architecture Software developer manual Vol 3B
+         */
+        .counter_bits           = 32,
+        .counter_mask           = (1ULL << 32) - 1,
+};
+
 static struct x86_pmu intel_pmu = {
         .name                   = "Intel",
         .handle_irq             = intel_pmu_handle_irq,
@@ -1363,6 +1564,7 @@ static struct x86_pmu intel_pmu = {
         .event_map              = intel_pmu_event_map,
         .raw_event              = intel_pmu_raw_event,
         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
+        .apic                   = 1,
         /*
          * Intel PMCs cannot be accessed sanely above 32 bit width,
          * so we install an artificial 1<<31 period regardless of
@@ -1386,10 +1588,43 @@ static struct x86_pmu amd_pmu = {
         .num_counters           = 4,
         .counter_bits           = 48,
         .counter_mask           = (1ULL << 48) - 1,
+        .apic                   = 1,
         /* use highest bit to detect overflow */
         .max_period             = (1ULL << 47) - 1,
 };
 
+static int p6_pmu_init(void)
+{
+        switch (boot_cpu_data.x86_model) {
+        case 1:
+        case 3:  /* Pentium Pro */
+        case 5:
+        case 6:  /* Pentium II */
+        case 7:
+        case 8:
+        case 11: /* Pentium III */
+                break;
+        case 9:
+        case 13:
+                /* Pentium M */
+                break;
+        default:
+                pr_cont("unsupported p6 CPU model %d ",
+                        boot_cpu_data.x86_model);
+                return -ENODEV;
+        }
+
+        x86_pmu = p6_pmu;
+
+        if (!cpu_has_apic) {
+                pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
+                pr_info("no hardware sampling interrupt available.\n");
+                x86_pmu.apic = 0;
+        }
+
+        return 0;
+}
+
 static int intel_pmu_init(void)
 {
         union cpuid10_edx edx;
@@ -1398,8 +1633,14 @@ static int intel_pmu_init(void)
         unsigned int ebx;
         int version;
 
-        if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
+        if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
+                /* check for P6 processor family */
+           if (boot_cpu_data.x86 == 6) {
+                return p6_pmu_init();
+           } else {
                 return -ENODEV;
+           }
+        }
 
         /*
          * Check whether the Architectural PerfMon supports
@@ -1425,8 +1666,6 @@ static int intel_pmu_init(void)
          */
         x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
 
-        rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
-
         /*
          * Install the hw-cache-events table:
          */
@@ -1459,18 +1698,16 @@ static int intel_pmu_init(void)
 
 static int amd_pmu_init(void)
 {
+        /* Performance-monitoring supported from K7 and later: */
+        if (boot_cpu_data.x86 < 6)
+                return -ENODEV;
+
         x86_pmu = amd_pmu;
 
-        switch (boot_cpu_data.x86) {
-        case 0x0f:
-        case 0x10:
-        case 0x11:
-                memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
-                       sizeof(hw_cache_event_ids));
+        /* Events are common for all AMDs */
+        memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
+               sizeof(hw_cache_event_ids));
 
-                pr_cont("AMD Family 0f/10/11 events, ");
-                break;
-        }
         return 0;
 }
 
@@ -1498,21 +1735,22 @@ void __init init_hw_perf_counters(void)
         pr_cont("%s PMU driver.\n", x86_pmu.name);
 
         if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
-                x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
                      x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
+                x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
         }
         perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
         perf_max_counters = x86_pmu.num_counters;
 
         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
-                x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
                      x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
+                x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
         }
 
         perf_counter_mask |=
                 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
+        x86_pmu.intel_ctrl = perf_counter_mask;
 
         perf_counters_lapic_init();
         register_die_notifier(&perf_counter_nmi_notifier);
@@ -1554,14 +1792,15 @@ const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
  */
 
 static inline
-void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
+void callchain_store(struct perf_callchain_entry *entry, u64 ip)
 {
-        if (entry->nr < MAX_STACK_DEPTH)
+        if (entry->nr < PERF_MAX_STACK_DEPTH)
                 entry->ip[entry->nr++] = ip;
 }
 
 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
+static DEFINE_PER_CPU(int, in_nmi_frame);
 
 
 static void
@@ -1577,14 +1816,19 @@ static void backtrace_warning(void *data, char *msg)
 
 static int backtrace_stack(void *data, char *name)
 {
-        /* Don't bother with IRQ stacks for now */
-        return -1;
+        per_cpu(in_nmi_frame, smp_processor_id()) =
+                        x86_is_stack_id(NMI_STACK, name);
+
+        return 0;
 }
 
 static void backtrace_address(void *data, unsigned long addr, int reliable)
 {
         struct perf_callchain_entry *entry = data;
 
+        if (per_cpu(in_nmi_frame, smp_processor_id()))
+                return;
+
         if (reliable)
                 callchain_store(entry, addr);
 }
@@ -1596,47 +1840,59 @@ static const struct stacktrace_ops backtrace_ops = {
         .address                = backtrace_address,
 };
 
+#include "../dumpstack.h"
+
 static void
 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
 {
-        unsigned long bp;
-        char *stack;
-        int nr = entry->nr;
+        callchain_store(entry, PERF_CONTEXT_KERNEL);
+        callchain_store(entry, regs->ip);
 
-        callchain_store(entry, instruction_pointer(regs));
+        dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
+}
 
-        stack = ((char *)regs + sizeof(struct pt_regs));
-#ifdef CONFIG_FRAME_POINTER
-        bp = frame_pointer(regs);
-#else
-        bp = 0;
-#endif
+/*
+ * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
+ */
+static unsigned long
+copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
+{
+        unsigned long offset, addr = (unsigned long)from;
+        int type = in_nmi() ? KM_NMI : KM_IRQ0;
+        unsigned long size, len = 0;
+        struct page *page;
+        void *map;
+        int ret;
 
-        dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
+        do {
+                ret = __get_user_pages_fast(addr, 1, 0, &page);
+                if (!ret)
+                        break;
 
-        entry->kernel = entry->nr - nr;
-}
+                offset = addr & (PAGE_SIZE - 1);
+                size = min(PAGE_SIZE - offset, n - len);
 
+                map = kmap_atomic(page, type);
+                memcpy(to, map+offset, size);
+                kunmap_atomic(map, type);
+                put_page(page);
 
-struct stack_frame {
-        const void __user       *next_fp;
-        unsigned long           return_address;
-};
+                len  += size;
+                to   += size;
+                addr += size;
+
+        } while (len < n);
+
+        return len;
+}
 
 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
 {
-        int ret;
-
-        if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
-                return 0;
+        unsigned long bytes;
 
-        ret = 1;
-        pagefault_disable();
-        if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
-                ret = 0;
-        pagefault_enable();
+        bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
 
-        return ret;
+        return bytes == sizeof(*frame);
 }
 
 static void
@@ -1644,28 +1900,28 @@ perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
 {
         struct stack_frame frame;
         const void __user *fp;
-        int nr = entry->nr;
 
-        regs = (struct pt_regs *)current->thread.sp0 - 1;
-        fp   = (void __user *)regs->bp;
+        if (!user_mode(regs))
+                regs = task_pt_regs(current);
+
+        fp = (void __user *)regs->bp;
 
+        callchain_store(entry, PERF_CONTEXT_USER);
         callchain_store(entry, regs->ip);
 
-        while (entry->nr < MAX_STACK_DEPTH) {
-                frame.next_fp        = NULL;
+        while (entry->nr < PERF_MAX_STACK_DEPTH) {
+                frame.next_frame             = NULL;
                 frame.return_address = 0;
 
                 if (!copy_stack_frame(fp, &frame))
                         break;
 
-                if ((unsigned long)fp < user_stack_pointer(regs))
+                if ((unsigned long)fp < regs->sp)
                         break;
 
                 callchain_store(entry, frame.return_address);
-                fp = frame.next_fp;
+                fp = frame.next_frame;
         }
-
-        entry->user = entry->nr - nr;
 }
 
 static void
@@ -1701,9 +1957,6 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
                 entry = &__get_cpu_var(irq_entry);
 
         entry->nr = 0;
-        entry->hv = 0;
-        entry->kernel = 0;
-        entry->user = 0;
 
         perf_do_callchain(regs, entry);
 
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index d6f5b9fbde32..e60ed740d2b3 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -716,11 +716,15 @@ static void probe_nmi_watchdog(void)
                 wd_ops = &k7_wd_ops;
                 break;
         case X86_VENDOR_INTEL:
-                /*
-                 * Work around Core Duo (Yonah) errata AE49 where perfctr1
-                 * doesn't have a working enable bit.
+                /* Work around where perfctr1 doesn't have a working enable
+                 * bit as described in the following errata:
+                 * AE49 Core Duo and Intel Core Solo 65 nm
+                 * AN49 Intel Pentium Dual-Core
+                 * AF49 Dual-Core Intel Xeon Processor LV
                  */
-                if (boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 14) {
+                if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 14) ||
+                    ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 15 &&
+                     boot_cpu_data.x86_mask == 4))) {
                         intel_arch_wd_ops.perfctr = MSR_ARCH_PERFMON_PERFCTR0;
                         intel_arch_wd_ops.evntsel = MSR_ARCH_PERFMON_EVENTSEL0;
                 }
@@ -799,8 +803,3 @@ int __kprobes lapic_wd_event(unsigned nmi_hz)
         wd_ops->rearm(wd, nmi_hz);
         return 1;
 }
-
-int lapic_watchdog_ok(void)
-{
-        return wd_ops != NULL;
-}