Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (184 commits)
  perf probe: Clean up probe_point_lazy_walker() return value
  tracing: Fix irqoff selftest expanding max buffer
  tracing: Align 4 byte ints together in struct tracer
  tracing: Export trace_set_clr_event()
  tracing: Explain about unstable clock on resume with ring buffer warning
  ftrace/graph: Trace function entry before updating index
  ftrace: Add .ref.text as one of the safe areas to trace
  tracing: Adjust conditional expression latency formatting.
  tracing: Fix event alignment: skb:kfree_skb
  tracing: Fix event alignment: mce:mce_record
  tracing: Fix event alignment: kvm:kvm_hv_hypercall
  tracing: Fix event alignment: module:module_request
  tracing: Fix event alignment: ftrace:context_switch and ftrace:wakeup
  tracing: Remove lock_depth from event entry
  perf header: Stop using 'self'
  perf session: Use evlist/evsel for managing perf.data attributes
  perf top: Don't let events to eat up whole header line
  perf top: Fix events overflow in top command
  ring-buffer: Remove unused #include <linux/trace_irq.h>
  tracing: Add an 'overwrite' trace_option.
  ...

14 files changed, 814 insertions, 131 deletions

diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 9d977a2ea693..26604188aa49 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -30,6 +30,7 @@
 #include <asm/stacktrace.h>
 #include <asm/nmi.h>
 #include <asm/compat.h>
+#include <asm/smp.h>
 
 #if 0
 #undef wrmsrl
@@ -93,6 +94,8 @@ struct amd_nb {
         struct event_constraint event_constraints[X86_PMC_IDX_MAX];
 };
 
+struct intel_percore;
+
 #define MAX_LBR_ENTRIES         16
 
 struct cpu_hw_events {
@@ -128,6 +131,13 @@ struct cpu_hw_events {
         struct perf_branch_entry        lbr_entries[MAX_LBR_ENTRIES];
 
         /*
+         * Intel percore register state.
+         * Coordinate shared resources between HT threads.
+         */
+        int                             percore_used; /* Used by this CPU? */
+        struct intel_percore            *per_core;
+
+        /*
          * AMD specific bits
          */
         struct amd_nb           *amd_nb;
@@ -166,8 +176,10 @@ struct cpu_hw_events {
 /*
  * Constraint on the Event code + UMask
  */
-#define PEBS_EVENT_CONSTRAINT(c, n)     \
+#define INTEL_UEVENT_CONSTRAINT(c, n)   \
         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
+#define PEBS_EVENT_CONSTRAINT(c, n)     \
+        INTEL_UEVENT_CONSTRAINT(c, n)
 
 #define EVENT_CONSTRAINT_END            \
         EVENT_CONSTRAINT(0, 0, 0)
@@ -175,6 +187,28 @@ struct cpu_hw_events {
 #define for_each_event_constraint(e, c) \
         for ((e) = (c); (e)->weight; (e)++)
 
+/*
+ * Extra registers for specific events.
+ * Some events need large masks and require external MSRs.
+ * Define a mapping to these extra registers.
+ */
+struct extra_reg {
+        unsigned int            event;
+        unsigned int            msr;
+        u64                     config_mask;
+        u64                     valid_mask;
+};
+
+#define EVENT_EXTRA_REG(e, ms, m, vm) { \
+        .event = (e),           \
+        .msr = (ms),            \
+        .config_mask = (m),     \
+        .valid_mask = (vm),     \
+        }
+#define INTEL_EVENT_EXTRA_REG(event, msr, vm)   \
+        EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm)
+#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0)
+
 union perf_capabilities {
         struct {
                 u64     lbr_format    : 6;
@@ -219,6 +253,7 @@ struct x86_pmu {
         void            (*put_event_constraints)(struct cpu_hw_events *cpuc,
                                                  struct perf_event *event);
         struct event_constraint *event_constraints;
+        struct event_constraint *percore_constraints;
         void            (*quirks)(void);
         int             perfctr_second_write;
 
@@ -247,6 +282,11 @@ struct x86_pmu {
          */
         unsigned long   lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
         int             lbr_nr;                    /* hardware stack size */
+
+        /*
+         * Extra registers for events
+         */
+        struct extra_reg *extra_regs;
 };
 
 static struct x86_pmu x86_pmu __read_mostly;
@@ -271,6 +311,10 @@ static u64 __read_mostly hw_cache_event_ids
                                 [PERF_COUNT_HW_CACHE_MAX]
                                 [PERF_COUNT_HW_CACHE_OP_MAX]
                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
+static u64 __read_mostly hw_cache_extra_regs
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX];
 
 /*
  * Propagate event elapsed time into the generic event.
@@ -298,7 +342,7 @@ x86_perf_event_update(struct perf_event *event)
          */
 again:
         prev_raw_count = local64_read(&hwc->prev_count);
-        rdmsrl(hwc->event_base + idx, new_raw_count);
+        rdmsrl(hwc->event_base, new_raw_count);
 
         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
                                         new_raw_count) != prev_raw_count)
@@ -321,6 +365,49 @@ again:
         return new_raw_count;
 }
 
+/* using X86_FEATURE_PERFCTR_CORE to later implement ALTERNATIVE() here */
+static inline int x86_pmu_addr_offset(int index)
+{
+        if (boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
+                return index << 1;
+        return index;
+}
+
+static inline unsigned int x86_pmu_config_addr(int index)
+{
+        return x86_pmu.eventsel + x86_pmu_addr_offset(index);
+}
+
+static inline unsigned int x86_pmu_event_addr(int index)
+{
+        return x86_pmu.perfctr + x86_pmu_addr_offset(index);
+}
+
+/*
+ * Find and validate any extra registers to set up.
+ */
+static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
+{
+        struct extra_reg *er;
+
+        event->hw.extra_reg = 0;
+        event->hw.extra_config = 0;
+
+        if (!x86_pmu.extra_regs)
+                return 0;
+
+        for (er = x86_pmu.extra_regs; er->msr; er++) {
+                if (er->event != (config & er->config_mask))
+                        continue;
+                if (event->attr.config1 & ~er->valid_mask)
+                        return -EINVAL;
+                event->hw.extra_reg = er->msr;
+                event->hw.extra_config = event->attr.config1;
+                break;
+        }
+        return 0;
+}
+
 static atomic_t active_events;
 static DEFINE_MUTEX(pmc_reserve_mutex);
 
@@ -331,12 +418,12 @@ static bool reserve_pmc_hardware(void)
         int i;
 
         for (i = 0; i < x86_pmu.num_counters; i++) {
-                if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
+                if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
                         goto perfctr_fail;
         }
 
         for (i = 0; i < x86_pmu.num_counters; i++) {
-                if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
+                if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
                         goto eventsel_fail;
         }
 
@@ -344,13 +431,13 @@ static bool reserve_pmc_hardware(void)
 
 eventsel_fail:
         for (i--; i >= 0; i--)
-                release_evntsel_nmi(x86_pmu.eventsel + i);
+                release_evntsel_nmi(x86_pmu_config_addr(i));
 
         i = x86_pmu.num_counters;
 
 perfctr_fail:
         for (i--; i >= 0; i--)
-                release_perfctr_nmi(x86_pmu.perfctr + i);
+                release_perfctr_nmi(x86_pmu_event_addr(i));
 
         return false;
 }
@@ -360,8 +447,8 @@ static void release_pmc_hardware(void)
         int i;
 
         for (i = 0; i < x86_pmu.num_counters; i++) {
-                release_perfctr_nmi(x86_pmu.perfctr + i);
-                release_evntsel_nmi(x86_pmu.eventsel + i);
+                release_perfctr_nmi(x86_pmu_event_addr(i));
+                release_evntsel_nmi(x86_pmu_config_addr(i));
         }
 }
 
@@ -382,7 +469,7 @@ static bool check_hw_exists(void)
          * complain and bail.
          */
         for (i = 0; i < x86_pmu.num_counters; i++) {
-                reg = x86_pmu.eventsel + i;
+                reg = x86_pmu_config_addr(i);
                 ret = rdmsrl_safe(reg, &val);
                 if (ret)
                         goto msr_fail;
@@ -407,8 +494,8 @@ static bool check_hw_exists(void)
          * that don't trap on the MSR access and always return 0s.
          */
         val = 0xabcdUL;
-        ret = checking_wrmsrl(x86_pmu.perfctr, val);
-        ret |= rdmsrl_safe(x86_pmu.perfctr, &val_new);
+        ret = checking_wrmsrl(x86_pmu_event_addr(0), val);
+        ret |= rdmsrl_safe(x86_pmu_event_addr(0), &val_new);
         if (ret || val != val_new)
                 goto msr_fail;
 
@@ -442,8 +529,9 @@ static inline int x86_pmu_initialized(void)
 }
 
 static inline int
-set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
+set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
 {
+        struct perf_event_attr *attr = &event->attr;
         unsigned int cache_type, cache_op, cache_result;
         u64 config, val;
 
@@ -470,8 +558,8 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
                 return -EINVAL;
 
         hwc->config |= val;
-
-        return 0;
+        attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
+        return x86_pmu_extra_regs(val, event);
 }
 
 static int x86_setup_perfctr(struct perf_event *event)
@@ -496,10 +584,10 @@ static int x86_setup_perfctr(struct perf_event *event)
         }
 
         if (attr->type == PERF_TYPE_RAW)
-                return 0;
+                return x86_pmu_extra_regs(event->attr.config, event);
 
         if (attr->type == PERF_TYPE_HW_CACHE)
-                return set_ext_hw_attr(hwc, attr);
+                return set_ext_hw_attr(hwc, event);
 
         if (attr->config >= x86_pmu.max_events)
                 return -EINVAL;
@@ -617,11 +705,11 @@ static void x86_pmu_disable_all(void)
 
                 if (!test_bit(idx, cpuc->active_mask))
                         continue;
-                rdmsrl(x86_pmu.eventsel + idx, val);
+                rdmsrl(x86_pmu_config_addr(idx), val);
                 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
                         continue;
                 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
-                wrmsrl(x86_pmu.eventsel + idx, val);
+                wrmsrl(x86_pmu_config_addr(idx), val);
         }
 }
 
@@ -642,21 +730,26 @@ static void x86_pmu_disable(struct pmu *pmu)
         x86_pmu.disable_all();
 }
 
+static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
+                                          u64 enable_mask)
+{
+        if (hwc->extra_reg)
+                wrmsrl(hwc->extra_reg, hwc->extra_config);
+        wrmsrl(hwc->config_base, hwc->config | enable_mask);
+}
+
 static void x86_pmu_enable_all(int added)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         int idx;
 
         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
-                struct perf_event *event = cpuc->events[idx];
-                u64 val;
+                struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
 
                 if (!test_bit(idx, cpuc->active_mask))
                         continue;
 
-                val = event->hw.config;
-                val |= ARCH_PERFMON_EVENTSEL_ENABLE;
-                wrmsrl(x86_pmu.eventsel + idx, val);
+                __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
         }
 }
 
@@ -821,15 +914,10 @@ static inline void x86_assign_hw_event(struct perf_event *event,
                 hwc->event_base = 0;
         } else if (hwc->idx >= X86_PMC_IDX_FIXED) {
                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
-                /*
-                 * We set it so that event_base + idx in wrmsr/rdmsr maps to
-                 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
-                 */
-                hwc->event_base =
-                        MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
+                hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0;
         } else {
-                hwc->config_base = x86_pmu.eventsel;
-                hwc->event_base  = x86_pmu.perfctr;
+                hwc->config_base = x86_pmu_config_addr(hwc->idx);
+                hwc->event_base  = x86_pmu_event_addr(hwc->idx);
         }
 }
 
@@ -915,17 +1003,11 @@ static void x86_pmu_enable(struct pmu *pmu)
         x86_pmu.enable_all(added);
 }
 
-static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
-                                          u64 enable_mask)
-{
-        wrmsrl(hwc->config_base + hwc->idx, hwc->config | enable_mask);
-}
-
 static inline void x86_pmu_disable_event(struct perf_event *event)
 {
         struct hw_perf_event *hwc = &event->hw;
 
-        wrmsrl(hwc->config_base + hwc->idx, hwc->config);
+        wrmsrl(hwc->config_base, hwc->config);
 }
 
 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -978,7 +1060,7 @@ x86_perf_event_set_period(struct perf_event *event)
          */
         local64_set(&hwc->prev_count, (u64)-left);
 
-        wrmsrl(hwc->event_base + idx, (u64)(-left) & x86_pmu.cntval_mask);
+        wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
 
         /*
          * Due to erratum on certan cpu we need
@@ -986,7 +1068,7 @@ x86_perf_event_set_period(struct perf_event *event)
          * is updated properly
          */
         if (x86_pmu.perfctr_second_write) {
-                wrmsrl(hwc->event_base + idx,
+                wrmsrl(hwc->event_base,
                         (u64)(-left) & x86_pmu.cntval_mask);
         }
 
@@ -1113,8 +1195,8 @@ void perf_event_print_debug(void)
         pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
 
         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
-                rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
-                rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
+                rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
+                rdmsrl(x86_pmu_event_addr(idx), pmc_count);
 
                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
 
@@ -1389,7 +1471,7 @@ static void __init pmu_check_apic(void)
         pr_info("no hardware sampling interrupt available.\n");
 }
 
-int __init init_hw_perf_events(void)
+static int __init init_hw_perf_events(void)
 {
         struct event_constraint *c;
         int err;
@@ -1608,7 +1690,7 @@ out:
         return ret;
 }
 
-int x86_pmu_event_init(struct perf_event *event)
+static int x86_pmu_event_init(struct perf_event *event)
 {
         struct pmu *tmp;
         int err;
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index 67e2202a6039..461f62bbd774 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -127,6 +127,11 @@ static int amd_pmu_hw_config(struct perf_event *event)
 /*
  * AMD64 events are detected based on their event codes.
  */
+static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
+{
+        return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff);
+}
+
 static inline int amd_is_nb_event(struct hw_perf_event *hwc)
 {
         return (hwc->config & 0xe0) == 0xe0;
@@ -385,13 +390,181 @@ static __initconst const struct x86_pmu amd_pmu = {
         .cpu_dead               = amd_pmu_cpu_dead,
 };
 
+/* AMD Family 15h */
+
+#define AMD_EVENT_TYPE_MASK     0x000000F0ULL
+
+#define AMD_EVENT_FP            0x00000000ULL ... 0x00000010ULL
+#define AMD_EVENT_LS            0x00000020ULL ... 0x00000030ULL
+#define AMD_EVENT_DC            0x00000040ULL ... 0x00000050ULL
+#define AMD_EVENT_CU            0x00000060ULL ... 0x00000070ULL
+#define AMD_EVENT_IC_DE         0x00000080ULL ... 0x00000090ULL
+#define AMD_EVENT_EX_LS         0x000000C0ULL
+#define AMD_EVENT_DE            0x000000D0ULL
+#define AMD_EVENT_NB            0x000000E0ULL ... 0x000000F0ULL
+
+/*
+ * AMD family 15h event code/PMC mappings:
+ *
+ * type = event_code & 0x0F0:
+ *
+ * 0x000        FP      PERF_CTL[5:3]
+ * 0x010        FP      PERF_CTL[5:3]
+ * 0x020        LS      PERF_CTL[5:0]
+ * 0x030        LS      PERF_CTL[5:0]
+ * 0x040        DC      PERF_CTL[5:0]
+ * 0x050        DC      PERF_CTL[5:0]
+ * 0x060        CU      PERF_CTL[2:0]
+ * 0x070        CU      PERF_CTL[2:0]
+ * 0x080        IC/DE   PERF_CTL[2:0]
+ * 0x090        IC/DE   PERF_CTL[2:0]
+ * 0x0A0        ---
+ * 0x0B0        ---
+ * 0x0C0        EX/LS   PERF_CTL[5:0]
+ * 0x0D0        DE      PERF_CTL[2:0]
+ * 0x0E0        NB      NB_PERF_CTL[3:0]
+ * 0x0F0        NB      NB_PERF_CTL[3:0]
+ *
+ * Exceptions:
+ *
+ * 0x003        FP      PERF_CTL[3]
+ * 0x00B        FP      PERF_CTL[3]
+ * 0x00D        FP      PERF_CTL[3]
+ * 0x023        DE      PERF_CTL[2:0]
+ * 0x02D        LS      PERF_CTL[3]
+ * 0x02E        LS      PERF_CTL[3,0]
+ * 0x043        CU      PERF_CTL[2:0]
+ * 0x045        CU      PERF_CTL[2:0]
+ * 0x046        CU      PERF_CTL[2:0]
+ * 0x054        CU      PERF_CTL[2:0]
+ * 0x055        CU      PERF_CTL[2:0]
+ * 0x08F        IC      PERF_CTL[0]
+ * 0x187        DE      PERF_CTL[0]
+ * 0x188        DE      PERF_CTL[0]
+ * 0x0DB        EX      PERF_CTL[5:0]
+ * 0x0DC        LS      PERF_CTL[5:0]
+ * 0x0DD        LS      PERF_CTL[5:0]
+ * 0x0DE        LS      PERF_CTL[5:0]
+ * 0x0DF        LS      PERF_CTL[5:0]
+ * 0x1D6        EX      PERF_CTL[5:0]
+ * 0x1D8        EX      PERF_CTL[5:0]
+ */
+
+static struct event_constraint amd_f15_PMC0  = EVENT_CONSTRAINT(0, 0x01, 0);
+static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0);
+static struct event_constraint amd_f15_PMC3  = EVENT_CONSTRAINT(0, 0x08, 0);
+static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT(0, 0x09, 0);
+static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
+static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
+
+static struct event_constraint *
+amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+        unsigned int event_code = amd_get_event_code(&event->hw);
+
+        switch (event_code & AMD_EVENT_TYPE_MASK) {
+        case AMD_EVENT_FP:
+                switch (event_code) {
+                case 0x003:
+                case 0x00B:
+                case 0x00D:
+                        return &amd_f15_PMC3;
+                default:
+                        return &amd_f15_PMC53;
+                }
+        case AMD_EVENT_LS:
+        case AMD_EVENT_DC:
+        case AMD_EVENT_EX_LS:
+                switch (event_code) {
+                case 0x023:
+                case 0x043:
+                case 0x045:
+                case 0x046:
+                case 0x054:
+                case 0x055:
+                        return &amd_f15_PMC20;
+                case 0x02D:
+                        return &amd_f15_PMC3;
+                case 0x02E:
+                        return &amd_f15_PMC30;
+                default:
+                        return &amd_f15_PMC50;
+                }
+        case AMD_EVENT_CU:
+        case AMD_EVENT_IC_DE:
+        case AMD_EVENT_DE:
+                switch (event_code) {
+                case 0x08F:
+                case 0x187:
+                case 0x188:
+                        return &amd_f15_PMC0;
+                case 0x0DB ... 0x0DF:
+                case 0x1D6:
+                case 0x1D8:
+                        return &amd_f15_PMC50;
+                default:
+                        return &amd_f15_PMC20;
+                }
+        case AMD_EVENT_NB:
+                /* not yet implemented */
+                return &emptyconstraint;
+        default:
+                return &emptyconstraint;
+        }
+}
+
+static __initconst const struct x86_pmu amd_pmu_f15h = {
+        .name                   = "AMD Family 15h",
+        .handle_irq             = x86_pmu_handle_irq,
+        .disable_all            = x86_pmu_disable_all,
+        .enable_all             = x86_pmu_enable_all,
+        .enable                 = x86_pmu_enable_event,
+        .disable                = x86_pmu_disable_event,
+        .hw_config              = amd_pmu_hw_config,
+        .schedule_events        = x86_schedule_events,
+        .eventsel               = MSR_F15H_PERF_CTL,
+        .perfctr                = MSR_F15H_PERF_CTR,
+        .event_map              = amd_pmu_event_map,
+        .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
+        .num_counters           = 6,
+        .cntval_bits            = 48,
+        .cntval_mask            = (1ULL << 48) - 1,
+        .apic                   = 1,
+        /* use highest bit to detect overflow */
+        .max_period             = (1ULL << 47) - 1,
+        .get_event_constraints  = amd_get_event_constraints_f15h,
+        /* nortbridge counters not yet implemented: */
+#if 0
+        .put_event_constraints  = amd_put_event_constraints,
+
+        .cpu_prepare            = amd_pmu_cpu_prepare,
+        .cpu_starting           = amd_pmu_cpu_starting,
+        .cpu_dead               = amd_pmu_cpu_dead,
+#endif
+};
+
 static __init int amd_pmu_init(void)
 {
         /* Performance-monitoring supported from K7 and later: */
         if (boot_cpu_data.x86 < 6)
                 return -ENODEV;
 
-        x86_pmu = amd_pmu;
+        /*
+         * If core performance counter extensions exists, it must be
+         * family 15h, otherwise fail. See x86_pmu_addr_offset().
+         */
+        switch (boot_cpu_data.x86) {
+        case 0x15:
+                if (!cpu_has_perfctr_core)
+                        return -ENODEV;
+                x86_pmu = amd_pmu_f15h;
+                break;
+        default:
+                if (cpu_has_perfctr_core)
+                        return -ENODEV;
+                x86_pmu = amd_pmu;
+                break;
+        }
 
         /* Events are common for all AMDs */
         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 008835c1d79c..8fc2b2cee1da 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -1,5 +1,27 @@
 #ifdef CONFIG_CPU_SUP_INTEL
 
+#define MAX_EXTRA_REGS 2
+
+/*
+ * Per register state.
+ */
+struct er_account {
+        int                     ref;            /* reference count */
+        unsigned int            extra_reg;      /* extra MSR number */
+        u64                     extra_config;   /* extra MSR config */
+};
+
+/*
+ * Per core state
+ * This used to coordinate shared registers for HT threads.
+ */
+struct intel_percore {
+        raw_spinlock_t          lock;           /* protect structure */
+        struct er_account       regs[MAX_EXTRA_REGS];
+        int                     refcnt;         /* number of threads */
+        unsigned                core_id;
+};
+
 /*
  * Intel PerfMon, used on Core and later.
  */
@@ -64,6 +86,18 @@ static struct event_constraint intel_nehalem_event_constraints[] =
         EVENT_CONSTRAINT_END
 };
 
+static struct extra_reg intel_nehalem_extra_regs[] =
+{
+        INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
+        EVENT_EXTRA_END
+};
+
+static struct event_constraint intel_nehalem_percore_constraints[] =
+{
+        INTEL_EVENT_CONSTRAINT(0xb7, 0),
+        EVENT_CONSTRAINT_END
+};
+
 static struct event_constraint intel_westmere_event_constraints[] =
 {
         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
@@ -76,6 +110,33 @@ static struct event_constraint intel_westmere_event_constraints[] =
         EVENT_CONSTRAINT_END
 };
 
+static struct event_constraint intel_snb_event_constraints[] =
+{
+        FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+        FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
+        /* FIXED_EVENT_CONSTRAINT(0x013c, 2), CPU_CLK_UNHALTED.REF */
+        INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
+        INTEL_EVENT_CONSTRAINT(0xb7, 0x1), /* OFF_CORE_RESPONSE_0 */
+        INTEL_EVENT_CONSTRAINT(0xbb, 0x8), /* OFF_CORE_RESPONSE_1 */
+        INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
+        INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
+        EVENT_CONSTRAINT_END
+};
+
+static struct extra_reg intel_westmere_extra_regs[] =
+{
+        INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
+        INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff),
+        EVENT_EXTRA_END
+};
+
+static struct event_constraint intel_westmere_percore_constraints[] =
+{
+        INTEL_EVENT_CONSTRAINT(0xb7, 0),
+        INTEL_EVENT_CONSTRAINT(0xbb, 0),
+        EVENT_CONSTRAINT_END
+};
+
 static struct event_constraint intel_gen_event_constraints[] =
 {
         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
@@ -89,6 +150,106 @@ static u64 intel_pmu_event_map(int hw_event)
         return intel_perfmon_event_map[hw_event];
 }
 
+static __initconst const u64 snb_hw_cache_event_ids
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS        */
+                [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPLACEMENT              */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES       */
+                [ C(RESULT_MISS)   ] = 0x0851, /* L1D.ALL_M_REPLACEMENT        */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0x024e, /* HW_PRE_REQ.DL1_MISS          */
+        },
+ },
+ [ C(L1I ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0x0280, /* ICACHE.MISSES */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0x0,
+        },
+ },
+ [ C(LL  ) ] = {
+        /*
+         * TBD: Need Off-core Response Performance Monitoring support
+         */
+        [ C(OP_READ) ] = {
+                /* OFFCORE_RESPONSE_0.ANY_DATA.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE_1.ANY_DATA.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01bb,
+        },
+        [ C(OP_WRITE) ] = {
+                /* OFFCORE_RESPONSE_0.ANY_RFO.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE_1.ANY_RFO.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01bb,
+        },
+        [ C(OP_PREFETCH) ] = {
+                /* OFFCORE_RESPONSE_0.PREFETCH.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE_1.PREFETCH.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01bb,
+        },
+ },
+ [ C(DTLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
+                [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
+                [ C(RESULT_MISS)   ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0x0,
+        },
+ },
+ [ C(ITLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT         */
+                [ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK    */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+ [ C(BPU ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
+                [ C(RESULT_MISS)   ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+};
+
 static __initconst const u64 westmere_hw_cache_event_ids
                                 [PERF_COUNT_HW_CACHE_MAX]
                                 [PERF_COUNT_HW_CACHE_OP_MAX]
@@ -124,16 +285,26 @@ static __initconst const u64 westmere_hw_cache_event_ids
  },
  [ C(LL  ) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
-                [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
+                /* OFFCORE_RESPONSE_0.ANY_DATA.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE_1.ANY_DATA.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01bb,
         },
+        /*
+         * Use RFO, not WRITEBACK, because a write miss would typically occur
+         * on RFO.
+         */
         [ C(OP_WRITE) ] = {
-                [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
-                [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
+                /* OFFCORE_RESPONSE_1.ANY_RFO.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01bb,
+                /* OFFCORE_RESPONSE_0.ANY_RFO.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
         [ C(OP_PREFETCH) ] = {
-                [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
-                [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
+                /* OFFCORE_RESPONSE_0.PREFETCH.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE_1.PREFETCH.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01bb,
         },
  },
  [ C(DTLB) ] = {
@@ -180,6 +351,39 @@ static __initconst const u64 westmere_hw_cache_event_ids
  },
 };
 
+/*
+ * OFFCORE_RESPONSE MSR bits (subset), See IA32 SDM Vol 3 30.6.1.3
+ */
+
+#define DMND_DATA_RD     (1 << 0)
+#define DMND_RFO         (1 << 1)
+#define DMND_WB          (1 << 3)
+#define PF_DATA_RD       (1 << 4)
+#define PF_DATA_RFO      (1 << 5)
+#define RESP_UNCORE_HIT  (1 << 8)
+#define RESP_MISS        (0xf600) /* non uncore hit */
+
+static __initconst const u64 nehalem_hw_cache_extra_regs
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL  ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = DMND_DATA_RD|RESP_UNCORE_HIT,
+                [ C(RESULT_MISS)   ] = DMND_DATA_RD|RESP_MISS,
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = DMND_RFO|DMND_WB|RESP_UNCORE_HIT,
+                [ C(RESULT_MISS)   ] = DMND_RFO|DMND_WB|RESP_MISS,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = PF_DATA_RD|PF_DATA_RFO|RESP_UNCORE_HIT,
+                [ C(RESULT_MISS)   ] = PF_DATA_RD|PF_DATA_RFO|RESP_MISS,
+        },
+ }
+};
+
 static __initconst const u64 nehalem_hw_cache_event_ids
                                 [PERF_COUNT_HW_CACHE_MAX]
                                 [PERF_COUNT_HW_CACHE_OP_MAX]
@@ -215,16 +419,26 @@ static __initconst const u64 nehalem_hw_cache_event_ids
  },
  [ C(LL  ) ] = {
         [ C(OP_READ) ] = {
-                [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
-                [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
+                /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
+        /*
+         * Use RFO, not WRITEBACK, because a write miss would typically occur
+         * on RFO.
+         */
         [ C(OP_WRITE) ] = {
-                [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
-                [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
+                /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
         [ C(OP_PREFETCH) ] = {
-                [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
-                [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
+                /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
+                [ C(RESULT_ACCESS) ] = 0x01b7,
+                /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
+                [ C(RESULT_MISS)   ] = 0x01b7,
         },
  },
  [ C(DTLB) ] = {
@@ -691,8 +905,8 @@ static void intel_pmu_reset(void)
         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
 
         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
-                checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
-                checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
+                checking_wrmsrl(x86_pmu_config_addr(idx), 0ull);
+                checking_wrmsrl(x86_pmu_event_addr(idx),  0ull);
         }
         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
@@ -794,6 +1008,67 @@ intel_bts_constraints(struct perf_event *event)
 }
 
 static struct event_constraint *
+intel_percore_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+        struct hw_perf_event *hwc = &event->hw;
+        unsigned int e = hwc->config & ARCH_PERFMON_EVENTSEL_EVENT;
+        struct event_constraint *c;
+        struct intel_percore *pc;
+        struct er_account *era;
+        int i;
+        int free_slot;
+        int found;
+
+        if (!x86_pmu.percore_constraints || hwc->extra_alloc)
+                return NULL;
+
+        for (c = x86_pmu.percore_constraints; c->cmask; c++) {
+                if (e != c->code)
+                        continue;
+
+                /*
+                 * Allocate resource per core.
+                 */
+                pc = cpuc->per_core;
+                if (!pc)
+                        break;
+                c = &emptyconstraint;
+                raw_spin_lock(&pc->lock);
+                free_slot = -1;
+                found = 0;
+                for (i = 0; i < MAX_EXTRA_REGS; i++) {
+                        era = &pc->regs[i];
+                        if (era->ref > 0 && hwc->extra_reg == era->extra_reg) {
+                                /* Allow sharing same config */
+                                if (hwc->extra_config == era->extra_config) {
+                                        era->ref++;
+                                        cpuc->percore_used = 1;
+                                        hwc->extra_alloc = 1;
+                                        c = NULL;
+                                }
+                                /* else conflict */
+                                found = 1;
+                                break;
+                        } else if (era->ref == 0 && free_slot == -1)
+                                free_slot = i;
+                }
+                if (!found && free_slot != -1) {
+                        era = &pc->regs[free_slot];
+                        era->ref = 1;
+                        era->extra_reg = hwc->extra_reg;
+                        era->extra_config = hwc->extra_config;
+                        cpuc->percore_used = 1;
+                        hwc->extra_alloc = 1;
+                        c = NULL;
+                }
+                raw_spin_unlock(&pc->lock);
+                return c;
+        }
+
+        return NULL;
+}
+
+static struct event_constraint *
 intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
 {
         struct event_constraint *c;
@@ -806,9 +1081,51 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event
         if (c)
                 return c;
 
+        c = intel_percore_constraints(cpuc, event);
+        if (c)
+                return c;
+
         return x86_get_event_constraints(cpuc, event);
 }
 
+static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
+                                        struct perf_event *event)
+{
+        struct extra_reg *er;
+        struct intel_percore *pc;
+        struct er_account *era;
+        struct hw_perf_event *hwc = &event->hw;
+        int i, allref;
+
+        if (!cpuc->percore_used)
+                return;
+
+        for (er = x86_pmu.extra_regs; er->msr; er++) {
+                if (er->event != (hwc->config & er->config_mask))
+                        continue;
+
+                pc = cpuc->per_core;
+                raw_spin_lock(&pc->lock);
+                for (i = 0; i < MAX_EXTRA_REGS; i++) {
+                        era = &pc->regs[i];
+                        if (era->ref > 0 &&
+                            era->extra_config == hwc->extra_config &&
+                            era->extra_reg == er->msr) {
+                                era->ref--;
+                                hwc->extra_alloc = 0;
+                                break;
+                        }
+                }
+                allref = 0;
+                for (i = 0; i < MAX_EXTRA_REGS; i++)
+                        allref += pc->regs[i].ref;
+                if (allref == 0)
+                        cpuc->percore_used = 0;
+                raw_spin_unlock(&pc->lock);
+                break;
+        }
+}
+
 static int intel_pmu_hw_config(struct perf_event *event)
 {
         int ret = x86_pmu_hw_config(event);
@@ -880,20 +1197,67 @@ static __initconst const struct x86_pmu core_pmu = {
          */
         .max_period             = (1ULL << 31) - 1,
         .get_event_constraints  = intel_get_event_constraints,
+        .put_event_constraints  = intel_put_event_constraints,
         .event_constraints      = intel_core_event_constraints,
 };
 
+static int intel_pmu_cpu_prepare(int cpu)
+{
+        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
+
+        if (!cpu_has_ht_siblings())
+                return NOTIFY_OK;
+
+        cpuc->per_core = kzalloc_node(sizeof(struct intel_percore),
+                                      GFP_KERNEL, cpu_to_node(cpu));
+        if (!cpuc->per_core)
+                return NOTIFY_BAD;
+
+        raw_spin_lock_init(&cpuc->per_core->lock);
+        cpuc->per_core->core_id = -1;
+        return NOTIFY_OK;
+}
+
 static void intel_pmu_cpu_starting(int cpu)
 {
+        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
+        int core_id = topology_core_id(cpu);
+        int i;
+
         init_debug_store_on_cpu(cpu);
         /*
          * Deal with CPUs that don't clear their LBRs on power-up.
          */
         intel_pmu_lbr_reset();
+
+        if (!cpu_has_ht_siblings())
+                return;
+
+        for_each_cpu(i, topology_thread_cpumask(cpu)) {
+                struct intel_percore *pc = per_cpu(cpu_hw_events, i).per_core;
+
+                if (pc && pc->core_id == core_id) {
+                        kfree(cpuc->per_core);
+                        cpuc->per_core = pc;
+                        break;
+                }
+        }
+
+        cpuc->per_core->core_id = core_id;
+        cpuc->per_core->refcnt++;
 }
 
 static void intel_pmu_cpu_dying(int cpu)
 {
+        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
+        struct intel_percore *pc = cpuc->per_core;
+
+        if (pc) {
+                if (pc->core_id == -1 || --pc->refcnt == 0)
+                        kfree(pc);
+                cpuc->per_core = NULL;
+        }
+
         fini_debug_store_on_cpu(cpu);
 }
 
@@ -918,7 +1282,9 @@ static __initconst const struct x86_pmu intel_pmu = {
          */
         .max_period             = (1ULL << 31) - 1,
         .get_event_constraints  = intel_get_event_constraints,
+        .put_event_constraints  = intel_put_event_constraints,
 
+        .cpu_prepare            = intel_pmu_cpu_prepare,
         .cpu_starting           = intel_pmu_cpu_starting,
         .cpu_dying              = intel_pmu_cpu_dying,
 };
@@ -1024,6 +1390,7 @@ static __init int intel_pmu_init(void)
                 intel_pmu_lbr_init_core();
 
                 x86_pmu.event_constraints = intel_core2_event_constraints;
+                x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
                 pr_cont("Core2 events, ");
                 break;
 
@@ -1032,11 +1399,16 @@ static __init int intel_pmu_init(void)
         case 46: /* 45 nm nehalem-ex, "Beckton" */
                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
                        sizeof(hw_cache_event_ids));
+                memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
+                       sizeof(hw_cache_extra_regs));
 
                 intel_pmu_lbr_init_nhm();
 
                 x86_pmu.event_constraints = intel_nehalem_event_constraints;
+                x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
+                x86_pmu.percore_constraints = intel_nehalem_percore_constraints;
                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
+                x86_pmu.extra_regs = intel_nehalem_extra_regs;
                 pr_cont("Nehalem events, ");
                 break;
 
@@ -1047,6 +1419,7 @@ static __init int intel_pmu_init(void)
                 intel_pmu_lbr_init_atom();
 
                 x86_pmu.event_constraints = intel_gen_event_constraints;
+                x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
                 pr_cont("Atom events, ");
                 break;
 
@@ -1054,14 +1427,30 @@ static __init int intel_pmu_init(void)
         case 44: /* 32 nm nehalem, "Gulftown" */
                 memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
                        sizeof(hw_cache_event_ids));
+                memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
+                       sizeof(hw_cache_extra_regs));
 
                 intel_pmu_lbr_init_nhm();
 
                 x86_pmu.event_constraints = intel_westmere_event_constraints;
+                x86_pmu.percore_constraints = intel_westmere_percore_constraints;
                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
+                x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
+                x86_pmu.extra_regs = intel_westmere_extra_regs;
                 pr_cont("Westmere events, ");
                 break;
 
+        case 42: /* SandyBridge */
+                memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
+                       sizeof(hw_cache_event_ids));
+
+                intel_pmu_lbr_init_nhm();
+
+                x86_pmu.event_constraints = intel_snb_event_constraints;
+                x86_pmu.pebs_constraints = intel_snb_pebs_events;
+                pr_cont("SandyBridge events, ");
+                break;
+
         default:
                 /*
                  * default constraints for v2 and up
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index b7dcd9f2b8a0..b95c66ae4a2a 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -361,30 +361,88 @@ static int intel_pmu_drain_bts_buffer(void)
 /*
  * PEBS
  */
-
-static struct event_constraint intel_core_pebs_events[] = {
-        PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
+static struct event_constraint intel_core2_pebs_event_constraints[] = {
+        PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
         PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
         PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
         PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
-        PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
-        PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
-        PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
-        PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
-        PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
+        INTEL_EVENT_CONSTRAINT(0xcb, 0x1),  /* MEM_LOAD_RETIRED.* */
+        EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_atom_pebs_event_constraints[] = {
+        PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
+        PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* MISPREDICTED_BRANCH_RETIRED */
+        INTEL_EVENT_CONSTRAINT(0xcb, 0x1),  /* MEM_LOAD_RETIRED.* */
         EVENT_CONSTRAINT_END
 };
 
-static struct event_constraint intel_nehalem_pebs_events[] = {
-        PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
-        PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
-        PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
-        PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
-        PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
-        PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
-        PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
-        PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
-        PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
+static struct event_constraint intel_nehalem_pebs_event_constraints[] = {
+        INTEL_EVENT_CONSTRAINT(0x0b, 0xf),  /* MEM_INST_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0x0f, 0xf),  /* MEM_UNCORE_RETIRED.* */
+        PEBS_EVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
+        INTEL_EVENT_CONSTRAINT(0xc0, 0xf),  /* INST_RETIRED.ANY */
+        INTEL_EVENT_CONSTRAINT(0xc2, 0xf),  /* UOPS_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xc4, 0xf),  /* BR_INST_RETIRED.* */
+        PEBS_EVENT_CONSTRAINT(0x02c5, 0xf), /* BR_MISP_RETIRED.NEAR_CALL */
+        INTEL_EVENT_CONSTRAINT(0xc7, 0xf),  /* SSEX_UOPS_RETIRED.* */
+        PEBS_EVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
+        INTEL_EVENT_CONSTRAINT(0xcb, 0xf),  /* MEM_LOAD_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xf7, 0xf),  /* FP_ASSIST.* */
+        EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_westmere_pebs_event_constraints[] = {
+        INTEL_EVENT_CONSTRAINT(0x0b, 0xf),  /* MEM_INST_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0x0f, 0xf),  /* MEM_UNCORE_RETIRED.* */
+        PEBS_EVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
+        INTEL_EVENT_CONSTRAINT(0xc0, 0xf),  /* INSTR_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xc2, 0xf),  /* UOPS_RETIRED.* */
+
+        INTEL_EVENT_CONSTRAINT(0xc4, 0xf),  /* BR_INST_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xc5, 0xf),  /* BR_MISP_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xc7, 0xf),  /* SSEX_UOPS_RETIRED.* */
+        PEBS_EVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
+        INTEL_EVENT_CONSTRAINT(0xcb, 0xf),  /* MEM_LOAD_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xf7, 0xf),  /* FP_ASSIST.* */
+        EVENT_CONSTRAINT_END
+};
+
+static struct event_constraint intel_snb_pebs_events[] = {
+        PEBS_EVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
+        PEBS_EVENT_CONSTRAINT(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
+        PEBS_EVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
+        PEBS_EVENT_CONSTRAINT(0x01c4, 0xf), /* BR_INST_RETIRED.CONDITIONAL */
+        PEBS_EVENT_CONSTRAINT(0x02c4, 0xf), /* BR_INST_RETIRED.NEAR_CALL */
+        PEBS_EVENT_CONSTRAINT(0x04c4, 0xf), /* BR_INST_RETIRED.ALL_BRANCHES */
+        PEBS_EVENT_CONSTRAINT(0x08c4, 0xf), /* BR_INST_RETIRED.NEAR_RETURN */
+        PEBS_EVENT_CONSTRAINT(0x10c4, 0xf), /* BR_INST_RETIRED.NOT_TAKEN */
+        PEBS_EVENT_CONSTRAINT(0x20c4, 0xf), /* BR_INST_RETIRED.NEAR_TAKEN */
+        PEBS_EVENT_CONSTRAINT(0x40c4, 0xf), /* BR_INST_RETIRED.FAR_BRANCH */
+        PEBS_EVENT_CONSTRAINT(0x01c5, 0xf), /* BR_MISP_RETIRED.CONDITIONAL */
+        PEBS_EVENT_CONSTRAINT(0x02c5, 0xf), /* BR_MISP_RETIRED.NEAR_CALL */
+        PEBS_EVENT_CONSTRAINT(0x04c5, 0xf), /* BR_MISP_RETIRED.ALL_BRANCHES */
+        PEBS_EVENT_CONSTRAINT(0x10c5, 0xf), /* BR_MISP_RETIRED.NOT_TAKEN */
+        PEBS_EVENT_CONSTRAINT(0x20c5, 0xf), /* BR_MISP_RETIRED.TAKEN */
+        PEBS_EVENT_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
+        PEBS_EVENT_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORE */
+        PEBS_EVENT_CONSTRAINT(0x11d0, 0xf), /* MEM_UOP_RETIRED.STLB_MISS_LOADS */
+        PEBS_EVENT_CONSTRAINT(0x12d0, 0xf), /* MEM_UOP_RETIRED.STLB_MISS_STORES */
+        PEBS_EVENT_CONSTRAINT(0x21d0, 0xf), /* MEM_UOP_RETIRED.LOCK_LOADS */
+        PEBS_EVENT_CONSTRAINT(0x22d0, 0xf), /* MEM_UOP_RETIRED.LOCK_STORES */
+        PEBS_EVENT_CONSTRAINT(0x41d0, 0xf), /* MEM_UOP_RETIRED.SPLIT_LOADS */
+        PEBS_EVENT_CONSTRAINT(0x42d0, 0xf), /* MEM_UOP_RETIRED.SPLIT_STORES */
+        PEBS_EVENT_CONSTRAINT(0x81d0, 0xf), /* MEM_UOP_RETIRED.ANY_LOADS */
+        PEBS_EVENT_CONSTRAINT(0x82d0, 0xf), /* MEM_UOP_RETIRED.ANY_STORES */
+        PEBS_EVENT_CONSTRAINT(0x01d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.L1_HIT */
+        PEBS_EVENT_CONSTRAINT(0x02d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.L2_HIT */
+        PEBS_EVENT_CONSTRAINT(0x04d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.LLC_HIT */
+        PEBS_EVENT_CONSTRAINT(0x40d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.HIT_LFB */
+        PEBS_EVENT_CONSTRAINT(0x01d2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS */
+        PEBS_EVENT_CONSTRAINT(0x02d2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT */
+        PEBS_EVENT_CONSTRAINT(0x04d2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM */
+        PEBS_EVENT_CONSTRAINT(0x08d2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_NONE */
+        PEBS_EVENT_CONSTRAINT(0x02d4, 0xf), /* MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS */
         EVENT_CONSTRAINT_END
 };
 
@@ -695,20 +753,17 @@ static void intel_ds_init(void)
                         printk(KERN_CONT "PEBS fmt0%c, ", pebs_type);
                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
-                        x86_pmu.pebs_constraints = intel_core_pebs_events;
                         break;
 
                 case 1:
                         printk(KERN_CONT "PEBS fmt1%c, ", pebs_type);
                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
-                        x86_pmu.pebs_constraints = intel_nehalem_pebs_events;
                         break;
 
                 default:
                         printk(KERN_CONT "no PEBS fmt%d%c, ", format, pebs_type);
                         x86_pmu.pebs = 0;
-                        break;
                 }
         }
 }
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c
index ff751a9f182b..3769ac822f96 100644
--- a/arch/x86/kernel/cpu/perf_event_p4.c
+++ b/arch/x86/kernel/cpu/perf_event_p4.c
@@ -764,9 +764,9 @@ static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
         u64 v;
 
         /* an official way for overflow indication */
-        rdmsrl(hwc->config_base + hwc->idx, v);
+        rdmsrl(hwc->config_base, v);
         if (v & P4_CCCR_OVF) {
-                wrmsrl(hwc->config_base + hwc->idx, v & ~P4_CCCR_OVF);
+                wrmsrl(hwc->config_base, v & ~P4_CCCR_OVF);
                 return 1;
         }
 
@@ -815,7 +815,7 @@ static inline void p4_pmu_disable_event(struct perf_event *event)
          * state we need to clear P4_CCCR_OVF, otherwise interrupt get
          * asserted again and again
          */
-        (void)checking_wrmsrl(hwc->config_base + hwc->idx,
+        (void)checking_wrmsrl(hwc->config_base,
                 (u64)(p4_config_unpack_cccr(hwc->config)) &
                         ~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED);
 }
@@ -885,7 +885,7 @@ static void p4_pmu_enable_event(struct perf_event *event)
         p4_pmu_enable_pebs(hwc->config);
 
         (void)checking_wrmsrl(escr_addr, escr_conf);
-        (void)checking_wrmsrl(hwc->config_base + hwc->idx,
+        (void)checking_wrmsrl(hwc->config_base,
                                 (cccr & ~P4_CCCR_RESERVED) | P4_CCCR_ENABLE);
 }
 
diff --git a/arch/x86/kernel/cpu/perf_event_p6.c b/arch/x86/kernel/cpu/perf_event_p6.c
index 34ba07be2cda..20c097e33860 100644
--- a/arch/x86/kernel/cpu/perf_event_p6.c
+++ b/arch/x86/kernel/cpu/perf_event_p6.c
@@ -68,7 +68,7 @@ p6_pmu_disable_event(struct perf_event *event)
         if (cpuc->enabled)
                 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
 
-        (void)checking_wrmsrl(hwc->config_base + hwc->idx, val);
+        (void)checking_wrmsrl(hwc->config_base, val);
 }
 
 static void p6_pmu_enable_event(struct perf_event *event)
@@ -81,7 +81,7 @@ static void p6_pmu_enable_event(struct perf_event *event)
         if (cpuc->enabled)
                 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
 
-        (void)checking_wrmsrl(hwc->config_base + hwc->idx, val);
+        (void)checking_wrmsrl(hwc->config_base, val);
 }
 
 static __initconst const struct x86_pmu p6_pmu = {
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index d5a236615501..966512b2cacf 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -46,6 +46,8 @@ static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
         /* returns the bit offset of the performance counter register */
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_AMD:
+                if (msr >= MSR_F15H_PERF_CTR)
+                        return (msr - MSR_F15H_PERF_CTR) >> 1;
                 return msr - MSR_K7_PERFCTR0;
         case X86_VENDOR_INTEL:
                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
@@ -70,6 +72,8 @@ static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
         /* returns the bit offset of the event selection register */
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_AMD:
+                if (msr >= MSR_F15H_PERF_CTL)
+                        return (msr - MSR_F15H_PERF_CTL) >> 1;
                 return msr - MSR_K7_EVNTSEL0;
         case X86_VENDOR_INTEL:
                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index df20723a6a1b..220a1c11cfde 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -320,31 +320,6 @@ void die(const char *str, struct pt_regs *regs, long err)
         oops_end(flags, regs, sig);
 }
 
-void notrace __kprobes
-die_nmi(char *str, struct pt_regs *regs, int do_panic)
-{
-        unsigned long flags;
-
-        if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, 2, SIGINT) == NOTIFY_STOP)
-                return;
-
-        /*
-         * We are in trouble anyway, lets at least try
-         * to get a message out.
-         */
-        flags = oops_begin();
-        printk(KERN_EMERG "%s", str);
-        printk(" on CPU%d, ip %08lx, registers:\n",
-                smp_processor_id(), regs->ip);
-        show_registers(regs);
-        oops_end(flags, regs, 0);
-        if (do_panic || panic_on_oops)
-                panic("Non maskable interrupt");
-        nmi_exit();
-        local_irq_enable();
-        do_exit(SIGBUS);
-}
-
 static int __init oops_setup(char *s)
 {
         if (!s)
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index c8b4efad7ebb..f5accf8eaa78 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -65,6 +65,8 @@
 #define sysexit_audit   syscall_exit_work
 #endif
 
+        .section .entry.text, "ax"
+
 /*
  * We use macros for low-level operations which need to be overridden
  * for paravirtualization.  The following will never clobber any registers:
@@ -788,7 +790,7 @@ ENDPROC(ptregs_clone)
  */
 .section .init.rodata,"a"
 ENTRY(interrupt)
-.text
+.section .entry.text, "ax"
         .p2align 5
         .p2align CONFIG_X86_L1_CACHE_SHIFT
 ENTRY(irq_entries_start)
@@ -807,7 +809,7 @@ vector=FIRST_EXTERNAL_VECTOR
       .endif
       .previous
         .long 1b
-      .text
+      .section .entry.text, "ax"
 vector=vector+1
     .endif
   .endr
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index aed1ffbeb0c9..0a0ed794edb2 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -61,6 +61,8 @@
 #define __AUDIT_ARCH_LE    0x40000000
 
         .code64
+        .section .entry.text, "ax"
+
 #ifdef CONFIG_FUNCTION_TRACER
 #ifdef CONFIG_DYNAMIC_FTRACE
 ENTRY(mcount)
@@ -744,7 +746,7 @@ END(stub_rt_sigreturn)
  */
         .section .init.rodata,"a"
 ENTRY(interrupt)
-        .text
+        .section .entry.text
         .p2align 5
         .p2align CONFIG_X86_L1_CACHE_SHIFT
 ENTRY(irq_entries_start)
@@ -763,7 +765,7 @@ vector=FIRST_EXTERNAL_VECTOR
       .endif
       .previous
         .quad 1b
-      .text
+      .section .entry.text
 vector=vector+1
     .endif
   .endr
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 382eb2936d4d..a93742a57468 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -437,18 +437,19 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
                 return;
         }
 
-        if (ftrace_push_return_trace(old, self_addr, &trace.depth,
-                    frame_pointer) == -EBUSY) {
-                *parent = old;
-                return;
-        }
-
         trace.func = self_addr;
+        trace.depth = current->curr_ret_stack + 1;
 
         /* Only trace if the calling function expects to */
         if (!ftrace_graph_entry(&trace)) {
-                current->curr_ret_stack--;
                 *parent = old;
+                return;
+        }
+
+        if (ftrace_push_return_trace(old, self_addr, &trace.depth,
+                    frame_pointer) == -EBUSY) {
+                *parent = old;
+                return;
         }
 }
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c
index a4130005028a..7c64c420a9f6 100644
--- a/arch/x86/kernel/kgdb.c
+++ b/arch/x86/kernel/kgdb.c
@@ -533,15 +533,6 @@ static int __kgdb_notify(struct die_args *args, unsigned long cmd)
                 }
                 return NOTIFY_DONE;
 
-        case DIE_NMIWATCHDOG:
-                if (atomic_read(&kgdb_active) != -1) {
-                        /* KGDB CPU roundup: */
-                        kgdb_nmicallback(raw_smp_processor_id(), regs);
-                        return NOTIFY_STOP;
-                }
-                /* Enter debugger: */
-                break;
-
         case DIE_DEBUG:
                 if (atomic_read(&kgdb_cpu_doing_single_step) != -1) {
                         if (user_mode(regs))
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index d91c477b3f62..c969fd9d1566 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -1276,6 +1276,14 @@ static int __kprobes can_optimize(unsigned long paddr)
         if (!kallsyms_lookup_size_offset(paddr, &size, &offset))
                 return 0;
 
+        /*
+         * Do not optimize in the entry code due to the unstable
+         * stack handling.
+         */
+        if ((paddr >= (unsigned long )__entry_text_start) &&
+            (paddr <  (unsigned long )__entry_text_end))
+                return 0;
+
         /* Check there is enough space for a relative jump. */
         if (size - offset < RELATIVEJUMP_SIZE)
                 return 0;
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index bf4700755184..6d4341d5c52a 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -105,6 +105,7 @@ SECTIONS
                 SCHED_TEXT
                 LOCK_TEXT
                 KPROBES_TEXT
+                ENTRY_TEXT
                 IRQENTRY_TEXT
                 *(.fixup)
                 *(.gnu.warning)