15 files changed, 398 insertions, 180 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 36b05ed0bb3c..a2570490eab6 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -39,10 +39,12 @@ config X86
         select HAVE_DMA_CONTIGUOUS if !SWIOTLB
         select HAVE_KRETPROBES
         select HAVE_OPTPROBES
+        select HAVE_KPROBES_ON_FTRACE
         select HAVE_FTRACE_MCOUNT_RECORD
         select HAVE_FENTRY if X86_64
         select HAVE_C_RECORDMCOUNT
         select HAVE_DYNAMIC_FTRACE
+        select HAVE_DYNAMIC_FTRACE_WITH_REGS
         select HAVE_FUNCTION_TRACER
         select HAVE_FUNCTION_GRAPH_TRACER
         select HAVE_FUNCTION_GRAPH_FP_TEST
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 2d9075e863a0..93fe929d1cee 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -167,6 +167,7 @@
 #define X86_FEATURE_TBM         (6*32+21) /* trailing bit manipulations */
 #define X86_FEATURE_TOPOEXT     (6*32+22) /* topology extensions CPUID leafs */
 #define X86_FEATURE_PERFCTR_CORE (6*32+23) /* core performance counter extensions */
+#define X86_FEATURE_PERFCTR_NB  (6*32+24) /* NB performance counter extensions */
 
 /*
  * Auxiliary flags: Linux defined - For features scattered in various
@@ -309,6 +310,7 @@ extern const char * const x86_power_flags[32];
 #define cpu_has_hypervisor      boot_cpu_has(X86_FEATURE_HYPERVISOR)
 #define cpu_has_pclmulqdq       boot_cpu_has(X86_FEATURE_PCLMULQDQ)
 #define cpu_has_perfctr_core    boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
+#define cpu_has_perfctr_nb      boot_cpu_has(X86_FEATURE_PERFCTR_NB)
 #define cpu_has_cx8             boot_cpu_has(X86_FEATURE_CX8)
 #define cpu_has_cx16            boot_cpu_has(X86_FEATURE_CX16)
 #define cpu_has_eager_fpu       boot_cpu_has(X86_FEATURE_EAGER_FPU)
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index 9a25b522d377..86cb51e1ca96 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -44,7 +44,6 @@
 
 #ifdef CONFIG_DYNAMIC_FTRACE
 #define ARCH_SUPPORTS_FTRACE_OPS 1
-#define ARCH_SUPPORTS_FTRACE_SAVE_REGS
 #endif
 
 #ifndef __ASSEMBLY__
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 4fabcdf1cfa7..57cb63402213 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -29,8 +29,13 @@
 #define ARCH_PERFMON_EVENTSEL_INV                       (1ULL << 23)
 #define ARCH_PERFMON_EVENTSEL_CMASK                     0xFF000000ULL
 
-#define AMD_PERFMON_EVENTSEL_GUESTONLY                  (1ULL << 40)
-#define AMD_PERFMON_EVENTSEL_HOSTONLY                   (1ULL << 41)
+#define AMD64_EVENTSEL_INT_CORE_ENABLE                  (1ULL << 36)
+#define AMD64_EVENTSEL_GUESTONLY                        (1ULL << 40)
+#define AMD64_EVENTSEL_HOSTONLY                         (1ULL << 41)
+
+#define AMD64_EVENTSEL_INT_CORE_SEL_SHIFT               37
+#define AMD64_EVENTSEL_INT_CORE_SEL_MASK                \
+        (0xFULL << AMD64_EVENTSEL_INT_CORE_SEL_SHIFT)
 
 #define AMD64_EVENTSEL_EVENT    \
         (ARCH_PERFMON_EVENTSEL_EVENT | (0x0FULL << 32))
@@ -46,8 +51,12 @@
 #define AMD64_RAW_EVENT_MASK            \
         (X86_RAW_EVENT_MASK          |  \
          AMD64_EVENTSEL_EVENT)
+#define AMD64_RAW_EVENT_MASK_NB         \
+        (AMD64_EVENTSEL_EVENT        |  \
+         ARCH_PERFMON_EVENTSEL_UMASK)
 #define AMD64_NUM_COUNTERS                              4
 #define AMD64_NUM_COUNTERS_CORE                         6
+#define AMD64_NUM_COUNTERS_NB                           4
 
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL           0x3c
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK         (0x00 << 8)
diff --git a/arch/x86/include/uapi/asm/msr-index.h b/arch/x86/include/uapi/asm/msr-index.h
index 433a59fb1a74..075a40255591 100644
--- a/arch/x86/include/uapi/asm/msr-index.h
+++ b/arch/x86/include/uapi/asm/msr-index.h
@@ -194,6 +194,8 @@
 /* Fam 15h MSRs */
 #define MSR_F15H_PERF_CTL               0xc0010200
 #define MSR_F15H_PERF_CTR               0xc0010201
+#define MSR_F15H_NB_PERF_CTL            0xc0010240
+#define MSR_F15H_NB_PERF_CTR            0xc0010241
 
 /* Fam 10h MSRs */
 #define MSR_FAM10H_MMIO_CONF_BASE       0xc0010058
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 34e923a53762..ac3b3d002833 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -65,8 +65,7 @@ obj-$(CONFIG_X86_TSC)		+= trace_clock.o
 obj-$(CONFIG_KEXEC)             += machine_kexec_$(BITS).o
 obj-$(CONFIG_KEXEC)             += relocate_kernel_$(BITS).o crash.o
 obj-$(CONFIG_CRASH_DUMP)        += crash_dump_$(BITS).o
-obj-$(CONFIG_KPROBES)           += kprobes.o
-obj-$(CONFIG_OPTPROBES)         += kprobes-opt.o
+obj-y                           += kprobes/
 obj-$(CONFIG_MODULES)           += module.o
 obj-$(CONFIG_DOUBLEFAULT)       += doublefault_32.o
 obj-$(CONFIG_KGDB)              += kgdb.o
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 6774c17a5576..bf0f01aea994 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -829,7 +829,7 @@ static inline void x86_assign_hw_event(struct perf_event *event,
         } else {
                 hwc->config_base = x86_pmu_config_addr(hwc->idx);
                 hwc->event_base  = x86_pmu_event_addr(hwc->idx);
-                hwc->event_base_rdpmc = hwc->idx;
+                hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
         }
 }
 
@@ -1310,11 +1310,6 @@ static struct attribute_group x86_pmu_format_group = {
         .attrs = NULL,
 };
 
-struct perf_pmu_events_attr {
-        struct device_attribute attr;
-        u64 id;
-};
-
 /*
  * Remove all undefined events (x86_pmu.event_map(id) == 0)
  * out of events_attr attributes.
@@ -1348,11 +1343,9 @@ static ssize_t events_sysfs_show(struct device *dev, struct device_attribute *at
 #define EVENT_VAR(_id)  event_attr_##_id
 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
 
-#define EVENT_ATTR(_name, _id)                                  \
-static struct perf_pmu_events_attr EVENT_VAR(_id) = {           \
-        .attr = __ATTR(_name, 0444, events_sysfs_show, NULL),   \
-        .id   =  PERF_COUNT_HW_##_id,                           \
-};
+#define EVENT_ATTR(_name, _id)                                          \
+        PMU_EVENT_ATTR(_name, EVENT_VAR(_id), PERF_COUNT_HW_##_id,      \
+                        events_sysfs_show)
 
 EVENT_ATTR(cpu-cycles,                  CPU_CYCLES              );
 EVENT_ATTR(instructions,                INSTRUCTIONS            );
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 115c1ea97746..7f5c75c2afdd 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -325,6 +325,8 @@ struct x86_pmu {
         int             (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
         unsigned        eventsel;
         unsigned        perfctr;
+        int             (*addr_offset)(int index, bool eventsel);
+        int             (*rdpmc_index)(int index);
         u64             (*event_map)(int);
         int             max_events;
         int             num_counters;
@@ -446,28 +448,21 @@ extern u64 __read_mostly hw_cache_extra_regs
 
 u64 x86_perf_event_update(struct perf_event *event);
 
-static inline int x86_pmu_addr_offset(int index)
+static inline unsigned int x86_pmu_config_addr(int index)
 {
-        int offset;
-
-        /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */
-        alternative_io(ASM_NOP2,
-                       "shll $1, %%eax",
-                       X86_FEATURE_PERFCTR_CORE,
-                       "=a" (offset),
-                       "a"  (index));
-
-        return offset;
+        return x86_pmu.eventsel + (x86_pmu.addr_offset ?
+                                   x86_pmu.addr_offset(index, true) : index);
 }
 
-static inline unsigned int x86_pmu_config_addr(int index)
+static inline unsigned int x86_pmu_event_addr(int index)
 {
-        return x86_pmu.eventsel + x86_pmu_addr_offset(index);
+        return x86_pmu.perfctr + (x86_pmu.addr_offset ?
+                                  x86_pmu.addr_offset(index, false) : index);
 }
 
-static inline unsigned int x86_pmu_event_addr(int index)
+static inline int x86_pmu_rdpmc_index(int index)
 {
-        return x86_pmu.perfctr + x86_pmu_addr_offset(index);
+        return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
 }
 
 int x86_setup_perfctr(struct perf_event *event);
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index c93bc4e813a0..dfdab42aed27 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -132,21 +132,102 @@ static u64 amd_pmu_event_map(int hw_event)
         return amd_perfmon_event_map[hw_event];
 }
 
-static int amd_pmu_hw_config(struct perf_event *event)
+static struct event_constraint *amd_nb_event_constraint;
+
+/*
+ * Previously calculated offsets
+ */
+static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
+static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
+static unsigned int rdpmc_indexes[X86_PMC_IDX_MAX] __read_mostly;
+
+/*
+ * Legacy CPUs:
+ *   4 counters starting at 0xc0010000 each offset by 1
+ *
+ * CPUs with core performance counter extensions:
+ *   6 counters starting at 0xc0010200 each offset by 2
+ *
+ * CPUs with north bridge performance counter extensions:
+ *   4 additional counters starting at 0xc0010240 each offset by 2
+ *   (indexed right above either one of the above core counters)
+ */
+static inline int amd_pmu_addr_offset(int index, bool eventsel)
 {
-        int ret;
+        int offset, first, base;
 
-        /* pass precise event sampling to ibs: */
-        if (event->attr.precise_ip && get_ibs_caps())
-                return -ENOENT;
+        if (!index)
+                return index;
+
+        if (eventsel)
+                offset = event_offsets[index];
+        else
+                offset = count_offsets[index];
+
+        if (offset)
+                return offset;
+
+        if (amd_nb_event_constraint &&
+            test_bit(index, amd_nb_event_constraint->idxmsk)) {
+                /*
+                 * calculate the offset of NB counters with respect to
+                 * base eventsel or perfctr
+                 */
+
+                first = find_first_bit(amd_nb_event_constraint->idxmsk,
+                                       X86_PMC_IDX_MAX);
+
+                if (eventsel)
+                        base = MSR_F15H_NB_PERF_CTL - x86_pmu.eventsel;
+                else
+                        base = MSR_F15H_NB_PERF_CTR - x86_pmu.perfctr;
+
+                offset = base + ((index - first) << 1);
+        } else if (!cpu_has_perfctr_core)
+                offset = index;
+        else
+                offset = index << 1;
+
+        if (eventsel)
+                event_offsets[index] = offset;
+        else
+                count_offsets[index] = offset;
+
+        return offset;
+}
+
+static inline int amd_pmu_rdpmc_index(int index)
+{
+        int ret, first;
+
+        if (!index)
+                return index;
+
+        ret = rdpmc_indexes[index];
 
-        ret = x86_pmu_hw_config(event);
         if (ret)
                 return ret;
 
-        if (has_branch_stack(event))
-                return -EOPNOTSUPP;
+        if (amd_nb_event_constraint &&
+            test_bit(index, amd_nb_event_constraint->idxmsk)) {
+                /*
+                 * according to the mnual, ECX value of the NB counters is
+                 * the index of the NB counter (0, 1, 2 or 3) plus 6
+                 */
+
+                first = find_first_bit(amd_nb_event_constraint->idxmsk,
+                                       X86_PMC_IDX_MAX);
+                ret = index - first + 6;
+        } else
+                ret = index;
+
+        rdpmc_indexes[index] = ret;
+
+        return ret;
+}
 
+static int amd_core_hw_config(struct perf_event *event)
+{
         if (event->attr.exclude_host && event->attr.exclude_guest)
                 /*
                  * When HO == GO == 1 the hardware treats that as GO == HO == 0
@@ -156,14 +237,37 @@ static int amd_pmu_hw_config(struct perf_event *event)
                 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
                                       ARCH_PERFMON_EVENTSEL_OS);
         else if (event->attr.exclude_host)
-                event->hw.config |= AMD_PERFMON_EVENTSEL_GUESTONLY;
+                event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
         else if (event->attr.exclude_guest)
-                event->hw.config |= AMD_PERFMON_EVENTSEL_HOSTONLY;
+                event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
+
+        return 0;
+}
+
+/*
+ * NB counters do not support the following event select bits:
+ *   Host/Guest only
+ *   Counter mask
+ *   Invert counter mask
+ *   Edge detect
+ *   OS/User mode
+ */
+static int amd_nb_hw_config(struct perf_event *event)
+{
+        /* for NB, we only allow system wide counting mode */
+        if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+                return -EINVAL;
+
+        if (event->attr.exclude_user || event->attr.exclude_kernel ||
+            event->attr.exclude_host || event->attr.exclude_guest)
+                return -EINVAL;
 
-        if (event->attr.type != PERF_TYPE_RAW)
-                return 0;
+        event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
+                              ARCH_PERFMON_EVENTSEL_OS);
 
-        event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
+        if (event->hw.config & ~(AMD64_RAW_EVENT_MASK_NB |
+                                 ARCH_PERFMON_EVENTSEL_INT))
+                return -EINVAL;
 
         return 0;
 }
@@ -181,6 +285,11 @@ static inline int amd_is_nb_event(struct hw_perf_event *hwc)
         return (hwc->config & 0xe0) == 0xe0;
 }
 
+static inline int amd_is_perfctr_nb_event(struct hw_perf_event *hwc)
+{
+        return amd_nb_event_constraint && amd_is_nb_event(hwc);
+}
+
 static inline int amd_has_nb(struct cpu_hw_events *cpuc)
 {
         struct amd_nb *nb = cpuc->amd_nb;
@@ -188,20 +297,37 @@ static inline int amd_has_nb(struct cpu_hw_events *cpuc)
         return nb && nb->nb_id != -1;
 }
 
-static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
-                                      struct perf_event *event)
+static int amd_pmu_hw_config(struct perf_event *event)
+{
+        int ret;
+
+        /* pass precise event sampling to ibs: */
+        if (event->attr.precise_ip && get_ibs_caps())
+                return -ENOENT;
+
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
+        ret = x86_pmu_hw_config(event);
+        if (ret)
+                return ret;
+
+        if (event->attr.type == PERF_TYPE_RAW)
+                event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
+
+        if (amd_is_perfctr_nb_event(&event->hw))
+                return amd_nb_hw_config(event);
+
+        return amd_core_hw_config(event);
+}
+
+static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
+                                           struct perf_event *event)
 {
-        struct hw_perf_event *hwc = &event->hw;
         struct amd_nb *nb = cpuc->amd_nb;
         int i;
 
         /*
-         * only care about NB events
-         */
-        if (!(amd_has_nb(cpuc) && amd_is_nb_event(hwc)))
-                return;
-
-        /*
          * need to scan whole list because event may not have
          * been assigned during scheduling
          *
@@ -215,6 +341,19 @@ static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
         }
 }
 
+static void amd_nb_interrupt_hw_config(struct hw_perf_event *hwc)
+{
+        int core_id = cpu_data(smp_processor_id()).cpu_core_id;
+
+        /* deliver interrupts only to this core */
+        if (hwc->config & ARCH_PERFMON_EVENTSEL_INT) {
+                hwc->config |= AMD64_EVENTSEL_INT_CORE_ENABLE;
+                hwc->config &= ~AMD64_EVENTSEL_INT_CORE_SEL_MASK;
+                hwc->config |= (u64)(core_id) <<
+                        AMD64_EVENTSEL_INT_CORE_SEL_SHIFT;
+        }
+}
+
  /*
   * AMD64 NorthBridge events need special treatment because
   * counter access needs to be synchronized across all cores
@@ -247,24 +386,24 @@ static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
   *
   * Given that resources are allocated (cmpxchg), they must be
   * eventually freed for others to use. This is accomplished by
-  * calling amd_put_event_constraints().
+  * calling __amd_put_nb_event_constraints()
   *
   * Non NB events are not impacted by this restriction.
   */
 static struct event_constraint *
-amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+__amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
+                               struct event_constraint *c)
 {
         struct hw_perf_event *hwc = &event->hw;
         struct amd_nb *nb = cpuc->amd_nb;
-        struct perf_event *old = NULL;
-        int max = x86_pmu.num_counters;
-        int i, j, k = -1;
+        struct perf_event *old;
+        int idx, new = -1;
 
-        /*
-         * if not NB event or no NB, then no constraints
-         */
-        if (!(amd_has_nb(cpuc) && amd_is_nb_event(hwc)))
-                return &unconstrained;
+        if (!c)
+                c = &unconstrained;
+
+        if (cpuc->is_fake)
+                return c;
 
         /*
          * detect if already present, if so reuse
@@ -276,48 +415,36 @@ amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
          * because of successive calls to x86_schedule_events() from
          * hw_perf_group_sched_in() without hw_perf_enable()
          */
-        for (i = 0; i < max; i++) {
-                /*
-                 * keep track of first free slot
-                 */
-                if (k == -1 && !nb->owners[i])
-                        k = i;
+        for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
+                if (new == -1 || hwc->idx == idx)
+                        /* assign free slot, prefer hwc->idx */
+                        old = cmpxchg(nb->owners + idx, NULL, event);
+                else if (nb->owners[idx] == event)
+                        /* event already present */
+                        old = event;
+                else
+                        continue;
+
+                if (old && old != event)
+                        continue;
+
+                /* reassign to this slot */
+                if (new != -1)
+                        cmpxchg(nb->owners + new, event, NULL);
+                new = idx;
 
                 /* already present, reuse */
-                if (nb->owners[i] == event)
-                        goto done;
-        }
-        /*
-         * not present, so grab a new slot
-         * starting either at:
-         */
-        if (hwc->idx != -1) {
-                /* previous assignment */
-                i = hwc->idx;
-        } else if (k != -1) {
-                /* start from free slot found */
-                i = k;
-        } else {
-                /*
-                 * event not found, no slot found in
-                 * first pass, try again from the
-                 * beginning
-                 */
-                i = 0;
-        }
-        j = i;
-        do {
-                old = cmpxchg(nb->owners+i, NULL, event);
-                if (!old)
+                if (old == event)
                         break;
-                if (++i == max)
-                        i = 0;
-        } while (i != j);
-done:
-        if (!old)
-                return &nb->event_constraints[i];
-
-        return &emptyconstraint;
+        }
+
+        if (new == -1)
+                return &emptyconstraint;
+
+        if (amd_is_perfctr_nb_event(hwc))
+                amd_nb_interrupt_hw_config(hwc);
+
+        return &nb->event_constraints[new];
 }
 
 static struct amd_nb *amd_alloc_nb(int cpu)
@@ -364,7 +491,7 @@ static void amd_pmu_cpu_starting(int cpu)
         struct amd_nb *nb;
         int i, nb_id;
 
-        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+        cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 
         if (boot_cpu_data.x86_max_cores < 2)
                 return;
@@ -407,6 +534,26 @@ static void amd_pmu_cpu_dead(int cpu)
         }
 }
 
+static struct event_constraint *
+amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+        /*
+         * if not NB event or no NB, then no constraints
+         */
+        if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
+                return &unconstrained;
+
+        return __amd_get_nb_event_constraints(cpuc, event,
+                                              amd_nb_event_constraint);
+}
+
+static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
+                                      struct perf_event *event)
+{
+        if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
+                __amd_put_nb_event_constraints(cpuc, event);
+}
+
 PMU_FORMAT_ATTR(event,  "config:0-7,32-35");
 PMU_FORMAT_ATTR(umask,  "config:8-15"   );
 PMU_FORMAT_ATTR(edge,   "config:18"     );
@@ -496,6 +643,9 @@ static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09,
 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
 
+static struct event_constraint amd_NBPMC96 = EVENT_CONSTRAINT(0, 0x3C0, 0);
+static struct event_constraint amd_NBPMC74 = EVENT_CONSTRAINT(0, 0xF0, 0);
+
 static struct event_constraint *
 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
 {
@@ -561,8 +711,8 @@ amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *ev
                         return &amd_f15_PMC20;
                 }
         case AMD_EVENT_NB:
-                /* not yet implemented */
-                return &emptyconstraint;
+                return __amd_get_nb_event_constraints(cpuc, event,
+                                                      amd_nb_event_constraint);
         default:
                 return &emptyconstraint;
         }
@@ -587,6 +737,8 @@ static __initconst const struct x86_pmu amd_pmu = {
         .schedule_events        = x86_schedule_events,
         .eventsel               = MSR_K7_EVNTSEL0,
         .perfctr                = MSR_K7_PERFCTR0,
+        .addr_offset            = amd_pmu_addr_offset,
+        .rdpmc_index            = amd_pmu_rdpmc_index,
         .event_map              = amd_pmu_event_map,
         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
         .num_counters           = AMD64_NUM_COUNTERS,
@@ -608,7 +760,7 @@ static __initconst const struct x86_pmu amd_pmu = {
 
 static int setup_event_constraints(void)
 {
-        if (boot_cpu_data.x86 >= 0x15)
+        if (boot_cpu_data.x86 == 0x15)
                 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
         return 0;
 }
@@ -638,6 +790,23 @@ static int setup_perfctr_core(void)
         return 0;
 }
 
+static int setup_perfctr_nb(void)
+{
+        if (!cpu_has_perfctr_nb)
+                return -ENODEV;
+
+        x86_pmu.num_counters += AMD64_NUM_COUNTERS_NB;
+
+        if (cpu_has_perfctr_core)
+                amd_nb_event_constraint = &amd_NBPMC96;
+        else
+                amd_nb_event_constraint = &amd_NBPMC74;
+
+        printk(KERN_INFO "perf: AMD northbridge performance counters detected\n");
+
+        return 0;
+}
+
 __init int amd_pmu_init(void)
 {
         /* Performance-monitoring supported from K7 and later: */
@@ -648,6 +817,7 @@ __init int amd_pmu_init(void)
 
         setup_event_constraints();
         setup_perfctr_core();
+        setup_perfctr_nb();
 
         /* Events are common for all AMDs */
         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
@@ -678,7 +848,7 @@ void amd_pmu_disable_virt(void)
          * SVM is disabled the Guest-only bits still gets set and the counter
          * will not count anything.
          */
-        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+        cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 
         /* Reload all events */
         x86_pmu_disable_all();
diff --git a/arch/x86/kernel/kprobes/Makefile b/arch/x86/kernel/kprobes/Makefile
new file mode 100644
index 000000000000..0d33169cc1a2
--- /dev/null
+++ b/arch/x86/kernel/kprobes/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for kernel probes
+#
+
+obj-$(CONFIG_KPROBES)           += core.o
+obj-$(CONFIG_OPTPROBES)         += opt.o
+obj-$(CONFIG_KPROBES_ON_FTRACE) += ftrace.o
diff --git a/arch/x86/kernel/kprobes-common.h b/arch/x86/kernel/kprobes/common.h
index 3230b68ef29a..2e9d4b5af036 100644
--- a/arch/x86/kernel/kprobes-common.h
+++ b/arch/x86/kernel/kprobes/common.h
@@ -99,4 +99,15 @@ static inline unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsig
         return addr;
 }
 #endif
+
+#ifdef CONFIG_KPROBES_ON_FTRACE
+extern int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                           struct kprobe_ctlblk *kcb);
+#else
+static inline int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                                  struct kprobe_ctlblk *kcb)
+{
+        return 0;
+}
+#endif
 #endif
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes/core.c
index 57916c0d3cf6..e124554598ee 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -58,7 +58,7 @@
 #include <asm/insn.h>
 #include <asm/debugreg.h>
 
-#include "kprobes-common.h"
+#include "common.h"
 
 void jprobe_return_end(void);
 
@@ -78,7 +78,7 @@ DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
          * Groups, and some special opcodes can not boost.
          * This is non-const and volatile to keep gcc from statically
          * optimizing it out, as variable_test_bit makes gcc think only
-         * *(unsigned long*) is used. 
+         * *(unsigned long*) is used.
          */
 static volatile u32 twobyte_is_boostable[256 / 32] = {
         /*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
@@ -117,7 +117,7 @@ static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
         struct __arch_relative_insn {
                 u8 op;
                 s32 raddr;
-        } __attribute__((packed)) *insn;
+        } __packed *insn;
 
         insn = (struct __arch_relative_insn *)from;
         insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
@@ -541,23 +541,6 @@ reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb
         return 1;
 }
 
-#ifdef KPROBES_CAN_USE_FTRACE
-static void __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs,
-                                      struct kprobe_ctlblk *kcb)
-{
-        /*
-         * Emulate singlestep (and also recover regs->ip)
-         * as if there is a 5byte nop
-         */
-        regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
-        if (unlikely(p->post_handler)) {
-                kcb->kprobe_status = KPROBE_HIT_SSDONE;
-                p->post_handler(p, regs, 0);
-        }
-        __this_cpu_write(current_kprobe, NULL);
-}
-#endif
-
 /*
  * Interrupts are disabled on entry as trap3 is an interrupt gate and they
  * remain disabled throughout this function.
@@ -616,13 +599,8 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
         } else if (kprobe_running()) {
                 p = __this_cpu_read(current_kprobe);
                 if (p->break_handler && p->break_handler(p, regs)) {
-#ifdef KPROBES_CAN_USE_FTRACE
-                        if (kprobe_ftrace(p)) {
-                                skip_singlestep(p, regs, kcb);
-                                return 1;
-                        }
-#endif
-                        setup_singlestep(p, regs, kcb, 0);
+                        if (!skip_singlestep(p, regs, kcb))
+                                setup_singlestep(p, regs, kcb, 0);
                         return 1;
                 }
         } /* else: not a kprobe fault; let the kernel handle it */
@@ -1075,50 +1053,6 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
         return 0;
 }
 
-#ifdef KPROBES_CAN_USE_FTRACE
-/* Ftrace callback handler for kprobes */
-void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
-                                     struct ftrace_ops *ops, struct pt_regs *regs)
-{
-        struct kprobe *p;
-        struct kprobe_ctlblk *kcb;
-        unsigned long flags;
-
-        /* Disable irq for emulating a breakpoint and avoiding preempt */
-        local_irq_save(flags);
-
-        p = get_kprobe((kprobe_opcode_t *)ip);
-        if (unlikely(!p) || kprobe_disabled(p))
-                goto end;
-
-        kcb = get_kprobe_ctlblk();
-        if (kprobe_running()) {
-                kprobes_inc_nmissed_count(p);
-        } else {
-                /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
-                regs->ip = ip + sizeof(kprobe_opcode_t);
-
-                __this_cpu_write(current_kprobe, p);
-                kcb->kprobe_status = KPROBE_HIT_ACTIVE;
-                if (!p->pre_handler || !p->pre_handler(p, regs))
-                        skip_singlestep(p, regs, kcb);
-                /*
-                 * If pre_handler returns !0, it sets regs->ip and
-                 * resets current kprobe.
-                 */
-        }
-end:
-        local_irq_restore(flags);
-}
-
-int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
-{
-        p->ainsn.insn = NULL;
-        p->ainsn.boostable = -1;
-        return 0;
-}
-#endif
-
 int __init arch_init_kprobes(void)
 {
         return arch_init_optprobes();
diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c
new file mode 100644
index 000000000000..23ef5c556f06
--- /dev/null
+++ b/arch/x86/kernel/kprobes/ftrace.c
@@ -0,0 +1,93 @@
+/*
+ * Dynamic Ftrace based Kprobes Optimization
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) Hitachi Ltd., 2012
+ */
+#include <linux/kprobes.h>
+#include <linux/ptrace.h>
+#include <linux/hardirq.h>
+#include <linux/preempt.h>
+#include <linux/ftrace.h>
+
+#include "common.h"
+
+static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                             struct kprobe_ctlblk *kcb)
+{
+        /*
+         * Emulate singlestep (and also recover regs->ip)
+         * as if there is a 5byte nop
+         */
+        regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
+        if (unlikely(p->post_handler)) {
+                kcb->kprobe_status = KPROBE_HIT_SSDONE;
+                p->post_handler(p, regs, 0);
+        }
+        __this_cpu_write(current_kprobe, NULL);
+        return 1;
+}
+
+int __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                              struct kprobe_ctlblk *kcb)
+{
+        if (kprobe_ftrace(p))
+                return __skip_singlestep(p, regs, kcb);
+        else
+                return 0;
+}
+
+/* Ftrace callback handler for kprobes */
+void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
+                                     struct ftrace_ops *ops, struct pt_regs *regs)
+{
+        struct kprobe *p;
+        struct kprobe_ctlblk *kcb;
+        unsigned long flags;
+
+        /* Disable irq for emulating a breakpoint and avoiding preempt */
+        local_irq_save(flags);
+
+        p = get_kprobe((kprobe_opcode_t *)ip);
+        if (unlikely(!p) || kprobe_disabled(p))
+                goto end;
+
+        kcb = get_kprobe_ctlblk();
+        if (kprobe_running()) {
+                kprobes_inc_nmissed_count(p);
+        } else {
+                /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
+                regs->ip = ip + sizeof(kprobe_opcode_t);
+
+                __this_cpu_write(current_kprobe, p);
+                kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+                if (!p->pre_handler || !p->pre_handler(p, regs))
+                        __skip_singlestep(p, regs, kcb);
+                /*
+                 * If pre_handler returns !0, it sets regs->ip and
+                 * resets current kprobe.
+                 */
+        }
+end:
+        local_irq_restore(flags);
+}
+
+int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
+{
+        p->ainsn.insn = NULL;
+        p->ainsn.boostable = -1;
+        return 0;
+}
diff --git a/arch/x86/kernel/kprobes-opt.c b/arch/x86/kernel/kprobes/opt.c
index c5e410eed403..76dc6f095724 100644
--- a/arch/x86/kernel/kprobes-opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -37,7 +37,7 @@
 #include <asm/insn.h>
 #include <asm/debugreg.h>
 
-#include "kprobes-common.h"
+#include "common.h"
 
 unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr)
 {
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index c71025b67462..0ba4cfb4f412 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -680,8 +680,10 @@ static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
                 if (auprobe->insn[i] == 0x66)
                         continue;
 
-                if (auprobe->insn[i] == 0x90)
+                if (auprobe->insn[i] == 0x90) {
+                        regs->ip += i + 1;
                         return true;
+                }
 
                 break;
         }