1 files changed, 115 insertions, 88 deletions

diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index ab3ba1c1b7dd..ae96cfa5eddd 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -12,6 +12,7 @@
 
 #define BTS_BUFFER_SIZE         (PAGE_SIZE << 4)
 #define PEBS_BUFFER_SIZE        PAGE_SIZE
+#define PEBS_FIXUP_SIZE         PAGE_SIZE
 
 /*
  * pebs_record_32 for p4 and core not supported
@@ -182,18 +183,32 @@ struct pebs_record_nhm {
  * Same as pebs_record_nhm, with two additional fields.
  */
 struct pebs_record_hsw {
-        struct pebs_record_nhm nhm;
-        /*
-         * Real IP of the event. In the Intel documentation this
-         * is called eventingrip.
-         */
-        u64 real_ip;
-        /*
-         * TSX tuning information field: abort cycles and abort flags.
-         */
-        u64 tsx_tuning;
+        u64 flags, ip;
+        u64 ax, bx, cx, dx;
+        u64 si, di, bp, sp;
+        u64 r8,  r9,  r10, r11;
+        u64 r12, r13, r14, r15;
+        u64 status, dla, dse, lat;
+        u64 real_ip, tsx_tuning;
+};
+
+union hsw_tsx_tuning {
+        struct {
+                u32 cycles_last_block     : 32,
+                    hle_abort             : 1,
+                    rtm_abort             : 1,
+                    instruction_abort     : 1,
+                    non_instruction_abort : 1,
+                    retry                 : 1,
+                    data_conflict         : 1,
+                    capacity_writes       : 1,
+                    capacity_reads        : 1;
+        };
+        u64         value;
 };
 
+#define PEBS_HSW_TSX_FLAGS      0xff00000000ULL
+
 void init_debug_store_on_cpu(int cpu)
 {
         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
@@ -214,12 +229,14 @@ void fini_debug_store_on_cpu(int cpu)
         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
 }
 
+static DEFINE_PER_CPU(void *, insn_buffer);
+
 static int alloc_pebs_buffer(int cpu)
 {
         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
         int node = cpu_to_node(cpu);
         int max, thresh = 1; /* always use a single PEBS record */
-        void *buffer;
+        void *buffer, *ibuffer;
 
         if (!x86_pmu.pebs)
                 return 0;
@@ -228,6 +245,19 @@ static int alloc_pebs_buffer(int cpu)
         if (unlikely(!buffer))
                 return -ENOMEM;
 
+        /*
+         * HSW+ already provides us the eventing ip; no need to allocate this
+         * buffer then.
+         */
+        if (x86_pmu.intel_cap.pebs_format < 2) {
+                ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
+                if (!ibuffer) {
+                        kfree(buffer);
+                        return -ENOMEM;
+                }
+                per_cpu(insn_buffer, cpu) = ibuffer;
+        }
+
         max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
 
         ds->pebs_buffer_base = (u64)(unsigned long)buffer;
@@ -248,6 +278,9 @@ static void release_pebs_buffer(int cpu)
         if (!ds || !x86_pmu.pebs)
                 return;
 
+        kfree(per_cpu(insn_buffer, cpu));
+        per_cpu(insn_buffer, cpu) = NULL;
+
         kfree((void *)(unsigned long)ds->pebs_buffer_base);
         ds->pebs_buffer_base = 0;
 }
@@ -715,6 +748,7 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
         unsigned long old_to, to = cpuc->lbr_entries[0].to;
         unsigned long ip = regs->ip;
         int is_64bit = 0;
+        void *kaddr;
 
         /*
          * We don't need to fixup if the PEBS assist is fault like
@@ -738,7 +772,7 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
          * unsigned math, either ip is before the start (impossible) or
          * the basic block is larger than 1 page (sanity)
          */
-        if ((ip - to) > PAGE_SIZE)
+        if ((ip - to) > PEBS_FIXUP_SIZE)
                 return 0;
 
         /*
@@ -749,29 +783,33 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
                 return 1;
         }
 
+        if (!kernel_ip(ip)) {
+                int size, bytes;
+                u8 *buf = this_cpu_read(insn_buffer);
+
+                size = ip - to; /* Must fit our buffer, see above */
+                bytes = copy_from_user_nmi(buf, (void __user *)to, size);
+                if (bytes != 0)
+                        return 0;
+
+                kaddr = buf;
+        } else {
+                kaddr = (void *)to;
+        }
+
         do {
                 struct insn insn;
-                u8 buf[MAX_INSN_SIZE];
-                void *kaddr;
 
                 old_to = to;
-                if (!kernel_ip(ip)) {
-                        int bytes, size = MAX_INSN_SIZE;
-
-                        bytes = copy_from_user_nmi(buf, (void __user *)to, size);
-                        if (bytes != size)
-                                return 0;
-
-                        kaddr = buf;
-                } else
-                        kaddr = (void *)to;
 
 #ifdef CONFIG_X86_64
                 is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
 #endif
                 insn_init(&insn, kaddr, is_64bit);
                 insn_get_length(&insn);
+
                 to += insn.length;
+                kaddr += insn.length;
         } while (to < ip);
 
         if (to == ip) {
@@ -786,16 +824,34 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
         return 0;
 }
 
+static inline u64 intel_hsw_weight(struct pebs_record_hsw *pebs)
+{
+        if (pebs->tsx_tuning) {
+                union hsw_tsx_tuning tsx = { .value = pebs->tsx_tuning };
+                return tsx.cycles_last_block;
+        }
+        return 0;
+}
+
+static inline u64 intel_hsw_transaction(struct pebs_record_hsw *pebs)
+{
+        u64 txn = (pebs->tsx_tuning & PEBS_HSW_TSX_FLAGS) >> 32;
+
+        /* For RTM XABORTs also log the abort code from AX */
+        if ((txn & PERF_TXN_TRANSACTION) && (pebs->ax & 1))
+                txn |= ((pebs->ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
+        return txn;
+}
+
 static void __intel_pmu_pebs_event(struct perf_event *event,
                                    struct pt_regs *iregs, void *__pebs)
 {
         /*
-         * We cast to pebs_record_nhm to get the load latency data
-         * if extra_reg MSR_PEBS_LD_LAT_THRESHOLD used
+         * We cast to the biggest pebs_record but are careful not to
+         * unconditionally access the 'extra' entries.
          */
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-        struct pebs_record_nhm *pebs = __pebs;
-        struct pebs_record_hsw *pebs_hsw = __pebs;
+        struct pebs_record_hsw *pebs = __pebs;
         struct perf_sample_data data;
         struct pt_regs regs;
         u64 sample_type;
@@ -854,7 +910,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
         regs.sp = pebs->sp;
 
         if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format >= 2) {
-                regs.ip = pebs_hsw->real_ip;
+                regs.ip = pebs->real_ip;
                 regs.flags |= PERF_EFLAGS_EXACT;
         } else if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(&regs))
                 regs.flags |= PERF_EFLAGS_EXACT;
@@ -862,9 +918,18 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
                 regs.flags &= ~PERF_EFLAGS_EXACT;
 
         if ((event->attr.sample_type & PERF_SAMPLE_ADDR) &&
-                x86_pmu.intel_cap.pebs_format >= 1)
+            x86_pmu.intel_cap.pebs_format >= 1)
                 data.addr = pebs->dla;
 
+        if (x86_pmu.intel_cap.pebs_format >= 2) {
+                /* Only set the TSX weight when no memory weight. */
+                if ((event->attr.sample_type & PERF_SAMPLE_WEIGHT) && !fll)
+                        data.weight = intel_hsw_weight(pebs);
+
+                if (event->attr.sample_type & PERF_SAMPLE_TRANSACTION)
+                        data.txn = intel_hsw_transaction(pebs);
+        }
+
         if (has_branch_stack(event))
                 data.br_stack = &cpuc->lbr_stack;
 
@@ -913,17 +978,34 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
         __intel_pmu_pebs_event(event, iregs, at);
 }
 
-static void __intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, void *at,
-                                        void *top)
+static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct debug_store *ds = cpuc->ds;
         struct perf_event *event = NULL;
+        void *at, *top;
         u64 status = 0;
         int bit;
 
+        if (!x86_pmu.pebs_active)
+                return;
+
+        at  = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
+        top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
+
         ds->pebs_index = ds->pebs_buffer_base;
 
+        if (unlikely(at > top))
+                return;
+
+        /*
+         * Should not happen, we program the threshold at 1 and do not
+         * set a reset value.
+         */
+        WARN_ONCE(top - at > x86_pmu.max_pebs_events * x86_pmu.pebs_record_size,
+                  "Unexpected number of pebs records %ld\n",
+                  (long)(top - at) / x86_pmu.pebs_record_size);
+
         for (; at < top; at += x86_pmu.pebs_record_size) {
                 struct pebs_record_nhm *p = at;
 
@@ -951,61 +1033,6 @@ static void __intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, void *at,
         }
 }
 
-static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
-{
-        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-        struct debug_store *ds = cpuc->ds;
-        struct pebs_record_nhm *at, *top;
-        int n;
-
-        if (!x86_pmu.pebs_active)
-                return;
-
-        at  = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
-        top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
-
-        ds->pebs_index = ds->pebs_buffer_base;
-
-        n = top - at;
-        if (n <= 0)
-                return;
-
-        /*
-         * Should not happen, we program the threshold at 1 and do not
-         * set a reset value.
-         */
-        WARN_ONCE(n > x86_pmu.max_pebs_events,
-                  "Unexpected number of pebs records %d\n", n);
-
-        return __intel_pmu_drain_pebs_nhm(iregs, at, top);
-}
-
-static void intel_pmu_drain_pebs_hsw(struct pt_regs *iregs)
-{
-        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
-        struct debug_store *ds = cpuc->ds;
-        struct pebs_record_hsw *at, *top;
-        int n;
-
-        if (!x86_pmu.pebs_active)
-                return;
-
-        at  = (struct pebs_record_hsw *)(unsigned long)ds->pebs_buffer_base;
-        top = (struct pebs_record_hsw *)(unsigned long)ds->pebs_index;
-
-        n = top - at;
-        if (n <= 0)
-                return;
-        /*
-         * Should not happen, we program the threshold at 1 and do not
-         * set a reset value.
-         */
-        WARN_ONCE(n > x86_pmu.max_pebs_events,
-                  "Unexpected number of pebs records %d\n", n);
-
-        return __intel_pmu_drain_pebs_nhm(iregs, at, top);
-}
-
 /*
  * BTS, PEBS probe and setup
  */
@@ -1040,7 +1067,7 @@ void intel_ds_init(void)
                 case 2:
                         pr_cont("PEBS fmt2%c, ", pebs_type);
                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_hsw);
-                        x86_pmu.drain_pebs = intel_pmu_drain_pebs_hsw;
+                        x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
                         break;
 
                 default: