Merge branch 'perf/hw-branch-sampling' into perf/core

Merge reason: The 'perf record -b' hardware branch sampling feature is ready for upstream.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

13 files changed, 734 insertions, 72 deletions

diff --git a/arch/alpha/kernel/perf_event.c b/arch/alpha/kernel/perf_event.c
index 8143cd7cdbfb..0dae252f7a33 100644
--- a/arch/alpha/kernel/perf_event.c
+++ b/arch/alpha/kernel/perf_event.c
@@ -685,6 +685,10 @@ static int alpha_pmu_event_init(struct perf_event *event)
 {
         int err;
 
+        /* does not support taken branch sampling */
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         switch (event->attr.type) {
         case PERF_TYPE_RAW:
         case PERF_TYPE_HARDWARE:
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index b2abfa18f137..8a89d3b7626b 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -539,6 +539,10 @@ static int armpmu_event_init(struct perf_event *event)
         int err = 0;
         atomic_t *active_events = &armpmu->active_events;
 
+        /* does not support taken branch sampling */
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         if (armpmu->map_event(event) == -ENOENT)
                 return -ENOENT;
 
diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c
index e3b897acfbc0..811084f4e422 100644
--- a/arch/mips/kernel/perf_event_mipsxx.c
+++ b/arch/mips/kernel/perf_event_mipsxx.c
@@ -606,6 +606,10 @@ static int mipspmu_event_init(struct perf_event *event)
 {
         int err = 0;
 
+        /* does not support taken branch sampling */
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         switch (event->attr.type) {
         case PERF_TYPE_RAW:
         case PERF_TYPE_HARDWARE:
diff --git a/arch/powerpc/kernel/perf_event.c b/arch/powerpc/kernel/perf_event.c
index f04c2301725e..c2e27ede07ec 100644
--- a/arch/powerpc/kernel/perf_event.c
+++ b/arch/powerpc/kernel/perf_event.c
@@ -1084,6 +1084,10 @@ static int power_pmu_event_init(struct perf_event *event)
         if (!ppmu)
                 return -ENOENT;
 
+        /* does not support taken branch sampling */
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         switch (event->attr.type) {
         case PERF_TYPE_HARDWARE:
                 ev = event->attr.config;
diff --git a/arch/sh/kernel/perf_event.c b/arch/sh/kernel/perf_event.c
index 10b14e3a7eb8..068b8a2759b5 100644
--- a/arch/sh/kernel/perf_event.c
+++ b/arch/sh/kernel/perf_event.c
@@ -310,6 +310,10 @@ static int sh_pmu_event_init(struct perf_event *event)
 {
         int err;
 
+        /* does not support taken branch sampling */
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         switch (event->attr.type) {
         case PERF_TYPE_RAW:
         case PERF_TYPE_HW_CACHE:
diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c
index 614da624330c..8e16a4a21582 100644
--- a/arch/sparc/kernel/perf_event.c
+++ b/arch/sparc/kernel/perf_event.c
@@ -1105,6 +1105,10 @@ static int sparc_pmu_event_init(struct perf_event *event)
         if (atomic_read(&nmi_active) < 0)
                 return -ENODEV;
 
+        /* does not support taken branch sampling */
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         switch (attr->type) {
         case PERF_TYPE_HARDWARE:
                 if (attr->config >= sparc_pmu->max_events)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index a6962d9161a0..ccb805966f68 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -56,6 +56,13 @@
 #define MSR_OFFCORE_RSP_0               0x000001a6
 #define MSR_OFFCORE_RSP_1               0x000001a7
 
+#define MSR_LBR_SELECT                  0x000001c8
+#define MSR_LBR_TOS                     0x000001c9
+#define MSR_LBR_NHM_FROM                0x00000680
+#define MSR_LBR_NHM_TO                  0x000006c0
+#define MSR_LBR_CORE_FROM               0x00000040
+#define MSR_LBR_CORE_TO                 0x00000060
+
 #define MSR_IA32_PEBS_ENABLE            0x000003f1
 #define MSR_IA32_DS_AREA                0x00000600
 #define MSR_IA32_PERF_CAPABILITIES      0x00000345
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index f8bddb5b0600..0a18d16cb58d 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -353,6 +353,36 @@ int x86_setup_perfctr(struct perf_event *event)
         return 0;
 }
 
+/*
+ * check that branch_sample_type is compatible with
+ * settings needed for precise_ip > 1 which implies
+ * using the LBR to capture ALL taken branches at the
+ * priv levels of the measurement
+ */
+static inline int precise_br_compat(struct perf_event *event)
+{
+        u64 m = event->attr.branch_sample_type;
+        u64 b = 0;
+
+        /* must capture all branches */
+        if (!(m & PERF_SAMPLE_BRANCH_ANY))
+                return 0;
+
+        m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
+
+        if (!event->attr.exclude_user)
+                b |= PERF_SAMPLE_BRANCH_USER;
+
+        if (!event->attr.exclude_kernel)
+                b |= PERF_SAMPLE_BRANCH_KERNEL;
+
+        /*
+         * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
+         */
+
+        return m == b;
+}
+
 int x86_pmu_hw_config(struct perf_event *event)
 {
         if (event->attr.precise_ip) {
@@ -369,6 +399,36 @@ int x86_pmu_hw_config(struct perf_event *event)
 
                 if (event->attr.precise_ip > precise)
                         return -EOPNOTSUPP;
+                /*
+                 * check that PEBS LBR correction does not conflict with
+                 * whatever the user is asking with attr->branch_sample_type
+                 */
+                if (event->attr.precise_ip > 1) {
+                        u64 *br_type = &event->attr.branch_sample_type;
+
+                        if (has_branch_stack(event)) {
+                                if (!precise_br_compat(event))
+                                        return -EOPNOTSUPP;
+
+                                /* branch_sample_type is compatible */
+
+                        } else {
+                                /*
+                                 * user did not specify  branch_sample_type
+                                 *
+                                 * For PEBS fixups, we capture all
+                                 * the branches at the priv level of the
+                                 * event.
+                                 */
+                                *br_type = PERF_SAMPLE_BRANCH_ANY;
+
+                                if (!event->attr.exclude_user)
+                                        *br_type |= PERF_SAMPLE_BRANCH_USER;
+
+                                if (!event->attr.exclude_kernel)
+                                        *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
+                        }
+                }
         }
 
         /*
@@ -426,6 +486,10 @@ static int __x86_pmu_event_init(struct perf_event *event)
         /* mark unused */
         event->hw.extra_reg.idx = EXTRA_REG_NONE;
 
+        /* mark not used */
+        event->hw.extra_reg.idx = EXTRA_REG_NONE;
+        event->hw.branch_reg.idx = EXTRA_REG_NONE;
+
         return x86_pmu.hw_config(event);
 }
 
@@ -1607,25 +1671,32 @@ static const struct attribute_group *x86_pmu_attr_groups[] = {
         NULL,
 };
 
+static void x86_pmu_flush_branch_stack(void)
+{
+        if (x86_pmu.flush_branch_stack)
+                x86_pmu.flush_branch_stack();
+}
+
 static struct pmu pmu = {
-        .pmu_enable     = x86_pmu_enable,
-        .pmu_disable    = x86_pmu_disable,
+        .pmu_enable             = x86_pmu_enable,
+        .pmu_disable            = x86_pmu_disable,
 
         .attr_groups    = x86_pmu_attr_groups,
 
         .event_init     = x86_pmu_event_init,
 
-        .add            = x86_pmu_add,
-        .del            = x86_pmu_del,
-        .start          = x86_pmu_start,
-        .stop           = x86_pmu_stop,
-        .read           = x86_pmu_read,
+        .add                    = x86_pmu_add,
+        .del                    = x86_pmu_del,
+        .start                  = x86_pmu_start,
+        .stop                   = x86_pmu_stop,
+        .read                   = x86_pmu_read,
 
         .start_txn      = x86_pmu_start_txn,
         .cancel_txn     = x86_pmu_cancel_txn,
         .commit_txn     = x86_pmu_commit_txn,
 
         .event_idx      = x86_pmu_event_idx,
+        .flush_branch_stack     = x86_pmu_flush_branch_stack,
 };
 
 void perf_update_user_clock(struct perf_event_mmap_page *userpg, u64 now)
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 66fda0c26402..8484e77c211e 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -33,6 +33,7 @@ enum extra_reg_type {
 
         EXTRA_REG_RSP_0 = 0,    /* offcore_response_0 */
         EXTRA_REG_RSP_1 = 1,    /* offcore_response_1 */
+        EXTRA_REG_LBR   = 2,    /* lbr_select */
 
         EXTRA_REG_MAX           /* number of entries needed */
 };
@@ -130,6 +131,8 @@ struct cpu_hw_events {
         void                            *lbr_context;
         struct perf_branch_stack        lbr_stack;
         struct perf_branch_entry        lbr_entries[MAX_LBR_ENTRIES];
+        struct er_account               *lbr_sel;
+        u64                             br_sel;
 
         /*
          * Intel host/guest exclude bits
@@ -344,6 +347,7 @@ struct x86_pmu {
         void            (*cpu_starting)(int cpu);
         void            (*cpu_dying)(int cpu);
         void            (*cpu_dead)(int cpu);
+        void            (*flush_branch_stack)(void);
 
         /*
          * Intel Arch Perfmon v2+
@@ -365,6 +369,8 @@ struct x86_pmu {
          */
         unsigned long   lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
         int             lbr_nr;                    /* hardware stack size */
+        u64             lbr_sel_mask;              /* LBR_SELECT valid bits */
+        const int       *lbr_sel_map;              /* lbr_select mappings */
 
         /*
          * Extra registers for events
@@ -478,6 +484,15 @@ extern struct event_constraint emptyconstraint;
 
 extern struct event_constraint unconstrained;
 
+static inline bool kernel_ip(unsigned long ip)
+{
+#ifdef CONFIG_X86_32
+        return ip > PAGE_OFFSET;
+#else
+        return (long)ip < 0;
+#endif
+}
+
 #ifdef CONFIG_CPU_SUP_AMD
 
 int amd_pmu_init(void);
@@ -558,6 +573,10 @@ void intel_pmu_lbr_init_nhm(void);
 
 void intel_pmu_lbr_init_atom(void);
 
+void intel_pmu_lbr_init_snb(void);
+
+int intel_pmu_setup_lbr_filter(struct perf_event *event);
+
 int p4_pmu_init(void);
 
 int p6_pmu_init(void);
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index 67250a52430b..dd002faff7a6 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -139,6 +139,9 @@ static int amd_pmu_hw_config(struct perf_event *event)
         if (ret)
                 return ret;
 
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
         if (event->attr.exclude_host && event->attr.exclude_guest)
                 /*
                  * When HO == GO == 1 the hardware treats that as GO == HO == 0
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 4bd9c9ef9d42..6a84e7f28f05 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -728,6 +728,19 @@ static __initconst const u64 atom_hw_cache_event_ids
  },
 };
 
+static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
+{
+        /* user explicitly requested branch sampling */
+        if (has_branch_stack(event))
+                return true;
+
+        /* implicit branch sampling to correct PEBS skid */
+        if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
+                return true;
+
+        return false;
+}
+
 static void intel_pmu_disable_all(void)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -882,6 +895,13 @@ static void intel_pmu_disable_event(struct perf_event *event)
         cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
         cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
 
+        /*
+         * must disable before any actual event
+         * because any event may be combined with LBR
+         */
+        if (intel_pmu_needs_lbr_smpl(event))
+                intel_pmu_lbr_disable(event);
+
         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
                 intel_pmu_disable_fixed(hwc);
                 return;
@@ -936,6 +956,12 @@ static void intel_pmu_enable_event(struct perf_event *event)
                 intel_pmu_enable_bts(hwc->config);
                 return;
         }
+        /*
+         * must enabled before any actual event
+         * because any event may be combined with LBR
+         */
+        if (intel_pmu_needs_lbr_smpl(event))
+                intel_pmu_lbr_enable(event);
 
         if (event->attr.exclude_host)
                 cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
@@ -1058,6 +1084,9 @@ again:
 
                 data.period = event->hw.last_period;
 
+                if (has_branch_stack(event))
+                        data.br_stack = &cpuc->lbr_stack;
+
                 if (perf_event_overflow(event, &data, regs))
                         x86_pmu_stop(event, 0);
         }
@@ -1124,17 +1153,17 @@ static bool intel_try_alt_er(struct perf_event *event, int orig_idx)
  */
 static struct event_constraint *
 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
-                                   struct perf_event *event)
+                                   struct perf_event *event,
+                                   struct hw_perf_event_extra *reg)
 {
         struct event_constraint *c = &emptyconstraint;
-        struct hw_perf_event_extra *reg = &event->hw.extra_reg;
         struct er_account *era;
         unsigned long flags;
         int orig_idx = reg->idx;
 
         /* already allocated shared msr */
         if (reg->alloc)
-                return &unconstrained;
+                return NULL; /* call x86_get_event_constraint() */
 
 again:
         era = &cpuc->shared_regs->regs[reg->idx];
@@ -1157,14 +1186,10 @@ again:
                 reg->alloc = 1;
 
                 /*
-                 * All events using extra_reg are unconstrained.
-                 * Avoids calling x86_get_event_constraints()
-                 *
-                 * Must revisit if extra_reg controlling events
-                 * ever have constraints. Worst case we go through
-                 * the regular event constraint table.
+                 * need to call x86_get_event_constraint()
+                 * to check if associated event has constraints
                  */
-                c = &unconstrained;
+                c = NULL;
         } else if (intel_try_alt_er(event, orig_idx)) {
                 raw_spin_unlock_irqrestore(&era->lock, flags);
                 goto again;
@@ -1201,11 +1226,23 @@ static struct event_constraint *
 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
                               struct perf_event *event)
 {
-        struct event_constraint *c = NULL;
-
-        if (event->hw.extra_reg.idx != EXTRA_REG_NONE)
-                c = __intel_shared_reg_get_constraints(cpuc, event);
-
+        struct event_constraint *c = NULL, *d;
+        struct hw_perf_event_extra *xreg, *breg;
+
+        xreg = &event->hw.extra_reg;
+        if (xreg->idx != EXTRA_REG_NONE) {
+                c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
+                if (c == &emptyconstraint)
+                        return c;
+        }
+        breg = &event->hw.branch_reg;
+        if (breg->idx != EXTRA_REG_NONE) {
+                d = __intel_shared_reg_get_constraints(cpuc, event, breg);
+                if (d == &emptyconstraint) {
+                        __intel_shared_reg_put_constraints(cpuc, xreg);
+                        c = d;
+                }
+        }
         return c;
 }
 
@@ -1253,6 +1290,10 @@ intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
         reg = &event->hw.extra_reg;
         if (reg->idx != EXTRA_REG_NONE)
                 __intel_shared_reg_put_constraints(cpuc, reg);
+
+        reg = &event->hw.branch_reg;
+        if (reg->idx != EXTRA_REG_NONE)
+                __intel_shared_reg_put_constraints(cpuc, reg);
 }
 
 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
@@ -1295,6 +1336,12 @@ static int intel_pmu_hw_config(struct perf_event *event)
                 event->hw.config = alt_config;
         }
 
+        if (intel_pmu_needs_lbr_smpl(event)) {
+                ret = intel_pmu_setup_lbr_filter(event);
+                if (ret)
+                        return ret;
+        }
+
         if (event->attr.type != PERF_TYPE_RAW)
                 return 0;
 
@@ -1433,7 +1480,7 @@ static int intel_pmu_cpu_prepare(int cpu)
 {
         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 
-        if (!x86_pmu.extra_regs)
+        if (!(x86_pmu.extra_regs || x86_pmu.lbr_sel_map))
                 return NOTIFY_OK;
 
         cpuc->shared_regs = allocate_shared_regs(cpu);
@@ -1455,22 +1502,28 @@ static void intel_pmu_cpu_starting(int cpu)
          */
         intel_pmu_lbr_reset();
 
-        if (!cpuc->shared_regs || (x86_pmu.er_flags & ERF_NO_HT_SHARING))
+        cpuc->lbr_sel = NULL;
+
+        if (!cpuc->shared_regs)
                 return;
 
-        for_each_cpu(i, topology_thread_cpumask(cpu)) {
-                struct intel_shared_regs *pc;
+        if (!(x86_pmu.er_flags & ERF_NO_HT_SHARING)) {
+                for_each_cpu(i, topology_thread_cpumask(cpu)) {
+                        struct intel_shared_regs *pc;
 
-                pc = per_cpu(cpu_hw_events, i).shared_regs;
-                if (pc && pc->core_id == core_id) {
-                        cpuc->kfree_on_online = cpuc->shared_regs;
-                        cpuc->shared_regs = pc;
-                        break;
+                        pc = per_cpu(cpu_hw_events, i).shared_regs;
+                        if (pc && pc->core_id == core_id) {
+                                cpuc->kfree_on_online = cpuc->shared_regs;
+                                cpuc->shared_regs = pc;
+                                break;
+                        }
                 }
+                cpuc->shared_regs->core_id = core_id;
+                cpuc->shared_regs->refcnt++;
         }
 
-        cpuc->shared_regs->core_id = core_id;
-        cpuc->shared_regs->refcnt++;
+        if (x86_pmu.lbr_sel_map)
+                cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
 }
 
 static void intel_pmu_cpu_dying(int cpu)
@@ -1488,6 +1541,18 @@ static void intel_pmu_cpu_dying(int cpu)
         fini_debug_store_on_cpu(cpu);
 }
 
+static void intel_pmu_flush_branch_stack(void)
+{
+        /*
+         * Intel LBR does not tag entries with the
+         * PID of the current task, then we need to
+         * flush it on ctxsw
+         * For now, we simply reset it
+         */
+        if (x86_pmu.lbr_nr)
+                intel_pmu_lbr_reset();
+}
+
 static __initconst const struct x86_pmu intel_pmu = {
         .name                   = "Intel",
         .handle_irq             = intel_pmu_handle_irq,
@@ -1515,6 +1580,7 @@ static __initconst const struct x86_pmu intel_pmu = {
         .cpu_starting           = intel_pmu_cpu_starting,
         .cpu_dying              = intel_pmu_cpu_dying,
         .guest_get_msrs         = intel_guest_get_msrs,
+        .flush_branch_stack     = intel_pmu_flush_branch_stack,
 };
 
 static __init void intel_clovertown_quirk(void)
@@ -1745,7 +1811,7 @@ __init int intel_pmu_init(void)
                 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
                        sizeof(hw_cache_event_ids));
 
-                intel_pmu_lbr_init_nhm();
+                intel_pmu_lbr_init_snb();
 
                 x86_pmu.event_constraints = intel_snb_event_constraints;
                 x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index d6bd49faa40c..7f64df19e7dd 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -3,6 +3,7 @@
 #include <linux/slab.h>
 
 #include <asm/perf_event.h>
+#include <asm/insn.h>
 
 #include "perf_event.h"
 
@@ -439,9 +440,6 @@ void intel_pmu_pebs_enable(struct perf_event *event)
         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
 
         cpuc->pebs_enabled |= 1ULL << hwc->idx;
-
-        if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
-                intel_pmu_lbr_enable(event);
 }
 
 void intel_pmu_pebs_disable(struct perf_event *event)
@@ -454,9 +452,6 @@ void intel_pmu_pebs_disable(struct perf_event *event)
                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
 
         hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
-
-        if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
-                intel_pmu_lbr_disable(event);
 }
 
 void intel_pmu_pebs_enable_all(void)
@@ -475,17 +470,6 @@ void intel_pmu_pebs_disable_all(void)
                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
 }
 
-#include <asm/insn.h>
-
-static inline bool kernel_ip(unsigned long ip)
-{
-#ifdef CONFIG_X86_32
-        return ip > PAGE_OFFSET;
-#else
-        return (long)ip < 0;
-#endif
-}
-
 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -572,6 +556,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
          * both formats and we don't use the other fields in this
          * routine.
          */
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct pebs_record_core *pebs = __pebs;
         struct perf_sample_data data;
         struct pt_regs regs;
@@ -602,6 +587,9 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
         else
                 regs.flags &= ~PERF_EFLAGS_EXACT;
 
+        if (has_branch_stack(event))
+                data.br_stack = &cpuc->lbr_stack;
+
         if (perf_event_overflow(event, &data, &regs))
                 x86_pmu_stop(event, 0);
 }
diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index 47a7e63bfe54..520b4265fcd2 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -3,6 +3,7 @@
 
 #include <asm/perf_event.h>
 #include <asm/msr.h>
+#include <asm/insn.h>
 
 #include "perf_event.h"
 
@@ -14,6 +15,100 @@ enum {
 };
 
 /*
+ * Intel LBR_SELECT bits
+ * Intel Vol3a, April 2011, Section 16.7 Table 16-10
+ *
+ * Hardware branch filter (not available on all CPUs)
+ */
+#define LBR_KERNEL_BIT          0 /* do not capture at ring0 */
+#define LBR_USER_BIT            1 /* do not capture at ring > 0 */
+#define LBR_JCC_BIT             2 /* do not capture conditional branches */
+#define LBR_REL_CALL_BIT        3 /* do not capture relative calls */
+#define LBR_IND_CALL_BIT        4 /* do not capture indirect calls */
+#define LBR_RETURN_BIT          5 /* do not capture near returns */
+#define LBR_IND_JMP_BIT         6 /* do not capture indirect jumps */
+#define LBR_REL_JMP_BIT         7 /* do not capture relative jumps */
+#define LBR_FAR_BIT             8 /* do not capture far branches */
+
+#define LBR_KERNEL      (1 << LBR_KERNEL_BIT)
+#define LBR_USER        (1 << LBR_USER_BIT)
+#define LBR_JCC         (1 << LBR_JCC_BIT)
+#define LBR_REL_CALL    (1 << LBR_REL_CALL_BIT)
+#define LBR_IND_CALL    (1 << LBR_IND_CALL_BIT)
+#define LBR_RETURN      (1 << LBR_RETURN_BIT)
+#define LBR_REL_JMP     (1 << LBR_REL_JMP_BIT)
+#define LBR_IND_JMP     (1 << LBR_IND_JMP_BIT)
+#define LBR_FAR         (1 << LBR_FAR_BIT)
+
+#define LBR_PLM (LBR_KERNEL | LBR_USER)
+
+#define LBR_SEL_MASK    0x1ff   /* valid bits in LBR_SELECT */
+#define LBR_NOT_SUPP    -1      /* LBR filter not supported */
+#define LBR_IGN         0       /* ignored */
+
+#define LBR_ANY          \
+        (LBR_JCC        |\
+         LBR_REL_CALL   |\
+         LBR_IND_CALL   |\
+         LBR_RETURN     |\
+         LBR_REL_JMP    |\
+         LBR_IND_JMP    |\
+         LBR_FAR)
+
+#define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
+
+#define for_each_branch_sample_type(x) \
+        for ((x) = PERF_SAMPLE_BRANCH_USER; \
+             (x) < PERF_SAMPLE_BRANCH_MAX; (x) <<= 1)
+
+/*
+ * x86control flow change classification
+ * x86control flow changes include branches, interrupts, traps, faults
+ */
+enum {
+        X86_BR_NONE     = 0,      /* unknown */
+
+        X86_BR_USER     = 1 << 0, /* branch target is user */
+        X86_BR_KERNEL   = 1 << 1, /* branch target is kernel */
+
+        X86_BR_CALL     = 1 << 2, /* call */
+        X86_BR_RET      = 1 << 3, /* return */
+        X86_BR_SYSCALL  = 1 << 4, /* syscall */
+        X86_BR_SYSRET   = 1 << 5, /* syscall return */
+        X86_BR_INT      = 1 << 6, /* sw interrupt */
+        X86_BR_IRET     = 1 << 7, /* return from interrupt */
+        X86_BR_JCC      = 1 << 8, /* conditional */
+        X86_BR_JMP      = 1 << 9, /* jump */
+        X86_BR_IRQ      = 1 << 10,/* hw interrupt or trap or fault */
+        X86_BR_IND_CALL = 1 << 11,/* indirect calls */
+};
+
+#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
+
+#define X86_BR_ANY       \
+        (X86_BR_CALL    |\
+         X86_BR_RET     |\
+         X86_BR_SYSCALL |\
+         X86_BR_SYSRET  |\
+         X86_BR_INT     |\
+         X86_BR_IRET    |\
+         X86_BR_JCC     |\
+         X86_BR_JMP      |\
+         X86_BR_IRQ      |\
+         X86_BR_IND_CALL)
+
+#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
+
+#define X86_BR_ANY_CALL          \
+        (X86_BR_CALL            |\
+         X86_BR_IND_CALL        |\
+         X86_BR_SYSCALL         |\
+         X86_BR_IRQ             |\
+         X86_BR_INT)
+
+static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
+
+/*
  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  * otherwise it becomes near impossible to get a reliable stack.
  */
@@ -21,6 +116,10 @@ enum {
 static void __intel_pmu_lbr_enable(void)
 {
         u64 debugctl;
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        if (cpuc->lbr_sel)
+                wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config);
 
         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
         debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
@@ -76,11 +175,11 @@ void intel_pmu_lbr_enable(struct perf_event *event)
          * Reset the LBR stack if we changed task context to
          * avoid data leaks.
          */
-
         if (event->ctx->task && cpuc->lbr_context != event->ctx) {
                 intel_pmu_lbr_reset();
                 cpuc->lbr_context = event->ctx;
         }
+        cpuc->br_sel = event->hw.branch_reg.reg;
 
         cpuc->lbr_users++;
 }
@@ -95,8 +194,11 @@ void intel_pmu_lbr_disable(struct perf_event *event)
         cpuc->lbr_users--;
         WARN_ON_ONCE(cpuc->lbr_users < 0);
 
-        if (cpuc->enabled && !cpuc->lbr_users)
+        if (cpuc->enabled && !cpuc->lbr_users) {
                 __intel_pmu_lbr_disable();
+                /* avoid stale pointer */
+                cpuc->lbr_context = NULL;
+        }
 }
 
 void intel_pmu_lbr_enable_all(void)
@@ -115,6 +217,9 @@ void intel_pmu_lbr_disable_all(void)
                 __intel_pmu_lbr_disable();
 }
 
+/*
+ * TOS = most recently recorded branch
+ */
 static inline u64 intel_pmu_lbr_tos(void)
 {
         u64 tos;
@@ -142,15 +247,15 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 
                 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 
-                cpuc->lbr_entries[i].from  = msr_lastbranch.from;
-                cpuc->lbr_entries[i].to    = msr_lastbranch.to;
-                cpuc->lbr_entries[i].flags = 0;
+                cpuc->lbr_entries[i].from       = msr_lastbranch.from;
+                cpuc->lbr_entries[i].to         = msr_lastbranch.to;
+                cpuc->lbr_entries[i].mispred    = 0;
+                cpuc->lbr_entries[i].predicted  = 0;
+                cpuc->lbr_entries[i].reserved   = 0;
         }
         cpuc->lbr_stack.nr = i;
 }
 
-#define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
-
 /*
  * Due to lack of segmentation in Linux the effective address (offset)
  * is the same as the linear address, allowing us to merge the LIP and EIP
@@ -165,19 +270,22 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 
         for (i = 0; i < x86_pmu.lbr_nr; i++) {
                 unsigned long lbr_idx = (tos - i) & mask;
-                u64 from, to, flags = 0;
+                u64 from, to, mis = 0, pred = 0;
 
                 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
                 rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);
 
                 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
-                        flags = !!(from & LBR_FROM_FLAG_MISPRED);
+                        mis = !!(from & LBR_FROM_FLAG_MISPRED);
+                        pred = !mis;
                         from = (u64)((((s64)from) << 1) >> 1);
                 }
 
-                cpuc->lbr_entries[i].from  = from;
-                cpuc->lbr_entries[i].to    = to;
-                cpuc->lbr_entries[i].flags = flags;
+                cpuc->lbr_entries[i].from       = from;
+                cpuc->lbr_entries[i].to         = to;
+                cpuc->lbr_entries[i].mispred    = mis;
+                cpuc->lbr_entries[i].predicted  = pred;
+                cpuc->lbr_entries[i].reserved   = 0;
         }
         cpuc->lbr_stack.nr = i;
 }
@@ -193,28 +301,404 @@ void intel_pmu_lbr_read(void)
                 intel_pmu_lbr_read_32(cpuc);
         else
                 intel_pmu_lbr_read_64(cpuc);
+
+        intel_pmu_lbr_filter(cpuc);
+}
+
+/*
+ * SW filter is used:
+ * - in case there is no HW filter
+ * - in case the HW filter has errata or limitations
+ */
+static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
+{
+        u64 br_type = event->attr.branch_sample_type;
+        int mask = 0;
+
+        if (br_type & PERF_SAMPLE_BRANCH_USER)
+                mask |= X86_BR_USER;
+
+        if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
+                mask |= X86_BR_KERNEL;
+
+        /* we ignore BRANCH_HV here */
+
+        if (br_type & PERF_SAMPLE_BRANCH_ANY)
+                mask |= X86_BR_ANY;
+
+        if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
+                mask |= X86_BR_ANY_CALL;
+
+        if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
+                mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
+
+        if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
+                mask |= X86_BR_IND_CALL;
+        /*
+         * stash actual user request into reg, it may
+         * be used by fixup code for some CPU
+         */
+        event->hw.branch_reg.reg = mask;
+}
+
+/*
+ * setup the HW LBR filter
+ * Used only when available, may not be enough to disambiguate
+ * all branches, may need the help of the SW filter
+ */
+static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
+{
+        struct hw_perf_event_extra *reg;
+        u64 br_type = event->attr.branch_sample_type;
+        u64 mask = 0, m;
+        u64 v;
+
+        for_each_branch_sample_type(m) {
+                if (!(br_type & m))
+                        continue;
+
+                v = x86_pmu.lbr_sel_map[m];
+                if (v == LBR_NOT_SUPP)
+                        return -EOPNOTSUPP;
+
+                if (v != LBR_IGN)
+                        mask |= v;
+        }
+        reg = &event->hw.branch_reg;
+        reg->idx = EXTRA_REG_LBR;
+
+        /* LBR_SELECT operates in suppress mode so invert mask */
+        reg->config = ~mask & x86_pmu.lbr_sel_mask;
+
+        return 0;
+}
+
+int intel_pmu_setup_lbr_filter(struct perf_event *event)
+{
+        int ret = 0;
+
+        /*
+         * no LBR on this PMU
+         */
+        if (!x86_pmu.lbr_nr)
+                return -EOPNOTSUPP;
+
+        /*
+         * setup SW LBR filter
+         */
+        intel_pmu_setup_sw_lbr_filter(event);
+
+        /*
+         * setup HW LBR filter, if any
+         */
+        if (x86_pmu.lbr_sel_map)
+                ret = intel_pmu_setup_hw_lbr_filter(event);
+
+        return ret;
 }
 
+/*
+ * return the type of control flow change at address "from"
+ * intruction is not necessarily a branch (in case of interrupt).
+ *
+ * The branch type returned also includes the priv level of the
+ * target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
+ *
+ * If a branch type is unknown OR the instruction cannot be
+ * decoded (e.g., text page not present), then X86_BR_NONE is
+ * returned.
+ */
+static int branch_type(unsigned long from, unsigned long to)
+{
+        struct insn insn;
+        void *addr;
+        int bytes, size = MAX_INSN_SIZE;
+        int ret = X86_BR_NONE;
+        int ext, to_plm, from_plm;
+        u8 buf[MAX_INSN_SIZE];
+        int is64 = 0;
+
+        to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
+        from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
+
+        /*
+         * maybe zero if lbr did not fill up after a reset by the time
+         * we get a PMU interrupt
+         */
+        if (from == 0 || to == 0)
+                return X86_BR_NONE;
+
+        if (from_plm == X86_BR_USER) {
+                /*
+                 * can happen if measuring at the user level only
+                 * and we interrupt in a kernel thread, e.g., idle.
+                 */
+                if (!current->mm)
+                        return X86_BR_NONE;
+
+                /* may fail if text not present */
+                bytes = copy_from_user_nmi(buf, (void __user *)from, size);
+                if (bytes != size)
+                        return X86_BR_NONE;
+
+                addr = buf;
+        } else
+                addr = (void *)from;
+
+        /*
+         * decoder needs to know the ABI especially
+         * on 64-bit systems running 32-bit apps
+         */
+#ifdef CONFIG_X86_64
+        is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32);
+#endif
+        insn_init(&insn, addr, is64);
+        insn_get_opcode(&insn);
+
+        switch (insn.opcode.bytes[0]) {
+        case 0xf:
+                switch (insn.opcode.bytes[1]) {
+                case 0x05: /* syscall */
+                case 0x34: /* sysenter */
+                        ret = X86_BR_SYSCALL;
+                        break;
+                case 0x07: /* sysret */
+                case 0x35: /* sysexit */
+                        ret = X86_BR_SYSRET;
+                        break;
+                case 0x80 ... 0x8f: /* conditional */
+                        ret = X86_BR_JCC;
+                        break;
+                default:
+                        ret = X86_BR_NONE;
+                }
+                break;
+        case 0x70 ... 0x7f: /* conditional */
+                ret = X86_BR_JCC;
+                break;
+        case 0xc2: /* near ret */
+        case 0xc3: /* near ret */
+        case 0xca: /* far ret */
+        case 0xcb: /* far ret */
+                ret = X86_BR_RET;
+                break;
+        case 0xcf: /* iret */
+                ret = X86_BR_IRET;
+                break;
+        case 0xcc ... 0xce: /* int */
+                ret = X86_BR_INT;
+                break;
+        case 0xe8: /* call near rel */
+        case 0x9a: /* call far absolute */
+                ret = X86_BR_CALL;
+                break;
+        case 0xe0 ... 0xe3: /* loop jmp */
+                ret = X86_BR_JCC;
+                break;
+        case 0xe9 ... 0xeb: /* jmp */
+                ret = X86_BR_JMP;
+                break;
+        case 0xff: /* call near absolute, call far absolute ind */
+                insn_get_modrm(&insn);
+                ext = (insn.modrm.bytes[0] >> 3) & 0x7;
+                switch (ext) {
+                case 2: /* near ind call */
+                case 3: /* far ind call */
+                        ret = X86_BR_IND_CALL;
+                        break;
+                case 4:
+                case 5:
+                        ret = X86_BR_JMP;
+                        break;
+                }
+                break;
+        default:
+                ret = X86_BR_NONE;
+        }
+        /*
+         * interrupts, traps, faults (and thus ring transition) may
+         * occur on any instructions. Thus, to classify them correctly,
+         * we need to first look at the from and to priv levels. If they
+         * are different and to is in the kernel, then it indicates
+         * a ring transition. If the from instruction is not a ring
+         * transition instr (syscall, systenter, int), then it means
+         * it was a irq, trap or fault.
+         *
+         * we have no way of detecting kernel to kernel faults.
+         */
+        if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
+            && ret != X86_BR_SYSCALL && ret != X86_BR_INT)
+                ret = X86_BR_IRQ;
+
+        /*
+         * branch priv level determined by target as
+         * is done by HW when LBR_SELECT is implemented
+         */
+        if (ret != X86_BR_NONE)
+                ret |= to_plm;
+
+        return ret;
+}
+
+/*
+ * implement actual branch filter based on user demand.
+ * Hardware may not exactly satisfy that request, thus
+ * we need to inspect opcodes. Mismatched branches are
+ * discarded. Therefore, the number of branches returned
+ * in PERF_SAMPLE_BRANCH_STACK sample may vary.
+ */
+static void
+intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
+{
+        u64 from, to;
+        int br_sel = cpuc->br_sel;
+        int i, j, type;
+        bool compress = false;
+
+        /* if sampling all branches, then nothing to filter */
+        if ((br_sel & X86_BR_ALL) == X86_BR_ALL)
+                return;
+
+        for (i = 0; i < cpuc->lbr_stack.nr; i++) {
+
+                from = cpuc->lbr_entries[i].from;
+                to = cpuc->lbr_entries[i].to;
+
+                type = branch_type(from, to);
+
+                /* if type does not correspond, then discard */
+                if (type == X86_BR_NONE || (br_sel & type) != type) {
+                        cpuc->lbr_entries[i].from = 0;
+                        compress = true;
+                }
+        }
+
+        if (!compress)
+                return;
+
+        /* remove all entries with from=0 */
+        for (i = 0; i < cpuc->lbr_stack.nr; ) {
+                if (!cpuc->lbr_entries[i].from) {
+                        j = i;
+                        while (++j < cpuc->lbr_stack.nr)
+                                cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
+                        cpuc->lbr_stack.nr--;
+                        if (!cpuc->lbr_entries[i].from)
+                                continue;
+                }
+                i++;
+        }
+}
+
+/*
+ * Map interface branch filters onto LBR filters
+ */
+static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
+        [PERF_SAMPLE_BRANCH_ANY]        = LBR_ANY,
+        [PERF_SAMPLE_BRANCH_USER]       = LBR_USER,
+        [PERF_SAMPLE_BRANCH_KERNEL]     = LBR_KERNEL,
+        [PERF_SAMPLE_BRANCH_HV]         = LBR_IGN,
+        [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_REL_JMP
+                                        | LBR_IND_JMP | LBR_FAR,
+        /*
+         * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
+         */
+        [PERF_SAMPLE_BRANCH_ANY_CALL] =
+         LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
+        /*
+         * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
+         */
+        [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP,
+};
+
+static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
+        [PERF_SAMPLE_BRANCH_ANY]        = LBR_ANY,
+        [PERF_SAMPLE_BRANCH_USER]       = LBR_USER,
+        [PERF_SAMPLE_BRANCH_KERNEL]     = LBR_KERNEL,
+        [PERF_SAMPLE_BRANCH_HV]         = LBR_IGN,
+        [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_FAR,
+        [PERF_SAMPLE_BRANCH_ANY_CALL]   = LBR_REL_CALL | LBR_IND_CALL
+                                        | LBR_FAR,
+        [PERF_SAMPLE_BRANCH_IND_CALL]   = LBR_IND_CALL,
+};
+
+/* core */
 void intel_pmu_lbr_init_core(void)
 {
         x86_pmu.lbr_nr     = 4;
-        x86_pmu.lbr_tos    = 0x01c9;
-        x86_pmu.lbr_from   = 0x40;
-        x86_pmu.lbr_to     = 0x60;
+        x86_pmu.lbr_tos    = MSR_LBR_TOS;
+        x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
+        x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
+
+        /*
+         * SW branch filter usage:
+         * - compensate for lack of HW filter
+         */
+        pr_cont("4-deep LBR, ");
 }
 
+/* nehalem/westmere */
 void intel_pmu_lbr_init_nhm(void)
 {
         x86_pmu.lbr_nr     = 16;
-        x86_pmu.lbr_tos    = 0x01c9;
-        x86_pmu.lbr_from   = 0x680;
-        x86_pmu.lbr_to     = 0x6c0;
+        x86_pmu.lbr_tos    = MSR_LBR_TOS;
+        x86_pmu.lbr_from   = MSR_LBR_NHM_FROM;
+        x86_pmu.lbr_to     = MSR_LBR_NHM_TO;
+
+        x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
+        x86_pmu.lbr_sel_map  = nhm_lbr_sel_map;
+
+        /*
+         * SW branch filter usage:
+         * - workaround LBR_SEL errata (see above)
+         * - support syscall, sysret capture.
+         *   That requires LBR_FAR but that means far
+         *   jmp need to be filtered out
+         */
+        pr_cont("16-deep LBR, ");
+}
+
+/* sandy bridge */
+void intel_pmu_lbr_init_snb(void)
+{
+        x86_pmu.lbr_nr   = 16;
+        x86_pmu.lbr_tos  = MSR_LBR_TOS;
+        x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
+        x86_pmu.lbr_to   = MSR_LBR_NHM_TO;
+
+        x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
+        x86_pmu.lbr_sel_map  = snb_lbr_sel_map;
+
+        /*
+         * SW branch filter usage:
+         * - support syscall, sysret capture.
+         *   That requires LBR_FAR but that means far
+         *   jmp need to be filtered out
+         */
+        pr_cont("16-deep LBR, ");
 }
 
+/* atom */
 void intel_pmu_lbr_init_atom(void)
 {
+        /*
+         * only models starting at stepping 10 seems
+         * to have an operational LBR which can freeze
+         * on PMU interrupt
+         */
+        if (boot_cpu_data.x86_mask < 10) {
+                pr_cont("LBR disabled due to erratum");
+                return;
+        }
+
         x86_pmu.lbr_nr     = 8;
-        x86_pmu.lbr_tos    = 0x01c9;
-        x86_pmu.lbr_from   = 0x40;
-        x86_pmu.lbr_to     = 0x60;
+        x86_pmu.lbr_tos    = MSR_LBR_TOS;
+        x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
+        x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
+
+        /*
+         * SW branch filter usage:
+         * - compensate for lack of HW filter
+         */
+        pr_cont("8-deep LBR, ");
 }