5 files changed, 540 insertions, 3 deletions

diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index e6b353f10e09..9bff68798836 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -40,7 +40,7 @@ endif
 obj-$(CONFIG_CPU_SUP_INTEL)             += perf_event_p6.o perf_event_knc.o perf_event_p4.o
 obj-$(CONFIG_CPU_SUP_INTEL)             += perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o
 obj-$(CONFIG_CPU_SUP_INTEL)             += perf_event_intel_rapl.o perf_event_intel_cqm.o
-obj-$(CONFIG_CPU_SUP_INTEL)             += perf_event_intel_pt.o
+obj-$(CONFIG_CPU_SUP_INTEL)             += perf_event_intel_pt.o perf_event_intel_bts.o
 
 obj-$(CONFIG_PERF_EVENTS_INTEL_UNCORE)  += perf_event_intel_uncore.o \
                                            perf_event_intel_uncore_snb.o \
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index f04729ac3290..eaebfd707016 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -410,6 +410,7 @@ union x86_pmu_config {
 
 enum {
         x86_lbr_exclusive_lbr,
+        x86_lbr_exclusive_bts,
         x86_lbr_exclusive_pt,
         x86_lbr_exclusive_max,
 };
@@ -810,6 +811,12 @@ int intel_pmu_setup_lbr_filter(struct perf_event *event);
 
 void intel_pt_interrupt(void);
 
+int intel_bts_interrupt(void);
+
+void intel_bts_enable_local(void);
+
+void intel_bts_disable_local(void);
+
 int p4_pmu_init(void);
 
 int p6_pmu_init(void);
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 8eb22ce26303..b9861e19cd3d 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -1242,6 +1242,8 @@ static void intel_pmu_disable_all(void)
 
         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
                 intel_pmu_disable_bts();
+        else
+                intel_bts_disable_local();
 
         intel_pmu_pebs_disable_all();
         intel_pmu_lbr_disable_all();
@@ -1264,7 +1266,8 @@ static void intel_pmu_enable_all(int added)
                         return;
 
                 intel_pmu_enable_bts(event->hw.config);
-        }
+        } else
+                intel_bts_enable_local();
 }
 
 /*
@@ -1550,6 +1553,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
                 apic_write(APIC_LVTPC, APIC_DM_NMI);
         intel_pmu_disable_all();
         handled = intel_pmu_drain_bts_buffer();
+        handled += intel_bts_interrupt();
         status = intel_pmu_get_status();
         if (!status)
                 goto done;
diff --git a/arch/x86/kernel/cpu/perf_event_intel_bts.c b/arch/x86/kernel/cpu/perf_event_intel_bts.c
new file mode 100644
index 000000000000..fb1a4c28f3e1
--- /dev/null
+++ b/arch/x86/kernel/cpu/perf_event_intel_bts.c
@@ -0,0 +1,525 @@
+/*
+ * BTS PMU driver for perf
+ * Copyright (c) 2013-2014, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#undef DEBUG
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/bitops.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/coredump.h>
+
+#include <asm-generic/sizes.h>
+#include <asm/perf_event.h>
+
+#include "perf_event.h"
+
+struct bts_ctx {
+        struct perf_output_handle       handle;
+        struct debug_store              ds_back;
+        int                             started;
+};
+
+static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
+
+#define BTS_RECORD_SIZE         24
+#define BTS_SAFETY_MARGIN       4080
+
+struct bts_phys {
+        struct page     *page;
+        unsigned long   size;
+        unsigned long   offset;
+        unsigned long   displacement;
+};
+
+struct bts_buffer {
+        size_t          real_size;      /* multiple of BTS_RECORD_SIZE */
+        unsigned int    nr_pages;
+        unsigned int    nr_bufs;
+        unsigned int    cur_buf;
+        bool            snapshot;
+        local_t         data_size;
+        local_t         lost;
+        local_t         head;
+        unsigned long   end;
+        void            **data_pages;
+        struct bts_phys buf[0];
+};
+
+struct pmu bts_pmu;
+
+void intel_pmu_enable_bts(u64 config);
+void intel_pmu_disable_bts(void);
+
+static size_t buf_size(struct page *page)
+{
+        return 1 << (PAGE_SHIFT + page_private(page));
+}
+
+static void *
+bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
+{
+        struct bts_buffer *buf;
+        struct page *page;
+        int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
+        unsigned long offset;
+        size_t size = nr_pages << PAGE_SHIFT;
+        int pg, nbuf, pad;
+
+        /* count all the high order buffers */
+        for (pg = 0, nbuf = 0; pg < nr_pages;) {
+                page = virt_to_page(pages[pg]);
+                if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
+                        return NULL;
+                pg += 1 << page_private(page);
+                nbuf++;
+        }
+
+        /*
+         * to avoid interrupts in overwrite mode, only allow one physical
+         */
+        if (overwrite && nbuf > 1)
+                return NULL;
+
+        buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
+        if (!buf)
+                return NULL;
+
+        buf->nr_pages = nr_pages;
+        buf->nr_bufs = nbuf;
+        buf->snapshot = overwrite;
+        buf->data_pages = pages;
+        buf->real_size = size - size % BTS_RECORD_SIZE;
+
+        for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
+                unsigned int __nr_pages;
+
+                page = virt_to_page(pages[pg]);
+                __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
+                buf->buf[nbuf].page = page;
+                buf->buf[nbuf].offset = offset;
+                buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
+                buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
+                pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
+                buf->buf[nbuf].size -= pad;
+
+                pg += __nr_pages;
+                offset += __nr_pages << PAGE_SHIFT;
+        }
+
+        return buf;
+}
+
+static void bts_buffer_free_aux(void *data)
+{
+        kfree(data);
+}
+
+static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
+{
+        return buf->buf[idx].offset + buf->buf[idx].displacement;
+}
+
+static void
+bts_config_buffer(struct bts_buffer *buf)
+{
+        int cpu = raw_smp_processor_id();
+        struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
+        struct bts_phys *phys = &buf->buf[buf->cur_buf];
+        unsigned long index, thresh = 0, end = phys->size;
+        struct page *page = phys->page;
+
+        index = local_read(&buf->head);
+
+        if (!buf->snapshot) {
+                if (buf->end < phys->offset + buf_size(page))
+                        end = buf->end - phys->offset - phys->displacement;
+
+                index -= phys->offset + phys->displacement;
+
+                if (end - index > BTS_SAFETY_MARGIN)
+                        thresh = end - BTS_SAFETY_MARGIN;
+                else if (end - index > BTS_RECORD_SIZE)
+                        thresh = end - BTS_RECORD_SIZE;
+                else
+                        thresh = end;
+        }
+
+        ds->bts_buffer_base = (u64)page_address(page) + phys->displacement;
+        ds->bts_index = ds->bts_buffer_base + index;
+        ds->bts_absolute_maximum = ds->bts_buffer_base + end;
+        ds->bts_interrupt_threshold = !buf->snapshot
+                ? ds->bts_buffer_base + thresh
+                : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
+}
+
+static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
+{
+        unsigned long index = head - phys->offset;
+
+        memset(page_address(phys->page) + index, 0, phys->size - index);
+}
+
+static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)
+{
+        if (buf->snapshot)
+                return false;
+
+        if (local_read(&buf->data_size) >= bts->handle.size ||
+            bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)
+                return true;
+
+        return false;
+}
+
+static void bts_update(struct bts_ctx *bts)
+{
+        int cpu = raw_smp_processor_id();
+        struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
+        struct bts_buffer *buf = perf_get_aux(&bts->handle);
+        unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
+
+        if (!buf)
+                return;
+
+        head = index + bts_buffer_offset(buf, buf->cur_buf);
+        old = local_xchg(&buf->head, head);
+
+        if (!buf->snapshot) {
+                if (old == head)
+                        return;
+
+                if (ds->bts_index >= ds->bts_absolute_maximum)
+                        local_inc(&buf->lost);
+
+                /*
+                 * old and head are always in the same physical buffer, so we
+                 * can subtract them to get the data size.
+                 */
+                local_add(head - old, &buf->data_size);
+        } else {
+                local_set(&buf->data_size, head);
+        }
+}
+
+static void __bts_event_start(struct perf_event *event)
+{
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+        struct bts_buffer *buf = perf_get_aux(&bts->handle);
+        u64 config = 0;
+
+        if (!buf || bts_buffer_is_full(buf, bts))
+                return;
+
+        event->hw.state = 0;
+
+        if (!buf->snapshot)
+                config |= ARCH_PERFMON_EVENTSEL_INT;
+        if (!event->attr.exclude_kernel)
+                config |= ARCH_PERFMON_EVENTSEL_OS;
+        if (!event->attr.exclude_user)
+                config |= ARCH_PERFMON_EVENTSEL_USR;
+
+        bts_config_buffer(buf);
+
+        /*
+         * local barrier to make sure that ds configuration made it
+         * before we enable BTS
+         */
+        wmb();
+
+        intel_pmu_enable_bts(config);
+}
+
+static void bts_event_start(struct perf_event *event, int flags)
+{
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+
+        __bts_event_start(event);
+
+        /* PMI handler: this counter is running and likely generating PMIs */
+        ACCESS_ONCE(bts->started) = 1;
+}
+
+static void __bts_event_stop(struct perf_event *event)
+{
+        /*
+         * No extra synchronization is mandated by the documentation to have
+         * BTS data stores globally visible.
+         */
+        intel_pmu_disable_bts();
+
+        if (event->hw.state & PERF_HES_STOPPED)
+                return;
+
+        ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
+}
+
+static void bts_event_stop(struct perf_event *event, int flags)
+{
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+
+        /* PMI handler: don't restart this counter */
+        ACCESS_ONCE(bts->started) = 0;
+
+        __bts_event_stop(event);
+
+        if (flags & PERF_EF_UPDATE)
+                bts_update(bts);
+}
+
+void intel_bts_enable_local(void)
+{
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+
+        if (bts->handle.event && bts->started)
+                __bts_event_start(bts->handle.event);
+}
+
+void intel_bts_disable_local(void)
+{
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+
+        if (bts->handle.event)
+                __bts_event_stop(bts->handle.event);
+}
+
+static int
+bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
+{
+        unsigned long head, space, next_space, pad, gap, skip, wakeup;
+        unsigned int next_buf;
+        struct bts_phys *phys, *next_phys;
+        int ret;
+
+        if (buf->snapshot)
+                return 0;
+
+        head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
+        if (WARN_ON_ONCE(head != local_read(&buf->head)))
+                return -EINVAL;
+
+        phys = &buf->buf[buf->cur_buf];
+        space = phys->offset + phys->displacement + phys->size - head;
+        pad = space;
+        if (space > handle->size) {
+                space = handle->size;
+                space -= space % BTS_RECORD_SIZE;
+        }
+        if (space <= BTS_SAFETY_MARGIN) {
+                /* See if next phys buffer has more space */
+                next_buf = buf->cur_buf + 1;
+                if (next_buf >= buf->nr_bufs)
+                        next_buf = 0;
+                next_phys = &buf->buf[next_buf];
+                gap = buf_size(phys->page) - phys->displacement - phys->size +
+                      next_phys->displacement;
+                skip = pad + gap;
+                if (handle->size >= skip) {
+                        next_space = next_phys->size;
+                        if (next_space + skip > handle->size) {
+                                next_space = handle->size - skip;
+                                next_space -= next_space % BTS_RECORD_SIZE;
+                        }
+                        if (next_space > space || !space) {
+                                if (pad)
+                                        bts_buffer_pad_out(phys, head);
+                                ret = perf_aux_output_skip(handle, skip);
+                                if (ret)
+                                        return ret;
+                                /* Advance to next phys buffer */
+                                phys = next_phys;
+                                space = next_space;
+                                head = phys->offset + phys->displacement;
+                                /*
+                                 * After this, cur_buf and head won't match ds
+                                 * anymore, so we must not be racing with
+                                 * bts_update().
+                                 */
+                                buf->cur_buf = next_buf;
+                                local_set(&buf->head, head);
+                        }
+                }
+        }
+
+        /* Don't go far beyond wakeup watermark */
+        wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
+                 handle->head;
+        if (space > wakeup) {
+                space = wakeup;
+                space -= space % BTS_RECORD_SIZE;
+        }
+
+        buf->end = head + space;
+
+        /*
+         * If we have no space, the lost notification would have been sent when
+         * we hit absolute_maximum - see bts_update()
+         */
+        if (!space)
+                return -ENOSPC;
+
+        return 0;
+}
+
+int intel_bts_interrupt(void)
+{
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+        struct perf_event *event = bts->handle.event;
+        struct bts_buffer *buf;
+        s64 old_head;
+        int err;
+
+        if (!event || !bts->started)
+                return 0;
+
+        buf = perf_get_aux(&bts->handle);
+        /*
+         * Skip snapshot counters: they don't use the interrupt, but
+         * there's no other way of telling, because the pointer will
+         * keep moving
+         */
+        if (!buf || buf->snapshot)
+                return 0;
+
+        old_head = local_read(&buf->head);
+        bts_update(bts);
+
+        /* no new data */
+        if (old_head == local_read(&buf->head))
+                return 0;
+
+        perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
+                            !!local_xchg(&buf->lost, 0));
+
+        buf = perf_aux_output_begin(&bts->handle, event);
+        if (!buf)
+                return 1;
+
+        err = bts_buffer_reset(buf, &bts->handle);
+        if (err)
+                perf_aux_output_end(&bts->handle, 0, false);
+
+        return 1;
+}
+
+static void bts_event_del(struct perf_event *event, int mode)
+{
+        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+        struct bts_buffer *buf = perf_get_aux(&bts->handle);
+
+        bts_event_stop(event, PERF_EF_UPDATE);
+
+        if (buf) {
+                if (buf->snapshot)
+                        bts->handle.head =
+                                local_xchg(&buf->data_size,
+                                           buf->nr_pages << PAGE_SHIFT);
+                perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
+                                    !!local_xchg(&buf->lost, 0));
+        }
+
+        cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
+        cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
+        cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
+        cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
+}
+
+static int bts_event_add(struct perf_event *event, int mode)
+{
+        struct bts_buffer *buf;
+        struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
+        int ret = -EBUSY;
+
+        event->hw.state = PERF_HES_STOPPED;
+
+        if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
+                return -EBUSY;
+
+        if (bts->handle.event)
+                return -EBUSY;
+
+        buf = perf_aux_output_begin(&bts->handle, event);
+        if (!buf)
+                return -EINVAL;
+
+        ret = bts_buffer_reset(buf, &bts->handle);
+        if (ret) {
+                perf_aux_output_end(&bts->handle, 0, false);
+                return ret;
+        }
+
+        bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
+        bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
+        bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
+
+        if (mode & PERF_EF_START) {
+                bts_event_start(event, 0);
+                if (hwc->state & PERF_HES_STOPPED) {
+                        bts_event_del(event, 0);
+                        return -EBUSY;
+                }
+        }
+
+        return 0;
+}
+
+static void bts_event_destroy(struct perf_event *event)
+{
+        x86_del_exclusive(x86_lbr_exclusive_bts);
+}
+
+static int bts_event_init(struct perf_event *event)
+{
+        if (event->attr.type != bts_pmu.type)
+                return -ENOENT;
+
+        if (x86_add_exclusive(x86_lbr_exclusive_bts))
+                return -EBUSY;
+
+        event->destroy = bts_event_destroy;
+
+        return 0;
+}
+
+static void bts_event_read(struct perf_event *event)
+{
+}
+
+static __init int bts_init(void)
+{
+        if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
+                return -ENODEV;
+
+        bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE;
+        bts_pmu.task_ctx_nr     = perf_sw_context;
+        bts_pmu.event_init      = bts_event_init;
+        bts_pmu.add             = bts_event_add;
+        bts_pmu.del             = bts_event_del;
+        bts_pmu.start           = bts_event_start;
+        bts_pmu.stop            = bts_event_stop;
+        bts_pmu.read            = bts_event_read;
+        bts_pmu.setup_aux       = bts_buffer_setup_aux;
+        bts_pmu.free_aux        = bts_buffer_free_aux;
+
+        return perf_pmu_register(&bts_pmu, "intel_bts", -1);
+}
+
+module_init(bts_init);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 073983398364..a5149c7abe73 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -461,7 +461,8 @@ void intel_pmu_enable_bts(u64 config)
 
         debugctlmsr |= DEBUGCTLMSR_TR;
         debugctlmsr |= DEBUGCTLMSR_BTS;
-        debugctlmsr |= DEBUGCTLMSR_BTINT;
+        if (config & ARCH_PERFMON_EVENTSEL_INT)
+                debugctlmsr |= DEBUGCTLMSR_BTINT;
 
         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;