1 files changed, 73 insertions, 35 deletions

diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 9b983a474253..0319311dbdbb 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -806,9 +806,65 @@ struct event_constraint *intel_pebs_constraints(struct perf_event *event)
         return &emptyconstraint;
 }
 
-static inline bool pebs_is_enabled(struct cpu_hw_events *cpuc)
+/*
+ * We need the sched_task callback even for per-cpu events when we use
+ * the large interrupt threshold, such that we can provide PID and TID
+ * to PEBS samples.
+ */
+static inline bool pebs_needs_sched_cb(struct cpu_hw_events *cpuc)
+{
+        return cpuc->n_pebs && (cpuc->n_pebs == cpuc->n_large_pebs);
+}
+
+static inline void pebs_update_threshold(struct cpu_hw_events *cpuc)
+{
+        struct debug_store *ds = cpuc->ds;
+        u64 threshold;
+
+        if (cpuc->n_pebs == cpuc->n_large_pebs) {
+                threshold = ds->pebs_absolute_maximum -
+                        x86_pmu.max_pebs_events * x86_pmu.pebs_record_size;
+        } else {
+                threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size;
+        }
+
+        ds->pebs_interrupt_threshold = threshold;
+}
+
+static void
+pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc, struct pmu *pmu)
+{
+        /*
+         * Make sure we get updated with the first PEBS
+         * event. It will trigger also during removal, but
+         * that does not hurt:
+         */
+        bool update = cpuc->n_pebs == 1;
+
+        if (needed_cb != pebs_needs_sched_cb(cpuc)) {
+                if (!needed_cb)
+                        perf_sched_cb_inc(pmu);
+                else
+                        perf_sched_cb_dec(pmu);
+
+                update = true;
+        }
+
+        if (update)
+                pebs_update_threshold(cpuc);
+}
+
+void intel_pmu_pebs_add(struct perf_event *event)
 {
-        return (cpuc->pebs_enabled & ((1ULL << MAX_PEBS_EVENTS) - 1));
+        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
+        bool needed_cb = pebs_needs_sched_cb(cpuc);
+
+        cpuc->n_pebs++;
+        if (hwc->flags & PERF_X86_EVENT_FREERUNNING)
+                cpuc->n_large_pebs++;
+
+        pebs_update_state(needed_cb, cpuc, event->ctx->pmu);
 }
 
 void intel_pmu_pebs_enable(struct perf_event *event)
@@ -816,12 +872,9 @@ void intel_pmu_pebs_enable(struct perf_event *event)
         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
         struct debug_store *ds = cpuc->ds;
-        bool first_pebs;
-        u64 threshold;
 
         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
 
-        first_pebs = !pebs_is_enabled(cpuc);
         cpuc->pebs_enabled |= 1ULL << hwc->idx;
 
         if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
@@ -830,46 +883,34 @@ void intel_pmu_pebs_enable(struct perf_event *event)
                 cpuc->pebs_enabled |= 1ULL << 63;
 
         /*
-         * When the event is constrained enough we can use a larger
-         * threshold and run the event with less frequent PMI.
+         * Use auto-reload if possible to save a MSR write in the PMI.
+         * This must be done in pmu::start(), because PERF_EVENT_IOC_PERIOD.
          */
-        if (hwc->flags & PERF_X86_EVENT_FREERUNNING) {
-                threshold = ds->pebs_absolute_maximum -
-                        x86_pmu.max_pebs_events * x86_pmu.pebs_record_size;
-
-                if (first_pebs)
-                        perf_sched_cb_inc(event->ctx->pmu);
-        } else {
-                threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size;
-
-                /*
-                 * If not all events can use larger buffer,
-                 * roll back to threshold = 1
-                 */
-                if (!first_pebs &&
-                    (ds->pebs_interrupt_threshold > threshold))
-                        perf_sched_cb_dec(event->ctx->pmu);
-        }
-
-        /* Use auto-reload if possible to save a MSR write in the PMI */
         if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
                 ds->pebs_event_reset[hwc->idx] =
                         (u64)(-hwc->sample_period) & x86_pmu.cntval_mask;
         }
+}
+
+void intel_pmu_pebs_del(struct perf_event *event)
+{
+        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
+        bool needed_cb = pebs_needs_sched_cb(cpuc);
 
-        if (first_pebs || ds->pebs_interrupt_threshold > threshold)
-                ds->pebs_interrupt_threshold = threshold;
+        cpuc->n_pebs--;
+        if (hwc->flags & PERF_X86_EVENT_FREERUNNING)
+                cpuc->n_large_pebs--;
+
+        pebs_update_state(needed_cb, cpuc, event->ctx->pmu);
 }
 
 void intel_pmu_pebs_disable(struct perf_event *event)
 {
         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
-        struct debug_store *ds = cpuc->ds;
-        bool large_pebs = ds->pebs_interrupt_threshold >
-                ds->pebs_buffer_base + x86_pmu.pebs_record_size;
 
-        if (large_pebs)
+        if (cpuc->n_pebs == cpuc->n_large_pebs)
                 intel_pmu_drain_pebs_buffer();
 
         cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
@@ -879,9 +920,6 @@ void intel_pmu_pebs_disable(struct perf_event *event)
         else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
                 cpuc->pebs_enabled &= ~(1ULL << 63);
 
-        if (large_pebs && !pebs_is_enabled(cpuc))
-                perf_sched_cb_dec(event->ctx->pmu);
-
         if (cpuc->enabled)
                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);