4 files changed, 47 insertions, 5 deletions

diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index a73dfc97226b..74089bcb6d74 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -92,9 +92,11 @@ struct amd_nb {
 /*
  * Flags PEBS can handle without an PMI.
  *
+ * TID can only be handled by flushing at context switch.
+ *
  */
 #define PEBS_FREERUNNING_FLAGS \
-        (PERF_SAMPLE_IP | PERF_SAMPLE_ADDR | \
+        (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
         PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
         PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
         PERF_SAMPLE_TRANSACTION)
@@ -877,6 +879,8 @@ void intel_pmu_pebs_enable_all(void);
 
 void intel_pmu_pebs_disable_all(void);
 
+void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in);
+
 void intel_ds_init(void);
 
 void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 6985f43c5eb9..d455e2a61287 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -2642,6 +2642,15 @@ static void intel_pmu_cpu_dying(int cpu)
         fini_debug_store_on_cpu(cpu);
 }
 
+static void intel_pmu_sched_task(struct perf_event_context *ctx,
+                                 bool sched_in)
+{
+        if (x86_pmu.pebs_active)
+                intel_pmu_pebs_sched_task(ctx, sched_in);
+        if (x86_pmu.lbr_nr)
+                intel_pmu_lbr_sched_task(ctx, sched_in);
+}
+
 PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
 
 PMU_FORMAT_ATTR(ldlat, "config1:0-15");
@@ -2731,7 +2740,7 @@ static __initconst const struct x86_pmu intel_pmu = {
         .cpu_starting           = intel_pmu_cpu_starting,
         .cpu_dying              = intel_pmu_cpu_dying,
         .guest_get_msrs         = intel_guest_get_msrs,
-        .sched_task             = intel_pmu_lbr_sched_task,
+        .sched_task             = intel_pmu_sched_task,
 };
 
 static __init void intel_clovertown_quirk(void)
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 0ce455d958b8..62852470ccbb 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -546,6 +546,19 @@ int intel_pmu_drain_bts_buffer(void)
         return 1;
 }
 
+static inline void intel_pmu_drain_pebs_buffer(void)
+{
+        struct pt_regs regs;
+
+        x86_pmu.drain_pebs(&regs);
+}
+
+void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in)
+{
+        if (!sched_in)
+                intel_pmu_drain_pebs_buffer();
+}
+
 /*
  * PEBS
  */
@@ -711,8 +724,19 @@ void intel_pmu_pebs_enable(struct perf_event *event)
         if (hwc->flags & PERF_X86_EVENT_FREERUNNING) {
                 threshold = ds->pebs_absolute_maximum -
                         x86_pmu.max_pebs_events * x86_pmu.pebs_record_size;
+
+                if (first_pebs)
+                        perf_sched_cb_inc(event->ctx->pmu);
         } else {
                 threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size;
+
+                /*
+                 * If not all events can use larger buffer,
+                 * roll back to threshold = 1
+                 */
+                if (!first_pebs &&
+                    (ds->pebs_interrupt_threshold > threshold))
+                        perf_sched_cb_dec(event->ctx->pmu);
         }
 
         /* Use auto-reload if possible to save a MSR write in the PMI */
@@ -729,6 +753,7 @@ void intel_pmu_pebs_disable(struct perf_event *event)
 {
         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
+        struct debug_store *ds = cpuc->ds;
 
         cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
 
@@ -737,6 +762,13 @@ void intel_pmu_pebs_disable(struct perf_event *event)
         else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
                 cpuc->pebs_enabled &= ~(1ULL << 63);
 
+        if (ds->pebs_interrupt_threshold >
+            ds->pebs_buffer_base + x86_pmu.pebs_record_size) {
+                intel_pmu_drain_pebs_buffer();
+                if (!pebs_is_enabled(cpuc))
+                        perf_sched_cb_dec(event->ctx->pmu);
+        }
+
         if (cpuc->enabled)
                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
 
diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index 201e16f6655a..452a7bd2dedb 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -264,9 +264,6 @@ void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in)
         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
         struct x86_perf_task_context *task_ctx;
 
-        if (!x86_pmu.lbr_nr)
-                return;
-
         /*
          * If LBR callstack feature is enabled and the stack was saved when
          * the task was scheduled out, restore the stack. Otherwise flush