1 files changed, 85 insertions, 0 deletions

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 242bb51c67f2..c61234b1a988 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -137,6 +137,7 @@ enum event_type_t {
  */
 struct static_key_deferred perf_sched_events __read_mostly;
 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
+static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
 
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
@@ -888,6 +889,9 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx)
         if (is_cgroup_event(event))
                 ctx->nr_cgroups++;
 
+        if (has_branch_stack(event))
+                ctx->nr_branch_stack++;
+
         list_add_rcu(&event->event_entry, &ctx->event_list);
         if (!ctx->nr_events)
                 perf_pmu_rotate_start(ctx->pmu);
@@ -1027,6 +1031,9 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx)
                         cpuctx->cgrp = NULL;
         }
 
+        if (has_branch_stack(event))
+                ctx->nr_branch_stack--;
+
         ctx->nr_events--;
         if (event->attr.inherit_stat)
                 ctx->nr_stat--;
@@ -2202,6 +2209,66 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx,
 }
 
 /*
+ * When sampling the branck stack in system-wide, it may be necessary
+ * to flush the stack on context switch. This happens when the branch
+ * stack does not tag its entries with the pid of the current task.
+ * Otherwise it becomes impossible to associate a branch entry with a
+ * task. This ambiguity is more likely to appear when the branch stack
+ * supports priv level filtering and the user sets it to monitor only
+ * at the user level (which could be a useful measurement in system-wide
+ * mode). In that case, the risk is high of having a branch stack with
+ * branch from multiple tasks. Flushing may mean dropping the existing
+ * entries or stashing them somewhere in the PMU specific code layer.
+ *
+ * This function provides the context switch callback to the lower code
+ * layer. It is invoked ONLY when there is at least one system-wide context
+ * with at least one active event using taken branch sampling.
+ */
+static void perf_branch_stack_sched_in(struct task_struct *prev,
+                                       struct task_struct *task)
+{
+        struct perf_cpu_context *cpuctx;
+        struct pmu *pmu;
+        unsigned long flags;
+
+        /* no need to flush branch stack if not changing task */
+        if (prev == task)
+                return;
+
+        local_irq_save(flags);
+
+        rcu_read_lock();
+
+        list_for_each_entry_rcu(pmu, &pmus, entry) {
+                cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
+
+                /*
+                 * check if the context has at least one
+                 * event using PERF_SAMPLE_BRANCH_STACK
+                 */
+                if (cpuctx->ctx.nr_branch_stack > 0
+                    && pmu->flush_branch_stack) {
+
+                        pmu = cpuctx->ctx.pmu;
+
+                        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
+
+                        perf_pmu_disable(pmu);
+
+                        pmu->flush_branch_stack();
+
+                        perf_pmu_enable(pmu);
+
+                        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
+                }
+        }
+
+        rcu_read_unlock();
+
+        local_irq_restore(flags);
+}
+
+/*
  * Called from scheduler to add the events of the current task
  * with interrupts disabled.
  *
@@ -2232,6 +2299,10 @@ void __perf_event_task_sched_in(struct task_struct *prev,
          */
         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
                 perf_cgroup_sched_in(prev, task);
+
+        /* check for system-wide branch_stack events */
+        if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
+                perf_branch_stack_sched_in(prev, task);
 }
 
 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
@@ -2798,6 +2869,14 @@ static void free_event(struct perf_event *event)
                         atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
                         static_key_slow_dec_deferred(&perf_sched_events);
                 }
+
+                if (has_branch_stack(event)) {
+                        static_key_slow_dec_deferred(&perf_sched_events);
+                        /* is system-wide event */
+                        if (!(event->attach_state & PERF_ATTACH_TASK))
+                                atomic_dec(&per_cpu(perf_branch_stack_events,
+                                                    event->cpu));
+                }
         }
 
         if (event->rb) {
@@ -5924,6 +6003,12 @@ done:
                                 return ERR_PTR(err);
                         }
                 }
+                if (has_branch_stack(event)) {
+                        static_key_slow_inc(&perf_sched_events.key);
+                        if (!(event->attach_state & PERF_ATTACH_TASK))
+                                atomic_inc(&per_cpu(perf_branch_stack_events,
+                                                    event->cpu));
+                }
         }
 
         return event;