1 files changed, 48 insertions, 11 deletions

diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index e928e1af7b71..43c1dfb7b386 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -2612,11 +2612,26 @@ static void perf_pending_event(struct perf_pending_entry *entry)
                 __perf_event_disable(event);
         }
 
+#ifndef CONFIG_PREEMPT_RT
         if (event->pending_wakeup) {
                 event->pending_wakeup = 0;
                 perf_event_wakeup(event);
         }
+#endif
+}
+
+#ifdef CONFIG_PREEMPT_RT
+static void perf_pending_counter_softirq(struct perf_pending_entry *entry)
+{
+        struct perf_event *counter = container_of(entry,
+                        struct perf_event, pending_softirq);
+
+        if (counter->pending_wakeup) {
+                counter->pending_wakeup = 0;
+                perf_event_wakeup(counter);
+        }
 }
+#endif
 
 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
 
@@ -2624,33 +2639,42 @@ static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
         PENDING_TAIL,
 };
 
-static void perf_pending_queue(struct perf_pending_entry *entry,
-                               void (*func)(struct perf_pending_entry *))
-{
-        struct perf_pending_entry **head;
+static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_softirq_head) = {
+        PENDING_TAIL,
+};
 
+static void __perf_pending_queue(struct perf_pending_entry **head,
+                                 struct perf_pending_entry *entry,
+                                 void (*func)(struct perf_pending_entry *))
+{
         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
                 return;
 
         entry->func = func;
 
-        head = &get_cpu_var(perf_pending_head);
-
         do {
                 entry->next = *head;
         } while (cmpxchg(head, entry->next, entry) != entry->next);
+}
 
-        set_perf_event_pending();
+static void perf_pending_queue(struct perf_pending_entry *entry,
+                               void (*func)(struct perf_pending_entry *))
+{
+        struct perf_pending_entry **head;
 
+        head = &get_cpu_var(perf_pending_head);
+        __perf_pending_queue(head, entry, func);
         put_cpu_var(perf_pending_head);
+
+        set_perf_event_pending();
 }
 
-static int __perf_pending_run(void)
+static int __perf_pending_run(struct perf_pending_entry **head)
 {
         struct perf_pending_entry *list;
         int nr = 0;
 
-        list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
+        list = xchg(head, PENDING_TAIL);
         while (list != PENDING_TAIL) {
                 void (*func)(struct perf_pending_entry *);
                 struct perf_pending_entry *entry = list;
@@ -2680,7 +2704,8 @@ static inline int perf_not_pending(struct perf_event *event)
          * need to wait.
          */
         get_cpu();
-        __perf_pending_run();
+        __perf_pending_run(&__get_cpu_var(perf_pending_head));
+        __perf_pending_run(&__get_cpu_var(perf_pending_softirq_head));
         put_cpu();
 
         /*
@@ -2698,7 +2723,13 @@ static void perf_pending_sync(struct perf_event *event)
 
 void perf_event_do_pending(void)
 {
-        __perf_pending_run();
+        __perf_pending_run(&__get_cpu_var(perf_pending_head));
+}
+
+void perf_event_do_pending_softirq(void)
+{
+        __perf_pending_run(&__get_cpu_var(perf_pending_head));
+        __perf_pending_run(&__get_cpu_var(perf_pending_softirq_head));
 }
 
 /*
@@ -2736,12 +2767,18 @@ static void perf_output_wakeup(struct perf_output_handle *handle)
 {
         atomic_set(&handle->data->poll, POLL_IN);
 
+#ifndef CONFIG_PREEMPT_RT
         if (handle->nmi) {
                 handle->event->pending_wakeup = 1;
                 perf_pending_queue(&handle->event->pending,
                                    perf_pending_event);
         } else
                 perf_event_wakeup(handle->event);
+#else
+        __perf_pending_queue(&__get_cpu_var(perf_pending_softirq_head),
+                        &handle->event->pending_softirq,
+                        perf_pending_counter_softirq);
+#endif
 }
 
 /*