perf_counter: unify and fix delayed counter wakeup

While going over the wakeup code I noticed delayed wakeups only work
for hardware counters but basically all software counters rely on
them.

This patch unifies and generalizes the delayed wakeup to fix this
issue.

Since we're dealing with NMI context bits here, use a cmpxchg() based
single link list implementation to track counters that have pending
wakeups.

[ This should really be generic code for delayed wakeups, but since we
  cannot use cmpxchg()/xchg() in generic code, I've let it live in the
  perf_counter code. -- Eric Dumazet could use it to aggregate the
  network wakeups. ]

Furthermore, the x86 method of using TIF flags was flawed in that its
quite possible to end up setting the bit on the idle task, loosing the
wakeup.

The powerpc method uses per-cpu storage and does appear to be
sufficient.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Orig-LKML-Reference: <20090330171023.153932974@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 10 insertions, 5 deletions

diff --git a/include/linux/perf_counter.h b/include/linux/perf_counter.h
index 6bf67ce17625..0d833228eee5 100644
--- a/include/linux/perf_counter.h
+++ b/include/linux/perf_counter.h
@@ -275,6 +275,10 @@ struct perf_mmap_data {
         void                            *data_pages[0];
 };
 
+struct perf_wakeup_entry {
+        struct perf_wakeup_entry *next;
+};
+
 /**
  * struct perf_counter - performance counter kernel representation:
  */
@@ -350,7 +354,7 @@ struct perf_counter {
         /* poll related */
         wait_queue_head_t               waitq;
         /* optional: for NMIs */
-        int                             wakeup_pending;
+        struct perf_wakeup_entry        wakeup;
 
         void (*destroy)(struct perf_counter *);
         struct rcu_head                 rcu_head;
@@ -427,7 +431,7 @@ extern void perf_counter_task_sched_out(struct task_struct *task, int cpu);
 extern void perf_counter_task_tick(struct task_struct *task, int cpu);
 extern void perf_counter_init_task(struct task_struct *child);
 extern void perf_counter_exit_task(struct task_struct *child);
-extern void perf_counter_notify(struct pt_regs *regs);
+extern void perf_counter_do_pending(void);
 extern void perf_counter_print_debug(void);
 extern void perf_counter_unthrottle(void);
 extern u64 hw_perf_save_disable(void);
@@ -461,7 +465,7 @@ static inline void
 perf_counter_task_tick(struct task_struct *task, int cpu)               { }
 static inline void perf_counter_init_task(struct task_struct *child)    { }
 static inline void perf_counter_exit_task(struct task_struct *child)    { }
-static inline void perf_counter_notify(struct pt_regs *regs)            { }
+static inline void perf_counter_do_pending(void)                        { }
 static inline void perf_counter_print_debug(void)                       { }
 static inline void perf_counter_unthrottle(void)                        { }
 static inline void hw_perf_restore(u64 ctrl)                            { }
@@ -469,8 +473,9 @@ static inline u64 hw_perf_save_disable(void)		      { return 0; }
 static inline int perf_counter_task_disable(void)       { return -EINVAL; }
 static inline int perf_counter_task_enable(void)        { return -EINVAL; }
 
-static inline void perf_swcounter_event(u32 event, u64 nr,
-                                        int nmi, struct pt_regs *regs)  { }
+static inline void
+perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs)  { }
+
 #endif
 
 #endif /* __KERNEL__ */