hrtimer: fix rq->lock inversion (again)

It appears I inadvertly introduced rq->lock recursion to the
hrtimer_start() path when I delegated running already expired
timers to softirq context.

This patch fixes it by introducing a __hrtimer_start_range_ns()
method that will not use raise_softirq_irqoff() but
__raise_softirq_irqoff() which avoids the wakeup.

It then also changes schedule() to check for pending softirqs and
do the wakeup then, I'm not quite sure I like this last bit, nor
am I convinced its really needed.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: paulus@samba.org
LKML-Reference: <20090313112301.096138802@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 34 insertions, 21 deletions

diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index f394d2a42ca3..cb8a15c19583 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -651,14 +651,20 @@ static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
  * and expiry check is done in the hrtimer_interrupt or in the softirq.
  */
 static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
-                                            struct hrtimer_clock_base *base)
+                                            struct hrtimer_clock_base *base,
+                                            int wakeup)
 {
         if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
-                spin_unlock(&base->cpu_base->lock);
-                raise_softirq_irqoff(HRTIMER_SOFTIRQ);
-                spin_lock(&base->cpu_base->lock);
+                if (wakeup) {
+                        spin_unlock(&base->cpu_base->lock);
+                        raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+                        spin_lock(&base->cpu_base->lock);
+                } else
+                        __raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+
                 return 1;
         }
+
         return 0;
 }
 
@@ -703,7 +709,8 @@ static inline int hrtimer_is_hres_enabled(void) { return 0; }
 static inline int hrtimer_switch_to_hres(void) { return 0; }
 static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { }
 static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
-                                            struct hrtimer_clock_base *base)
+                                            struct hrtimer_clock_base *base,
+                                            int wakeup)
 {
         return 0;
 }
@@ -886,20 +893,9 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
         return 0;
 }
 
-/**
- * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU
- * @timer:      the timer to be added
- * @tim:        expiry time
- * @delta_ns:   "slack" range for the timer
- * @mode:       expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
- *
- * Returns:
- *  0 on success
- *  1 when the timer was active
- */
-int
-hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns,
-                        const enum hrtimer_mode mode)
+int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
+                unsigned long delta_ns, const enum hrtimer_mode mode,
+                int wakeup)
 {
         struct hrtimer_clock_base *base, *new_base;
         unsigned long flags;
@@ -940,12 +936,29 @@ hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_n
          * XXX send_remote_softirq() ?
          */
         if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases))
-                hrtimer_enqueue_reprogram(timer, new_base);
+                hrtimer_enqueue_reprogram(timer, new_base, wakeup);
 
         unlock_hrtimer_base(timer, &flags);
 
         return ret;
 }
+
+/**
+ * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU
+ * @timer:      the timer to be added
+ * @tim:        expiry time
+ * @delta_ns:   "slack" range for the timer
+ * @mode:       expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
+ *
+ * Returns:
+ *  0 on success
+ *  1 when the timer was active
+ */
+int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
+                unsigned long delta_ns, const enum hrtimer_mode mode)
+{
+        return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, 1);
+}
 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
 
 /**
@@ -961,7 +974,7 @@ EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
 int
 hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
 {
-        return hrtimer_start_range_ns(timer, tim, 0, mode);
+        return __hrtimer_start_range_ns(timer, tim, 0, mode, 1);
 }
 EXPORT_SYMBOL_GPL(hrtimer_start);