hrtimer: fix rq->lock inversion (again)

It appears I inadvertly introduced rq->lock recursion to the
hrtimer_start() path when I delegated running already expired
timers to softirq context.

This patch fixes it by introducing a __hrtimer_start_range_ns()
method that will not use raise_softirq_irqoff() but
__raise_softirq_irqoff() which avoids the wakeup.

It then also changes schedule() to check for pending softirqs and
do the wakeup then, I'm not quite sure I like this last bit, nor
am I convinced its really needed.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: paulus@samba.org
LKML-Reference: <20090313112301.096138802@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 11 insertions, 3 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 196d48babbef..63256e3ede2a 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
 
         spin_lock(&rt_b->rt_runtime_lock);
         for (;;) {
+                unsigned long delta;
+                ktime_t soft, hard;
+
                 if (hrtimer_active(&rt_b->rt_period_timer))
                         break;
 
                 now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
                 hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
-                hrtimer_start_expires(&rt_b->rt_period_timer,
-                                HRTIMER_MODE_ABS);
+
+                soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
+                hard = hrtimer_get_expires(&rt_b->rt_period_timer);
+                delta = ktime_to_ns(ktime_sub(hard, soft));
+                __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
+                                HRTIMER_MODE_ABS, 0);
         }
         spin_unlock(&rt_b->rt_runtime_lock);
 }
@@ -1146,7 +1153,8 @@ static __init void init_hrtick(void)
  */
 static void hrtick_start(struct rq *rq, u64 delay)
 {
-        hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
+        __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
+                        HRTIMER_MODE_REL, 0);
 }
 
 static inline void init_hrtick(void)