hrtimer: fixup hrtimer callback changes for preempt-rt

In preempt-rt we can not call the callbacks which take sleeping locks
from the timer interrupt context.

Bring back the softirq split for now, until we fixed the signal
delivery problem for real.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

1 files changed, 126 insertions, 15 deletions

diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 2e318d9d73e0..aecb304768ed 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -535,15 +535,24 @@ static int hrtimer_reprogram(struct hrtimer *timer,
 
         WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
 
+#ifndef CONFIG_PREEMPT_RT
         /*
          * When the callback is running, we do not reprogram the clock event
          * device. The timer callback is either running on a different CPU or
          * the callback is executed in the hrtimer_interrupt context. The
-         * reprogramming is handled either by the softirq, which called the
-         * callback or at the end of the hrtimer_interrupt.
+         * reprogramming is handled at the end of the hrtimer_interrupt.
          */
         if (hrtimer_callback_running(timer))
                 return 0;
+#else
+        /*
+         * preempt-rt changes the rules here as long as we have not
+         * solved the callback problem. For softirq based timers we
+         * need to allow reprogramming.
+         */
+        if (hrtimer_callback_running(timer) && timer->irqsafe)
+                return 0;
+#endif
 
         /*
          * CLOCK_REALTIME timer might be requested with an absolute
@@ -642,6 +651,8 @@ static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
 {
 }
 
+static void __run_hrtimer(struct hrtimer *timer);
+static int hrtimer_rt_defer(struct hrtimer *timer);
 
 /*
  * When High resolution timers are active, try to reprogram. Note, that in case
@@ -654,6 +665,17 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
                                             int wakeup)
 {
         if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
+#ifdef CONFIG_PREEMPT_RT
+                /*
+                 * Move softirq based timers away from the rbtree in
+                 * case it expired already. Otherwise we would have a
+                 * stale base->first entry until the softirq runs.
+                 */
+                if (!hrtimer_rt_defer(timer)) {
+                        __run_hrtimer(timer);
+                        return 1;
+                }
+#endif
                 if (wakeup) {
                         atomic_spin_unlock(&base->cpu_base->lock);
                         raise_softirq_irqoff(HRTIMER_SOFTIRQ);
@@ -878,6 +900,11 @@ static void __remove_hrtimer(struct hrtimer *timer,
                              unsigned long newstate, int reprogram)
 {
         if (timer->state & HRTIMER_STATE_ENQUEUED) {
+
+                if (unlikely(!list_empty(&timer->cb_entry))) {
+                        list_del_init(&timer->cb_entry);
+                        goto out;
+                }
                 /*
                  * Remove the timer from the rbtree and replace the
                  * first entry pointer if necessary.
@@ -890,6 +917,7 @@ static void __remove_hrtimer(struct hrtimer *timer,
                 }
                 rb_erase(&timer->node, &base->active);
         }
+out:
         timer->state = newstate;
 }
 
@@ -1203,6 +1231,77 @@ static void __run_hrtimer(struct hrtimer *timer)
         timer->state &= ~HRTIMER_STATE_CALLBACK;
 }
 
+#ifdef CONFIG_PREEMPT_RT
+
+/*
+ * The changes in mainline which removed the callback modes from
+ * hrtimer are not yet working with -rt. The non wakeup_process()
+ * based callbacks which involve sleeping locks need to be treated
+ * seperately.
+ */
+static void hrtimer_rt_run_pending(void)
+{
+        enum hrtimer_restart (*fn)(struct hrtimer *);
+        struct hrtimer_cpu_base *cpu_base;
+        struct hrtimer_clock_base *base;
+        struct hrtimer *timer;
+        int index, restart;
+
+        local_irq_disable();
+        cpu_base = &per_cpu(hrtimer_bases, smp_processor_id());
+
+        atomic_spin_lock(&cpu_base->lock);
+
+        for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) {
+                base = &cpu_base->clock_base[index];
+
+                while (!list_empty(&base->expired)) {
+                        timer = list_first_entry(&base->expired,
+                                                 struct hrtimer, cb_entry);
+
+                        /*
+                         * Same as the above __run_hrtimer function
+                         * just we run with interrupts enabled.
+                         */
+                        debug_hrtimer_deactivate(timer);
+                        __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
+                        timer_stats_account_hrtimer(timer);
+                        fn = timer->function;
+
+                        atomic_spin_unlock_irq(&cpu_base->lock);
+                        restart = fn(timer);
+                        atomic_spin_lock_irq(&cpu_base->lock);
+
+                        if (restart != HRTIMER_NORESTART) {
+                                BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);
+                                enqueue_hrtimer(timer, base);
+                        }
+                        timer->state &= ~HRTIMER_STATE_CALLBACK;
+                }
+        }
+
+        atomic_spin_unlock_irq(&cpu_base->lock);
+
+        wake_up_timer_waiters(cpu_base);
+}
+
+static int hrtimer_rt_defer(struct hrtimer *timer)
+{
+        if (timer->irqsafe)
+                return 0;
+
+        __remove_hrtimer(timer, timer->base, timer->state, 0);
+        list_add_tail(&timer->cb_entry, &timer->base->expired);
+        return 1;
+}
+
+#else
+
+static inline void hrtimer_rt_run_pending(void) { }
+static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; }
+
+#endif
+
 #ifdef CONFIG_HIGH_RES_TIMERS
 
 static int force_clock_reprogram;
@@ -1238,7 +1337,7 @@ void hrtimer_interrupt(struct clock_event_device *dev)
         struct hrtimer_clock_base *base;
         ktime_t expires_next, now;
         int nr_retries = 0;
-        int i;
+        int i, raise = 0;
 
         BUG_ON(!cpu_base->hres_active);
         cpu_base->nr_events++;
@@ -1299,7 +1398,10 @@ void hrtimer_interrupt(struct clock_event_device *dev)
                                 break;
                         }
 
-                        __run_hrtimer(timer);
+                        if (!hrtimer_rt_defer(timer))
+                                __run_hrtimer(timer);
+                        else
+                                raise = 1;
                 }
                 base++;
         }
@@ -1316,6 +1418,9 @@ void hrtimer_interrupt(struct clock_event_device *dev)
                 if (tick_program_event(expires_next, force_clock_reprogram))
                         goto retry;
         }
+
+        if (raise)
+                raise_softirq_irqoff(HRTIMER_SOFTIRQ);
 }
 
 /*
@@ -1354,17 +1459,18 @@ void hrtimer_peek_ahead_timers(void)
         local_irq_restore(flags);
 }
 
-static void run_hrtimer_softirq(struct softirq_action *h)
-{
-        hrtimer_peek_ahead_timers();
-}
-
 #else /* CONFIG_HIGH_RES_TIMERS */
 
 static inline void __hrtimer_peek_ahead_timers(void) { }
 
 #endif  /* !CONFIG_HIGH_RES_TIMERS */
 
+static void run_hrtimer_softirq(struct softirq_action *h)
+{
+        hrtimer_peek_ahead_timers();
+        hrtimer_rt_run_pending();
+}
+
 /*
  * Called from timer softirq every jiffy, expire hrtimers:
  *
@@ -1399,7 +1505,7 @@ void hrtimer_run_queues(void)
         struct rb_node *node;
         struct hrtimer_cpu_base *cpu_base;
         struct hrtimer_clock_base *base;
-        int index, gettime = 1;
+        int index, gettime = 1, raise = 0;
 
         cpu_base = &per_cpu(hrtimer_bases, raw_smp_processor_id());
         if (hrtimer_hres_active(cpu_base))
@@ -1426,12 +1532,16 @@ void hrtimer_run_queues(void)
                                         hrtimer_get_expires_tv64(timer))
                                 break;
 
-                        __run_hrtimer(timer);
+                        if (!hrtimer_rt_defer(timer))
+                                __run_hrtimer(timer);
+                        else
+                                raise = 1;
                 }
                 atomic_spin_unlock(&cpu_base->lock);
         }
 
-        wake_up_timer_waiters(cpu_base);
+        if (raise)
+                raise_softirq_irqoff(HRTIMER_SOFTIRQ);
 }
 
 /*
@@ -1453,6 +1563,7 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
 void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
 {
         sl->timer.function = hrtimer_wakeup;
+        sl->timer.irqsafe = 1;
         sl->task = task;
 }
 
@@ -1587,8 +1698,10 @@ static void __cpuinit init_hrtimers_cpu(int cpu)
 
         atomic_spin_lock_init(&cpu_base->lock);
 
-        for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
+        for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
                 cpu_base->clock_base[i].cpu_base = cpu_base;
+                INIT_LIST_HEAD(&cpu_base->clock_base[i].expired);
+        }
 
         hrtimer_init_hres(cpu_base);
 #ifdef CONFIG_PREEMPT_SOFTIRQS
@@ -1706,9 +1819,7 @@ void __init hrtimers_init(void)
         hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
                           (void *)(long)smp_processor_id());
         register_cpu_notifier(&hrtimers_nb);
-#ifdef CONFIG_HIGH_RES_TIMERS
         open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
-#endif
 }
 
 /**