Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull core timer updates from Thomas Gleixner:
 "Timers and timekeeping updates:

   - A large overhaul of the posix CPU timer code which is a preparation
     for moving the CPU timer expiry out into task work so it can be
     properly accounted on the task/process.

     An update to the bogus permission checks will come later during the
     merge window as feedback was not complete before heading of for
     travel.

   - Switch the timerqueue code to use cached rbtrees and get rid of the
     homebrewn caching of the leftmost node.

   - Consolidate hrtimer_init() + hrtimer_init_sleeper() calls into a
     single function

   - Implement the separation of hrtimers to be forced to expire in hard
     interrupt context even when PREEMPT_RT is enabled and mark the
     affected timers accordingly.

   - Implement a mechanism for hrtimers and the timer wheel to protect
     RT against priority inversion and live lock issues when a (hr)timer
     which should be canceled is currently executing the callback.
     Instead of infinitely spinning, the task which tries to cancel the
     timer blocks on a per cpu base expiry lock which is held and
     released by the (hr)timer expiry code.

   - Enable the Hyper-V TSC page based sched_clock for Hyper-V guests
     resulting in faster access to timekeeping functions.

   - Updates to various clocksource/clockevent drivers and their device
     tree bindings.

   - The usual small improvements all over the place"

* 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits)
  posix-cpu-timers: Fix permission check regression
  posix-cpu-timers: Always clear head pointer on dequeue
  hrtimer: Add a missing bracket and hide `migration_base' on !SMP
  posix-cpu-timers: Make expiry_active check actually work correctly
  posix-timers: Unbreak CONFIG_POSIX_TIMERS=n build
  tick: Mark sched_timer to expire in hard interrupt context
  hrtimer: Add kernel doc annotation for HRTIMER_MODE_HARD
  x86/hyperv: Hide pv_ops access for CONFIG_PARAVIRT=n
  posix-cpu-timers: Utilize timerqueue for storage
  posix-cpu-timers: Move state tracking to struct posix_cputimers
  posix-cpu-timers: Deduplicate rlimit handling
  posix-cpu-timers: Remove pointless comparisons
  posix-cpu-timers: Get rid of 64bit divisions
  posix-cpu-timers: Consolidate timer expiry further
  posix-cpu-timers: Get rid of zero checks
  rlimit: Rewrite non-sensical RLIMIT_CPU comment
  posix-cpu-timers: Respect INFINITY for hard RTTIME limit
  posix-cpu-timers: Switch thread group sampling to array
  posix-cpu-timers: Restructure expiry array
  posix-cpu-timers: Remove cputime_expires
  ...

1 files changed, 96 insertions, 9 deletions

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 343c7ba33b1c..0e315a2e77ae 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -196,6 +196,10 @@ EXPORT_SYMBOL(jiffies_64);
 struct timer_base {
         raw_spinlock_t          lock;
         struct timer_list       *running_timer;
+#ifdef CONFIG_PREEMPT_RT
+        spinlock_t              expiry_lock;
+        atomic_t                timer_waiters;
+#endif
         unsigned long           clk;
         unsigned long           next_expiry;
         unsigned int            cpu;
@@ -1227,7 +1231,78 @@ int try_to_del_timer_sync(struct timer_list *timer)
 }
 EXPORT_SYMBOL(try_to_del_timer_sync);
 
-#ifdef CONFIG_SMP
+#ifdef CONFIG_PREEMPT_RT
+static __init void timer_base_init_expiry_lock(struct timer_base *base)
+{
+        spin_lock_init(&base->expiry_lock);
+}
+
+static inline void timer_base_lock_expiry(struct timer_base *base)
+{
+        spin_lock(&base->expiry_lock);
+}
+
+static inline void timer_base_unlock_expiry(struct timer_base *base)
+{
+        spin_unlock(&base->expiry_lock);
+}
+
+/*
+ * The counterpart to del_timer_wait_running().
+ *
+ * If there is a waiter for base->expiry_lock, then it was waiting for the
+ * timer callback to finish. Drop expiry_lock and reaquire it. That allows
+ * the waiter to acquire the lock and make progress.
+ */
+static void timer_sync_wait_running(struct timer_base *base)
+{
+        if (atomic_read(&base->timer_waiters)) {
+                spin_unlock(&base->expiry_lock);
+                spin_lock(&base->expiry_lock);
+        }
+}
+
+/*
+ * This function is called on PREEMPT_RT kernels when the fast path
+ * deletion of a timer failed because the timer callback function was
+ * running.
+ *
+ * This prevents priority inversion, if the softirq thread on a remote CPU
+ * got preempted, and it prevents a life lock when the task which tries to
+ * delete a timer preempted the softirq thread running the timer callback
+ * function.
+ */
+static void del_timer_wait_running(struct timer_list *timer)
+{
+        u32 tf;
+
+        tf = READ_ONCE(timer->flags);
+        if (!(tf & TIMER_MIGRATING)) {
+                struct timer_base *base = get_timer_base(tf);
+
+                /*
+                 * Mark the base as contended and grab the expiry lock,
+                 * which is held by the softirq across the timer
+                 * callback. Drop the lock immediately so the softirq can
+                 * expire the next timer. In theory the timer could already
+                 * be running again, but that's more than unlikely and just
+                 * causes another wait loop.
+                 */
+                atomic_inc(&base->timer_waiters);
+                spin_lock_bh(&base->expiry_lock);
+                atomic_dec(&base->timer_waiters);
+                spin_unlock_bh(&base->expiry_lock);
+        }
+}
+#else
+static inline void timer_base_init_expiry_lock(struct timer_base *base) { }
+static inline void timer_base_lock_expiry(struct timer_base *base) { }
+static inline void timer_base_unlock_expiry(struct timer_base *base) { }
+static inline void timer_sync_wait_running(struct timer_base *base) { }
+static inline void del_timer_wait_running(struct timer_list *timer) { }
+#endif
+
+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
 /**
  * del_timer_sync - deactivate a timer and wait for the handler to finish.
  * @timer: the timer to be deactivated
@@ -1266,6 +1341,8 @@ EXPORT_SYMBOL(try_to_del_timer_sync);
  */
 int del_timer_sync(struct timer_list *timer)
 {
+        int ret;
+
 #ifdef CONFIG_LOCKDEP
         unsigned long flags;
 
@@ -1283,12 +1360,17 @@ int del_timer_sync(struct timer_list *timer)
          * could lead to deadlock.
          */
         WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
-        for (;;) {
-                int ret = try_to_del_timer_sync(timer);
-                if (ret >= 0)
-                        return ret;
-                cpu_relax();
-        }
+
+        do {
+                ret = try_to_del_timer_sync(timer);
+
+                if (unlikely(ret < 0)) {
+                        del_timer_wait_running(timer);
+                        cpu_relax();
+                }
+        } while (ret < 0);
+
+        return ret;
 }
 EXPORT_SYMBOL(del_timer_sync);
 #endif
@@ -1360,10 +1442,13 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head)
                 if (timer->flags & TIMER_IRQSAFE) {
                         raw_spin_unlock(&base->lock);
                         call_timer_fn(timer, fn, baseclk);
+                        base->running_timer = NULL;
                         raw_spin_lock(&base->lock);
                 } else {
                         raw_spin_unlock_irq(&base->lock);
                         call_timer_fn(timer, fn, baseclk);
+                        base->running_timer = NULL;
+                        timer_sync_wait_running(base);
                         raw_spin_lock_irq(&base->lock);
                 }
         }
@@ -1643,7 +1728,7 @@ void update_process_times(int user_tick)
 #endif
         scheduler_tick();
         if (IS_ENABLED(CONFIG_POSIX_TIMERS))
-                run_posix_cpu_timers(p);
+                run_posix_cpu_timers();
 }
 
 /**
@@ -1658,6 +1743,7 @@ static inline void __run_timers(struct timer_base *base)
         if (!time_after_eq(jiffies, base->clk))
                 return;
 
+        timer_base_lock_expiry(base);
         raw_spin_lock_irq(&base->lock);
 
         /*
@@ -1684,8 +1770,8 @@ static inline void __run_timers(struct timer_base *base)
                 while (levels--)
                         expire_timers(base, heads + levels);
         }
-        base->running_timer = NULL;
         raw_spin_unlock_irq(&base->lock);
+        timer_base_unlock_expiry(base);
 }
 
 /*
@@ -1930,6 +2016,7 @@ static void __init init_timer_cpu(int cpu)
                 base->cpu = cpu;
                 raw_spin_lock_init(&base->lock);
                 base->clk = jiffies;
+                timer_base_init_expiry_lock(base);
         }
 }