5 files changed, 74 insertions, 50 deletions

diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c
index 10c83e73837a..e3e3b979f9bd 100644
--- a/kernel/sched/clock.c
+++ b/kernel/sched/clock.c
@@ -53,6 +53,7 @@
  *
  */
 #include "sched.h"
+#include <linux/sched_clock.h>
 
 /*
  * Scheduler clock - returns current time in nanosec units.
@@ -66,12 +67,7 @@ unsigned long long __weak sched_clock(void)
 }
 EXPORT_SYMBOL_GPL(sched_clock);
 
-__read_mostly int sched_clock_running;
-
-void sched_clock_init(void)
-{
-        sched_clock_running = 1;
-}
+static DEFINE_STATIC_KEY_FALSE(sched_clock_running);
 
 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
 /*
@@ -195,17 +191,40 @@ void clear_sched_clock_stable(void)
 
         smp_mb(); /* matches sched_clock_init_late() */
 
-        if (sched_clock_running == 2)
+        if (static_key_count(&sched_clock_running.key) == 2)
                 __clear_sched_clock_stable();
 }
 
+static void __sched_clock_gtod_offset(void)
+{
+        struct sched_clock_data *scd = this_scd();
+
+        __scd_stamp(scd);
+        __gtod_offset = (scd->tick_raw + __sched_clock_offset) - scd->tick_gtod;
+}
+
+void __init sched_clock_init(void)
+{
+        /*
+         * Set __gtod_offset such that once we mark sched_clock_running,
+         * sched_clock_tick() continues where sched_clock() left off.
+         *
+         * Even if TSC is buggered, we're still UP at this point so it
+         * can't really be out of sync.
+         */
+        local_irq_disable();
+        __sched_clock_gtod_offset();
+        local_irq_enable();
+
+        static_branch_inc(&sched_clock_running);
+}
 /*
  * We run this as late_initcall() such that it runs after all built-in drivers,
  * notably: acpi_processor and intel_idle, which can mark the TSC as unstable.
  */
 static int __init sched_clock_init_late(void)
 {
-        sched_clock_running = 2;
+        static_branch_inc(&sched_clock_running);
         /*
          * Ensure that it is impossible to not do a static_key update.
          *
@@ -350,8 +369,8 @@ u64 sched_clock_cpu(int cpu)
         if (sched_clock_stable())
                 return sched_clock() + __sched_clock_offset;
 
-        if (unlikely(!sched_clock_running))
-                return 0ull;
+        if (!static_branch_unlikely(&sched_clock_running))
+                return sched_clock();
 
         preempt_disable_notrace();
         scd = cpu_sdc(cpu);
@@ -373,7 +392,7 @@ void sched_clock_tick(void)
         if (sched_clock_stable())
                 return;
 
-        if (unlikely(!sched_clock_running))
+        if (!static_branch_unlikely(&sched_clock_running))
                 return;
 
         lockdep_assert_irqs_disabled();
@@ -385,8 +404,6 @@ void sched_clock_tick(void)
 
 void sched_clock_tick_stable(void)
 {
-        u64 gtod, clock;
-
         if (!sched_clock_stable())
                 return;
 
@@ -398,9 +415,7 @@ void sched_clock_tick_stable(void)
          * TSC to be unstable, any computation will be computing crap.
          */
         local_irq_disable();
-        gtod = ktime_get_ns();
-        clock = sched_clock();
-        __gtod_offset = (clock + __sched_clock_offset) - gtod;
+        __sched_clock_gtod_offset();
         local_irq_enable();
 }
 
@@ -434,9 +449,17 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
 
 #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
 
+void __init sched_clock_init(void)
+{
+        static_branch_inc(&sched_clock_running);
+        local_irq_disable();
+        generic_sched_clock_init();
+        local_irq_enable();
+}
+
 u64 sched_clock_cpu(int cpu)
 {
-        if (unlikely(!sched_clock_running))
+        if (!static_branch_unlikely(&sched_clock_running))
                 return 0;
 
         return sched_clock();
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index d17d99778356..c45de46fdf10 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5916,7 +5916,6 @@ void __init sched_init(void)
         int i, j;
         unsigned long alloc_size = 0, ptr;
 
-        sched_clock_init();
         wait_bit_init();
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 870d4f3da285..60caf1fb94e0 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -622,8 +622,6 @@ void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq)
 #undef PU
 }
 
-extern __read_mostly int sched_clock_running;
-
 static void print_cpu(struct seq_file *m, int cpu)
 {
         struct rq *rq = cpu_rq(cpu);
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
index 2d8f05aad442..cbc72c2c1fca 100644
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -237,7 +237,7 @@ sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
         pr_debug("Registered %pF as sched_clock source\n", read);
 }
 
-void __init sched_clock_postinit(void)
+void __init generic_sched_clock_init(void)
 {
         /*
          * If no sched_clock() function has been provided at that point,
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index d9e659a12c76..f3b22f456fac 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -17,6 +17,7 @@
 #include <linux/nmi.h>
 #include <linux/sched.h>
 #include <linux/sched/loadavg.h>
+#include <linux/sched/clock.h>
 #include <linux/syscore_ops.h>
 #include <linux/clocksource.h>
 #include <linux/jiffies.h>
@@ -1505,18 +1506,23 @@ void __weak read_persistent_clock64(struct timespec64 *ts64)
 }
 
 /**
- * read_boot_clock64 -  Return time of the system start.
+ * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
+ *                                        from the boot.
  *
  * Weak dummy function for arches that do not yet support it.
- * Function to read the exact time the system has been started.
- * Returns a timespec64 with tv_sec=0 and tv_nsec=0 if unsupported.
+ * wall_time    - current time as returned by persistent clock
+ * boot_offset  - offset that is defined as wall_time - boot_time