sched/irqtime: Consolidate accounting synchronization with u64_stats API

The irqtime accounting currently implement its own ad hoc implementation
of u64_stats API. Lets rather consolidate it with the appropriate
library.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wanpeng Li <wanpeng.li@hotmail.com>
Link: http://lkml.kernel.org/r/1474849761-12678-5-git-send-email-fweisbec@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

2 files changed, 29 insertions, 55 deletions

diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 94b1a72879ec..1cea2f100798 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -23,10 +23,8 @@
  * task when irq is in progress while we read rq->clock. That is a worthy
  * compromise in place of having locks on each irq in account_system_time.
  */
-DEFINE_PER_CPU(u64, cpu_hardirq_time);
-DEFINE_PER_CPU(u64, cpu_softirq_time);
+DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
 
-static DEFINE_PER_CPU(u64, irq_start_time);
 static int sched_clock_irqtime;
 
 void enable_sched_clock_irqtime(void)
@@ -39,16 +37,13 @@ void disable_sched_clock_irqtime(void)
         sched_clock_irqtime = 0;
 }
 
-#ifndef CONFIG_64BIT
-DEFINE_PER_CPU(seqcount_t, irq_time_seq);
-#endif /* CONFIG_64BIT */
-
 /*
  * Called before incrementing preempt_count on {soft,}irq_enter
  * and before decrementing preempt_count on {soft,}irq_exit.
  */
 void irqtime_account_irq(struct task_struct *curr)
 {
+        struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
         s64 delta;
         int cpu;
 
@@ -56,10 +51,10 @@ void irqtime_account_irq(struct task_struct *curr)
                 return;
 
         cpu = smp_processor_id();
-        delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
-        __this_cpu_add(irq_start_time, delta);
+        delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
+        irqtime->irq_start_time += delta;
 
-        irq_time_write_begin();
+        u64_stats_update_begin(&irqtime->sync);
         /*
          * We do not account for softirq time from ksoftirqd here.
          * We want to continue accounting softirq time to ksoftirqd thread
@@ -67,11 +62,11 @@ void irqtime_account_irq(struct task_struct *curr)
          * that do not consume any time, but still wants to run.
          */
         if (hardirq_count())
-                __this_cpu_add(cpu_hardirq_time, delta);
+                irqtime->hardirq_time += delta;
         else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
-                __this_cpu_add(cpu_softirq_time, delta);
+                irqtime->softirq_time += delta;
 
-        irq_time_write_end();
+        u64_stats_update_end(&irqtime->sync);
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
@@ -79,9 +74,10 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
         u64 *cpustat = kcpustat_this_cpu->cpustat;
         cputime_t irq_cputime;
+        u64 nsecs;
 
-        irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
-                      cpustat[CPUTIME_IRQ];
+        nsecs = __this_cpu_read(cpu_irqtime.hardirq_time);
+        irq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_IRQ];
         irq_cputime = min(irq_cputime, maxtime);
         cpustat[CPUTIME_IRQ] += irq_cputime;
 
@@ -92,9 +88,10 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
         u64 *cpustat = kcpustat_this_cpu->cpustat;
         cputime_t softirq_cputime;
+        u64 nsecs;
 
-        softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
-                          cpustat[CPUTIME_SOFTIRQ];
+        nsecs = __this_cpu_read(cpu_irqtime.softirq_time);
+        softirq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_SOFTIRQ];
         softirq_cputime = min(softirq_cputime, maxtime);
         cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 5489d07a4643..19b99869809d 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2,6 +2,7 @@
 #include <linux/sched.h>
 #include <linux/sched/sysctl.h>
 #include <linux/sched/rt.h>
+#include <linux/u64_stats_sync.h>
 #include <linux/sched/deadline.h>
 #include <linux/binfmts.h>
 #include <linux/mutex.h>
@@ -1735,52 +1736,28 @@ static inline void nohz_balance_exit_idle(unsigned int cpu) { }
 #endif
 
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
+struct irqtime {
+        u64                     hardirq_time;
+        u64                     softirq_time;
+        u64                     irq_start_time;
+        struct u64_stats_sync   sync;
+};
 
-DECLARE_PER_CPU(u64, cpu_hardirq_time);
-DECLARE_PER_CPU(u64, cpu_softirq_time);
-
-#ifndef CONFIG_64BIT
-DECLARE_PER_CPU(seqcount_t, irq_time_seq);
-
-static inline void irq_time_write_begin(void)
-{
-        __this_cpu_inc(irq_time_seq.sequence);
-        smp_wmb();
-}
-
-static inline void irq_time_write_end(void)
-{
-        smp_wmb();
-        __this_cpu_inc(irq_time_seq.sequence);
-}
+DECLARE_PER_CPU(struct irqtime, cpu_irqtime);
 
 static inline u64 irq_time_read(int cpu)
 {
-        u64 irq_time;
-        unsigned seq;
+        struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu);
+        unsigned int seq;
+        u64 total;
 
         do {
-                seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu));
-                irq_time = per_cpu(cpu_softirq_time, cpu) +
-                           per_cpu(cpu_hardirq_time, cpu);
-        } while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq));
-
-        return irq_time;
-}
-#else /* CONFIG_64BIT */
-static inline void irq_time_write_begin(void)
-{
-}
+                seq = __u64_stats_fetch_begin(&irqtime->sync);
+                total = irqtime->softirq_time + irqtime->hardirq_time;
+        } while (__u64_stats_fetch_retry(&irqtime->sync, seq));
 
-static inline void irq_time_write_end(void)
-{
-}
-
-static inline u64 irq_time_read(int cpu)
-{
-        return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu);
+        return total;
 }
-#endif /* CONFIG_64BIT */
 #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
 
 #ifdef CONFIG_CPU_FREQ