x86-64: Simplify and optimize vdso clock_gettime monotonic variants

We used to store the wall-to-monotonic offset and the realtime base.
It's faster to precompute the monotonic base.

This is about a 3% speedup on Sandy Bridge for CLOCK_MONOTONIC.
It's much more impressive for CLOCK_MONOTONIC_COARSE.

Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Signed-off-by: John Stultz <john.stultz@linaro.org>

1 files changed, 8 insertions, 30 deletions

diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 944c5e5d6b6a..6eea70b8f384 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -113,27 +113,17 @@ notrace static noinline int do_realtime(struct timespec *ts)
 
 notrace static noinline int do_monotonic(struct timespec *ts)
 {
-        unsigned long seq, ns, secs;
+        unsigned long seq, ns;
         int mode;
 
         do {
                 seq = read_seqcount_begin(&gtod->seq);
                 mode = gtod->clock.vclock_mode;
-                secs = gtod->wall_time_sec;
-                ns = gtod->wall_time_nsec + vgetns();
-                secs += gtod->wall_to_monotonic.tv_sec;
-                ns += gtod->wall_to_monotonic.tv_nsec;
+                ts->tv_sec = gtod->monotonic_time_sec;
+                ts->tv_nsec = gtod->monotonic_time_nsec;
+                ns = vgetns();
         } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-
-        /* wall_time_nsec, vgetns(), and wall_to_monotonic.tv_nsec
-         * are all guaranteed to be nonnegative.
-         */
-        while (ns >= NSEC_PER_SEC) {
-                ns -= NSEC_PER_SEC;
-                ++secs;
-        }
-        ts->tv_sec = secs;
-        ts->tv_nsec = ns;
+        timespec_add_ns(ts, ns);
 
         return mode;
 }
@@ -151,25 +141,13 @@ notrace static noinline int do_realtime_coarse(struct timespec *ts)
 
 notrace static noinline int do_monotonic_coarse(struct timespec *ts)
 {
-        unsigned long seq, ns, secs;
+        unsigned long seq;
         do {
                 seq = read_seqcount_begin(&gtod->seq);
-                secs = gtod->wall_time_coarse.tv_sec;
-                ns = gtod->wall_time_coarse.tv_nsec;
-                secs += gtod->wall_to_monotonic.tv_sec;
-                ns += gtod->wall_to_monotonic.tv_nsec;
+                ts->tv_sec = gtod->monotonic_time_coarse.tv_sec;
+                ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec;
         } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
 
-        /* wall_time_nsec and wall_to_monotonic.tv_nsec are
-         * guaranteed to be between 0 and NSEC_PER_SEC.
-         */
-        if (ns >= NSEC_PER_SEC) {
-                ns -= NSEC_PER_SEC;
-                ++secs;
-        }
-        ts->tv_sec = secs;
-        ts->tv_nsec = ns;
-
         return 0;
 }