arm64: vdso: defer shifting of nanosecond component of timespec

Shifting the nanosecond component of the computed timespec early can
lead to sub-ns inaccuracies when using the truncated value as input to
further arithmetic for things like conversions to monotonic time.

This patch defers the timespec shifting until after the final value has
been computed.

Reported-by: John Stultz <john.stultz@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

2 files changed, 7 insertions, 3 deletions

diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index ba457943a16b..c958cb84d75f 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -239,7 +239,7 @@ void update_vsyscall(struct timekeeper *tk)
         if (!use_syscall) {
                 vdso_data->cs_cycle_last        = tk->clock->cycle_last;
                 vdso_data->xtime_clock_sec      = tk->xtime_sec;
-                vdso_data->xtime_clock_nsec     = tk->xtime_nsec >> tk->shift;
+                vdso_data->xtime_clock_nsec     = tk->xtime_nsec;
                 vdso_data->cs_mult              = tk->mult;
                 vdso_data->cs_shift             = tk->shift;
                 vdso_data->wtm_clock_sec        = tk->wall_to_monotonic.tv_sec;
diff --git a/arch/arm64/kernel/vdso/gettimeofday.S b/arch/arm64/kernel/vdso/gettimeofday.S
index 99b7d405767c..6681f4032260 100644
--- a/arch/arm64/kernel/vdso/gettimeofday.S
+++ b/arch/arm64/kernel/vdso/gettimeofday.S
@@ -66,6 +66,7 @@ ENTRY(__kernel_gettimeofday)
 
         /* Convert ns to us. */
         mov     x13, #1000
+        lsl     x13, x13, x12
         udiv    x11, x11, x13
         stp     x10, x11, [x0, #TVAL_TV_SEC]
 2:
@@ -136,11 +137,13 @@ ENTRY(__kernel_clock_gettime)
 4:
         /* Add on wtm timespec. */
         add     x10, x10, x13
+        lsl     x14, x14, x12
         add     x11, x11, x14
 
         /* Normalise the new timespec. */
         mov     x15, #NSEC_PER_SEC_LO16
         movk    x15, #NSEC_PER_SEC_HI16, lsl #16
+        lsl     x15, x15, x12
         cmp     x11, x15
         b.lt    5f
         sub     x11, x11, x15
@@ -152,6 +155,7 @@ ENTRY(__kernel_clock_gettime)
         sub     x10, x10, #1
 
 6:      /* Store to the user timespec. */
+        lsr     x11, x11, x12
         stp     x10, x11, [x1, #TSPEC_TV_SEC]
         mov     x0, xzr
         ret     x2
@@ -204,7 +208,7 @@ ENDPROC(__kernel_clock_getres)
  * Clobbers the temporary registers (x9 - x15).
  * Returns:
  *  - w9                = vDSO sequence counter
- *  - (x10, x11)        = (ts->tv_sec, ts->tv_nsec)
+ *  - (x10, x11)        = (ts->tv_sec, shifted ts->tv_nsec)
  *  - w12               = cs_shift
  */
 ENTRY(__do_get_tspec)
@@ -226,11 +230,11 @@ ENTRY(__do_get_tspec)
         movn    x15, #0xff00, lsl #48
         and     x10, x15, x10
         mul     x10, x10, x11
-        lsr     x10, x10, x12
 
         /* Use the kernel time to calculate the new timespec. */
         mov     x11, #NSEC_PER_SEC_LO16
         movk    x11, #NSEC_PER_SEC_HI16, lsl #16
+        lsl     x11, x11, x12
         add     x15, x10, x14
         udiv    x14, x15, x11
         add     x10, x13, x14