x86_64: Add vDSO for x86-64 with gettimeofday/clock_gettime/getcpu

This implements new vDSO for x86-64.  The concept is similar
to the existing vDSOs on i386 and PPC.  x86-64 has had static
vsyscalls before,  but these are not flexible enough anymore.

A vDSO is a ELF shared library supplied by the kernel that is mapped into
user address space.  The vDSO mapping is randomized for each process
for security reasons.

Doing this was needed for clock_gettime, because clock_gettime
always needs a syscall fallback and having one at a fixed
address would have made buffer overflow exploits too easy to write.

The vdso can be disabled with vdso=0

It currently includes a new gettimeofday implemention and optimized
clock_gettime(). The gettimeofday implementation is slightly faster
than the one in the old vsyscall.  clock_gettime is significantly faster
than the syscall for CLOCK_MONOTONIC and CLOCK_REALTIME.

The new calls are generally faster than the old vsyscall.

Advantages over the old x86-64 vsyscalls:
- Extensible
- Randomized
- Cleaner
- Easier to virtualize (the old static address range previously causes
overhead e.g. for Xen because it has to create special page tables for it)

Weak points:
- glibc support still to be written

The VM interface is partly based on Ingo Molnar's i386 version.

Includes compile fix from Joachim Deguara

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 120 insertions, 0 deletions

diff --git a/arch/x86_64/vdso/vclock_gettime.c b/arch/x86_64/vdso/vclock_gettime.c
new file mode 100644
index 000000000000..17f6a00de712
--- /dev/null
+++ b/arch/x86_64/vdso/vclock_gettime.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2006 Andi Kleen, SUSE Labs.
+ * Subject to the GNU Public License, v.2
+ *
+ * Fast user context implementation of clock_gettime and gettimeofday.
+ *
+ * The code should have no internal unresolved relocations.
+ * Check with readelf after changing.
+ * Also alternative() doesn't work.
+ */
+
+#include <linux/kernel.h>
+#include <linux/posix-timers.h>
+#include <linux/time.h>
+#include <linux/string.h>
+#include <asm/vsyscall.h>
+#include <asm/vgtod.h>
+#include <asm/timex.h>
+#include <asm/hpet.h>
+#include <asm/unistd.h>
+#include <asm/io.h>
+#include <asm/vgtod.h>
+#include "vextern.h"
+
+#define gtod vdso_vsyscall_gtod_data
+
+static long vdso_fallback_gettime(long clock, struct timespec *ts)
+{
+        long ret;
+        asm("syscall" : "=a" (ret) :
+            "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory");
+        return ret;
+}
+
+static inline long vgetns(void)
+{
+        cycles_t (*vread)(void);
+        vread = gtod->clock.vread;
+        return ((vread() - gtod->clock.cycle_last) * gtod->clock.mult) >>
+                gtod->clock.shift;
+}
+
+static noinline int do_realtime(struct timespec *ts)
+{
+        unsigned long seq, ns;
+        do {
+                seq = read_seqbegin(&gtod->lock);
+                ts->tv_sec = gtod->wall_time_sec;
+                ts->tv_nsec = gtod->wall_time_nsec;
+                ns = vgetns();
+        } while (unlikely(read_seqretry(&gtod->lock, seq)));
+        timespec_add_ns(ts, ns);
+        return 0;
+}
+
+/* Copy of the version in kernel/time.c which we cannot directly access */
+static void vset_normalized_timespec(struct timespec *ts, long sec, long nsec)
+{
+        while (nsec >= NSEC_PER_SEC) {
+                nsec -= NSEC_PER_SEC;
+                ++sec;
+        }
+        while (nsec < 0) {
+                nsec += NSEC_PER_SEC;
+                --sec;
+        }
+        ts->tv_sec = sec;
+        ts->tv_nsec = nsec;
+}
+
+static noinline int do_monotonic(struct timespec *ts)
+{
+        unsigned long seq, ns, secs;
+        do {
+                seq = read_seqbegin(&gtod->lock);
+                secs = gtod->wall_time_sec;
+                ns = gtod->wall_time_nsec + vgetns();
+                secs += gtod->wall_to_monotonic.tv_sec;
+                ns += gtod->wall_to_monotonic.tv_nsec;
+        } while (unlikely(read_seqretry(&gtod->lock, seq)));
+        vset_normalized_timespec(ts, secs, ns);
+        return 0;
+}
+
+int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
+{
+        if (likely(gtod->sysctl_enabled && gtod->clock.vread))
+                switch (clock) {
+                case CLOCK_REALTIME:
+                        return do_realtime(ts);
+                case CLOCK_MONOTONIC:
+                        return do_monotonic(ts);
+                }
+        return vdso_fallback_gettime(clock, ts);
+}
+int clock_gettime(clockid_t, struct timespec *)
+        __attribute__((weak, alias("__vdso_clock_gettime")));
+
+int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
+{
+        long ret;
+        if (likely(gtod->sysctl_enabled && gtod->clock.vread)) {
+                BUILD_BUG_ON(offsetof(struct timeval, tv_usec) !=
+                             offsetof(struct timespec, tv_nsec) ||
+                             sizeof(*tv) != sizeof(struct timespec));
+                do_realtime((struct timespec *)tv);
+                tv->tv_usec /= 1000;
+                if (unlikely(tz != NULL)) {
+                        /* This relies on gcc inlining the memcpy. We'll notice
+                           if it ever fails to do so. */
+                        memcpy(tz, &gtod->sys_tz, sizeof(struct timezone));
+                }
+                return 0;
+        }
+        asm("syscall" : "=a" (ret) :
+            "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
+        return ret;
+}
+int gettimeofday(struct timeval *, struct timezone *)
+        __attribute__((weak, alias("__vdso_gettimeofday")));