x86-64: Clean up vdso/kernel shared variables

Variables that are shared between the vdso and the kernel are
currently a bit of a mess.  They are each defined with their own
magic, they are accessed differently in the kernel, the vsyscall page,
and the vdso, and one of them (vsyscall_clock) doesn't even really
exist.

This changes them all to use a common mechanism.  All of them are
delcared in vvar.h with a fixed address (validated by the linker
script).  In the kernel (as before), they look like ordinary
read-write variables.  In the vsyscall page and the vdso, they are
accessed through a new macro VVAR, which gives read-only access.

The vdso is now loaded verbatim into memory without any fixups.  As a
side bonus, access from the vdso is faster because a level of
indirection is removed.

While we're at it, pack jiffies and vgetcpu_mode into the same
cacheline.

Signed-off-by: Andy Lutomirski <luto@mit.edu>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Borislav Petkov <bp@amd64.org>
Link: http://lkml.kernel.org/r/%3C7357882fbb51fa30491636a7b6528747301b7ee9.1306156808.git.luto%40mit.edu%3E
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

1 files changed, 20 insertions, 26 deletions

diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index dcbb28c4b694..5f6ad032575a 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -49,15 +49,8 @@
                 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
 #define __syscall_clobber "r11","cx","memory"
 
-/*
- * vsyscall_gtod_data contains data that is :
- * - readonly from vsyscalls
- * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
- * Try to keep this structure as small as possible to avoid cache line ping pongs
- */
-int __vgetcpu_mode __section_vgetcpu_mode;
-
-struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
+DEFINE_VVAR(int, vgetcpu_mode);
+DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data) =
 {
         .lock = SEQLOCK_UNLOCKED,
         .sysctl_enabled = 1,
@@ -97,7 +90,7 @@ void update_vsyscall(struct timespec *wall_time, struct timespec *wtm,
  */
 static __always_inline void do_get_tz(struct timezone * tz)
 {
-        *tz = __vsyscall_gtod_data.sys_tz;
+        *tz = VVAR(vsyscall_gtod_data).sys_tz;
 }
 
 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
@@ -126,23 +119,24 @@ static __always_inline void do_vgettimeofday(struct timeval * tv)
         unsigned long mult, shift, nsec;
         cycle_t (*vread)(void);
         do {
-                seq = read_seqbegin(&__vsyscall_gtod_data.lock);
+                seq = read_seqbegin(&VVAR(vsyscall_gtod_data).lock);
 
-                vread = __vsyscall_gtod_data.clock.vread;
-                if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
+                vread = VVAR(vsyscall_gtod_data).clock.vread;
+                if (unlikely(!VVAR(vsyscall_gtod_data).sysctl_enabled ||
+                             !vread)) {
                         gettimeofday(tv,NULL);
                         return;
                 }
 
                 now = vread();
-                base = __vsyscall_gtod_data.clock.cycle_last;
-                mask = __vsyscall_gtod_data.clock.mask;
-                mult = __vsyscall_gtod_data.clock.mult;
-                shift = __vsyscall_gtod_data.clock.shift;
+                base = VVAR(vsyscall_gtod_data).clock.cycle_last;
+                mask = VVAR(vsyscall_gtod_data).clock.mask;
+                mult = VVAR(vsyscall_gtod_data).clock.mult;
+                shift = VVAR(vsyscall_gtod_data).clock.shift;
 
-                tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
-                nsec = __vsyscall_gtod_data.wall_time_nsec;
-        } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
+                tv->tv_sec = VVAR(vsyscall_gtod_data).wall_time_sec;
+                nsec = VVAR(vsyscall_gtod_data).wall_time_nsec;
+        } while (read_seqretry(&VVAR(vsyscall_gtod_data).lock, seq));
 
         /* calculate interval: */
         cycle_delta = (now - base) & mask;
@@ -171,15 +165,15 @@ time_t __vsyscall(1) vtime(time_t *t)
 {
         unsigned seq;
         time_t result;
-        if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
+        if (unlikely(!VVAR(vsyscall_gtod_data).sysctl_enabled))
                 return time_syscall(t);
 
         do {
-                seq = read_seqbegin(&__vsyscall_gtod_data.lock);
+                seq = read_seqbegin(&VVAR(vsyscall_gtod_data).lock);
 
-                result = __vsyscall_gtod_data.wall_time_sec;
+                result = VVAR(vsyscall_gtod_data).wall_time_sec;
 
-        } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
+        } while (read_seqretry(&VVAR(vsyscall_gtod_data).lock, seq));
 
         if (t)
                 *t = result;
@@ -208,9 +202,9 @@ vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
            We do this here because otherwise user space would do it on
            its own in a likely inferior way (no access to jiffies).
            If you don't like it pass NULL. */
-        if (tcache && tcache->blob[0] == (j = __jiffies)) {
+        if (tcache && tcache->blob[0] == (j = VVAR(jiffies))) {
                 p = tcache->blob[1];
-        } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
+        } else if (VVAR(vgetcpu_mode) == VGETCPU_RDTSCP) {
                 /* Load per CPU data from RDTSCP */
                 native_read_tscp(&p);
         } else {