s390/vdso: optimize getcpu system call

Add the CPU number to the per-cpu vdso data page and add the
__kernel_getcpu function to the vdso object to retrieve the
CPU number in user space.

Suggested-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

9 files changed, 101 insertions, 3 deletions

diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h
index f9b02d300d60..d0a2dbf2433d 100644
--- a/arch/s390/include/asm/vdso.h
+++ b/arch/s390/include/asm/vdso.h
@@ -38,6 +38,8 @@ struct vdso_data {
 struct vdso_per_cpu_data {
         __u64 ectg_timer_base;
         __u64 ectg_user_time;
+        __u32 cpu_nr;
+        __u32 node_id;
 };
 
 extern struct vdso_data *vdso_data;
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index d5916ef9c619..53bbc9e8b281 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -80,6 +80,8 @@ int main(void)
         OFFSET(__VDSO_TK_SHIFT, vdso_data, tk_shift);
         OFFSET(__VDSO_ECTG_BASE, vdso_per_cpu_data, ectg_timer_base);
         OFFSET(__VDSO_ECTG_USER, vdso_per_cpu_data, ectg_user_time);
+        OFFSET(__VDSO_CPU_NR, vdso_per_cpu_data, cpu_nr);
+        OFFSET(__VDSO_NODE_ID, vdso_per_cpu_data, node_id);
         BLANK();
         /* constants used by the vdso */
         DEFINE(__CLOCK_REALTIME, CLOCK_REALTIME);
diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c
index 604f4a68b524..94495cac8be3 100644
--- a/arch/s390/kernel/vdso.c
+++ b/arch/s390/kernel/vdso.c
@@ -80,7 +80,7 @@ struct vdso_data *vdso_data = &vdso_data_store.data;
 /*
  * Setup vdso data page.
  */
-static void vdso_init_data(struct vdso_data *vd)
+static void __init vdso_init_data(struct vdso_data *vd)
 {
         vd->ectg_available = test_facility(31);
 }
@@ -93,6 +93,7 @@ static void vdso_init_data(struct vdso_data *vd)
 int vdso_alloc_per_cpu(struct lowcore *lowcore)
 {
         unsigned long segment_table, page_table, page_frame;
+        struct vdso_per_cpu_data *vd;
         u32 *psal, *aste;
         int i;
 
@@ -107,6 +108,12 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore)
         if (!segment_table || !page_table || !page_frame)
                 goto out;
 
+        /* Initialize per-cpu vdso data page */
+        vd = (struct vdso_per_cpu_data *) page_frame;
+        vd->cpu_nr = lowcore->cpu_nr;
+        vd->node_id = cpu_to_node(vd->cpu_nr);
+
+        /* Set up access register mode page table */
         clear_table((unsigned long *) segment_table, _SEGMENT_ENTRY_EMPTY,
                     PAGE_SIZE << SEGMENT_ORDER);
         clear_table((unsigned long *) page_table, _PAGE_INVALID,
diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile
index ee8a18e50a25..f9c459586649 100644
--- a/arch/s390/kernel/vdso32/Makefile
+++ b/arch/s390/kernel/vdso32/Makefile
@@ -1,6 +1,6 @@
 # List of files in the vdso, has to be asm only for now
 
-obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o
+obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o
 
 # Build rules
 
diff --git a/arch/s390/kernel/vdso32/getcpu.S b/arch/s390/kernel/vdso32/getcpu.S
new file mode 100644
index 000000000000..c1ed0b72030f
--- /dev/null
+++ b/arch/s390/kernel/vdso32/getcpu.S
@@ -0,0 +1,43 @@
+/*
+ * Userland implementation of getcpu() for 32 bits processes in a
+ * s390 kernel for use in the vDSO
+ *
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+#include <asm/vdso.h>
+#include <asm/asm-offsets.h>
+
+        .text
+        .align 4
+        .globl __kernel_getcpu
+        .type  __kernel_getcpu,@function
+__kernel_getcpu:
+        .cfi_startproc
+        ear     %r1,%a4
+        lhi     %r4,1
+        sll     %r4,24
+        sar     %a4,%r4
+        la      %r4,0
+        epsw    %r0,0
+        sacf    512
+        l       %r5,__VDSO_CPU_NR(%r4)
+        l       %r4,__VDSO_NODE_ID(%r4)
+        tml     %r0,0x4000
+        jo      1f
+        tml     %r0,0x8000
+        jno     0f
+        sacf    256
+        j       1f
+0:      sacf    0
+1:      sar     %a4,%r1
+        ltr     %r2,%r2
+        jz      2f
+        st      %r5,0(%r2)
+2:      ltr     %r3,%r3
+        jz      3f
+        st      %r4,0(%r3)
+3:      lhi     %r2,0
+        br      %r14
+        .cfi_endproc
+        .size   __kernel_getcpu,.-__kernel_getcpu
diff --git a/arch/s390/kernel/vdso32/vdso32.lds.S b/arch/s390/kernel/vdso32/vdso32.lds.S
index a8c379fa1247..8f048c2d6d13 100644
--- a/arch/s390/kernel/vdso32/vdso32.lds.S
+++ b/arch/s390/kernel/vdso32/vdso32.lds.S
@@ -132,6 +132,7 @@ VERSION
                 __kernel_gettimeofday;
                 __kernel_clock_gettime;
                 __kernel_clock_getres;
+                __kernel_getcpu;
 
         local: *;
         };
diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile
index c4b03f9ed228..058659c1b8cf 100644
--- a/arch/s390/kernel/vdso64/Makefile
+++ b/arch/s390/kernel/vdso64/Makefile
@@ -1,6 +1,6 @@
 # List of files in the vdso, has to be asm only for now
 
-obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o
+obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o
 
 # Build rules
 
diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S
new file mode 100644
index 000000000000..4cbe98291931
--- /dev/null
+++ b/arch/s390/kernel/vdso64/getcpu.S
@@ -0,0 +1,42 @@
+/*
+ * Userland implementation of getcpu() for 64 bits processes in a
+ * s390 kernel for use in the vDSO
+ *
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+#include <asm/vdso.h>
+#include <asm/asm-offsets.h>
+
+        .text
+        .align 4
+        .globl __kernel_getcpu
+        .type  __kernel_getcpu,@function
+__kernel_getcpu:
+        .cfi_startproc
+        ear     %r1,%a4
+        llilh   %r4,0x0100
+        sar     %a4,%r4
+        la      %r4,0
+        epsw    %r0,0
+        sacf    512
+        l       %r5,__VDSO_CPU_NR(%r4)
+        l       %r4,__VDSO_NODE_ID(%r4)
+        tml     %r0,0x4000
+        jo      1f
+        tml     %r0,0x8000
+        jno     0f
+        sacf    256
+        j       1f
+0:      sacf    0
+1:      sar     %a4,%r1
+        ltgr    %r2,%r2
+        jz      2f
+        st      %r5,0(%r2)
+2:      ltgr    %r3,%r3
+        jz      3f
+        st      %r4,0(%r3)
+3:      lghi    %r2,0
+        br      %r14
+        .cfi_endproc
+        .size   __kernel_getcpu,.-__kernel_getcpu
diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S
index 9f5979d102a9..f35455d497fe 100644
--- a/arch/s390/kernel/vdso64/vdso64.lds.S
+++ b/arch/s390/kernel/vdso64/vdso64.lds.S
@@ -132,6 +132,7 @@ VERSION
                 __kernel_gettimeofday;
                 __kernel_clock_gettime;
                 __kernel_clock_getres;
+                __kernel_getcpu;
 
         local: *;
         };