s390/uaccess: get rid of indirect function calls

There are only two uaccess variants on s390 left: the version that is used
if the mvcos instruction is available, and the page table walk variant.
So there is no need for expensive indirect function calls.

By default the mvcos variant will be called. If the mvcos instruction is not
available it will call the page table walk variant.

For minimal performance impact the "if (mvcos_is_available)" is implemented
with a jump label, which will be a six byte nop on machines with mvcos.

Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

7 files changed, 152 insertions, 149 deletions

diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h
index 51bcaa0fdeef..fda46bd38c99 100644
--- a/arch/s390/include/asm/futex.h
+++ b/arch/s390/include/asm/futex.h
@@ -5,7 +5,10 @@
 #include <linux/uaccess.h>
 #include <asm/errno.h>
 
-static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
+int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval);
+int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old);
+
+static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
 {
         int op = (encoded_op >> 28) & 7;
         int cmp = (encoded_op >> 24) & 15;
@@ -17,7 +20,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
                 oparg = 1 << oparg;
 
         pagefault_disable();
-        ret = uaccess.futex_atomic_op(op, uaddr, oparg, &oldval);
+        ret = __futex_atomic_op_inuser(op, uaddr, oparg, &oldval);
         pagefault_enable();
 
         if (!ret) {
@@ -34,10 +37,4 @@ static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
         return ret;
 }
 
-static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
-                                                u32 oldval, u32 newval)
-{
-        return uaccess.futex_atomic_cmpxchg(uval, uaddr, oldval, newval);
-}
-
 #endif /* _ASM_S390_FUTEX_H */
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 73199636ba98..49885a518e5e 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -92,33 +92,58 @@ static inline unsigned long extable_fixup(const struct exception_table_entry *x)
 #define ARCH_HAS_SORT_EXTABLE
 #define ARCH_HAS_SEARCH_EXTABLE
 
-struct uaccess_ops {
-        size_t (*copy_from_user)(void *, const void __user *, size_t);
-        size_t (*copy_to_user)(void __user *, const void *, size_t);
-        size_t (*copy_in_user)(void __user *, const void __user *, size_t);
-        size_t (*clear_user)(void __user *, size_t);
-        size_t (*strnlen_user)(const char __user *, size_t);
-        size_t (*strncpy_from_user)(char *, const char __user *, size_t);
-        int (*futex_atomic_op)(int op, u32 __user *, int oparg, int *old);
-        int (*futex_atomic_cmpxchg)(u32 *, u32 __user *, u32 old, u32 new);
-};
+int __handle_fault(unsigned long, unsigned long, int);
 
-extern struct uaccess_ops uaccess;
-extern struct uaccess_ops uaccess_mvcos;
-extern struct uaccess_ops uaccess_pt;
+/**
+ * __copy_from_user: - Copy a block of data from user space, with less checking.
+ * @to:   Destination address, in kernel space.
+ * @from: Source address, in user space.
+ * @n:    Number of bytes to copy.
+ *
+ * Context: User context only.  This function may sleep.
+ *
+ * Copy data from user space to kernel space.  Caller must check
+ * the specified block with access_ok() before calling this function.
+ *
+ * Returns number of bytes that could not be copied.
+ * On success, this will be zero.
+ *
+ * If some data could not be copied, this function will pad the copied
+ * data to the requested size using zero bytes.
+ */
+size_t __must_check __copy_from_user(void *to, const void __user *from,
+                                     size_t n);
+
+/**
+ * __copy_to_user: - Copy a block of data into user space, with less checking.
+ * @to:   Destination address, in user space.
+ * @from: Source address, in kernel space.
+ * @n:    Number of bytes to copy.
+ *
+ * Context: User context only.  This function may sleep.
+ *
+ * Copy data from kernel space to user space.  Caller must check
+ * the specified block with access_ok() before calling this function.
+ *
+ * Returns number of bytes that could not be copied.
+ * On success, this will be zero.
+ */
+unsigned long __must_check __copy_to_user(void __user *to, const void *from,
+                                          unsigned long n);
 
-extern int __handle_fault(unsigned long, unsigned long, int);
+#define __copy_to_user_inatomic __copy_to_user
+#define __copy_from_user_inatomic __copy_from_user
 
 static inline int __put_user_fn(void *x, void __user *ptr, size_t size)
 {
-        size = uaccess.copy_to_user(ptr, x, size);
-        return size ? -EFAULT : size;
+        size = __copy_to_user(ptr, x, size);
+        return size ? -EFAULT : 0;
 }
 
 static inline int __get_user_fn(void *x, const void __user *ptr, size_t size)
 {
-        size = uaccess.copy_from_user(x, ptr, size);
-        return size ? -EFAULT : size;
+        size = __copy_from_user(x, ptr, size);
+        return size ? -EFAULT : 0;
 }
 
 /*
@@ -152,7 +177,7 @@ static inline int __get_user_fn(void *x, const void __user *ptr, size_t size)
 })
 
 
-extern int __put_user_bad(void) __attribute__((noreturn));
+int __put_user_bad(void) __attribute__((noreturn));
 
 #define __get_user(x, ptr)                                      \
 ({                                                              \
@@ -200,35 +225,12 @@ extern int __put_user_bad(void) __attribute__((noreturn));
         __get_user(x, ptr);                                     \
 })
 
-extern int __get_user_bad(void) __attribute__((noreturn));
+int __get_user_bad(void) __attribute__((noreturn));
 
 #define __put_user_unaligned __put_user
 #define __get_user_unaligned __get_user
 
 /**
- * __copy_to_user: - Copy a block of data into user space, with less checking.
- * @to:   Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n:    Number of bytes to copy.
- *
- * Context: User context only.  This function may sleep.
- *
- * Copy data from kernel space to user space.  Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
-{
-        return uaccess.copy_to_user(to, from, n);
-}
-
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
-
-/**
  * copy_to_user: - Copy a block of data into user space.
  * @to:   Destination address, in user space.
  * @from: Source address, in kernel space.
@@ -248,30 +250,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n)
         return __copy_to_user(to, from, n);
 }
 
-/**
- * __copy_from_user: - Copy a block of data from user space, with less checking.
- * @to:   Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n:    Number of bytes to copy.
- *
- * Context: User context only.  This function may sleep.
- *
- * Copy data from user space to kernel space.  Caller must check
- * the specified block with access_ok() before calling this function.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-static inline unsigned long __must_check
-__copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-        return uaccess.copy_from_user(to, from, n);
-}
-
-extern void copy_from_user_overflow(void)
+void copy_from_user_overflow(void)
 #ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
 __compiletime_warning("copy_from_user() buffer size is not provably correct")
 #endif
@@ -306,11 +285,8 @@ copy_from_user(void *to, const void __user *from, unsigned long n)
         return __copy_from_user(to, from, n);
 }
 
-static inline unsigned long __must_check
-__copy_in_user(void __user *to, const void __user *from, unsigned long n)
-{
-        return uaccess.copy_in_user(to, from, n);
-}
+unsigned long __must_check
+__copy_in_user(void __user *to, const void __user *from, unsigned long n);
 
 static inline unsigned long __must_check
 copy_in_user(void __user *to, const void __user *from, unsigned long n)
@@ -322,18 +298,22 @@ copy_in_user(void __user *to, const void __user *from, unsigned long n)
 /*
  * Copy a null terminated string from userspace.
  */
+
+long __strncpy_from_user(char *dst, const char __user *src, long count);
+
 static inline long __must_check
 strncpy_from_user(char *dst, const char __user *src, long count)
 {
         might_fault();
-        return uaccess.strncpy_from_user(dst, src, count);
+        return __strncpy_from_user(dst, src, count);
 }
 
-static inline unsigned long
-strnlen_user(const char __user * src, unsigned long n)
+size_t __must_check __strnlen_user(const char __user *src, size_t count);
+
+static inline size_t strnlen_user(const char __user *src, size_t n)
 {
         might_fault();
-        return uaccess.strnlen_user(src, n);
+        return __strnlen_user(src, n);
 }
 
 /**
@@ -355,21 +335,15 @@ strnlen_user(const char __user * src, unsigned long n)
 /*
  * Zero Userspace
  */
+size_t __must_check __clear_user(void __user *to, size_t size);
 
-static inline unsigned long __must_check
-__clear_user(void __user *to, unsigned long n)
-{
-        return uaccess.clear_user(to, n);
-}
-
-static inline unsigned long __must_check
-clear_user(void __user *to, unsigned long n)
+static inline size_t __must_check clear_user(void __user *to, size_t n)
 {
         might_fault();
-        return uaccess.clear_user(to, n);
+        return __clear_user(to, n);
 }
 
-extern int copy_to_user_real(void __user *dest, void *src, size_t count);
-extern int copy_from_user_real(void *dest, void __user *src, size_t count);
+int copy_to_user_real(void __user *dest, void *src, size_t count);
+int copy_from_user_real(void *dest, void __user *src, size_t count);
 
 #endif /* __S390_UACCESS_H */
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 09e2f468f48b..91ea00955db7 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -47,7 +47,6 @@
 #include <linux/compat.h>
 
 #include <asm/ipl.h>
-#include <asm/uaccess.h>
 #include <asm/facility.h>
 #include <asm/smp.h>
 #include <asm/mmu_context.h>
@@ -65,12 +64,6 @@
 #include "entry.h"
 
 /*
- * User copy operations.
- */
-struct uaccess_ops uaccess;
-EXPORT_SYMBOL(uaccess);
-
-/*
  * Machine setup..
  */
 unsigned int console_mode = 0;
@@ -1009,8 +1002,6 @@ void __init setup_arch(char **cmdline_p)
         init_mm.end_data = (unsigned long) &_edata;
         init_mm.brk = (unsigned long) &_end;
 
-        uaccess = MACHINE_HAS_MVCOS ? uaccess_mvcos : uaccess_pt;
-
         parse_early_param();
         detect_memory_layout(memory_chunk, memory_end);
         os_info_init();
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index b068729e50ac..e3fffe1dff51 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -2,8 +2,7 @@
 # Makefile for s390-specific library files..
 #
 
-lib-y += delay.o string.o uaccess_pt.o find.o
+lib-y += delay.o string.o uaccess_pt.o uaccess_mvcos.o find.o
 obj-$(CONFIG_32BIT) += div64.o qrnnd.o ucmpdi2.o mem32.o
 obj-$(CONFIG_64BIT) += mem64.o
-lib-$(CONFIG_64BIT) += uaccess_mvcos.o
 lib-$(CONFIG_SMP) += spinlock.o
diff --git a/arch/s390/lib/uaccess.h b/arch/s390/lib/uaccess.h
index b1a22173d027..e5b9c924b733 100644
--- a/arch/s390/lib/uaccess.h
+++ b/arch/s390/lib/uaccess.h
@@ -6,7 +6,11 @@
 #ifndef __ARCH_S390_LIB_UACCESS_H
 #define __ARCH_S390_LIB_UACCESS_H
 
-extern int futex_atomic_op_pt(int, u32 __user *, int, int *);
-extern int futex_atomic_cmpxchg_pt(u32 *, u32 __user *, u32, u32);
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n);
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n);
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n);
+size_t clear_user_pt(void __user *to, size_t n);
+size_t strnlen_user_pt(const char __user *src, size_t count);
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count);
 
 #endif /* __ARCH_S390_LIB_UACCESS_H */
diff --git a/arch/s390/lib/uaccess_mvcos.c b/arch/s390/lib/uaccess_mvcos.c
index 95123f57aaf8..66f35e15db2d 100644
--- a/arch/s390/lib/uaccess_mvcos.c
+++ b/arch/s390/lib/uaccess_mvcos.c
@@ -6,7 +6,9 @@
  *               Gerald Schaefer (gerald.schaefer@de.ibm.com)
  */
 
+#include <linux/jump_label.h>
 #include <linux/errno.h>
+#include <linux/init.h>
 #include <linux/mm.h>
 #include <asm/uaccess.h>
 #include <asm/futex.h>
@@ -26,7 +28,10 @@
 #define SLR     "slgr"
 #endif
 
-static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
+static struct static_key have_mvcos = STATIC_KEY_INIT_TRUE;
+
+static inline size_t copy_from_user_mvcos(void *x, const void __user *ptr,
+                                          size_t size)
 {
         register unsigned long reg0 asm("0") = 0x81UL;
         unsigned long tmp1, tmp2;
@@ -65,7 +70,16 @@ static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
         return size;
 }
 
-static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
+size_t __copy_from_user(void *to, const void __user *from, size_t n)
+{
+        if (static_key_true(&have_mvcos))
+                return copy_from_user_mvcos(to, from, n);
+        return copy_from_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_from_user);
+
+static inline size_t copy_to_user_mvcos(void __user *ptr, const void *x,
+                                        size_t size)
 {
         register unsigned long reg0 asm("0") = 0x810000UL;
         unsigned long tmp1, tmp2;
@@ -94,8 +108,16 @@ static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
         return size;
 }
 
-static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
-                                 size_t size)
+size_t __copy_to_user(void __user *to, const void *from, size_t n)
+{
+        if (static_key_true(&have_mvcos))
+                return copy_to_user_mvcos(to, from, n);
+        return copy_to_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_to_user);
+
+static inline size_t copy_in_user_mvcos(void __user *to, const void __user *from,
+                                        size_t size)
 {
         register unsigned long reg0 asm("0") = 0x810081UL;
         unsigned long tmp1, tmp2;
@@ -117,7 +139,15 @@ static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
         return size;
 }
 
-static size_t clear_user_mvcos(void __user *to, size_t size)
+size_t __copy_in_user(void __user *to, const void __user *from, size_t n)
+{
+        if (static_key_true(&have_mvcos))
+                return copy_in_user_mvcos(to, from, n);
+        return copy_in_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_in_user);
+
+static inline size_t clear_user_mvcos(void __user *to, size_t size)
 {
         register unsigned long reg0 asm("0") = 0x810000UL;
         unsigned long tmp1, tmp2;
@@ -145,7 +175,15 @@ static size_t clear_user_mvcos(void __user *to, size_t size)
         return size;
 }
 
-static size_t strnlen_user_mvcos(const char __user *src, size_t count)
+size_t __clear_user(void __user *to, size_t size)
+{
+        if (static_key_true(&have_mvcos))
+                return clear_user_mvcos(to, size);
+        return clear_user_pt(to, size);
+}
+EXPORT_SYMBOL(__clear_user);
+
+static inline size_t strnlen_user_mvcos(const char __user *src, size_t count)
 {
         size_t done, len, offset, len_str;
         char buf[256];
@@ -164,10 +202,18 @@ static size_t strnlen_user_mvcos(const char __user *src, size_t count)
         return done + 1;
 }
 
-static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
-                                      size_t count)
+size_t __strnlen_user(const char __user *src, size_t count)
 {
-        size_t done, len, offset, len_str;
+        if (static_key_true(&have_mvcos))
+                return strnlen_user_mvcos(src, count);
+        return strnlen_user_pt(src, count);
+}
+EXPORT_SYMBOL(__strnlen_user);
+
+static inline size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
+                                             size_t count)
+{
+        unsigned long done, len, offset, len_str;
 
         if (unlikely(!count))
                 return 0;
@@ -185,13 +231,18 @@ static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
         return done;
 }
 
-struct uaccess_ops uaccess_mvcos = {
-        .copy_from_user = copy_from_user_mvcos,
-        .copy_to_user = copy_to_user_mvcos,
-        .copy_in_user = copy_in_user_mvcos,
-        .clear_user = clear_user_mvcos,
-        .strnlen_user = strnlen_user_mvcos,
-        .strncpy_from_user = strncpy_from_user_mvcos,
-        .futex_atomic_op = futex_atomic_op_pt,
-        .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
-};
+long __strncpy_from_user(char *dst, const char __user *src, long count)
+{
+        if (static_key_true(&have_mvcos))
+                return strncpy_from_user_mvcos(dst, src, count);
+        return strncpy_from_user_pt(dst, src, count);
+}
+EXPORT_SYMBOL(__strncpy_from_user);
+
+static int __init uaccess_init(void)
+{
+        if (!MACHINE_HAS_MVCOS)
+                static_key_slow_dec(&have_mvcos);
+        return 0;
+}
+early_initcall(uaccess_init);
diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 2fa696b39b56..b49c3a440a24 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -211,7 +211,7 @@ fault:
         return 0;
 }
 
-static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
 {
         size_t rc;
 
@@ -223,14 +223,14 @@ static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
         return rc;
 }
 
-static size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
 {
         if (segment_eq(get_fs(), KERNEL_DS))
                 return copy_in_kernel(to, (void __user *) from, n);
         return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
 }
 
-static size_t clear_user_pt(void __user *to, size_t n)
+size_t clear_user_pt(void __user *to, size_t n)
 {
         void *zpage = (void *) empty_zero_page;
         long done, size, ret;
@@ -253,7 +253,7 @@ static size_t clear_user_pt(void __user *to, size_t n)
         return 0;
 }
 
-static size_t strnlen_user_pt(const char __user *src, size_t count)
+size_t strnlen_user_pt(const char __user *src, size_t count)
 {
         unsigned long uaddr = (unsigned long) src;
         struct mm_struct *mm = current->mm;
@@ -289,8 +289,7 @@ fault:
         goto retry;
 }
 
-static size_t strncpy_from_user_pt(char *dst, const char __user *src,
-                                   size_t count)
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count)
 {
         size_t done, len, offset, len_str;
 
@@ -315,8 +314,7 @@ static size_t strncpy_from_user_pt(char *dst, const char __user *src,
         return done;
 }
 
-static size_t copy_in_user_pt(void __user *to, const void __user *from,
-                              size_t n)
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n)
 {
         struct mm_struct *mm = current->mm;
         unsigned long offset_max, uaddr, done, size, error_code;
@@ -411,7 +409,7 @@ static int __futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
         return ret;
 }
 
-int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
+int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old)
 {
         int ret;
 
@@ -449,8 +447,8 @@ static int __futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
         return ret;
 }
 
-int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
-                            u32 oldval, u32 newval)
+int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
+                                  u32 oldval, u32 newval)
 {
         int ret;
 
@@ -471,14 +469,3 @@ int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
         put_page(virt_to_page(uaddr));
         return ret;
 }
-
-struct uaccess_ops uaccess_pt = {
-        .copy_from_user         = copy_from_user_pt,
-        .copy_to_user           = copy_to_user_pt,
-        .copy_in_user           = copy_in_user_pt,
-        .clear_user             = clear_user_pt,
-        .strnlen_user           = strnlen_user_pt,
-        .strncpy_from_user      = strncpy_from_user_pt,
-        .futex_atomic_op        = futex_atomic_op_pt,
-        .futex_atomic_cmpxchg   = futex_atomic_cmpxchg_pt,
-};