Patched in Tegra support.

16 files changed, 229 insertions, 112 deletions

diff --git a/arch/tile/lib/Makefile b/arch/tile/lib/Makefile
index 985f5985823..0c26086ecbe 100644
--- a/arch/tile/lib/Makefile
+++ b/arch/tile/lib/Makefile
@@ -7,7 +7,6 @@ lib-y = cacheflush.o checksum.o cpumask.o delay.o uaccess.o \
         strchr_$(BITS).o strlen_$(BITS).o
 
 ifeq ($(CONFIG_TILEGX),y)
-CFLAGS_REMOVE_memcpy_user_64.o = -fno-omit-frame-pointer
 lib-y += memcpy_user_64.o
 else
 lib-y += atomic_32.o atomic_asm_32.o memcpy_tile64.o
diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c
index f5cada70c3c..771b251b409 100644
--- a/arch/tile/lib/atomic_32.c
+++ b/arch/tile/lib/atomic_32.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/atomic.h>
+#include <asm/futex.h>
 #include <arch/chip.h>
 
 /* See <asm/atomic_32.h> */
@@ -49,7 +50,7 @@ int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss;
 
 #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
 
-int *__atomic_hashed_lock(volatile void *v)
+static inline int *__atomic_hashed_lock(volatile void *v)
 {
         /* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */
 #if ATOMIC_LOCKS_FOUND_VIA_TABLE()
@@ -190,6 +191,47 @@ u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
 EXPORT_SYMBOL(_atomic64_cmpxchg);
 
 
+static inline int *__futex_setup(int __user *v)
+{
+        /*
+         * Issue a prefetch to the counter to bring it into cache.
+         * As for __atomic_setup, but we can't do a read into the L1
+         * since it might fault; instead we do a prefetch into the L2.
+         */
+        __insn_prefetch(v);
+        return __atomic_hashed_lock((int __force *)v);
+}
+
+struct __get_user futex_set(u32 __user *v, int i)
+{
+        return __atomic_xchg((int __force *)v, __futex_setup(v), i);
+}
+
+struct __get_user futex_add(u32 __user *v, int n)
+{
+        return __atomic_xchg_add((int __force *)v, __futex_setup(v), n);
+}
+
+struct __get_user futex_or(u32 __user *v, int n)
+{
+        return __atomic_or((int __force *)v, __futex_setup(v), n);
+}
+
+struct __get_user futex_andn(u32 __user *v, int n)
+{
+        return __atomic_andn((int __force *)v, __futex_setup(v), n);
+}
+
+struct __get_user futex_xor(u32 __user *v, int n)
+{
+        return __atomic_xor((int __force *)v, __futex_setup(v), n);
+}
+
+struct __get_user futex_cmpxchg(u32 __user *v, int o, int n)
+{
+        return __atomic_cmpxchg((int __force *)v, __futex_setup(v), o, n);
+}
+
 /*
  * If any of the atomic or futex routines hit a bad address (not in
  * the page tables at kernel PL) this routine is called.  The futex
@@ -281,4 +323,7 @@ void __init __init_atomic_per_cpu(void)
         BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE);
 
 #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
+
+        /* The futex code makes this assumption, so we validate it here. */
+        BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int));
 }
diff --git a/arch/tile/lib/cacheflush.c b/arch/tile/lib/cacheflush.c
index db4fb89e12d..8928aace7a6 100644
--- a/arch/tile/lib/cacheflush.c
+++ b/arch/tile/lib/cacheflush.c
@@ -39,21 +39,7 @@ void finv_buffer_remote(void *buffer, size_t size, int hfh)
 {
         char *p, *base;
         size_t step_size, load_count;
-
-        /*
-         * On TILEPro the striping granularity is a fixed 8KB; on
-         * TILE-Gx it is configurable, and we rely on the fact that
-         * the hypervisor always configures maximum striping, so that
-         * bits 9 and 10 of the PA are part of the stripe function, so
-         * every 512 bytes we hit a striping boundary.
-         *
-         */
-#ifdef __tilegx__
-        const unsigned long STRIPE_WIDTH = 512;
-#else
         const unsigned long STRIPE_WIDTH = 8192;
-#endif
-
 #ifdef __tilegx__
         /*
          * On TILE-Gx, we must disable the dstream prefetcher before doing
@@ -88,7 +74,7 @@ void finv_buffer_remote(void *buffer, size_t size, int hfh)
          * memory, that one load would be sufficient, but since we may
          * be, we also need to back up to the last load issued to
          * another memory controller, which would be the point where
-         * we crossed a "striping" boundary (the granularity of striping
+         * we crossed an 8KB boundary (the granularity of striping
          * across memory controllers).  Keep backing up and doing this
          * until we are before the beginning of the buffer, or have
          * hit all the controllers.
@@ -102,22 +88,12 @@ void finv_buffer_remote(void *buffer, size_t size, int hfh)
          * every cache line on a full memory stripe on each
          * controller" that we simply do that, to simplify the logic.
          *
-         * On TILE-Gx the hash-for-home function is much more complex,
-         * with the upshot being we can't readily guarantee we have
-         * hit both entries in the 128-entry AMT that were hit by any
-         * load in the entire range, so we just re-load them all.
-         * With larger buffers, we may want to consider using a hypervisor
-         * trap to issue loads directly to each hash-for-home tile for
-         * each controller (doing it from Linux would trash the TLB).
+         * FIXME: See bug 9535 for some issues with this code.
          */
         if (hfh) {
                 step_size = L2_CACHE_BYTES;
-#ifdef __tilegx__
-                load_count = (size + L2_CACHE_BYTES - 1) / L2_CACHE_BYTES;
-#else
                 load_count = (STRIPE_WIDTH / L2_CACHE_BYTES) *
                               (1 << CHIP_LOG_NUM_MSHIMS());
-#endif
         } else {
                 step_size = STRIPE_WIDTH;
                 load_count = (1 << CHIP_LOG_NUM_MSHIMS());
@@ -133,7 +109,7 @@ void finv_buffer_remote(void *buffer, size_t size, int hfh)
 
         /* Figure out how far back we need to go. */
         base = p - (step_size * (load_count - 2));
-        if ((unsigned long)base < (unsigned long)buffer)
+        if ((long)base < (long)buffer)
                 base = buffer;
 
         /*
diff --git a/arch/tile/lib/checksum.c b/arch/tile/lib/checksum.c
index c3ca3e64d9d..e4bab5bd3f3 100644
--- a/arch/tile/lib/checksum.c
+++ b/arch/tile/lib/checksum.c
@@ -16,6 +16,19 @@
 #include <net/checksum.h>
 #include <linux/module.h>
 
+static inline unsigned int longto16(unsigned long x)
+{
+        unsigned long ret;
+#ifdef __tilegx__
+        ret = __insn_v2sadu(x, 0);
+        ret = __insn_v2sadu(ret, 0);
+#else
+        ret = __insn_sadh_u(x, 0);
+        ret = __insn_sadh_u(ret, 0);
+#endif
+        return ret;
+}
+
 __wsum do_csum(const unsigned char *buff, int len)
 {
         int odd, count;
@@ -81,7 +94,7 @@ __wsum do_csum(const unsigned char *buff, int len)
         }
         if (len & 1)
                 result += *buff;
-        result = csum_long(result);
+        result = longto16(result);
         if (odd)
                 result = swab16(result);
 out:
diff --git a/arch/tile/lib/exports.c b/arch/tile/lib/exports.c
index dd5f0a33fda..49284fae9d0 100644
--- a/arch/tile/lib/exports.c
+++ b/arch/tile/lib/exports.c
@@ -18,6 +18,14 @@
 
 /* arch/tile/lib/usercopy.S */
 #include <linux/uaccess.h>
+EXPORT_SYMBOL(__get_user_1);
+EXPORT_SYMBOL(__get_user_2);
+EXPORT_SYMBOL(__get_user_4);
+EXPORT_SYMBOL(__get_user_8);
+EXPORT_SYMBOL(__put_user_1);
+EXPORT_SYMBOL(__put_user_2);
+EXPORT_SYMBOL(__put_user_4);
+EXPORT_SYMBOL(__put_user_8);
 EXPORT_SYMBOL(strnlen_user_asm);
 EXPORT_SYMBOL(strncpy_from_user_asm);
 EXPORT_SYMBOL(clear_user_asm);
@@ -31,9 +39,6 @@ EXPORT_SYMBOL(finv_user_asm);
 EXPORT_SYMBOL(current_text_addr);
 EXPORT_SYMBOL(dump_stack);
 
-/* arch/tile/kernel/head.S */
-EXPORT_SYMBOL(empty_zero_page);
-
 /* arch/tile/lib/, various memcpy files */
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(__copy_to_user_inatomic);
@@ -74,6 +79,8 @@ EXPORT_SYMBOL(__umoddi3);
 int64_t __moddi3(int64_t dividend, int64_t divisor);
 EXPORT_SYMBOL(__moddi3);
 #ifndef __tilegx__
+uint64_t __ll_mul(uint64_t n0, uint64_t n1);
+EXPORT_SYMBOL(__ll_mul);
 int64_t __muldi3(int64_t, int64_t);
 EXPORT_SYMBOL(__muldi3);
 uint64_t __lshrdi3(uint64_t, unsigned int);
diff --git a/arch/tile/lib/memchr_64.c b/arch/tile/lib/memchr_64.c
index 6f867dbf7c5..84fdc8d8e73 100644
--- a/arch/tile/lib/memchr_64.c
+++ b/arch/tile/lib/memchr_64.c
@@ -15,7 +15,6 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/module.h>
-#include "string-endian.h"
 
 void *memchr(const void *s, int c, size_t n)
 {
@@ -40,8 +39,11 @@ void *memchr(const void *s, int c, size_t n)
 
         /* Read the first word, but munge it so that bytes before the array
          * will not match goal.
+         *
+         * Note that this shift count expression works because we know
+         * shift counts are taken mod 64.
          */
-        before_mask = MASK(s_int);
+        before_mask = (1ULL << (s_int << 3)) - 1;
         v = (*p | before_mask) ^ (goal & before_mask);
 
         /* Compute the address of the last byte. */
@@ -63,7 +65,7 @@ void *memchr(const void *s, int c, size_t n)
         /* We found a match, but it might be in a byte past the end
          * of the array.
          */
-        ret = ((char *)p) + (CFZ(bits) >> 3);
+        ret = ((char *)p) + (__insn_ctz(bits) >> 3);
         return (ret <= last_byte_ptr) ? ret : NULL;
 }
 EXPORT_SYMBOL(memchr);
diff --git a/arch/tile/lib/memcpy_64.c b/arch/tile/lib/memcpy_64.c
index c79b8e7c682..3fab9a6a2bb 100644
--- a/arch/tile/lib/memcpy_64.c
+++ b/arch/tile/lib/memcpy_64.c
@@ -15,6 +15,7 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/module.h>
+#define __memcpy memcpy
 /* EXPORT_SYMBOL() is in arch/tile/lib/exports.c since this should be asm. */
 
 /* Must be 8 bytes in size. */
@@ -187,7 +188,6 @@ int USERCOPY_FUNC(void *__restrict dstv, const void *__restrict srcv, size_t n)
 
         /* n != 0 if we get here.  Write out any trailing bytes. */
         dst1 = (char *)dst8;
-#ifndef __BIG_ENDIAN__
         if (n & 4) {
                 ST4((uint32_t *)dst1, final);
                 dst1 += 4;
@@ -202,30 +202,11 @@ int USERCOPY_FUNC(void *__restrict dstv, const void *__restrict srcv, size_t n)
         }
         if (n)
                 ST1((uint8_t *)dst1, final);
-#else
-        if (n & 4) {
-                ST4((uint32_t *)dst1, final >> 32);
-                dst1 += 4;
-        }
-        else
-        {
-                final >>= 32;
-        }
-        if (n & 2) {
-                ST2((uint16_t *)dst1, final >> 16);
-                dst1 += 2;
-        }
-        else
-        {
-                final >>= 16;
-        }
-        if (n & 1)
-                ST1((uint8_t *)dst1, final >> 8);
-#endif
 
         return RETVAL;
 }
 
+
 #ifdef USERCOPY_FUNC
 #undef ST1
 #undef ST2
diff --git a/arch/tile/lib/memcpy_tile64.c b/arch/tile/lib/memcpy_tile64.c
index 3bc4b4e40d9..b2fe15e0107 100644
--- a/arch/tile/lib/memcpy_tile64.c
+++ b/arch/tile/lib/memcpy_tile64.c
@@ -160,7 +160,7 @@ retry_source:
                         break;
                 if (get_remote_cache_cpu(src_pte) == smp_processor_id())
                         break;
-                src_page = pfn_to_page(pte_pfn(src_pte));
+                src_page = pfn_to_page(hv_pte_get_pfn(src_pte));
                 get_page(src_page);
                 if (pte_val(src_pte) != pte_val(*src_ptep)) {
                         put_page(src_page);
@@ -168,7 +168,7 @@ retry_source:
                 }
                 if (pte_huge(src_pte)) {
                         /* Adjust the PTE to correspond to a small page */
-                        int pfn = pte_pfn(src_pte);
+                        int pfn = hv_pte_get_pfn(src_pte);
                         pfn += (((unsigned long)source & (HPAGE_SIZE-1))
                                 >> PAGE_SHIFT);
                         src_pte = pfn_pte(pfn, src_pte);
@@ -188,7 +188,7 @@ retry_dest:
                         put_page(src_page);
                         break;
                 }
-                dst_page = pfn_to_page(pte_pfn(dst_pte));
+                dst_page = pfn_to_page(hv_pte_get_pfn(dst_pte));
                 if (dst_page == src_page) {
                         /*
                          * Source and dest are on the same page; this
@@ -206,7 +206,7 @@ retry_dest:
                 }
                 if (pte_huge(dst_pte)) {
                         /* Adjust the PTE to correspond to a small page */
-                        int pfn = pte_pfn(dst_pte);
+                        int pfn = hv_pte_get_pfn(dst_pte);
                         pfn += (((unsigned long)dest & (HPAGE_SIZE-1))
                                 >> PAGE_SHIFT);
                         dst_pte = pfn_pte(pfn, dst_pte);
diff --git a/arch/tile/lib/memcpy_user_64.c b/arch/tile/lib/memcpy_user_64.c
index 37440caa737..4763b3aff1c 100644
--- a/arch/tile/lib/memcpy_user_64.c
+++ b/arch/tile/lib/memcpy_user_64.c
@@ -14,13 +14,7 @@
  * Do memcpy(), but trap and return "n" when a load or store faults.
  *
  * Note: this idiom only works when memcpy() compiles to a leaf function.
- * Here leaf function not only means it does not have calls, but also
- * requires no stack operations (sp, stack frame pointer) and no
- * use of callee-saved registers, else "jrp lr" will be incorrect since
- * unwinding stack frame is bypassed. Since memcpy() is not complex so
- * these conditions are satisfied here, but we need to be careful when
- * modifying this file. This is not a clean solution but is the best
- * one so far.
+ * If "sp" is updated during memcpy, the "jrp lr" will be incorrect.
  *
  * Also note that we are capturing "n" from the containing scope here.
  */
diff --git a/arch/tile/lib/spinlock_32.c b/arch/tile/lib/spinlock_32.c
index b16ac49a968..cb0999fb64b 100644
--- a/arch/tile/lib/spinlock_32.c
+++ b/arch/tile/lib/spinlock_32.c
@@ -144,7 +144,7 @@ void arch_read_unlock(arch_rwlock_t *rwlock)
         for (;;) {
                 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1);
                 val = __insn_tns((int *)&rwlock->lock);
-                if (likely((val & 1) == 0)) {
+                if (likely(val & 1) == 0) {
                         rwlock->lock = val - (1 << _RD_COUNT_SHIFT);
                         __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);
                         break;
diff --git a/arch/tile/lib/spinlock_common.h b/arch/tile/lib/spinlock_common.h
index 6ac37509fac..c1010980913 100644
--- a/arch/tile/lib/spinlock_common.h
+++ b/arch/tile/lib/spinlock_common.h
@@ -60,5 +60,5 @@ static void delay_backoff(int iterations)
         loops += __insn_crc32_32(stack_pointer, get_cycles_low()) &
                 (loops - 1);
 
-        relax(loops);
+        relax(1 << exponent);
 }
diff --git a/arch/tile/lib/strchr_64.c b/arch/tile/lib/strchr_64.c
index f39f9dc422b..617a9273aaa 100644
--- a/arch/tile/lib/strchr_64.c
+++ b/arch/tile/lib/strchr_64.c
@@ -15,7 +15,8 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/module.h>
-#include "string-endian.h"
+
+#undef strchr
 
 char *strchr(const char *s, int c)
 {
@@ -32,9 +33,13 @@ char *strchr(const char *s, int c)
          * match neither zero nor goal (we make sure the high bit of each
          * byte is 1, and the low 7 bits are all the opposite of the goal
          * byte).
+         *
+         * Note that this shift count expression works because we know shift
+         * counts are taken mod 64.
          */
-        const uint64_t before_mask = MASK(s_int);
-        uint64_t v = (*p | before_mask) ^ (goal & __insn_v1shrui(before_mask, 1));
+        const uint64_t before_mask = (1ULL << (s_int << 3)) - 1;
+        uint64_t v = (*p | before_mask) ^
+                (goal & __insn_v1shrsi(before_mask, 1));
 
         uint64_t zero_matches, goal_matches;
         while (1) {
@@ -50,8 +55,8 @@ char *strchr(const char *s, int c)
                 v = *++p;
         }
 
-        z = CFZ(zero_matches);
-        g = CFZ(goal_matches);
+        z = __insn_ctz(zero_matches);
+        g = __insn_ctz(goal_matches);
 
         /* If we found c before '\0' we got a match. Note that if c == '\0'
          * then g == z, and we correctly return the address of the '\0'
diff --git a/arch/tile/lib/string-endian.h b/arch/tile/lib/string-endian.h
deleted file mode 100644
index c0eed7ce69c..00000000000
--- a/arch/tile/lib/string-endian.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2011 Tilera Corporation. All Rights Reserved.
- *
- *   This program is free software; you can redistribute it and/or
- *   modify it under the terms of the GNU General Public License
- *   as published by the Free Software Foundation, version 2.
- *
- *   This program is distributed in the hope that it will be useful, but
- *   WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *   NON INFRINGEMENT.  See the GNU General Public License for
- *   more details.
- *
- * Provide a mask based on the pointer alignment that
- * sets up non-zero bytes before the beginning of the string.
- * The MASK expression works because shift counts are taken mod 64.
- * Also, specify how to count "first" and "last" bits
- * when the bits have been read as a word.
- */
-
-#include <asm/byteorder.h>
-
-#ifdef __LITTLE_ENDIAN
-#define MASK(x) (__insn_shl(1ULL, (x << 3)) - 1)
-#define NULMASK(x) ((2ULL << x) - 1)
-#define CFZ(x) __insn_ctz(x)
-#define REVCZ(x) __insn_clz(x)
-#else
-#define MASK(x) (__insn_shl(-2LL, ((-x << 3) - 1)))
-#define NULMASK(x) (-2LL << (63 - x))
-#define CFZ(x) __insn_clz(x)
-#define REVCZ(x) __insn_ctz(x)
-#endif
diff --git a/arch/tile/lib/strlen_64.c b/arch/tile/lib/strlen_64.c
index 9583fc3361f..1c92d46202a 100644
--- a/arch/tile/lib/strlen_64.c
+++ b/arch/tile/lib/strlen_64.c
@@ -15,7 +15,8 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/module.h>
-#include "string-endian.h"
+
+#undef strlen
 
 size_t strlen(const char *s)
 {
@@ -23,13 +24,15 @@ size_t strlen(const char *s)
         const uintptr_t s_int = (uintptr_t) s;
         const uint64_t *p = (const uint64_t *)(s_int & -8);
 
-        /* Read and MASK the first word. */
-        uint64_t v = *p | MASK(s_int);
+        /* Read the first word, but force bytes before the string to be nonzero.
+         * This expression works because we know shift counts are taken mod 64.
+         */
+        uint64_t v = *p | ((1ULL << (s_int << 3)) - 1);
 
         uint64_t bits;
         while ((bits = __insn_v1cmpeqi(v, 0)) == 0)
                 v = *++p;
 
-        return ((const char *)p) + (CFZ(bits) >> 3) - s;
+        return ((const char *)p) + (__insn_ctz(bits) >> 3) - s;
 }
 EXPORT_SYMBOL(strlen);
diff --git a/arch/tile/lib/usercopy_32.S b/arch/tile/lib/usercopy_32.S
index b62d002af00..979f76d8374 100644
--- a/arch/tile/lib/usercopy_32.S
+++ b/arch/tile/lib/usercopy_32.S
@@ -19,6 +19,82 @@
 
 /* Access user memory, but use MMU to avoid propagating kernel exceptions. */
 
+        .pushsection .fixup,"ax"
+
+get_user_fault:
+        { move r0, zero; move r1, zero }
+        { movei r2, -EFAULT; jrp lr }
+        ENDPROC(get_user_fault)
+
+put_user_fault:
+        { movei r0, -EFAULT; jrp lr }
+        ENDPROC(put_user_fault)
+
+        .popsection
+
+/*
+ * __get_user_N functions take a pointer in r0, and return 0 in r2
+ * on success, with the value in r0; or else -EFAULT in r2.
+ */
+#define __get_user_N(bytes, LOAD) \
+        STD_ENTRY(__get_user_##bytes); \
+1:      { LOAD r0, r0; move r1, zero; move r2, zero }; \
+        jrp lr; \
+        STD_ENDPROC(__get_user_##bytes); \
+        .pushsection __ex_table,"a"; \
+        .word 1b, get_user_fault; \
+        .popsection
+
+__get_user_N(1, lb_u)
+__get_user_N(2, lh_u)
+__get_user_N(4, lw)
+
+/*
+ * __get_user_8 takes a pointer in r0, and returns 0 in r2
+ * on success, with the value in r0/r1; or else -EFAULT in r2.
+ */
+        STD_ENTRY(__get_user_8);
+1:      { lw r0, r0; addi r1, r0, 4 };
+2:      { lw r1, r1; move r2, zero };
+        jrp lr;
+        STD_ENDPROC(__get_user_8);
+        .pushsection __ex_table,"a";
+        .word 1b, get_user_fault;
+        .word 2b, get_user_fault;
+        .popsection
+
+/*
+ * __put_user_N functions take a value in r0 and a pointer in r1,
+ * and return 0 in r0 on success or -EFAULT on failure.
+ */
+#define __put_user_N(bytes, STORE) \
+        STD_ENTRY(__put_user_##bytes); \
+1:      { STORE r1, r0; move r0, zero }; \
+        jrp lr; \
+        STD_ENDPROC(__put_user_##bytes); \
+        .pushsection __ex_table,"a"; \
+        .word 1b, put_user_fault; \
+        .popsection
+
+__put_user_N(1, sb)
+__put_user_N(2, sh)
+__put_user_N(4, sw)
+
+/*
+ * __put_user_8 takes a value in r0/r1 and a pointer in r2,
+ * and returns 0 in r0 on success or -EFAULT on failure.
+ */
+STD_ENTRY(__put_user_8)
+1:      { sw r2, r0; addi r2, r2, 4 }
+2:      { sw r2, r1; move r0, zero }
+        jrp lr
+        STD_ENDPROC(__put_user_8)
+        .pushsection __ex_table,"a"
+        .word 1b, put_user_fault
+        .word 2b, put_user_fault
+        .popsection
+
+
 /*
  * strnlen_user_asm takes the pointer in r0, and the length bound in r1.
  * It returns the length, including the terminating NUL, or zero on exception.
diff --git a/arch/tile/lib/usercopy_64.S b/arch/tile/lib/usercopy_64.S
index adb2dbbc70c..2ff44f87b78 100644
--- a/arch/tile/lib/usercopy_64.S
+++ b/arch/tile/lib/usercopy_64.S
@@ -19,6 +19,55 @@
 
 /* Access user memory, but use MMU to avoid propagating kernel exceptions. */
 
+        .pushsection .fixup,"ax"
+
+get_user_fault:
+        { movei r1, -EFAULT; move r0, zero }
+        jrp lr
+        ENDPROC(get_user_fault)
+
+put_user_fault:
+        { movei r0, -EFAULT; jrp lr }
+        ENDPROC(put_user_fault)
+
+        .popsection
+
+/*
+ * __get_user_N functions take a pointer in r0, and return 0 in r1
+ * on success, with the value in r0; or else -EFAULT in r1.
+ */
+#define __get_user_N(bytes, LOAD) \
+        STD_ENTRY(__get_user_##bytes); \
+1:      { LOAD r0, r0; move r1, zero }; \
+        jrp lr; \
+        STD_ENDPROC(__get_user_##bytes); \
+        .pushsection __ex_table,"a"; \
+        .quad 1b, get_user_fault; \
+        .popsection
+
+__get_user_N(1, ld1u)
+__get_user_N(2, ld2u)
+__get_user_N(4, ld4u)
+__get_user_N(8, ld)
+
+/*
+ * __put_user_N functions take a value in r0 and a pointer in r1,
+ * and return 0 in r0 on success or -EFAULT on failure.
+ */
+#define __put_user_N(bytes, STORE) \
+        STD_ENTRY(__put_user_##bytes); \
+1:      { STORE r1, r0; move r0, zero }; \
+        jrp lr; \
+        STD_ENDPROC(__put_user_##bytes); \
+        .pushsection __ex_table,"a"; \
+        .quad 1b, put_user_fault; \
+        .popsection
+
+__put_user_N(1, st1)
+__put_user_N(2, st2)
+__put_user_N(4, st4)
+__put_user_N(8, st)
+
 /*
  * strnlen_user_asm takes the pointer in r0, and the length bound in r1.
  * It returns the length, including the terminating NUL, or zero on exception.