Merge branch 'for-next/futex' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux into for-next/core

4 files changed, 163 insertions, 99 deletions

diff --git a/Documentation/robust-futexes.txt b/Documentation/robust-futexes.txt
index 6c42c75103eb..6361fb01c9c1 100644
--- a/Documentation/robust-futexes.txt
+++ b/Documentation/robust-futexes.txt
@@ -218,5 +218,4 @@ All other architectures should build just fine too - but they won't have
 the new syscalls yet.
 
 Architectures need to implement the new futex_atomic_cmpxchg_inatomic()
-inline function before writing up the syscalls (that function returns
--ENOSYS right now).
+inline function before writing up the syscalls.
diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index cccb83ad7fa8..bdb3c05070a2 100644
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -23,26 +23,34 @@
 
 #include <asm/errno.h>
 
+#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
+
 #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)         \
 do {                                                                    \
+        unsigned int loops = FUTEX_MAX_LOOPS;                           \
+                                                                        \
         uaccess_enable();                                               \
         asm volatile(                                                   \
 "       prfm    pstl1strm, %2\n"                                        \
 "1:     ldxr    %w1, %2\n"                                              \
         insn "\n"                                                       \
-"2:     stlxr   %w3, %w0, %2\n"                                         \
-"       cbnz    %w3, 1b\n"                                              \
-"       dmb     ish\n"                                                  \
+"2:     stlxr   %w0, %w3, %2\n"                                         \
+"       cbz     %w0, 3f\n"                                              \
+"       sub     %w4, %w4, %w0\n"                                        \
+"       cbnz    %w4, 1b\n"                                              \
+"       mov     %w0, %w7\n"                                             \
 "3:\n"                                                                  \
+"       dmb     ish\n"                                                  \
 "       .pushsection .fixup,\"ax\"\n"                                   \
 "       .align  2\n"                                                    \
-"4:     mov     %w0, %w5\n"                                             \
+"4:     mov     %w0, %w6\n"                                             \
 "       b       3b\n"                                                   \
 "       .popsection\n"                                                  \
         _ASM_EXTABLE(1b, 4b)                                            \
         _ASM_EXTABLE(2b, 4b)                                            \
-        : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)       \
-        : "r" (oparg), "Ir" (-EFAULT)                                   \
+        : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),      \
+          "+r" (loops)                                                  \
+        : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN)                   \
         : "memory");                                                    \
         uaccess_disable();                                              \
 } while (0)
@@ -50,30 +58,30 @@ do {									\
 static inline int
 arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
 {
-        int oldval = 0, ret, tmp;
+        int oldval, ret, tmp;
         u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
 
         pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
-                __futex_atomic_op("mov  %w0, %w4",
+                __futex_atomic_op("mov  %w3, %w5",
                                   ret, oldval, uaddr, tmp, oparg);
                 break;
         case FUTEX_OP_ADD:
-                __futex_atomic_op("add  %w0, %w1, %w4",
+                __futex_atomic_op("add  %w3, %w1, %w5",
                                   ret, oldval, uaddr, tmp, oparg);
                 break;
         case FUTEX_OP_OR:
-                __futex_atomic_op("orr  %w0, %w1, %w4",
+                __futex_atomic_op("orr  %w3, %w1, %w5",
                                   ret, oldval, uaddr, tmp, oparg);
                 break;
         case FUTEX_OP_ANDN:
-                __futex_atomic_op("and  %w0, %w1, %w4",
+                __futex_atomic_op("and  %w3, %w1, %w5",
                                   ret, oldval, uaddr, tmp, ~oparg);
                 break;
         case FUTEX_OP_XOR:
-                __futex_atomic_op("eor  %w0, %w1, %w4",
+                __futex_atomic_op("eor  %w3, %w1, %w5",
                                   ret, oldval, uaddr, tmp, oparg);
                 break;
         default:
@@ -93,6 +101,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
                               u32 oldval, u32 newval)
 {
         int ret = 0;
+        unsigned int loops = FUTEX_MAX_LOOPS;
         u32 val, tmp;
         u32 __user *uaddr;
 
@@ -104,24 +113,30 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
         asm volatile("// futex_atomic_cmpxchg_inatomic\n"
 "       prfm    pstl1strm, %2\n"
 "1:     ldxr    %w1, %2\n"
-"       sub     %w3, %w1, %w4\n"
-"       cbnz    %w3, 3f\n"
-"2:     stlxr   %w3, %w5, %2\n"
-"       cbnz    %w3, 1b\n"
-"       dmb     ish\n"
+"       sub     %w3, %w1, %w5\n"
+"       cbnz    %w3, 4f\n"
+"2:     stlxr   %w3, %w6, %2\n"
+"       cbz     %w3, 3f\n"
+"       sub     %w4, %w4, %w3\n"
+"       cbnz    %w4, 1b\n"
+"       mov     %w0, %w8\n"
 "3:\n"
+"       dmb     ish\n"
+"4:\n"
 "       .pushsection .fixup,\"ax\"\n"
-"4:     mov     %w0, %w6\n"
-"       b       3b\n"
+"5:     mov     %w0, %w7\n"
+"       b       4b\n"
 "       .popsection\n"
-        _ASM_EXTABLE(1b, 4b)
-        _ASM_EXTABLE(2b, 4b)
-        : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
-        : "r" (oldval), "r" (newval), "Ir" (-EFAULT)
+        _ASM_EXTABLE(1b, 5b)
+        _ASM_EXTABLE(2b, 5b)
+        : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
+        : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
         : "memory");
         uaccess_disable();
 
-        *uval = val;
+        if (!ret)
+                *uval = val;
+
         return ret;
 }
 
diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index fcb61b4659b3..8666fe7f35d7 100644
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -23,7 +23,9 @@
  *
  * Return:
  * 0 - On success
- * <0 - On error
+ * -EFAULT - User access resulted in a page fault
+ * -EAGAIN - Atomic operation was unable to complete due to contention
+ * -ENOSYS - Operation not supported
  */
 static inline int
 arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr)
@@ -85,7 +87,9 @@ out_pagefault_enable:
  *
  * Return:
  * 0 - On success
- * <0 - On error
+ * -EFAULT - User access resulted in a page fault
+ * -EAGAIN - Atomic operation was unable to complete due to contention
+ * -ENOSYS - Function not implemented (only if !HAVE_FUTEX_CMPXCHG)
  */
 static inline int
 futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
diff --git a/kernel/futex.c b/kernel/futex.c
index 9e40cf7be606..6262f1534ac9 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1311,13 +1311,15 @@ static int lookup_pi_state(u32 __user *uaddr, u32 uval,
 
 static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
 {
+        int err;
         u32 uninitialized_var(curval);
 
         if (unlikely(should_fail_futex(true)))
                 return -EFAULT;
 
-        if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
-                return -EFAULT;
+        err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
+        if (unlikely(err))
+                return err;
 
         /* If user space value changed, let the caller retry */
         return curval != uval ? -EAGAIN : 0;
@@ -1502,10 +1504,8 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_
         if (unlikely(should_fail_futex(true)))
                 ret = -EFAULT;
 
-        if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)) {
-                ret = -EFAULT;
-
-        } else if (curval != uval) {
+        ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
+        if (!ret && (curval != uval)) {
                 /*
                  * If a unconditional UNLOCK_PI operation (user space did not
                  * try the TID->0 transition) raced with a waiter setting the
@@ -1700,32 +1700,32 @@ retry_private:
         double_lock_hb(hb1, hb2);
         op_ret = futex_atomic_op_inuser(op, uaddr2);
         if (unlikely(op_ret < 0)) {
-
                 double_unlock_hb(hb1, hb2);
 
-#ifndef CONFIG_MMU
-                /*
-                 * we don't get EFAULT from MMU faults if we don't have an MMU,
-                 * but we might get them from range checking
-                 */
-                ret = op_ret;
-                goto out_put_keys;
-#endif
-
-                if (unlikely(op_ret != -EFAULT)) {
+                if (!IS_ENABLED(CONFIG_MMU) ||
+                    unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
+                        /*
+                         * we don't get EFAULT from MMU faults if we don't have
+                         * an MMU, but we might get them from range checking
+                         */
                         ret = op_ret;
                         goto out_put_keys;
                 }
 
-                ret = fault_in_user_writeable(uaddr2);
-                if (ret)
-                        goto out_put_keys;
+                if (op_ret == -EFAULT) {
+                        ret = fault_in_user_writeable(uaddr2);
+                        if (ret)
+                                goto out_put_keys;
+                }
 
-                if (!(flags & FLAGS_SHARED))
+                if (!(flags & FLAGS_SHARED)) {
+                        cond_resched();
                         goto retry_private;
+                }
 
                 put_futex_key(&key2);
                 put_futex_key(&key1);
+                cond_resched();
                 goto retry;
         }
 
@@ -2350,7 +2350,7 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
         u32 uval, uninitialized_var(curval), newval;
         struct task_struct *oldowner, *newowner;
         u32 newtid;
-        int ret;
+        int ret, err = 0;
 
         lockdep_assert_held(q->lock_ptr);
 
@@ -2421,14 +2421,17 @@ retry:
         if (!pi_state->owner)
                 newtid |= FUTEX_OWNER_DIED;
 
-        if (get_futex_value_locked(&uval, uaddr))
-                goto handle_fault;
+        err = get_futex_value_locked(&uval, uaddr);
+        if (err)
+                goto handle_err;
 
         for (;;) {
                 newval = (uval & FUTEX_OWNER_DIED) | newtid;
 
-                if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
-                        goto handle_fault;
+                err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
+                if (err)
+                        goto handle_err;
+
                 if (curval == uval)
                         break;
                 uval = curval;
@@ -2456,23 +2459,37 @@ retry:
         return 0;
 
         /*
-         * To handle the page fault we need to drop the locks here. That gives
-         * the other task (either the highest priority waiter itself or the
-         * task which stole the rtmutex) the chance to try the fixup of the
-         * pi_state. So once we are back from handling the fault we need to
-         * check the pi_state after reacquiring the locks and before trying to
-         * do another fixup. When the fixup has been done already we simply
-         * return.
+         * In order to reschedule or handle a page fault, we need to drop the
+         * locks here. In the case of a fault, this gives the other task
+         * (either the highest priority waiter itself or the task which stole
+         * the rtmutex) the chance to try the fixup of the pi_state. So once we
+         * are back from handling the fault we need to check the pi_state after
+         * reacquiring the locks and before trying to do another fixup. When
+         * the fixup has been done already we simply return.
          *
          * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely
          * drop hb->lock since the caller owns the hb -> futex_q relation.
          * Dropping the pi_mutex->wait_lock requires the state revalidate.
          */
-handle_fault:
+handle_err:
         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
         spin_unlock(q->lock_ptr);
 
-        ret = fault_in_user_writeable(uaddr);
+        switch (err) {
+        case -EFAULT:
+                ret = fault_in_user_writeable(uaddr);
+                break;
+
+        case -EAGAIN:
+                cond_resched();
+                ret = 0;
+                break;
+
+        default:
+                WARN_ON_ONCE(1);
+                ret = err;
+                break;
+        }
 
         spin_lock(q->lock_ptr);
         raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
@@ -3041,10 +3058,8 @@ retry:
                  * A unconditional UNLOCK_PI op raced against a waiter
                  * setting the FUTEX_WAITERS bit. Try again.
                  */
-                if (ret == -EAGAIN) {
-                        put_futex_key(&key);
-                        goto retry;
-                }
+                if (ret == -EAGAIN)
+                        goto pi_retry;
                 /*
                  * wake_futex_pi has detected invalid state. Tell user
                  * space.
@@ -3059,9 +3074,19 @@ retry:
          * preserve the WAITERS bit not the OWNER_DIED one. We are the
          * owner.
          */
-        if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0)) {
+        if ((ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))) {
                 spin_unlock(&hb->lock);
-                goto pi_faulted;
+                switch (ret) {
+                case -EFAULT:
+                        goto pi_faulted;
+
+                case -EAGAIN:
+                        goto pi_retry;
+
+                default:
+                        WARN_ON_ONCE(1);
+                        goto out_putkey;
+                }
         }
 
         /*
@@ -3075,6 +3100,11 @@ out_putkey:
         put_futex_key(&key);
         return ret;
 
+pi_retry:
+        put_futex_key(&key);
+        cond_resched();
+        goto retry;
+
 pi_faulted:
         put_futex_key(&key);
 
@@ -3435,6 +3465,7 @@ err_unlock:
 static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi)
 {
         u32 uval, uninitialized_var(nval), mval;
+        int err;
 
         /* Futex address must be 32bit aligned */
         if ((((unsigned long)uaddr) % sizeof(*uaddr)) != 0)
@@ -3444,42 +3475,57 @@ retry:
         if (get_user(uval, uaddr))
                 return -1;
 
-        if ((uval & FUTEX_TID_MASK) == task_pid_vnr(curr)) {
-                /*
-                 * Ok, this dying thread is truly holding a futex
-                 * of interest. Set the OWNER_DIED bit atomically
-                 * via cmpxchg, and if the value had FUTEX_WAITERS
-                 * set, wake up a waiter (if any). (We have to do a
-                 * futex_wake() even if OWNER_DIED is already set -
-                 * to handle the rare but possible case of recursive
-                 * thread-death.) The rest of the cleanup is done in
-                 * userspace.
-                 */
-                mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
-                /*
-                 * We are not holding a lock here, but we want to have
-                 * the pagefault_disable/enable() protection because
-                 * we want to handle the fault gracefully. If the
-                 * access fails we try to fault in the futex with R/W
-                 * verification via get_user_pages. get_user() above
-                 * does not guarantee R/W access. If that fails we
-                 * give up and leave the futex locked.
-                 */
-                if (cmpxchg_futex_value_locked(&nval, uaddr, uval, mval)) {
+        if ((uval & FUTEX_TID_MASK) != task_pid_vnr(curr))
+                return 0;
+
+        /*
+         * Ok, this dying thread is truly holding a futex
+         * of interest. Set the OWNER_DIED bit atomically
+         * via cmpxchg, and if the value had FUTEX_WAITERS
+         * set, wake up a waiter (if any). (We have to do a
+         * futex_wake() even if OWNER_DIED is already set -
+         * to handle the rare but possible case of recursive
+         * thread-death.) The rest of the cleanup is done in
+         * userspace.
+         */
+        mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
+
+        /*
+         * We are not holding a lock here, but we want to have
+         * the pagefault_disable/enable() protection because
+         * we want to handle the fault gracefully. If the
+         * access fails we try to fault in the futex with R/W
+         * verification via get_user_pages. get_user() above
+         * does not guarantee R/W access. If that fails we
+         * give up and leave the futex locked.
+         */
+        if ((err = cmpxchg_futex_value_locked(&nval, uaddr, uval, mval))) {
+                switch (err) {
+                case -EFAULT:
                         if (fault_in_user_writeable(uaddr))
                                 return -1;
                         goto retry;
-                }
-                if (nval != uval)
+
+                case -EAGAIN:
+                        cond_resched();
                         goto retry;
 
-                /*
-                 * Wake robust non-PI futexes here. The wakeup of
-                 * PI futexes happens in exit_pi_state():
-                 */
-                if (!pi && (uval & FUTEX_WAITERS))
-                        futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
+                default:
+                        WARN_ON_ONCE(1);
+                        return err;
+                }
         }
+
+        if (nval != uval)
+                goto retry;
+
+        /*
+         * Wake robust non-PI futexes here. The wakeup of
+         * PI futexes happens in exit_pi_state():
+         */
+        if (!pi && (uval & FUTEX_WAITERS))
+                futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
+
         return 0;
 }