1 files changed, 53 insertions, 23 deletions

diff --git a/kernel/futex.c b/kernel/futex.c
index 794c862125f..248dd119a86 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -115,6 +115,9 @@ struct futex_q {
         /* rt_waiter storage for requeue_pi: */
         struct rt_mutex_waiter *rt_waiter;
 
+        /* The expected requeue pi target futex key: */
+        union futex_key *requeue_pi_key;
+
         /* Bitset for the optional bitmasked wakeup */
         u32 bitset;
 };
@@ -247,6 +250,7 @@ again:
         if (err < 0)
                 return err;
 
+        page = compound_head(page);
         lock_page(page);
         if (!page->mapping) {
                 unlock_page(page);
@@ -1009,15 +1013,19 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
  * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue
  * q:   the futex_q
  * key: the key of the requeue target futex
+ * hb:  the hash_bucket of the requeue target futex
  *
  * During futex_requeue, with requeue_pi=1, it is possible to acquire the
  * target futex if it is uncontended or via a lock steal.  Set the futex_q key
  * to the requeue target futex so the waiter can detect the wakeup on the right
  * futex, but remove it from the hb and NULL the rt_waiter so it can detect
- * atomic lock acquisition.  Must be called with the q->lock_ptr held.
+ * atomic lock acquisition.  Set the q->lock_ptr to the requeue target hb->lock
+ * to protect access to the pi_state to fixup the owner later.  Must be called
+ * with both q->lock_ptr and hb->lock held.
  */
 static inline
-void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key)
+void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
+                           struct futex_hash_bucket *hb)
 {
         drop_futex_key_refs(&q->key);
         get_futex_key_refs(key);
@@ -1029,6 +1037,11 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key)
         WARN_ON(!q->rt_waiter);
         q->rt_waiter = NULL;
 
+        q->lock_ptr = &hb->lock;
+#ifdef CONFIG_DEBUG_PI_LIST
+        q->list.plist.lock = &hb->lock;
+#endif
+
         wake_up_state(q->task, TASK_NORMAL);
 }
 
@@ -1079,6 +1092,10 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
         if (!top_waiter)
                 return 0;
 
+        /* Ensure we requeue to the expected futex. */
+        if (!match_futex(top_waiter->requeue_pi_key, key2))
+                return -EINVAL;
+
         /*
          * Try to take the lock for top_waiter.  Set the FUTEX_WAITERS bit in
          * the contended case or if set_waiters is 1.  The pi_state is returned
@@ -1087,7 +1104,7 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
         ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
                                    set_waiters);
         if (ret == 1)
-                requeue_pi_wake_futex(top_waiter, key2);
+                requeue_pi_wake_futex(top_waiter, key2, hb2);
 
         return ret;
 }
@@ -1246,8 +1263,15 @@ retry_private:
                 if (!match_futex(&this->key, &key1))
                         continue;
 
-                WARN_ON(!requeue_pi && this->rt_waiter);
-                WARN_ON(requeue_pi && !this->rt_waiter);
+                /*
+                 * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always
+                 * be paired with each other and no other futex ops.
+                 */
+                if ((requeue_pi && !this->rt_waiter) ||
+                    (!requeue_pi && this->rt_waiter)) {
+                        ret = -EINVAL;
+                        break;
+                }
 
                 /*
                  * Wake nr_wake waiters.  For requeue_pi, if we acquired the
@@ -1259,6 +1283,12 @@ retry_private:
                         continue;
                 }
 
+                /* Ensure we requeue to the expected futex for requeue_pi. */
+                if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) {
+                        ret = -EINVAL;
+                        break;
+                }
+
                 /*
                  * Requeue nr_requeue waiters and possibly one more in the case
                  * of requeue_pi if we couldn't acquire the lock atomically.
@@ -1272,7 +1302,7 @@ retry_private:
                                                         this->task, 1);
                         if (ret == 1) {
                                 /* We got the lock. */
-                                requeue_pi_wake_futex(this, &key2);
+                                requeue_pi_wake_futex(this, &key2, hb2);
                                 continue;
                         } else if (ret) {
                                 /* -EDEADLK */
@@ -1734,6 +1764,7 @@ static int futex_wait(u32 __user *uaddr, int fshared,
         q.pi_state = NULL;
         q.bitset = bitset;
         q.rt_waiter = NULL;
+        q.requeue_pi_key = NULL;
 
         if (abs_time) {
                 to = &timeout;
@@ -1841,6 +1872,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
 
         q.pi_state = NULL;
         q.rt_waiter = NULL;
+        q.requeue_pi_key = NULL;
 retry:
         q.key = FUTEX_KEY_INIT;
         ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE);
@@ -2101,11 +2133,11 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
  * We call schedule in futex_wait_queue_me() when we enqueue and return there
  * via the following:
  * 1) wakeup on uaddr2 after an atomic lock acquisition by futex_requeue()
- * 2) wakeup on uaddr2 after a requeue and subsequent unlock
- * 3) signal (before or after requeue)
- * 4) timeout (before or after requeue)
+ * 2) wakeup on uaddr2 after a requeue
+ * 3) signal
+ * 4) timeout
  *
- * If 3, we setup a restart_block with futex_wait_requeue_pi() as the function.
+ * If 3, cleanup and return -ERESTARTNOINTR.
  *
  * If 2, we may then block on trying to take the rt_mutex and return via:
  * 5) successful lock
@@ -2113,7 +2145,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
  * 7) timeout
  * 8) other lock acquisition failure
  *
- * If 6, we setup a restart_block with futex_lock_pi() as the function.
+ * If 6, return -EWOULDBLOCK (restarting the syscall would do the same).
  *
  * If 4 or 7, we cleanup and return with -ETIMEDOUT.
  *
@@ -2152,15 +2184,16 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
         debug_rt_mutex_init_waiter(&rt_waiter);
         rt_waiter.task = NULL;
 
-        q.pi_state = NULL;
-        q.bitset = bitset;
-        q.rt_waiter = &rt_waiter;
-
         key2 = FUTEX_KEY_INIT;
         ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE);
         if (unlikely(ret != 0))
                 goto out;
 
+        q.pi_state = NULL;
+        q.bitset = bitset;
+        q.rt_waiter = &rt_waiter;
+        q.requeue_pi_key = &key2;
+
         /* Prepare to wait on uaddr. */
         ret = futex_wait_setup(uaddr, val, fshared, &q, &hb);
         if (ret)
@@ -2231,14 +2264,11 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
                         rt_mutex_unlock(pi_mutex);
         } else if (ret == -EINTR) {
                 /*
-                 * We've already been requeued, but we have no way to
-                 * restart by calling futex_lock_pi() directly. We
-                 * could restart the syscall, but that will look at
-                 * the user space value and return right away. So we
-                 * drop back with EWOULDBLOCK to tell user space that
-                 * "val" has been changed. That's the same what the
-                 * restart of the syscall would do in
-                 * futex_wait_setup().
+                 * We've already been requeued, but cannot restart by calling
+                 * futex_lock_pi() directly. We could restart this syscall, but
+                 * it would detect that the user space "val" changed and return
+                 * -EWOULDBLOCK.  Save the overhead of the restart and return
+                 * -EWOULDBLOCK directly.
                  */
                 ret = -EWOULDBLOCK;
         }