1 files changed, 76 insertions, 125 deletions

diff --git a/kernel/futex.c b/kernel/futex.c
index 438701adce2..6b50a024bca 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -114,7 +114,9 @@ struct futex_q {
 };
 
 /*
- * Split the global futex_lock into every hash list lock.
+ * Hash buckets are shared by all the futex_keys that hash to the same
+ * location.  Each key may have multiple futex_q structures, one for each task
+ * waiting on a futex.
  */
 struct futex_hash_bucket {
         spinlock_t lock;
@@ -189,8 +191,7 @@ static void drop_futex_key_refs(union futex_key *key)
 /**
  * get_futex_key - Get parameters which are the keys for a futex.
  * @uaddr: virtual address of the futex
- * @shared: NULL for a PROCESS_PRIVATE futex,
- *      &current->mm->mmap_sem for a PROCESS_SHARED futex
+ * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED
  * @key: address where result is stored.
  *
  * Returns a negative error code or 0
@@ -200,9 +201,7 @@ static void drop_futex_key_refs(union futex_key *key)
  * offset_within_page).  For private mappings, it's (uaddr, current->mm).
  * We can usually work out the index without swapping in the page.
  *
- * fshared is NULL for PROCESS_PRIVATE futexes
- * For other futexes, it points to &current->mm->mmap_sem and
- * caller must have taken the reader lock. but NOT any spinlocks.
+ * lock_page() might sleep, the caller should not hold a spinlock.
  */
 static int get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key)
 {
@@ -299,41 +298,6 @@ static int get_futex_value_locked(u32 *dest, u32 __user *from)
         return ret ? -EFAULT : 0;
 }
 
-/*
- * Fault handling.
- */
-static int futex_handle_fault(unsigned long address, int attempt)
-{
-        struct vm_area_struct * vma;
-        struct mm_struct *mm = current->mm;
-        int ret = -EFAULT;
-
-        if (attempt > 2)
-                return ret;
-
-        down_read(&mm->mmap_sem);
-        vma = find_vma(mm, address);
-        if (vma && address >= vma->vm_start &&
-            (vma->vm_flags & VM_WRITE)) {
-                int fault;
-                fault = handle_mm_fault(mm, vma, address, 1);
-                if (unlikely((fault & VM_FAULT_ERROR))) {
-#if 0
-                        /* XXX: let's do this when we verify it is OK */
-                        if (ret & VM_FAULT_OOM)
-                                ret = -ENOMEM;
-#endif
-                } else {
-                        ret = 0;
-                        if (fault & VM_FAULT_MAJOR)
-                                current->maj_flt++;
-                        else
-                                current->min_flt++;
-                }
-        }
-        up_read(&mm->mmap_sem);
-        return ret;
-}
 
 /*
  * PI code:
@@ -589,10 +553,9 @@ static void wake_futex(struct futex_q *q)
          * The waiting task can free the futex_q as soon as this is written,
          * without taking any locks.  This must come last.
          *
-         * A memory barrier is required here to prevent the following store
-         * to lock_ptr from getting ahead of the wakeup. Clearing the lock
-         * at the end of wake_up_all() does not prevent this store from
-         * moving.
+         * A memory barrier is required here to prevent the following store to
+         * lock_ptr from getting ahead of the wakeup. Clearing the lock at the
+         * end of wake_up() does not prevent this store from moving.
          */
         smp_wmb();
         q->lock_ptr = NULL;
@@ -692,9 +655,16 @@ double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
         }
 }
 
+static inline void
+double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
+{
+        spin_unlock(&hb1->lock);
+        if (hb1 != hb2)
+                spin_unlock(&hb2->lock);
+}
+
 /*
- * Wake up all waiters hashed on the physical page that is mapped
- * to this virtual address:
+ * Wake up waiters matching bitset queued on this futex (uaddr).
  */
 static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset)
 {
@@ -750,9 +720,9 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
         struct futex_hash_bucket *hb1, *hb2;
         struct plist_head *head;
         struct futex_q *this, *next;
-        int ret, op_ret, attempt = 0;
+        int ret, op_ret;
 
-retryfull:
+retry:
         ret = get_futex_key(uaddr1, fshared, &key1);
         if (unlikely(ret != 0))
                 goto out;
@@ -763,16 +733,13 @@ retryfull:
         hb1 = hash_futex(&key1);
         hb2 = hash_futex(&key2);
 
-retry:
         double_lock_hb(hb1, hb2);
-
+retry_private:
         op_ret = futex_atomic_op_inuser(op, uaddr2);
         if (unlikely(op_ret < 0)) {
                 u32 dummy;
 
-                spin_unlock(&hb1->lock);
-                if (hb1 != hb2)
-                        spin_unlock(&hb2->lock);
+                double_unlock_hb(hb1, hb2);
 
 #ifndef CONFIG_MMU
                 /*
@@ -788,26 +755,16 @@ retry:
                         goto out_put_keys;
                 }
 
-                /*
-                 * futex_atomic_op_inuser needs to both read and write
-                 * *(int __user *)uaddr2, but we can't modify it
-                 * non-atomically.  Therefore, if get_user below is not
-                 * enough, we need to handle the fault ourselves, while
-                 * still holding the mmap_sem.
-                 */
-                if (attempt++) {
-                        ret = futex_handle_fault((unsigned long)uaddr2,
-                                                 attempt);
-                        if (ret)
-                                goto out_put_keys;
-                        goto retry;
-                }
-
                 ret = get_user(dummy, uaddr2);
                 if (ret)
-                        return ret;
+                        goto out_put_keys;
+
+                if (!fshared)
+                        goto retry_private;
 
-                goto retryfull;
+                put_futex_key(fshared, &key2);
+                put_futex_key(fshared, &key1);
+                goto retry;
         }
 
         head = &hb1->chain;
@@ -834,9 +791,7 @@ retry:
                 ret += op_ret;
         }
 
-        spin_unlock(&hb1->lock);
-        if (hb1 != hb2)
-                spin_unlock(&hb2->lock);
+        double_unlock_hb(hb1, hb2);
 out_put_keys:
         put_futex_key(fshared, &key2);
 out_put_key1:
@@ -869,6 +824,7 @@ retry:
         hb1 = hash_futex(&key1);
         hb2 = hash_futex(&key2);
 
+retry_private:
         double_lock_hb(hb1, hb2);
 
         if (likely(cmpval != NULL)) {
@@ -877,16 +833,18 @@ retry:
                 ret = get_futex_value_locked(&curval, uaddr1);