aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/futex.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/futex.c')
-rw-r--r--kernel/futex.c77
1 files changed, 37 insertions, 40 deletions
diff --git a/kernel/futex.c b/kernel/futex.c
index d73ef1f3e55d..d9b3a2228f9d 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -203,8 +203,6 @@ static void drop_futex_key_refs(union futex_key *key)
203 * @uaddr: virtual address of the futex 203 * @uaddr: virtual address of the futex
204 * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED 204 * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED
205 * @key: address where result is stored. 205 * @key: address where result is stored.
206 * @rw: mapping needs to be read/write (values: VERIFY_READ,
207 * VERIFY_WRITE)
208 * 206 *
209 * Returns a negative error code or 0 207 * Returns a negative error code or 0
210 * The key words are stored in *key on success. 208 * The key words are stored in *key on success.
@@ -216,7 +214,7 @@ static void drop_futex_key_refs(union futex_key *key)
216 * lock_page() might sleep, the caller should not hold a spinlock. 214 * lock_page() might sleep, the caller should not hold a spinlock.
217 */ 215 */
218static int 216static int
219get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) 217get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key)
220{ 218{
221 unsigned long address = (unsigned long)uaddr; 219 unsigned long address = (unsigned long)uaddr;
222 struct mm_struct *mm = current->mm; 220 struct mm_struct *mm = current->mm;
@@ -239,7 +237,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
239 * but access_ok() should be faster than find_vma() 237 * but access_ok() should be faster than find_vma()
240 */ 238 */
241 if (!fshared) { 239 if (!fshared) {
242 if (unlikely(!access_ok(rw, uaddr, sizeof(u32)))) 240 if (unlikely(!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))))
243 return -EFAULT; 241 return -EFAULT;
244 key->private.mm = mm; 242 key->private.mm = mm;
245 key->private.address = address; 243 key->private.address = address;
@@ -248,7 +246,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
248 } 246 }
249 247
250again: 248again:
251 err = get_user_pages_fast(address, 1, rw == VERIFY_WRITE, &page); 249 err = get_user_pages_fast(address, 1, 1, &page);
252 if (err < 0) 250 if (err < 0)
253 return err; 251 return err;
254 252
@@ -403,9 +401,9 @@ static void free_pi_state(struct futex_pi_state *pi_state)
403 * and has cleaned up the pi_state already 401 * and has cleaned up the pi_state already
404 */ 402 */
405 if (pi_state->owner) { 403 if (pi_state->owner) {
406 spin_lock_irq(&pi_state->owner->pi_lock); 404 raw_spin_lock_irq(&pi_state->owner->pi_lock);
407 list_del_init(&pi_state->list); 405 list_del_init(&pi_state->list);
408 spin_unlock_irq(&pi_state->owner->pi_lock); 406 raw_spin_unlock_irq(&pi_state->owner->pi_lock);
409 407
410 rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); 408 rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner);
411 } 409 }
@@ -470,18 +468,18 @@ void exit_pi_state_list(struct task_struct *curr)
470 * pi_state_list anymore, but we have to be careful 468 * pi_state_list anymore, but we have to be careful
471 * versus waiters unqueueing themselves: 469 * versus waiters unqueueing themselves:
472 */ 470 */
473 spin_lock_irq(&curr->pi_lock); 471 raw_spin_lock_irq(&curr->pi_lock);
474 while (!list_empty(head)) { 472 while (!list_empty(head)) {
475 473
476 next = head->next; 474 next = head->next;
477 pi_state = list_entry(next, struct futex_pi_state, list); 475 pi_state = list_entry(next, struct futex_pi_state, list);
478 key = pi_state->key; 476 key = pi_state->key;
479 hb = hash_futex(&key); 477 hb = hash_futex(&key);
480 spin_unlock_irq(&curr->pi_lock); 478 raw_spin_unlock_irq(&curr->pi_lock);
481 479
482 spin_lock(&hb->lock); 480 spin_lock(&hb->lock);
483 481
484 spin_lock_irq(&curr->pi_lock); 482 raw_spin_lock_irq(&curr->pi_lock);
485 /* 483 /*
486 * We dropped the pi-lock, so re-check whether this 484 * We dropped the pi-lock, so re-check whether this
487 * task still owns the PI-state: 485 * task still owns the PI-state:
@@ -495,15 +493,15 @@ void exit_pi_state_list(struct task_struct *curr)
495 WARN_ON(list_empty(&pi_state->list)); 493 WARN_ON(list_empty(&pi_state->list));
496 list_del_init(&pi_state->list); 494 list_del_init(&pi_state->list);
497 pi_state->owner = NULL; 495 pi_state->owner = NULL;
498 spin_unlock_irq(&curr->pi_lock); 496 raw_spin_unlock_irq(&curr->pi_lock);
499 497
500 rt_mutex_unlock(&pi_state->pi_mutex); 498 rt_mutex_unlock(&pi_state->pi_mutex);
501 499
502 spin_unlock(&hb->lock); 500 spin_unlock(&hb->lock);
503 501
504 spin_lock_irq(&curr->pi_lock); 502 raw_spin_lock_irq(&curr->pi_lock);
505 } 503 }
506 spin_unlock_irq(&curr->pi_lock); 504 raw_spin_unlock_irq(&curr->pi_lock);
507} 505}
508 506
509static int 507static int
@@ -558,7 +556,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
558 * change of the task flags, we do this protected by 556 * change of the task flags, we do this protected by
559 * p->pi_lock: 557 * p->pi_lock:
560 */ 558 */
561 spin_lock_irq(&p->pi_lock); 559 raw_spin_lock_irq(&p->pi_lock);
562 if (unlikely(p->flags & PF_EXITING)) { 560 if (unlikely(p->flags & PF_EXITING)) {
563 /* 561 /*
564 * The task is on the way out. When PF_EXITPIDONE is 562 * The task is on the way out. When PF_EXITPIDONE is
@@ -567,7 +565,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
567 */ 565 */
568 int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN; 566 int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN;
569 567
570 spin_unlock_irq(&p->pi_lock); 568 raw_spin_unlock_irq(&p->pi_lock);
571 put_task_struct(p); 569 put_task_struct(p);
572 return ret; 570 return ret;
573 } 571 }
@@ -586,7 +584,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
586 WARN_ON(!list_empty(&pi_state->list)); 584 WARN_ON(!list_empty(&pi_state->list));
587 list_add(&pi_state->list, &p->pi_state_list); 585 list_add(&pi_state->list, &p->pi_state_list);
588 pi_state->owner = p; 586 pi_state->owner = p;
589 spin_unlock_irq(&p->pi_lock); 587 raw_spin_unlock_irq(&p->pi_lock);
590 588
591 put_task_struct(p); 589 put_task_struct(p);
592 590
@@ -760,7 +758,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
760 if (!pi_state) 758 if (!pi_state)
761 return -EINVAL; 759 return -EINVAL;
762 760
763 spin_lock(&pi_state->pi_mutex.wait_lock); 761 raw_spin_lock(&pi_state->pi_mutex.wait_lock);
764 new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); 762 new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
765 763
766 /* 764 /*
@@ -789,23 +787,23 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
789 else if (curval != uval) 787 else if (curval != uval)
790 ret = -EINVAL; 788 ret = -EINVAL;
791 if (ret) { 789 if (ret) {
792 spin_unlock(&pi_state->pi_mutex.wait_lock); 790 raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
793 return ret; 791 return ret;
794 } 792 }
795 } 793 }
796 794
797 spin_lock_irq(&pi_state->owner->pi_lock); 795 raw_spin_lock_irq(&pi_state->owner->pi_lock);
798 WARN_ON(list_empty(&pi_state->list)); 796 WARN_ON(list_empty(&pi_state->list));
799 list_del_init(&pi_state->list); 797 list_del_init(&pi_state->list);
800 spin_unlock_irq(&pi_state->owner->pi_lock); 798 raw_spin_unlock_irq(&pi_state->owner->pi_lock);
801 799
802 spin_lock_irq(&new_owner->pi_lock); 800 raw_spin_lock_irq(&new_owner->pi_lock);
803 WARN_ON(!list_empty(&pi_state->list)); 801 WARN_ON(!list_empty(&pi_state->list));
804 list_add(&pi_state->list, &new_owner->pi_state_list); 802 list_add(&pi_state->list, &new_owner->pi_state_list);
805 pi_state->owner = new_owner; 803 pi_state->owner = new_owner;
806 spin_unlock_irq(&new_owner->pi_lock); 804 raw_spin_unlock_irq(&new_owner->pi_lock);
807 805
808 spin_unlock(&pi_state->pi_mutex.wait_lock); 806 raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
809 rt_mutex_unlock(&pi_state->pi_mutex); 807 rt_mutex_unlock(&pi_state->pi_mutex);
810 808
811 return 0; 809 return 0;
@@ -867,7 +865,7 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset)
867 if (!bitset) 865 if (!bitset)
868 return -EINVAL; 866 return -EINVAL;
869 867
870 ret = get_futex_key(uaddr, fshared, &key, VERIFY_READ); 868 ret = get_futex_key(uaddr, fshared, &key);
871 if (unlikely(ret != 0)) 869 if (unlikely(ret != 0))
872 goto out; 870 goto out;
873 871
@@ -913,10 +911,10 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
913 int ret, op_ret; 911 int ret, op_ret;
914 912
915retry: 913retry:
916 ret = get_futex_key(uaddr1, fshared, &key1, VERIFY_READ); 914 ret = get_futex_key(uaddr1, fshared, &key1);
917 if (unlikely(ret != 0)) 915 if (unlikely(ret != 0))
918 goto out; 916 goto out;
919 ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE); 917 ret = get_futex_key(uaddr2, fshared, &key2);
920 if (unlikely(ret != 0)) 918 if (unlikely(ret != 0))
921 goto out_put_key1; 919 goto out_put_key1;
922 920
@@ -1010,7 +1008,7 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
1010 plist_add(&q->list, &hb2->chain); 1008 plist_add(&q->list, &hb2->chain);
1011 q->lock_ptr = &hb2->lock; 1009 q->lock_ptr = &hb2->lock;
1012#ifdef CONFIG_DEBUG_PI_LIST 1010#ifdef CONFIG_DEBUG_PI_LIST
1013 q->list.plist.lock = &hb2->lock; 1011 q->list.plist.spinlock = &hb2->lock;
1014#endif 1012#endif
1015 } 1013 }
1016 get_futex_key_refs(key2); 1014 get_futex_key_refs(key2);
@@ -1046,7 +1044,7 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
1046 1044
1047 q->lock_ptr = &hb->lock; 1045 q->lock_ptr = &hb->lock;
1048#ifdef CONFIG_DEBUG_PI_LIST 1046#ifdef CONFIG_DEBUG_PI_LIST
1049 q->list.plist.lock = &hb->lock; 1047 q->list.plist.spinlock = &hb->lock;
1050#endif 1048#endif
1051 1049
1052 wake_up_state(q->task, TASK_NORMAL); 1050 wake_up_state(q->task, TASK_NORMAL);
@@ -1175,11 +1173,10 @@ retry:
1175 pi_state = NULL; 1173 pi_state = NULL;
1176 } 1174 }
1177 1175
1178 ret = get_futex_key(uaddr1, fshared, &key1, VERIFY_READ); 1176 ret = get_futex_key(uaddr1, fshared, &key1);
1179 if (unlikely(ret != 0)) 1177 if (unlikely(ret != 0))
1180 goto out; 1178 goto out;
1181 ret = get_futex_key(uaddr2, fshared, &key2, 1179 ret = get_futex_key(uaddr2, fshared, &key2);
1182 requeue_pi ? VERIFY_WRITE : VERIFY_READ);
1183 if (unlikely(ret != 0)) 1180 if (unlikely(ret != 0))
1184 goto out_put_key1; 1181 goto out_put_key1;
1185 1182
@@ -1394,7 +1391,7 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
1394 1391
1395 plist_node_init(&q->list, prio); 1392 plist_node_init(&q->list, prio);
1396#ifdef CONFIG_DEBUG_PI_LIST 1393#ifdef CONFIG_DEBUG_PI_LIST
1397 q->list.plist.lock = &hb->lock; 1394 q->list.plist.spinlock = &hb->lock;
1398#endif 1395#endif
1399 plist_add(&q->list, &hb->chain); 1396 plist_add(&q->list, &hb->chain);
1400 q->task = current; 1397 q->task = current;
@@ -1529,18 +1526,18 @@ retry:
1529 * itself. 1526 * itself.
1530 */ 1527 */
1531 if (pi_state->owner != NULL) { 1528 if (pi_state->owner != NULL) {
1532 spin_lock_irq(&pi_state->owner->pi_lock); 1529 raw_spin_lock_irq(&pi_state->owner->pi_lock);
1533 WARN_ON(list_empty(&pi_state->list)); 1530 WARN_ON(list_empty(&pi_state->list));
1534 list_del_init(&pi_state->list); 1531 list_del_init(&pi_state->list);
1535 spin_unlock_irq(&pi_state->owner->pi_lock); 1532 raw_spin_unlock_irq(&pi_state->owner->pi_lock);
1536 } 1533 }
1537 1534
1538 pi_state->owner = newowner; 1535 pi_state->owner = newowner;
1539 1536
1540 spin_lock_irq(&newowner->pi_lock); 1537 raw_spin_lock_irq(&newowner->pi_lock);
1541 WARN_ON(!list_empty(&pi_state->list)); 1538 WARN_ON(!list_empty(&pi_state->list));
1542 list_add(&pi_state->list, &newowner->pi_state_list); 1539 list_add(&pi_state->list, &newowner->pi_state_list);
1543 spin_unlock_irq(&newowner->pi_lock); 1540 raw_spin_unlock_irq(&newowner->pi_lock);
1544 return 0; 1541 return 0;
1545 1542
1546 /* 1543 /*
@@ -1738,7 +1735,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared,
1738 */ 1735 */
1739retry: 1736retry:
1740 q->key = FUTEX_KEY_INIT; 1737 q->key = FUTEX_KEY_INIT;
1741 ret = get_futex_key(uaddr, fshared, &q->key, VERIFY_READ); 1738 ret = get_futex_key(uaddr, fshared, &q->key);
1742 if (unlikely(ret != 0)) 1739 if (unlikely(ret != 0))
1743 return ret; 1740 return ret;
1744 1741
@@ -1904,7 +1901,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
1904 q.requeue_pi_key = NULL; 1901 q.requeue_pi_key = NULL;
1905retry: 1902retry:
1906 q.key = FUTEX_KEY_INIT; 1903 q.key = FUTEX_KEY_INIT;
1907 ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE); 1904 ret = get_futex_key(uaddr, fshared, &q.key);
1908 if (unlikely(ret != 0)) 1905 if (unlikely(ret != 0))
1909 goto out; 1906 goto out;
1910 1907
@@ -2023,7 +2020,7 @@ retry:
2023 if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current)) 2020 if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current))
2024 return -EPERM; 2021 return -EPERM;
2025 2022
2026 ret = get_futex_key(uaddr, fshared, &key, VERIFY_WRITE); 2023 ret = get_futex_key(uaddr, fshared, &key);
2027 if (unlikely(ret != 0)) 2024 if (unlikely(ret != 0))
2028 goto out; 2025 goto out;
2029 2026
@@ -2215,7 +2212,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
2215 rt_waiter.task = NULL; 2212 rt_waiter.task = NULL;
2216 2213
2217 key2 = FUTEX_KEY_INIT; 2214 key2 = FUTEX_KEY_INIT;
2218 ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE); 2215 ret = get_futex_key(uaddr2, fshared, &key2);
2219 if (unlikely(ret != 0)) 2216 if (unlikely(ret != 0))
2220 goto out; 2217 goto out;
2221 2218