diff options
Diffstat (limited to 'kernel/futex.c')
-rw-r--r-- | kernel/futex.c | 77 |
1 files changed, 37 insertions, 40 deletions
diff --git a/kernel/futex.c b/kernel/futex.c index d73ef1f3e55d..d9b3a2228f9d 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -203,8 +203,6 @@ static void drop_futex_key_refs(union futex_key *key) | |||
203 | * @uaddr: virtual address of the futex | 203 | * @uaddr: virtual address of the futex |
204 | * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED | 204 | * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED |
205 | * @key: address where result is stored. | 205 | * @key: address where result is stored. |
206 | * @rw: mapping needs to be read/write (values: VERIFY_READ, | ||
207 | * VERIFY_WRITE) | ||
208 | * | 206 | * |
209 | * Returns a negative error code or 0 | 207 | * Returns a negative error code or 0 |
210 | * The key words are stored in *key on success. | 208 | * The key words are stored in *key on success. |
@@ -216,7 +214,7 @@ static void drop_futex_key_refs(union futex_key *key) | |||
216 | * lock_page() might sleep, the caller should not hold a spinlock. | 214 | * lock_page() might sleep, the caller should not hold a spinlock. |
217 | */ | 215 | */ |
218 | static int | 216 | static int |
219 | get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) | 217 | get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key) |
220 | { | 218 | { |
221 | unsigned long address = (unsigned long)uaddr; | 219 | unsigned long address = (unsigned long)uaddr; |
222 | struct mm_struct *mm = current->mm; | 220 | struct mm_struct *mm = current->mm; |
@@ -239,7 +237,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) | |||
239 | * but access_ok() should be faster than find_vma() | 237 | * but access_ok() should be faster than find_vma() |
240 | */ | 238 | */ |
241 | if (!fshared) { | 239 | if (!fshared) { |
242 | if (unlikely(!access_ok(rw, uaddr, sizeof(u32)))) | 240 | if (unlikely(!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))) |
243 | return -EFAULT; | 241 | return -EFAULT; |
244 | key->private.mm = mm; | 242 | key->private.mm = mm; |
245 | key->private.address = address; | 243 | key->private.address = address; |
@@ -248,7 +246,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) | |||
248 | } | 246 | } |
249 | 247 | ||
250 | again: | 248 | again: |
251 | err = get_user_pages_fast(address, 1, rw == VERIFY_WRITE, &page); | 249 | err = get_user_pages_fast(address, 1, 1, &page); |
252 | if (err < 0) | 250 | if (err < 0) |
253 | return err; | 251 | return err; |
254 | 252 | ||
@@ -403,9 +401,9 @@ static void free_pi_state(struct futex_pi_state *pi_state) | |||
403 | * and has cleaned up the pi_state already | 401 | * and has cleaned up the pi_state already |
404 | */ | 402 | */ |
405 | if (pi_state->owner) { | 403 | if (pi_state->owner) { |
406 | spin_lock_irq(&pi_state->owner->pi_lock); | 404 | raw_spin_lock_irq(&pi_state->owner->pi_lock); |
407 | list_del_init(&pi_state->list); | 405 | list_del_init(&pi_state->list); |
408 | spin_unlock_irq(&pi_state->owner->pi_lock); | 406 | raw_spin_unlock_irq(&pi_state->owner->pi_lock); |
409 | 407 | ||
410 | rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); | 408 | rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); |
411 | } | 409 | } |
@@ -470,18 +468,18 @@ void exit_pi_state_list(struct task_struct *curr) | |||
470 | * pi_state_list anymore, but we have to be careful | 468 | * pi_state_list anymore, but we have to be careful |
471 | * versus waiters unqueueing themselves: | 469 | * versus waiters unqueueing themselves: |
472 | */ | 470 | */ |
473 | spin_lock_irq(&curr->pi_lock); | 471 | raw_spin_lock_irq(&curr->pi_lock); |
474 | while (!list_empty(head)) { | 472 | while (!list_empty(head)) { |
475 | 473 | ||
476 | next = head->next; | 474 | next = head->next; |
477 | pi_state = list_entry(next, struct futex_pi_state, list); | 475 | pi_state = list_entry(next, struct futex_pi_state, list); |
478 | key = pi_state->key; | 476 | key = pi_state->key; |
479 | hb = hash_futex(&key); | 477 | hb = hash_futex(&key); |
480 | spin_unlock_irq(&curr->pi_lock); | 478 | raw_spin_unlock_irq(&curr->pi_lock); |
481 | 479 | ||
482 | spin_lock(&hb->lock); | 480 | spin_lock(&hb->lock); |
483 | 481 | ||
484 | spin_lock_irq(&curr->pi_lock); | 482 | raw_spin_lock_irq(&curr->pi_lock); |
485 | /* | 483 | /* |
486 | * We dropped the pi-lock, so re-check whether this | 484 | * We dropped the pi-lock, so re-check whether this |
487 | * task still owns the PI-state: | 485 | * task still owns the PI-state: |
@@ -495,15 +493,15 @@ void exit_pi_state_list(struct task_struct *curr) | |||
495 | WARN_ON(list_empty(&pi_state->list)); | 493 | WARN_ON(list_empty(&pi_state->list)); |
496 | list_del_init(&pi_state->list); | 494 | list_del_init(&pi_state->list); |
497 | pi_state->owner = NULL; | 495 | pi_state->owner = NULL; |
498 | spin_unlock_irq(&curr->pi_lock); | 496 | raw_spin_unlock_irq(&curr->pi_lock); |
499 | 497 | ||
500 | rt_mutex_unlock(&pi_state->pi_mutex); | 498 | rt_mutex_unlock(&pi_state->pi_mutex); |
501 | 499 | ||
502 | spin_unlock(&hb->lock); | 500 | spin_unlock(&hb->lock); |
503 | 501 | ||
504 | spin_lock_irq(&curr->pi_lock); | 502 | raw_spin_lock_irq(&curr->pi_lock); |
505 | } | 503 | } |
506 | spin_unlock_irq(&curr->pi_lock); | 504 | raw_spin_unlock_irq(&curr->pi_lock); |
507 | } | 505 | } |
508 | 506 | ||
509 | static int | 507 | static int |
@@ -558,7 +556,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, | |||
558 | * change of the task flags, we do this protected by | 556 | * change of the task flags, we do this protected by |
559 | * p->pi_lock: | 557 | * p->pi_lock: |
560 | */ | 558 | */ |
561 | spin_lock_irq(&p->pi_lock); | 559 | raw_spin_lock_irq(&p->pi_lock); |
562 | if (unlikely(p->flags & PF_EXITING)) { | 560 | if (unlikely(p->flags & PF_EXITING)) { |
563 | /* | 561 | /* |
564 | * The task is on the way out. When PF_EXITPIDONE is | 562 | * The task is on the way out. When PF_EXITPIDONE is |
@@ -567,7 +565,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, | |||
567 | */ | 565 | */ |
568 | int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN; | 566 | int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN; |
569 | 567 | ||
570 | spin_unlock_irq(&p->pi_lock); | 568 | raw_spin_unlock_irq(&p->pi_lock); |
571 | put_task_struct(p); | 569 | put_task_struct(p); |
572 | return ret; | 570 | return ret; |
573 | } | 571 | } |
@@ -586,7 +584,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, | |||
586 | WARN_ON(!list_empty(&pi_state->list)); | 584 | WARN_ON(!list_empty(&pi_state->list)); |
587 | list_add(&pi_state->list, &p->pi_state_list); | 585 | list_add(&pi_state->list, &p->pi_state_list); |
588 | pi_state->owner = p; | 586 | pi_state->owner = p; |
589 | spin_unlock_irq(&p->pi_lock); | 587 | raw_spin_unlock_irq(&p->pi_lock); |
590 | 588 | ||
591 | put_task_struct(p); | 589 | put_task_struct(p); |
592 | 590 | ||
@@ -760,7 +758,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) | |||
760 | if (!pi_state) | 758 | if (!pi_state) |
761 | return -EINVAL; | 759 | return -EINVAL; |
762 | 760 | ||
763 | spin_lock(&pi_state->pi_mutex.wait_lock); | 761 | raw_spin_lock(&pi_state->pi_mutex.wait_lock); |
764 | new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); | 762 | new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); |
765 | 763 | ||
766 | /* | 764 | /* |
@@ -789,23 +787,23 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) | |||
789 | else if (curval != uval) | 787 | else if (curval != uval) |
790 | ret = -EINVAL; | 788 | ret = -EINVAL; |
791 | if (ret) { | 789 | if (ret) { |
792 | spin_unlock(&pi_state->pi_mutex.wait_lock); | 790 | raw_spin_unlock(&pi_state->pi_mutex.wait_lock); |
793 | return ret; | 791 | return ret; |
794 | } | 792 | } |
795 | } | 793 | } |
796 | 794 | ||
797 | spin_lock_irq(&pi_state->owner->pi_lock); | 795 | raw_spin_lock_irq(&pi_state->owner->pi_lock); |
798 | WARN_ON(list_empty(&pi_state->list)); | 796 | WARN_ON(list_empty(&pi_state->list)); |
799 | list_del_init(&pi_state->list); | 797 | list_del_init(&pi_state->list); |
800 | spin_unlock_irq(&pi_state->owner->pi_lock); | 798 | raw_spin_unlock_irq(&pi_state->owner->pi_lock); |
801 | 799 | ||
802 | spin_lock_irq(&new_owner->pi_lock); | 800 | raw_spin_lock_irq(&new_owner->pi_lock); |
803 | WARN_ON(!list_empty(&pi_state->list)); | 801 | WARN_ON(!list_empty(&pi_state->list)); |
804 | list_add(&pi_state->list, &new_owner->pi_state_list); | 802 | list_add(&pi_state->list, &new_owner->pi_state_list); |
805 | pi_state->owner = new_owner; | 803 | pi_state->owner = new_owner; |
806 | spin_unlock_irq(&new_owner->pi_lock); | 804 | raw_spin_unlock_irq(&new_owner->pi_lock); |
807 | 805 | ||
808 | spin_unlock(&pi_state->pi_mutex.wait_lock); | 806 | raw_spin_unlock(&pi_state->pi_mutex.wait_lock); |
809 | rt_mutex_unlock(&pi_state->pi_mutex); | 807 | rt_mutex_unlock(&pi_state->pi_mutex); |
810 | 808 | ||
811 | return 0; | 809 | return 0; |
@@ -867,7 +865,7 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset) | |||
867 | if (!bitset) | 865 | if (!bitset) |
868 | return -EINVAL; | 866 | return -EINVAL; |
869 | 867 | ||
870 | ret = get_futex_key(uaddr, fshared, &key, VERIFY_READ); | 868 | ret = get_futex_key(uaddr, fshared, &key); |
871 | if (unlikely(ret != 0)) | 869 | if (unlikely(ret != 0)) |
872 | goto out; | 870 | goto out; |
873 | 871 | ||
@@ -913,10 +911,10 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2, | |||
913 | int ret, op_ret; | 911 | int ret, op_ret; |
914 | 912 | ||
915 | retry: | 913 | retry: |
916 | ret = get_futex_key(uaddr1, fshared, &key1, VERIFY_READ); | 914 | ret = get_futex_key(uaddr1, fshared, &key1); |
917 | if (unlikely(ret != 0)) | 915 | if (unlikely(ret != 0)) |
918 | goto out; | 916 | goto out; |
919 | ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE); | 917 | ret = get_futex_key(uaddr2, fshared, &key2); |
920 | if (unlikely(ret != 0)) | 918 | if (unlikely(ret != 0)) |
921 | goto out_put_key1; | 919 | goto out_put_key1; |
922 | 920 | ||
@@ -1010,7 +1008,7 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1, | |||
1010 | plist_add(&q->list, &hb2->chain); | 1008 | plist_add(&q->list, &hb2->chain); |
1011 | q->lock_ptr = &hb2->lock; | 1009 | q->lock_ptr = &hb2->lock; |
1012 | #ifdef CONFIG_DEBUG_PI_LIST | 1010 | #ifdef CONFIG_DEBUG_PI_LIST |
1013 | q->list.plist.lock = &hb2->lock; | 1011 | q->list.plist.spinlock = &hb2->lock; |
1014 | #endif | 1012 | #endif |
1015 | } | 1013 | } |
1016 | get_futex_key_refs(key2); | 1014 | get_futex_key_refs(key2); |
@@ -1046,7 +1044,7 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, | |||
1046 | 1044 | ||
1047 | q->lock_ptr = &hb->lock; | 1045 | q->lock_ptr = &hb->lock; |
1048 | #ifdef CONFIG_DEBUG_PI_LIST | 1046 | #ifdef CONFIG_DEBUG_PI_LIST |
1049 | q->list.plist.lock = &hb->lock; | 1047 | q->list.plist.spinlock = &hb->lock; |
1050 | #endif | 1048 | #endif |
1051 | 1049 | ||
1052 | wake_up_state(q->task, TASK_NORMAL); | 1050 | wake_up_state(q->task, TASK_NORMAL); |
@@ -1175,11 +1173,10 @@ retry: | |||
1175 | pi_state = NULL; | 1173 | pi_state = NULL; |
1176 | } | 1174 | } |
1177 | 1175 | ||
1178 | ret = get_futex_key(uaddr1, fshared, &key1, VERIFY_READ); | 1176 | ret = get_futex_key(uaddr1, fshared, &key1); |
1179 | if (unlikely(ret != 0)) | 1177 | if (unlikely(ret != 0)) |
1180 | goto out; | 1178 | goto out; |
1181 | ret = get_futex_key(uaddr2, fshared, &key2, | 1179 | ret = get_futex_key(uaddr2, fshared, &key2); |
1182 | requeue_pi ? VERIFY_WRITE : VERIFY_READ); | ||
1183 | if (unlikely(ret != 0)) | 1180 | if (unlikely(ret != 0)) |
1184 | goto out_put_key1; | 1181 | goto out_put_key1; |
1185 | 1182 | ||
@@ -1394,7 +1391,7 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) | |||
1394 | 1391 | ||
1395 | plist_node_init(&q->list, prio); | 1392 | plist_node_init(&q->list, prio); |
1396 | #ifdef CONFIG_DEBUG_PI_LIST | 1393 | #ifdef CONFIG_DEBUG_PI_LIST |
1397 | q->list.plist.lock = &hb->lock; | 1394 | q->list.plist.spinlock = &hb->lock; |
1398 | #endif | 1395 | #endif |
1399 | plist_add(&q->list, &hb->chain); | 1396 | plist_add(&q->list, &hb->chain); |
1400 | q->task = current; | 1397 | q->task = current; |
@@ -1529,18 +1526,18 @@ retry: | |||
1529 | * itself. | 1526 | * itself. |
1530 | */ | 1527 | */ |
1531 | if (pi_state->owner != NULL) { | 1528 | if (pi_state->owner != NULL) { |
1532 | spin_lock_irq(&pi_state->owner->pi_lock); | 1529 | raw_spin_lock_irq(&pi_state->owner->pi_lock); |
1533 | WARN_ON(list_empty(&pi_state->list)); | 1530 | WARN_ON(list_empty(&pi_state->list)); |
1534 | list_del_init(&pi_state->list); | 1531 | list_del_init(&pi_state->list); |
1535 | spin_unlock_irq(&pi_state->owner->pi_lock); | 1532 | raw_spin_unlock_irq(&pi_state->owner->pi_lock); |
1536 | } | 1533 | } |
1537 | 1534 | ||
1538 | pi_state->owner = newowner; | 1535 | pi_state->owner = newowner; |
1539 | 1536 | ||
1540 | spin_lock_irq(&newowner->pi_lock); | 1537 | raw_spin_lock_irq(&newowner->pi_lock); |
1541 | WARN_ON(!list_empty(&pi_state->list)); | 1538 | WARN_ON(!list_empty(&pi_state->list)); |
1542 | list_add(&pi_state->list, &newowner->pi_state_list); | 1539 | list_add(&pi_state->list, &newowner->pi_state_list); |
1543 | spin_unlock_irq(&newowner->pi_lock); | 1540 | raw_spin_unlock_irq(&newowner->pi_lock); |
1544 | return 0; | 1541 | return 0; |
1545 | 1542 | ||
1546 | /* | 1543 | /* |
@@ -1738,7 +1735,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared, | |||
1738 | */ | 1735 | */ |
1739 | retry: | 1736 | retry: |
1740 | q->key = FUTEX_KEY_INIT; | 1737 | q->key = FUTEX_KEY_INIT; |
1741 | ret = get_futex_key(uaddr, fshared, &q->key, VERIFY_READ); | 1738 | ret = get_futex_key(uaddr, fshared, &q->key); |
1742 | if (unlikely(ret != 0)) | 1739 | if (unlikely(ret != 0)) |
1743 | return ret; | 1740 | return ret; |
1744 | 1741 | ||
@@ -1904,7 +1901,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1904 | q.requeue_pi_key = NULL; | 1901 | q.requeue_pi_key = NULL; |
1905 | retry: | 1902 | retry: |
1906 | q.key = FUTEX_KEY_INIT; | 1903 | q.key = FUTEX_KEY_INIT; |
1907 | ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE); | 1904 | ret = get_futex_key(uaddr, fshared, &q.key); |
1908 | if (unlikely(ret != 0)) | 1905 | if (unlikely(ret != 0)) |
1909 | goto out; | 1906 | goto out; |
1910 | 1907 | ||
@@ -2023,7 +2020,7 @@ retry: | |||
2023 | if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current)) | 2020 | if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current)) |
2024 | return -EPERM; | 2021 | return -EPERM; |
2025 | 2022 | ||
2026 | ret = get_futex_key(uaddr, fshared, &key, VERIFY_WRITE); | 2023 | ret = get_futex_key(uaddr, fshared, &key); |
2027 | if (unlikely(ret != 0)) | 2024 | if (unlikely(ret != 0)) |
2028 | goto out; | 2025 | goto out; |
2029 | 2026 | ||
@@ -2215,7 +2212,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2215 | rt_waiter.task = NULL; | 2212 | rt_waiter.task = NULL; |
2216 | 2213 | ||
2217 | key2 = FUTEX_KEY_INIT; | 2214 | key2 = FUTEX_KEY_INIT; |
2218 | ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE); | 2215 | ret = get_futex_key(uaddr2, fshared, &key2); |
2219 | if (unlikely(ret != 0)) | 2216 | if (unlikely(ret != 0)) |
2220 | goto out; | 2217 | goto out; |
2221 | 2218 | ||