diff options
Diffstat (limited to 'kernel/futex.c')
-rw-r--r-- | kernel/futex.c | 72 |
1 files changed, 40 insertions, 32 deletions
diff --git a/kernel/futex.c b/kernel/futex.c index 7c6cbabe52b3..002aa189eb09 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -170,8 +170,11 @@ static void get_futex_key_refs(union futex_key *key) | |||
170 | */ | 170 | */ |
171 | static void drop_futex_key_refs(union futex_key *key) | 171 | static void drop_futex_key_refs(union futex_key *key) |
172 | { | 172 | { |
173 | if (!key->both.ptr) | 173 | if (!key->both.ptr) { |
174 | /* If we're here then we tried to put a key we failed to get */ | ||
175 | WARN_ON_ONCE(1); | ||
174 | return; | 176 | return; |
177 | } | ||
175 | 178 | ||
176 | switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { | 179 | switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { |
177 | case FUT_OFF_INODE: | 180 | case FUT_OFF_INODE: |
@@ -730,8 +733,8 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset) | |||
730 | } | 733 | } |
731 | 734 | ||
732 | spin_unlock(&hb->lock); | 735 | spin_unlock(&hb->lock); |
733 | out: | ||
734 | put_futex_key(fshared, &key); | 736 | put_futex_key(fshared, &key); |
737 | out: | ||
735 | return ret; | 738 | return ret; |
736 | } | 739 | } |
737 | 740 | ||
@@ -755,7 +758,7 @@ retryfull: | |||
755 | goto out; | 758 | goto out; |
756 | ret = get_futex_key(uaddr2, fshared, &key2); | 759 | ret = get_futex_key(uaddr2, fshared, &key2); |
757 | if (unlikely(ret != 0)) | 760 | if (unlikely(ret != 0)) |
758 | goto out; | 761 | goto out_put_key1; |
759 | 762 | ||
760 | hb1 = hash_futex(&key1); | 763 | hb1 = hash_futex(&key1); |
761 | hb2 = hash_futex(&key2); | 764 | hb2 = hash_futex(&key2); |
@@ -777,12 +780,12 @@ retry: | |||
777 | * but we might get them from range checking | 780 | * but we might get them from range checking |
778 | */ | 781 | */ |
779 | ret = op_ret; | 782 | ret = op_ret; |
780 | goto out; | 783 | goto out_put_keys; |
781 | #endif | 784 | #endif |
782 | 785 | ||
783 | if (unlikely(op_ret != -EFAULT)) { | 786 | if (unlikely(op_ret != -EFAULT)) { |
784 | ret = op_ret; | 787 | ret = op_ret; |
785 | goto out; | 788 | goto out_put_keys; |
786 | } | 789 | } |
787 | 790 | ||
788 | /* | 791 | /* |
@@ -796,7 +799,7 @@ retry: | |||
796 | ret = futex_handle_fault((unsigned long)uaddr2, | 799 | ret = futex_handle_fault((unsigned long)uaddr2, |
797 | attempt); | 800 | attempt); |
798 | if (ret) | 801 | if (ret) |
799 | goto out; | 802 | goto out_put_keys; |
800 | goto retry; | 803 | goto retry; |
801 | } | 804 | } |
802 | 805 | ||
@@ -834,10 +837,11 @@ retry: | |||
834 | spin_unlock(&hb1->lock); | 837 | spin_unlock(&hb1->lock); |
835 | if (hb1 != hb2) | 838 | if (hb1 != hb2) |
836 | spin_unlock(&hb2->lock); | 839 | spin_unlock(&hb2->lock); |
837 | out: | 840 | out_put_keys: |
838 | put_futex_key(fshared, &key2); | 841 | put_futex_key(fshared, &key2); |
842 | out_put_key1: | ||
839 | put_futex_key(fshared, &key1); | 843 | put_futex_key(fshared, &key1); |
840 | 844 | out: | |
841 | return ret; | 845 | return ret; |
842 | } | 846 | } |
843 | 847 | ||
@@ -854,13 +858,13 @@ static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2, | |||
854 | struct futex_q *this, *next; | 858 | struct futex_q *this, *next; |
855 | int ret, drop_count = 0; | 859 | int ret, drop_count = 0; |
856 | 860 | ||
857 | retry: | 861 | retry: |
858 | ret = get_futex_key(uaddr1, fshared, &key1); | 862 | ret = get_futex_key(uaddr1, fshared, &key1); |
859 | if (unlikely(ret != 0)) | 863 | if (unlikely(ret != 0)) |
860 | goto out; | 864 | goto out; |
861 | ret = get_futex_key(uaddr2, fshared, &key2); | 865 | ret = get_futex_key(uaddr2, fshared, &key2); |
862 | if (unlikely(ret != 0)) | 866 | if (unlikely(ret != 0)) |
863 | goto out; | 867 | goto out_put_key1; |
864 | 868 | ||
865 | hb1 = hash_futex(&key1); | 869 | hb1 = hash_futex(&key1); |
866 | hb2 = hash_futex(&key2); | 870 | hb2 = hash_futex(&key2); |
@@ -882,7 +886,7 @@ static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2, | |||
882 | if (!ret) | 886 | if (!ret) |
883 | goto retry; | 887 | goto retry; |
884 | 888 | ||
885 | return ret; | 889 | goto out_put_keys; |
886 | } | 890 | } |
887 | if (curval != *cmpval) { | 891 | if (curval != *cmpval) { |
888 | ret = -EAGAIN; | 892 | ret = -EAGAIN; |
@@ -927,9 +931,11 @@ out_unlock: | |||
927 | while (--drop_count >= 0) | 931 | while (--drop_count >= 0) |
928 | drop_futex_key_refs(&key1); | 932 | drop_futex_key_refs(&key1); |
929 | 933 | ||
930 | out: | 934 | out_put_keys: |
931 | put_futex_key(fshared, &key2); | 935 | put_futex_key(fshared, &key2); |
936 | out_put_key1: | ||
932 | put_futex_key(fshared, &key1); | 937 | put_futex_key(fshared, &key1); |
938 | out: | ||
933 | return ret; | 939 | return ret; |
934 | } | 940 | } |
935 | 941 | ||
@@ -990,7 +996,7 @@ static int unqueue_me(struct futex_q *q) | |||
990 | int ret = 0; | 996 | int ret = 0; |
991 | 997 | ||
992 | /* In the common case we don't take the spinlock, which is nice. */ | 998 | /* In the common case we don't take the spinlock, which is nice. */ |
993 | retry: | 999 | retry: |
994 | lock_ptr = q->lock_ptr; | 1000 | lock_ptr = q->lock_ptr; |
995 | barrier(); | 1001 | barrier(); |
996 | if (lock_ptr != NULL) { | 1002 | if (lock_ptr != NULL) { |
@@ -1172,11 +1178,11 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1172 | 1178 | ||
1173 | q.pi_state = NULL; | 1179 | q.pi_state = NULL; |
1174 | q.bitset = bitset; | 1180 | q.bitset = bitset; |
1175 | retry: | 1181 | retry: |
1176 | q.key = FUTEX_KEY_INIT; | 1182 | q.key = FUTEX_KEY_INIT; |
1177 | ret = get_futex_key(uaddr, fshared, &q.key); | 1183 | ret = get_futex_key(uaddr, fshared, &q.key); |
1178 | if (unlikely(ret != 0)) | 1184 | if (unlikely(ret != 0)) |
1179 | goto out_release_sem; | 1185 | goto out; |
1180 | 1186 | ||
1181 | hb = queue_lock(&q); | 1187 | hb = queue_lock(&q); |
1182 | 1188 | ||
@@ -1204,6 +1210,7 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1204 | 1210 | ||
1205 | if (unlikely(ret)) { | 1211 | if (unlikely(ret)) { |
1206 | queue_unlock(&q, hb); | 1212 | queue_unlock(&q, hb); |
1213 | put_futex_key(fshared, &q.key); | ||
1207 | 1214 | ||
1208 | ret = get_user(uval, uaddr); | 1215 | ret = get_user(uval, uaddr); |
1209 | 1216 | ||
@@ -1213,7 +1220,7 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1213 | } | 1220 | } |
1214 | ret = -EWOULDBLOCK; | 1221 | ret = -EWOULDBLOCK; |
1215 | if (uval != val) | 1222 | if (uval != val) |
1216 | goto out_unlock_release_sem; | 1223 | goto out_unlock_put_key; |
1217 | 1224 | ||
1218 | /* Only actually queue if *uaddr contained val. */ | 1225 | /* Only actually queue if *uaddr contained val. */ |
1219 | queue_me(&q, hb); | 1226 | queue_me(&q, hb); |
@@ -1305,11 +1312,11 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1305 | return -ERESTART_RESTARTBLOCK; | 1312 | return -ERESTART_RESTARTBLOCK; |
1306 | } | 1313 | } |
1307 | 1314 | ||
1308 | out_unlock_release_sem: | 1315 | out_unlock_put_key: |
1309 | queue_unlock(&q, hb); | 1316 | queue_unlock(&q, hb); |
1310 | |||
1311 | out_release_sem: | ||
1312 | put_futex_key(fshared, &q.key); | 1317 | put_futex_key(fshared, &q.key); |
1318 | |||
1319 | out: | ||
1313 | return ret; | 1320 | return ret; |
1314 | } | 1321 | } |
1315 | 1322 | ||
@@ -1358,16 +1365,16 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1358 | } | 1365 | } |
1359 | 1366 | ||
1360 | q.pi_state = NULL; | 1367 | q.pi_state = NULL; |
1361 | retry: | 1368 | retry: |
1362 | q.key = FUTEX_KEY_INIT; | 1369 | q.key = FUTEX_KEY_INIT; |
1363 | ret = get_futex_key(uaddr, fshared, &q.key); | 1370 | ret = get_futex_key(uaddr, fshared, &q.key); |
1364 | if (unlikely(ret != 0)) | 1371 | if (unlikely(ret != 0)) |
1365 | goto out_release_sem; | 1372 | goto out; |
1366 | 1373 | ||
1367 | retry_unlocked: | 1374 | retry_unlocked: |
1368 | hb = queue_lock(&q); | 1375 | hb = queue_lock(&q); |
1369 | 1376 | ||
1370 | retry_locked: | 1377 | retry_locked: |
1371 | ret = lock_taken = 0; | 1378 | ret = lock_taken = 0; |
1372 | 1379 | ||
1373 | /* | 1380 | /* |
@@ -1388,14 +1395,14 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1388 | */ | 1395 | */ |
1389 | if (unlikely((curval & FUTEX_TID_MASK) == task_pid_vnr(current))) { | 1396 | if (unlikely((curval & FUTEX_TID_MASK) == task_pid_vnr(current))) { |
1390 | ret = -EDEADLK; | 1397 | ret = -EDEADLK; |
1391 | goto out_unlock_release_sem; | 1398 | goto out_unlock_put_key; |
1392 | } | 1399 | } |
1393 | 1400 | ||
1394 | /* | 1401 | /* |
1395 | * Surprise - we got the lock. Just return to userspace: | 1402 | * Surprise - we got the lock. Just return to userspace: |
1396 | */ | 1403 | */ |
1397 | if (unlikely(!curval)) | 1404 | if (unlikely(!curval)) |
1398 | goto out_unlock_release_sem; | 1405 | goto out_unlock_put_key; |
1399 | 1406 | ||
1400 | uval = curval; | 1407 | uval = curval; |
1401 | 1408 | ||
@@ -1431,7 +1438,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1431 | * We took the lock due to owner died take over. | 1438 | * We took the lock due to owner died take over. |
1432 | */ | 1439 | */ |
1433 | if (unlikely(lock_taken)) | 1440 | if (unlikely(lock_taken)) |
1434 | goto out_unlock_release_sem; | 1441 | goto out_unlock_put_key; |
1435 | 1442 | ||
1436 | /* | 1443 | /* |
1437 | * We dont have the lock. Look up the PI state (or create it if | 1444 | * We dont have the lock. Look up the PI state (or create it if |
@@ -1470,7 +1477,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1470 | goto retry_locked; | 1477 | goto retry_locked; |
1471 | } | 1478 | } |
1472 | default: | 1479 | default: |
1473 | goto out_unlock_release_sem; | 1480 | goto out_unlock_put_key; |
1474 | } | 1481 | } |
1475 | } | 1482 | } |
1476 | 1483 | ||
@@ -1561,16 +1568,17 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1561 | destroy_hrtimer_on_stack(&to->timer); | 1568 | destroy_hrtimer_on_stack(&to->timer); |
1562 | return ret != -EINTR ? ret : -ERESTARTNOINTR; | 1569 | return ret != -EINTR ? ret : -ERESTARTNOINTR; |
1563 | 1570 | ||
1564 | out_unlock_release_sem: | 1571 | out_unlock_put_key: |
1565 | queue_unlock(&q, hb); | 1572 | queue_unlock(&q, hb); |
1566 | 1573 | ||
1567 | out_release_sem: | 1574 | out_put_key: |
1568 | put_futex_key(fshared, &q.key); | 1575 | put_futex_key(fshared, &q.key); |
1576 | out: | ||
1569 | if (to) | 1577 | if (to) |
1570 | destroy_hrtimer_on_stack(&to->timer); | 1578 | destroy_hrtimer_on_stack(&to->timer); |
1571 | return ret; | 1579 | return ret; |
1572 | 1580 | ||
1573 | uaddr_faulted: | 1581 | uaddr_faulted: |
1574 | /* | 1582 | /* |
1575 | * We have to r/w *(int __user *)uaddr, and we have to modify it | 1583 | * We have to r/w *(int __user *)uaddr, and we have to modify it |
1576 | * atomically. Therefore, if we continue to fault after get_user() | 1584 | * atomically. Therefore, if we continue to fault after get_user() |
@@ -1583,7 +1591,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1583 | if (attempt++) { | 1591 | if (attempt++) { |
1584 | ret = futex_handle_fault((unsigned long)uaddr, attempt); | 1592 | ret = futex_handle_fault((unsigned long)uaddr, attempt); |
1585 | if (ret) | 1593 | if (ret) |
1586 | goto out_release_sem; | 1594 | goto out_put_key; |
1587 | goto retry_unlocked; | 1595 | goto retry_unlocked; |
1588 | } | 1596 | } |
1589 | 1597 | ||
@@ -1675,9 +1683,9 @@ retry_unlocked: | |||
1675 | 1683 | ||
1676 | out_unlock: | 1684 | out_unlock: |
1677 | spin_unlock(&hb->lock); | 1685 | spin_unlock(&hb->lock); |
1678 | out: | ||
1679 | put_futex_key(fshared, &key); | 1686 | put_futex_key(fshared, &key); |
1680 | 1687 | ||
1688 | out: | ||
1681 | return ret; | 1689 | return ret; |
1682 | 1690 | ||
1683 | pi_faulted: | 1691 | pi_faulted: |