diff options
Diffstat (limited to 'kernel/futex.c')
-rw-r--r-- | kernel/futex.c | 235 |
1 files changed, 110 insertions, 125 deletions
diff --git a/kernel/futex.c b/kernel/futex.c index 40a8777a27d0..3019b92e6917 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -69,6 +69,14 @@ int __read_mostly futex_cmpxchg_enabled; | |||
69 | #define FUTEX_HASHBITS (CONFIG_BASE_SMALL ? 4 : 8) | 69 | #define FUTEX_HASHBITS (CONFIG_BASE_SMALL ? 4 : 8) |
70 | 70 | ||
71 | /* | 71 | /* |
72 | * Futex flags used to encode options to functions and preserve them across | ||
73 | * restarts. | ||
74 | */ | ||
75 | #define FLAGS_SHARED 0x01 | ||
76 | #define FLAGS_CLOCKRT 0x02 | ||
77 | #define FLAGS_HAS_TIMEOUT 0x04 | ||
78 | |||
79 | /* | ||
72 | * Priority Inheritance state: | 80 | * Priority Inheritance state: |
73 | */ | 81 | */ |
74 | struct futex_pi_state { | 82 | struct futex_pi_state { |
@@ -123,6 +131,12 @@ struct futex_q { | |||
123 | u32 bitset; | 131 | u32 bitset; |
124 | }; | 132 | }; |
125 | 133 | ||
134 | static const struct futex_q futex_q_init = { | ||
135 | /* list gets initialized in queue_me()*/ | ||
136 | .key = FUTEX_KEY_INIT, | ||
137 | .bitset = FUTEX_BITSET_MATCH_ANY | ||
138 | }; | ||
139 | |||
126 | /* | 140 | /* |
127 | * Hash buckets are shared by all the futex_keys that hash to the same | 141 | * Hash buckets are shared by all the futex_keys that hash to the same |
128 | * location. Each key may have multiple futex_q structures, one for each task | 142 | * location. Each key may have multiple futex_q structures, one for each task |
@@ -283,8 +297,7 @@ again: | |||
283 | return 0; | 297 | return 0; |
284 | } | 298 | } |
285 | 299 | ||
286 | static inline | 300 | static inline void put_futex_key(union futex_key *key) |
287 | void put_futex_key(int fshared, union futex_key *key) | ||
288 | { | 301 | { |
289 | drop_futex_key_refs(key); | 302 | drop_futex_key_refs(key); |
290 | } | 303 | } |
@@ -870,7 +883,8 @@ double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) | |||
870 | /* | 883 | /* |
871 | * Wake up waiters matching bitset queued on this futex (uaddr). | 884 | * Wake up waiters matching bitset queued on this futex (uaddr). |
872 | */ | 885 | */ |
873 | static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset) | 886 | static int |
887 | futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) | ||
874 | { | 888 | { |
875 | struct futex_hash_bucket *hb; | 889 | struct futex_hash_bucket *hb; |
876 | struct futex_q *this, *next; | 890 | struct futex_q *this, *next; |
@@ -881,7 +895,7 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset) | |||
881 | if (!bitset) | 895 | if (!bitset) |
882 | return -EINVAL; | 896 | return -EINVAL; |
883 | 897 | ||
884 | ret = get_futex_key(uaddr, fshared, &key); | 898 | ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key); |
885 | if (unlikely(ret != 0)) | 899 | if (unlikely(ret != 0)) |
886 | goto out; | 900 | goto out; |
887 | 901 | ||
@@ -907,7 +921,7 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset) | |||
907 | } | 921 | } |
908 | 922 | ||
909 | spin_unlock(&hb->lock); | 923 | spin_unlock(&hb->lock); |
910 | put_futex_key(fshared, &key); | 924 | put_futex_key(&key); |
911 | out: | 925 | out: |
912 | return ret; | 926 | return ret; |
913 | } | 927 | } |
@@ -917,7 +931,7 @@ out: | |||
917 | * to this virtual address: | 931 | * to this virtual address: |
918 | */ | 932 | */ |
919 | static int | 933 | static int |
920 | futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2, | 934 | futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, |
921 | int nr_wake, int nr_wake2, int op) | 935 | int nr_wake, int nr_wake2, int op) |
922 | { | 936 | { |
923 | union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT; | 937 | union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT; |
@@ -927,10 +941,10 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2, | |||
927 | int ret, op_ret; | 941 | int ret, op_ret; |
928 | 942 | ||
929 | retry: | 943 | retry: |
930 | ret = get_futex_key(uaddr1, fshared, &key1); | 944 | ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1); |
931 | if (unlikely(ret != 0)) | 945 | if (unlikely(ret != 0)) |
932 | goto out; | 946 | goto out; |
933 | ret = get_futex_key(uaddr2, fshared, &key2); | 947 | ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2); |
934 | if (unlikely(ret != 0)) | 948 | if (unlikely(ret != 0)) |
935 | goto out_put_key1; | 949 | goto out_put_key1; |
936 | 950 | ||
@@ -962,11 +976,11 @@ retry_private: | |||
962 | if (ret) | 976 | if (ret) |
963 | goto out_put_keys; | 977 | goto out_put_keys; |
964 | 978 | ||
965 | if (!fshared) | 979 | if (!(flags & FLAGS_SHARED)) |
966 | goto retry_private; | 980 | goto retry_private; |
967 | 981 | ||
968 | put_futex_key(fshared, &key2); | 982 | put_futex_key(&key2); |
969 | put_futex_key(fshared, &key1); | 983 | put_futex_key(&key1); |
970 | goto retry; | 984 | goto retry; |
971 | } | 985 | } |
972 | 986 | ||
@@ -996,9 +1010,9 @@ retry_private: | |||
996 | 1010 | ||
997 | double_unlock_hb(hb1, hb2); | 1011 | double_unlock_hb(hb1, hb2); |
998 | out_put_keys: | 1012 | out_put_keys: |
999 | put_futex_key(fshared, &key2); | 1013 | put_futex_key(&key2); |
1000 | out_put_key1: | 1014 | out_put_key1: |
1001 | put_futex_key(fshared, &key1); | 1015 | put_futex_key(&key1); |
1002 | out: | 1016 | out: |
1003 | return ret; | 1017 | return ret; |
1004 | } | 1018 | } |
@@ -1133,13 +1147,13 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex, | |||
1133 | /** | 1147 | /** |
1134 | * futex_requeue() - Requeue waiters from uaddr1 to uaddr2 | 1148 | * futex_requeue() - Requeue waiters from uaddr1 to uaddr2 |
1135 | * @uaddr1: source futex user address | 1149 | * @uaddr1: source futex user address |
1136 | * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED | 1150 | * @flags: futex flags (FLAGS_SHARED, etc.) |
1137 | * @uaddr2: target futex user address | 1151 | * @uaddr2: target futex user address |
1138 | * @nr_wake: number of waiters to wake (must be 1 for requeue_pi) | 1152 | * @nr_wake: number of waiters to wake (must be 1 for requeue_pi) |
1139 | * @nr_requeue: number of waiters to requeue (0-INT_MAX) | 1153 | * @nr_requeue: number of waiters to requeue (0-INT_MAX) |
1140 | * @cmpval: @uaddr1 expected value (or %NULL) | 1154 | * @cmpval: @uaddr1 expected value (or %NULL) |
1141 | * @requeue_pi: if we are attempting to requeue from a non-pi futex to a | 1155 | * @requeue_pi: if we are attempting to requeue from a non-pi futex to a |
1142 | * pi futex (pi to pi requeue is not supported) | 1156 | * pi futex (pi to pi requeue is not supported) |
1143 | * | 1157 | * |
1144 | * Requeue waiters on uaddr1 to uaddr2. In the requeue_pi case, try to acquire | 1158 | * Requeue waiters on uaddr1 to uaddr2. In the requeue_pi case, try to acquire |
1145 | * uaddr2 atomically on behalf of the top waiter. | 1159 | * uaddr2 atomically on behalf of the top waiter. |
@@ -1148,9 +1162,9 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex, | |||
1148 | * >=0 - on success, the number of tasks requeued or woken | 1162 | * >=0 - on success, the number of tasks requeued or woken |
1149 | * <0 - on error | 1163 | * <0 - on error |
1150 | */ | 1164 | */ |
1151 | static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2, | 1165 | static int futex_requeue(u32 __user *uaddr1, unsigned int flags, |
1152 | int nr_wake, int nr_requeue, u32 *cmpval, | 1166 | u32 __user *uaddr2, int nr_wake, int nr_requeue, |
1153 | int requeue_pi) | 1167 | u32 *cmpval, int requeue_pi) |
1154 | { | 1168 | { |
1155 | union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT; | 1169 | union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT; |
1156 | int drop_count = 0, task_count = 0, ret; | 1170 | int drop_count = 0, task_count = 0, ret; |
@@ -1191,10 +1205,10 @@ retry: | |||
1191 | pi_state = NULL; | 1205 | pi_state = NULL; |
1192 | } | 1206 | } |
1193 | 1207 | ||
1194 | ret = get_futex_key(uaddr1, fshared, &key1); | 1208 | ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1); |
1195 | if (unlikely(ret != 0)) | 1209 | if (unlikely(ret != 0)) |
1196 | goto out; | 1210 | goto out; |
1197 | ret = get_futex_key(uaddr2, fshared, &key2); | 1211 | ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2); |
1198 | if (unlikely(ret != 0)) | 1212 | if (unlikely(ret != 0)) |
1199 | goto out_put_key1; | 1213 | goto out_put_key1; |
1200 | 1214 | ||
@@ -1216,11 +1230,11 @@ retry_private: | |||
1216 | if (ret) | 1230 | if (ret) |
1217 | goto out_put_keys; | 1231 | goto out_put_keys; |
1218 | 1232 | ||
1219 | if (!fshared) | 1233 | if (!(flags & FLAGS_SHARED)) |
1220 | goto retry_private; | 1234 | goto retry_private; |
1221 | 1235 | ||
1222 | put_futex_key(fshared, &key2); | 1236 | put_futex_key(&key2); |
1223 | put_futex_key(fshared, &key1); | 1237 | put_futex_key(&key1); |
1224 | goto retry; | 1238 | goto retry; |
1225 | } | 1239 | } |
1226 | if (curval != *cmpval) { | 1240 | if (curval != *cmpval) { |
@@ -1260,8 +1274,8 @@ retry_private: | |||
1260 | break; | 1274 | break; |
1261 | case -EFAULT: | 1275 | case -EFAULT: |
1262 | double_unlock_hb(hb1, hb2); | 1276 | double_unlock_hb(hb1, hb2); |
1263 | put_futex_key(fshared, &key2); | 1277 | put_futex_key(&key2); |
1264 | put_futex_key(fshared, &key1); | 1278 | put_futex_key(&key1); |
1265 | ret = fault_in_user_writeable(uaddr2); | 1279 | ret = fault_in_user_writeable(uaddr2); |
1266 | if (!ret) | 1280 | if (!ret) |
1267 | goto retry; | 1281 | goto retry; |
@@ -1269,8 +1283,8 @@ retry_private: | |||
1269 | case -EAGAIN: | 1283 | case -EAGAIN: |
1270 | /* The owner was exiting, try again. */ | 1284 | /* The owner was exiting, try again. */ |
1271 | double_unlock_hb(hb1, hb2); | 1285 | double_unlock_hb(hb1, hb2); |
1272 | put_futex_key(fshared, &key2); | 1286 | put_futex_key(&key2); |
1273 | put_futex_key(fshared, &key1); | 1287 | put_futex_key(&key1); |
1274 | cond_resched(); | 1288 | cond_resched(); |
1275 | goto retry; | 1289 | goto retry; |
1276 | default: | 1290 | default: |
@@ -1352,9 +1366,9 @@ out_unlock: | |||
1352 | drop_futex_key_refs(&key1); | 1366 | drop_futex_key_refs(&key1); |
1353 | 1367 | ||
1354 | out_put_keys: | 1368 | out_put_keys: |
1355 | put_futex_key(fshared, &key2); | 1369 | put_futex_key(&key2); |
1356 | out_put_key1: | 1370 | out_put_key1: |
1357 | put_futex_key(fshared, &key1); | 1371 | put_futex_key(&key1); |
1358 | out: | 1372 | out: |
1359 | if (pi_state != NULL) | 1373 | if (pi_state != NULL) |
1360 | free_pi_state(pi_state); | 1374 | free_pi_state(pi_state); |
@@ -1494,7 +1508,7 @@ static void unqueue_me_pi(struct futex_q *q) | |||
1494 | * private futexes. | 1508 | * private futexes. |
1495 | */ | 1509 | */ |
1496 | static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, | 1510 | static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, |
1497 | struct task_struct *newowner, int fshared) | 1511 | struct task_struct *newowner) |
1498 | { | 1512 | { |
1499 | u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS; | 1513 | u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS; |
1500 | struct futex_pi_state *pi_state = q->pi_state; | 1514 | struct futex_pi_state *pi_state = q->pi_state; |
@@ -1587,20 +1601,11 @@ handle_fault: | |||
1587 | goto retry; | 1601 | goto retry; |
1588 | } | 1602 | } |
1589 | 1603 | ||
1590 | /* | ||
1591 | * In case we must use restart_block to restart a futex_wait, | ||
1592 | * we encode in the 'flags' shared capability | ||
1593 | */ | ||
1594 | #define FLAGS_SHARED 0x01 | ||
1595 | #define FLAGS_CLOCKRT 0x02 | ||
1596 | #define FLAGS_HAS_TIMEOUT 0x04 | ||
1597 | |||
1598 | static long futex_wait_restart(struct restart_block *restart); | 1604 | static long futex_wait_restart(struct restart_block *restart); |
1599 | 1605 | ||
1600 | /** | 1606 | /** |
1601 | * fixup_owner() - Post lock pi_state and corner case management | 1607 | * fixup_owner() - Post lock pi_state and corner case management |
1602 | * @uaddr: user address of the futex | 1608 | * @uaddr: user address of the futex |
1603 | * @fshared: whether the futex is shared (1) or not (0) | ||
1604 | * @q: futex_q (contains pi_state and access to the rt_mutex) | 1609 | * @q: futex_q (contains pi_state and access to the rt_mutex) |
1605 | * @locked: if the attempt to take the rt_mutex succeeded (1) or not (0) | 1610 | * @locked: if the attempt to take the rt_mutex succeeded (1) or not (0) |
1606 | * | 1611 | * |
@@ -1613,8 +1618,7 @@ static long futex_wait_restart(struct restart_block *restart); | |||
1613 | * 0 - success, lock not taken | 1618 | * 0 - success, lock not taken |
1614 | * <0 - on error (-EFAULT) | 1619 | * <0 - on error (-EFAULT) |
1615 | */ | 1620 | */ |
1616 | static int fixup_owner(u32 __user *uaddr, int fshared, struct futex_q *q, | 1621 | static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked) |
1617 | int locked) | ||
1618 | { | 1622 | { |
1619 | struct task_struct *owner; | 1623 | struct task_struct *owner; |
1620 | int ret = 0; | 1624 | int ret = 0; |
@@ -1625,7 +1629,7 @@ static int fixup_owner(u32 __user *uaddr, int fshared, struct futex_q *q, | |||
1625 | * did a lock-steal - fix up the PI-state in that case: | 1629 | * did a lock-steal - fix up the PI-state in that case: |
1626 | */ | 1630 | */ |
1627 | if (q->pi_state->owner != current) | 1631 | if (q->pi_state->owner != current) |
1628 | ret = fixup_pi_state_owner(uaddr, q, current, fshared); | 1632 | ret = fixup_pi_state_owner(uaddr, q, current); |
1629 | goto out; | 1633 | goto out; |
1630 | } | 1634 | } |
1631 | 1635 | ||
@@ -1652,7 +1656,7 @@ static int fixup_owner(u32 __user *uaddr, int fshared, struct futex_q *q, | |||
1652 | * lock. Fix the state up. | 1656 | * lock. Fix the state up. |
1653 | */ | 1657 | */ |
1654 | owner = rt_mutex_owner(&q->pi_state->pi_mutex); | 1658 | owner = rt_mutex_owner(&q->pi_state->pi_mutex); |
1655 | ret = fixup_pi_state_owner(uaddr, q, owner, fshared); | 1659 | ret = fixup_pi_state_owner(uaddr, q, owner); |
1656 | goto out; | 1660 | goto out; |
1657 | } | 1661 | } |
1658 | 1662 | ||
@@ -1715,7 +1719,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, | |||
1715 | * futex_wait_setup() - Prepare to wait on a futex | 1719 | * futex_wait_setup() - Prepare to wait on a futex |
1716 | * @uaddr: the futex userspace address | 1720 | * @uaddr: the futex userspace address |
1717 | * @val: the expected value | 1721 | * @val: the expected value |
1718 | * @fshared: whether the futex is shared (1) or not (0) | 1722 | * @flags: futex flags (FLAGS_SHARED, etc.) |
1719 | * @q: the associated futex_q | 1723 | * @q: the associated futex_q |
1720 | * @hb: storage for hash_bucket pointer to be returned to caller | 1724 | * @hb: storage for hash_bucket pointer to be returned to caller |
1721 | * | 1725 | * |
@@ -1728,7 +1732,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, | |||
1728 | * 0 - uaddr contains val and hb has been locked | 1732 | * 0 - uaddr contains val and hb has been locked |
1729 | * <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlcoked | 1733 | * <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlcoked |
1730 | */ | 1734 | */ |
1731 | static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared, | 1735 | static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, |
1732 | struct futex_q *q, struct futex_hash_bucket **hb) | 1736 | struct futex_q *q, struct futex_hash_bucket **hb) |
1733 | { | 1737 | { |
1734 | u32 uval; | 1738 | u32 uval; |
@@ -1752,8 +1756,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared, | |||
1752 | * rare, but normal. | 1756 | * rare, but normal. |
1753 | */ | 1757 | */ |
1754 | retry: | 1758 | retry: |
1755 | q->key = FUTEX_KEY_INIT; | 1759 | ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key); |
1756 | ret = get_futex_key(uaddr, fshared, &q->key); | ||
1757 | if (unlikely(ret != 0)) | 1760 | if (unlikely(ret != 0)) |
1758 | return ret; | 1761 | return ret; |
1759 | 1762 | ||
@@ -1769,10 +1772,10 @@ retry_private: | |||
1769 | if (ret) | 1772 | if (ret) |
1770 | goto out; | 1773 | goto out; |
1771 | 1774 | ||
1772 | if (!fshared) | 1775 | if (!(flags & FLAGS_SHARED)) |
1773 | goto retry_private; | 1776 | goto retry_private; |
1774 | 1777 | ||
1775 | put_futex_key(fshared, &q->key); | 1778 | put_futex_key(&q->key); |
1776 | goto retry; | 1779 | goto retry; |
1777 | } | 1780 | } |
1778 | 1781 | ||
@@ -1783,32 +1786,29 @@ retry_private: | |||
1783 | 1786 | ||
1784 | out: | 1787 | out: |
1785 | if (ret) | 1788 | if (ret) |
1786 | put_futex_key(fshared, &q->key); | 1789 | put_futex_key(&q->key); |
1787 | return ret; | 1790 | return ret; |
1788 | } | 1791 | } |
1789 | 1792 | ||
1790 | static int futex_wait(u32 __user *uaddr, int fshared, | 1793 | static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, |
1791 | u32 val, ktime_t *abs_time, u32 bitset, int clockrt) | 1794 | ktime_t *abs_time, u32 bitset) |
1792 | { | 1795 | { |
1793 | struct hrtimer_sleeper timeout, *to = NULL; | 1796 | struct hrtimer_sleeper timeout, *to = NULL; |
1794 | struct restart_block *restart; | 1797 | struct restart_block *restart; |
1795 | struct futex_hash_bucket *hb; | 1798 | struct futex_hash_bucket *hb; |
1796 | struct futex_q q; | 1799 | struct futex_q q = futex_q_init; |
1797 | int ret; | 1800 | int ret; |
1798 | 1801 | ||
1799 | if (!bitset) | 1802 | if (!bitset) |
1800 | return -EINVAL; | 1803 | return -EINVAL; |
1801 | |||
1802 | q.pi_state = NULL; | ||
1803 | q.bitset = bitset; | 1804 | q.bitset = bitset; |
1804 | q.rt_waiter = NULL; | ||
1805 | q.requeue_pi_key = NULL; | ||
1806 | 1805 | ||
1807 | if (abs_time) { | 1806 | if (abs_time) { |
1808 | to = &timeout; | 1807 | to = &timeout; |
1809 | 1808 | ||
1810 | hrtimer_init_on_stack(&to->timer, clockrt ? CLOCK_REALTIME : | 1809 | hrtimer_init_on_stack(&to->timer, (flags & FLAGS_CLOCKRT) ? |
1811 | CLOCK_MONOTONIC, HRTIMER_MODE_ABS); | 1810 | CLOCK_REALTIME : CLOCK_MONOTONIC, |
1811 | HRTIMER_MODE_ABS); | ||
1812 | hrtimer_init_sleeper(to, current); | 1812 | hrtimer_init_sleeper(to, current); |
1813 | hrtimer_set_expires_range_ns(&to->timer, *abs_time, | 1813 | hrtimer_set_expires_range_ns(&to->timer, *abs_time, |
1814 | current->timer_slack_ns); | 1814 | current->timer_slack_ns); |
@@ -1819,7 +1819,7 @@ retry: | |||
1819 | * Prepare to wait on uaddr. On success, holds hb lock and increments | 1819 | * Prepare to wait on uaddr. On success, holds hb lock and increments |
1820 | * q.key refs. | 1820 | * q.key refs. |
1821 | */ | 1821 | */ |
1822 | ret = futex_wait_setup(uaddr, val, fshared, &q, &hb); | 1822 | ret = futex_wait_setup(uaddr, val, flags, &q, &hb); |
1823 | if (ret) | 1823 | if (ret) |
1824 | goto out; | 1824 | goto out; |
1825 | 1825 | ||
@@ -1852,12 +1852,7 @@ retry: | |||
1852 | restart->futex.val = val; | 1852 | restart->futex.val = val; |
1853 | restart->futex.time = abs_time->tv64; | 1853 | restart->futex.time = abs_time->tv64; |
1854 | restart->futex.bitset = bitset; | 1854 | restart->futex.bitset = bitset; |
1855 | restart->futex.flags = FLAGS_HAS_TIMEOUT; | 1855 | restart->futex.flags = flags; |
1856 | |||
1857 | if (fshared) | ||
1858 | restart->futex.flags |= FLAGS_SHARED; | ||
1859 | if (clockrt) | ||
1860 | restart->futex.flags |= FLAGS_CLOCKRT; | ||
1861 | 1856 | ||
1862 | ret = -ERESTART_RESTARTBLOCK; | 1857 | ret = -ERESTART_RESTARTBLOCK; |
1863 | 1858 | ||
@@ -1873,7 +1868,6 @@ out: | |||
1873 | static long futex_wait_restart(struct restart_block *restart) | 1868 | static long futex_wait_restart(struct restart_block *restart) |
1874 | { | 1869 | { |
1875 | u32 __user *uaddr = restart->futex.uaddr; | 1870 | u32 __user *uaddr = restart->futex.uaddr; |
1876 | int fshared = 0; | ||
1877 | ktime_t t, *tp = NULL; | 1871 | ktime_t t, *tp = NULL; |
1878 | 1872 | ||
1879 | if (restart->futex.flags & FLAGS_HAS_TIMEOUT) { | 1873 | if (restart->futex.flags & FLAGS_HAS_TIMEOUT) { |
@@ -1881,11 +1875,9 @@ static long futex_wait_restart(struct restart_block *restart) | |||
1881 | tp = &t; | 1875 | tp = &t; |
1882 | } | 1876 | } |
1883 | restart->fn = do_no_restart_syscall; | 1877 | restart->fn = do_no_restart_syscall; |
1884 | if (restart->futex.flags & FLAGS_SHARED) | 1878 | |
1885 | fshared = 1; | 1879 | return (long)futex_wait(uaddr, restart->futex.flags, |
1886 | return (long)futex_wait(uaddr, fshared, restart->futex.val, tp, | 1880 | restart->futex.val, tp, restart->futex.bitset); |
1887 | restart->futex.bitset, | ||
1888 | restart->futex.flags & FLAGS_CLOCKRT); | ||
1889 | } | 1881 | } |
1890 | 1882 | ||
1891 | 1883 | ||
@@ -1895,12 +1887,12 @@ static long futex_wait_restart(struct restart_block *restart) | |||
1895 | * if there are waiters then it will block, it does PI, etc. (Due to | 1887 | * if there are waiters then it will block, it does PI, etc. (Due to |
1896 | * races the kernel might see a 0 value of the futex too.) | 1888 | * races the kernel might see a 0 value of the futex too.) |
1897 | */ | 1889 | */ |
1898 | static int futex_lock_pi(u32 __user *uaddr, int fshared, | 1890 | static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, int detect, |
1899 | int detect, ktime_t *time, int trylock) | 1891 | ktime_t *time, int trylock) |
1900 | { | 1892 | { |
1901 | struct hrtimer_sleeper timeout, *to = NULL; | 1893 | struct hrtimer_sleeper timeout, *to = NULL; |
1902 | struct futex_hash_bucket *hb; | 1894 | struct futex_hash_bucket *hb; |
1903 | struct futex_q q; | 1895 | struct futex_q q = futex_q_init; |
1904 | int res, ret; | 1896 | int res, ret; |
1905 | 1897 | ||
1906 | if (refill_pi_state_cache()) | 1898 | if (refill_pi_state_cache()) |
@@ -1914,12 +1906,8 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1914 | hrtimer_set_expires(&to->timer, *time); | 1906 | hrtimer_set_expires(&to->timer, *time); |
1915 | } | 1907 | } |
1916 | 1908 | ||
1917 | q.pi_state = NULL; | ||
1918 | q.rt_waiter = NULL; | ||
1919 | q.requeue_pi_key = NULL; | ||
1920 | retry: | 1909 | retry: |
1921 | q.key = FUTEX_KEY_INIT; | 1910 | ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key); |
1922 | ret = get_futex_key(uaddr, fshared, &q.key); | ||
1923 | if (unlikely(ret != 0)) | 1911 | if (unlikely(ret != 0)) |
1924 | goto out; | 1912 | goto out; |
1925 | 1913 | ||
@@ -1941,7 +1929,7 @@ retry_private: | |||
1941 | * exit to complete. | 1929 | * exit to complete. |
1942 | */ | 1930 | */ |
1943 | queue_unlock(&q, hb); | 1931 | queue_unlock(&q, hb); |
1944 | put_futex_key(fshared, &q.key); | 1932 | put_futex_key(&q.key); |
1945 | cond_resched(); | 1933 | cond_resched(); |
1946 | goto retry; | 1934 | goto retry; |
1947 | default: | 1935 | default: |
@@ -1971,7 +1959,7 @@ retry_private: | |||
1971 | * Fixup the pi_state owner and possibly acquire the lock if we | 1959 | * Fixup the pi_state owner and possibly acquire the lock if we |
1972 | * haven't already. | 1960 | * haven't already. |
1973 | */ | 1961 | */ |
1974 | res = fixup_owner(uaddr, fshared, &q, !ret); | 1962 | res = fixup_owner(uaddr, &q, !ret); |
1975 | /* | 1963 | /* |
1976 | * If fixup_owner() returned an error, proprogate that. If it acquired | 1964 | * If fixup_owner() returned an error, proprogate that. If it acquired |
1977 | * the lock, clear our -ETIMEDOUT or -EINTR. | 1965 | * the lock, clear our -ETIMEDOUT or -EINTR. |
@@ -1995,7 +1983,7 @@ out_unlock_put_key: | |||
1995 | queue_unlock(&q, hb); | 1983 | queue_unlock(&q, hb); |
1996 | 1984 | ||
1997 | out_put_key: | 1985 | out_put_key: |
1998 | put_futex_key(fshared, &q.key); | 1986 | put_futex_key(&q.key); |
1999 | out: | 1987 | out: |
2000 | if (to) | 1988 | if (to) |
2001 | destroy_hrtimer_on_stack(&to->timer); | 1989 | destroy_hrtimer_on_stack(&to->timer); |
@@ -2008,10 +1996,10 @@ uaddr_faulted: | |||
2008 | if (ret) | 1996 | if (ret) |
2009 | goto out_put_key; | 1997 | goto out_put_key; |
2010 | 1998 | ||
2011 | if (!fshared) | 1999 | if (!(flags & FLAGS_SHARED)) |
2012 | goto retry_private; | 2000 | goto retry_private; |
2013 | 2001 | ||
2014 | put_futex_key(fshared, &q.key); | 2002 | put_futex_key(&q.key); |
2015 | goto retry; | 2003 | goto retry; |
2016 | } | 2004 | } |
2017 | 2005 | ||
@@ -2020,7 +2008,7 @@ uaddr_faulted: | |||
2020 | * This is the in-kernel slowpath: we look up the PI state (if any), | 2008 | * This is the in-kernel slowpath: we look up the PI state (if any), |
2021 | * and do the rt-mutex unlock. | 2009 | * and do the rt-mutex unlock. |
2022 | */ | 2010 | */ |
2023 | static int futex_unlock_pi(u32 __user *uaddr, int fshared) | 2011 | static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) |
2024 | { | 2012 | { |
2025 | struct futex_hash_bucket *hb; | 2013 | struct futex_hash_bucket *hb; |
2026 | struct futex_q *this, *next; | 2014 | struct futex_q *this, *next; |
@@ -2038,7 +2026,7 @@ retry: | |||
2038 | if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current)) | 2026 | if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current)) |
2039 | return -EPERM; | 2027 | return -EPERM; |
2040 | 2028 | ||
2041 | ret = get_futex_key(uaddr, fshared, &key); | 2029 | ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key); |
2042 | if (unlikely(ret != 0)) | 2030 | if (unlikely(ret != 0)) |
2043 | goto out; | 2031 | goto out; |
2044 | 2032 | ||
@@ -2093,14 +2081,14 @@ retry: | |||
2093 | 2081 | ||
2094 | out_unlock: | 2082 | out_unlock: |
2095 | spin_unlock(&hb->lock); | 2083 | spin_unlock(&hb->lock); |
2096 | put_futex_key(fshared, &key); | 2084 | put_futex_key(&key); |
2097 | 2085 | ||
2098 | out: | 2086 | out: |
2099 | return ret; | 2087 | return ret; |
2100 | 2088 | ||
2101 | pi_faulted: | 2089 | pi_faulted: |
2102 | spin_unlock(&hb->lock); | 2090 | spin_unlock(&hb->lock); |
2103 | put_futex_key(fshared, &key); | 2091 | put_futex_key(&key); |
2104 | 2092 | ||
2105 | ret = fault_in_user_writeable(uaddr); | 2093 | ret = fault_in_user_writeable(uaddr); |
2106 | if (!ret) | 2094 | if (!ret) |
@@ -2160,7 +2148,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, | |||
2160 | /** | 2148 | /** |
2161 | * futex_wait_requeue_pi() - Wait on uaddr and take uaddr2 | 2149 | * futex_wait_requeue_pi() - Wait on uaddr and take uaddr2 |
2162 | * @uaddr: the futex we initially wait on (non-pi) | 2150 | * @uaddr: the futex we initially wait on (non-pi) |
2163 | * @fshared: whether the futexes are shared (1) or not (0). They must be | 2151 | * @flags: futex flags (FLAGS_SHARED, FLAGS_CLOCKRT, etc.), they must be |
2164 | * the same type, no requeueing from private to shared, etc. | 2152 | * the same type, no requeueing from private to shared, etc. |
2165 | * @val: the expected value of uaddr | 2153 | * @val: the expected value of uaddr |
2166 | * @abs_time: absolute timeout | 2154 | * @abs_time: absolute timeout |
@@ -2198,16 +2186,16 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, | |||
2198 | * 0 - On success | 2186 | * 0 - On success |
2199 | * <0 - On error | 2187 | * <0 - On error |
2200 | */ | 2188 | */ |
2201 | static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | 2189 | static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, |
2202 | u32 val, ktime_t *abs_time, u32 bitset, | 2190 | u32 val, ktime_t *abs_time, u32 bitset, |
2203 | int clockrt, u32 __user *uaddr2) | 2191 | u32 __user *uaddr2) |
2204 | { | 2192 | { |
2205 | struct hrtimer_sleeper timeout, *to = NULL; | 2193 | struct hrtimer_sleeper timeout, *to = NULL; |
2206 | struct rt_mutex_waiter rt_waiter; | 2194 | struct rt_mutex_waiter rt_waiter; |
2207 | struct rt_mutex *pi_mutex = NULL; | 2195 | struct rt_mutex *pi_mutex = NULL; |
2208 | struct futex_hash_bucket *hb; | 2196 | struct futex_hash_bucket *hb; |
2209 | union futex_key key2; | 2197 | union futex_key key2 = FUTEX_KEY_INIT; |
2210 | struct futex_q q; | 2198 | struct futex_q q = futex_q_init; |
2211 | int res, ret; | 2199 | int res, ret; |
2212 | 2200 | ||
2213 | if (!bitset) | 2201 | if (!bitset) |
@@ -2215,8 +2203,9 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2215 | 2203 | ||
2216 | if (abs_time) { | 2204 | if (abs_time) { |
2217 | to = &timeout; | 2205 | to = &timeout; |
2218 | hrtimer_init_on_stack(&to->timer, clockrt ? CLOCK_REALTIME : | 2206 | hrtimer_init_on_stack(&to->timer, (flags & FLAGS_CLOCKRT) ? |
2219 | CLOCK_MONOTONIC, HRTIMER_MODE_ABS); | 2207 | CLOCK_REALTIME : CLOCK_MONOTONIC, |
2208 | HRTIMER_MODE_ABS); | ||
2220 | hrtimer_init_sleeper(to, current); | 2209 | hrtimer_init_sleeper(to, current); |
2221 | hrtimer_set_expires_range_ns(&to->timer, *abs_time, | 2210 | hrtimer_set_expires_range_ns(&to->timer, *abs_time, |
2222 | current->timer_slack_ns); | 2211 | current->timer_slack_ns); |
@@ -2229,12 +2218,10 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2229 | debug_rt_mutex_init_waiter(&rt_waiter); | 2218 | debug_rt_mutex_init_waiter(&rt_waiter); |
2230 | rt_waiter.task = NULL; | 2219 | rt_waiter.task = NULL; |
2231 | 2220 | ||
2232 | key2 = FUTEX_KEY_INIT; | 2221 | ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2); |
2233 | ret = get_futex_key(uaddr2, fshared, &key2); | ||
2234 | if (unlikely(ret != 0)) | 2222 | if (unlikely(ret != 0)) |
2235 | goto out; | 2223 | goto out; |
2236 | 2224 | ||
2237 | q.pi_state = NULL; | ||
2238 | q.bitset = bitset; | 2225 | q.bitset = bitset; |
2239 | q.rt_waiter = &rt_waiter; | 2226 | q.rt_waiter = &rt_waiter; |
2240 | q.requeue_pi_key = &key2; | 2227 | q.requeue_pi_key = &key2; |
@@ -2243,7 +2230,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2243 | * Prepare to wait on uaddr. On success, increments q.key (key1) ref | 2230 | * Prepare to wait on uaddr. On success, increments q.key (key1) ref |
2244 | * count. | 2231 | * count. |
2245 | */ | 2232 | */ |
2246 | ret = futex_wait_setup(uaddr, val, fshared, &q, &hb); | 2233 | ret = futex_wait_setup(uaddr, val, flags, &q, &hb); |
2247 | if (ret) | 2234 | if (ret) |
2248 | goto out_key2; | 2235 | goto out_key2; |
2249 | 2236 | ||
@@ -2273,8 +2260,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2273 | */ | 2260 | */ |
2274 | if (q.pi_state && (q.pi_state->owner != current)) { | 2261 | if (q.pi_state && (q.pi_state->owner != current)) { |
2275 | spin_lock(q.lock_ptr); | 2262 | spin_lock(q.lock_ptr); |
2276 | ret = fixup_pi_state_owner(uaddr2, &q, current, | 2263 | ret = fixup_pi_state_owner(uaddr2, &q, current); |
2277 | fshared); | ||
2278 | spin_unlock(q.lock_ptr); | 2264 | spin_unlock(q.lock_ptr); |
2279 | } | 2265 | } |
2280 | } else { | 2266 | } else { |
@@ -2293,7 +2279,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2293 | * Fixup the pi_state owner and possibly acquire the lock if we | 2279 | * Fixup the pi_state owner and possibly acquire the lock if we |
2294 | * haven't already. | 2280 | * haven't already. |
2295 | */ | 2281 | */ |
2296 | res = fixup_owner(uaddr2, fshared, &q, !ret); | 2282 | res = fixup_owner(uaddr2, &q, !ret); |
2297 | /* | 2283 | /* |
2298 | * If fixup_owner() returned an error, proprogate that. If it | 2284 | * If fixup_owner() returned an error, proprogate that. If it |
2299 | * acquired the lock, clear -ETIMEDOUT or -EINTR. | 2285 | * acquired the lock, clear -ETIMEDOUT or -EINTR. |
@@ -2324,9 +2310,9 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2324 | } | 2310 | } |
2325 | 2311 | ||
2326 | out_put_keys: | 2312 | out_put_keys: |
2327 | put_futex_key(fshared, &q.key); | 2313 | put_futex_key(&q.key); |
2328 | out_key2: | 2314 | out_key2: |
2329 | put_futex_key(fshared, &key2); | 2315 | put_futex_key(&key2); |
2330 | 2316 | ||
2331 | out: | 2317 | out: |
2332 | if (to) { | 2318 | if (to) { |
@@ -2551,58 +2537,57 @@ void exit_robust_list(struct task_struct *curr) | |||
2551 | long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, | 2537 | long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, |
2552 | u32 __user *uaddr2, u32 val2, u32 val3) | 2538 | u32 __user *uaddr2, u32 val2, u32 val3) |
2553 | { | 2539 | { |
2554 | int clockrt, ret = -ENOSYS; | 2540 | int ret = -ENOSYS, cmd = op & FUTEX_CMD_MASK; |
2555 | int cmd = op & FUTEX_CMD_MASK; | 2541 | unsigned int flags = 0; |
2556 | int fshared = 0; | ||
2557 | 2542 | ||
2558 | if (!(op & FUTEX_PRIVATE_FLAG)) | 2543 | if (!(op & FUTEX_PRIVATE_FLAG)) |
2559 | fshared = 1; | 2544 | flags |= FLAGS_SHARED; |
2560 | 2545 | ||
2561 | clockrt = op & FUTEX_CLOCK_REALTIME; | 2546 | if (op & FUTEX_CLOCK_REALTIME) { |
2562 | if (clockrt && cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI) | 2547 | flags |= FLAGS_CLOCKRT; |
2563 | return -ENOSYS; | 2548 | if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI) |
2549 | return -ENOSYS; | ||
2550 | } | ||
2564 | 2551 | ||
2565 | switch (cmd) { | 2552 | switch (cmd) { |
2566 | case FUTEX_WAIT: | 2553 | case FUTEX_WAIT: |
2567 | val3 = FUTEX_BITSET_MATCH_ANY; | 2554 | val3 = FUTEX_BITSET_MATCH_ANY; |
2568 | case FUTEX_WAIT_BITSET: | 2555 | case FUTEX_WAIT_BITSET: |
2569 | ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt); | 2556 | ret = futex_wait(uaddr, flags, val, timeout, val3); |
2570 | break; | 2557 | break; |
2571 | case FUTEX_WAKE: | 2558 | case FUTEX_WAKE: |
2572 | val3 = FUTEX_BITSET_MATCH_ANY; | 2559 | val3 = FUTEX_BITSET_MATCH_ANY; |
2573 | case FUTEX_WAKE_BITSET: | 2560 | case FUTEX_WAKE_BITSET: |
2574 | ret = futex_wake(uaddr, fshared, val, val3); | 2561 | ret = futex_wake(uaddr, flags, val, val3); |
2575 | break; | 2562 | break; |
2576 | case FUTEX_REQUEUE: | 2563 | case FUTEX_REQUEUE: |
2577 | ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, NULL, 0); | 2564 | ret = futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0); |
2578 | break; | 2565 | break; |
2579 | case FUTEX_CMP_REQUEUE: | 2566 | case FUTEX_CMP_REQUEUE: |
2580 | ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, &val3, | 2567 | ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0); |
2581 | 0); | ||
2582 | break; | 2568 | break; |
2583 | case FUTEX_WAKE_OP: | 2569 | case FUTEX_WAKE_OP: |
2584 | ret = futex_wake_op(uaddr, fshared, uaddr2, val, val2, val3); | 2570 | ret = futex_wake_op(uaddr, flags, uaddr2, val, val2, val3); |
2585 | break; | 2571 | break; |
2586 | case FUTEX_LOCK_PI: | 2572 | case FUTEX_LOCK_PI: |
2587 | if (futex_cmpxchg_enabled) | 2573 | if (futex_cmpxchg_enabled) |
2588 | ret = futex_lock_pi(uaddr, fshared, val, timeout, 0); | 2574 | ret = futex_lock_pi(uaddr, flags, val, timeout, 0); |
2589 | break; | 2575 | break; |
2590 | case FUTEX_UNLOCK_PI: | 2576 | case FUTEX_UNLOCK_PI: |
2591 | if (futex_cmpxchg_enabled) | 2577 | if (futex_cmpxchg_enabled) |
2592 | ret = futex_unlock_pi(uaddr, fshared); | 2578 | ret = futex_unlock_pi(uaddr, flags); |
2593 | break; | 2579 | break; |
2594 | case FUTEX_TRYLOCK_PI: | 2580 | case FUTEX_TRYLOCK_PI: |
2595 | if (futex_cmpxchg_enabled) | 2581 | if (futex_cmpxchg_enabled) |
2596 | ret = futex_lock_pi(uaddr, fshared, 0, timeout, 1); | 2582 | ret = futex_lock_pi(uaddr, flags, 0, timeout, 1); |
2597 | break; | 2583 | break; |
2598 | case FUTEX_WAIT_REQUEUE_PI: | 2584 | case FUTEX_WAIT_REQUEUE_PI: |
2599 | val3 = FUTEX_BITSET_MATCH_ANY; | 2585 | val3 = FUTEX_BITSET_MATCH_ANY; |
2600 | ret = futex_wait_requeue_pi(uaddr, fshared, val, timeout, val3, | 2586 | ret = futex_wait_requeue_pi(uaddr, flags, val, timeout, val3, |
2601 | clockrt, uaddr2); | 2587 | uaddr2); |
2602 | break; | 2588 | break; |
2603 | case FUTEX_CMP_REQUEUE_PI: | 2589 | case FUTEX_CMP_REQUEUE_PI: |
2604 | ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, &val3, | 2590 | ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1); |
2605 | 1); | ||
2606 | break; | 2591 | break; |
2607 | default: | 2592 | default: |
2608 | ret = -ENOSYS; | 2593 | ret = -ENOSYS; |