diff options
Diffstat (limited to 'kernel/futex.c')
-rw-r--r-- | kernel/futex.c | 47 |
1 files changed, 30 insertions, 17 deletions
diff --git a/kernel/futex.c b/kernel/futex.c index e18cfbdc7190..248dd119a86e 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -115,6 +115,9 @@ struct futex_q { | |||
115 | /* rt_waiter storage for requeue_pi: */ | 115 | /* rt_waiter storage for requeue_pi: */ |
116 | struct rt_mutex_waiter *rt_waiter; | 116 | struct rt_mutex_waiter *rt_waiter; |
117 | 117 | ||
118 | /* The expected requeue pi target futex key: */ | ||
119 | union futex_key *requeue_pi_key; | ||
120 | |||
118 | /* Bitset for the optional bitmasked wakeup */ | 121 | /* Bitset for the optional bitmasked wakeup */ |
119 | u32 bitset; | 122 | u32 bitset; |
120 | }; | 123 | }; |
@@ -1089,6 +1092,10 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex, | |||
1089 | if (!top_waiter) | 1092 | if (!top_waiter) |
1090 | return 0; | 1093 | return 0; |
1091 | 1094 | ||
1095 | /* Ensure we requeue to the expected futex. */ | ||
1096 | if (!match_futex(top_waiter->requeue_pi_key, key2)) | ||
1097 | return -EINVAL; | ||
1098 | |||
1092 | /* | 1099 | /* |
1093 | * Try to take the lock for top_waiter. Set the FUTEX_WAITERS bit in | 1100 | * Try to take the lock for top_waiter. Set the FUTEX_WAITERS bit in |
1094 | * the contended case or if set_waiters is 1. The pi_state is returned | 1101 | * the contended case or if set_waiters is 1. The pi_state is returned |
@@ -1276,6 +1283,12 @@ retry_private: | |||
1276 | continue; | 1283 | continue; |
1277 | } | 1284 | } |
1278 | 1285 | ||
1286 | /* Ensure we requeue to the expected futex for requeue_pi. */ | ||
1287 | if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) { | ||
1288 | ret = -EINVAL; | ||
1289 | break; | ||
1290 | } | ||
1291 | |||
1279 | /* | 1292 | /* |
1280 | * Requeue nr_requeue waiters and possibly one more in the case | 1293 | * Requeue nr_requeue waiters and possibly one more in the case |
1281 | * of requeue_pi if we couldn't acquire the lock atomically. | 1294 | * of requeue_pi if we couldn't acquire the lock atomically. |
@@ -1751,6 +1764,7 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1751 | q.pi_state = NULL; | 1764 | q.pi_state = NULL; |
1752 | q.bitset = bitset; | 1765 | q.bitset = bitset; |
1753 | q.rt_waiter = NULL; | 1766 | q.rt_waiter = NULL; |
1767 | q.requeue_pi_key = NULL; | ||
1754 | 1768 | ||
1755 | if (abs_time) { | 1769 | if (abs_time) { |
1756 | to = &timeout; | 1770 | to = &timeout; |
@@ -1858,6 +1872,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared, | |||
1858 | 1872 | ||
1859 | q.pi_state = NULL; | 1873 | q.pi_state = NULL; |
1860 | q.rt_waiter = NULL; | 1874 | q.rt_waiter = NULL; |
1875 | q.requeue_pi_key = NULL; | ||
1861 | retry: | 1876 | retry: |
1862 | q.key = FUTEX_KEY_INIT; | 1877 | q.key = FUTEX_KEY_INIT; |
1863 | ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE); | 1878 | ret = get_futex_key(uaddr, fshared, &q.key, VERIFY_WRITE); |
@@ -2118,11 +2133,11 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, | |||
2118 | * We call schedule in futex_wait_queue_me() when we enqueue and return there | 2133 | * We call schedule in futex_wait_queue_me() when we enqueue and return there |
2119 | * via the following: | 2134 | * via the following: |
2120 | * 1) wakeup on uaddr2 after an atomic lock acquisition by futex_requeue() | 2135 | * 1) wakeup on uaddr2 after an atomic lock acquisition by futex_requeue() |
2121 | * 2) wakeup on uaddr2 after a requeue and subsequent unlock | 2136 | * 2) wakeup on uaddr2 after a requeue |
2122 | * 3) signal (before or after requeue) | 2137 | * 3) signal |
2123 | * 4) timeout (before or after requeue) | 2138 | * 4) timeout |
2124 | * | 2139 | * |
2125 | * If 3, we setup a restart_block with futex_wait_requeue_pi() as the function. | 2140 | * If 3, cleanup and return -ERESTARTNOINTR. |
2126 | * | 2141 | * |
2127 | * If 2, we may then block on trying to take the rt_mutex and return via: | 2142 | * If 2, we may then block on trying to take the rt_mutex and return via: |
2128 | * 5) successful lock | 2143 | * 5) successful lock |
@@ -2130,7 +2145,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, | |||
2130 | * 7) timeout | 2145 | * 7) timeout |
2131 | * 8) other lock acquisition failure | 2146 | * 8) other lock acquisition failure |
2132 | * | 2147 | * |
2133 | * If 6, we setup a restart_block with futex_lock_pi() as the function. | 2148 | * If 6, return -EWOULDBLOCK (restarting the syscall would do the same). |
2134 | * | 2149 | * |
2135 | * If 4 or 7, we cleanup and return with -ETIMEDOUT. | 2150 | * If 4 or 7, we cleanup and return with -ETIMEDOUT. |
2136 | * | 2151 | * |
@@ -2169,15 +2184,16 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2169 | debug_rt_mutex_init_waiter(&rt_waiter); | 2184 | debug_rt_mutex_init_waiter(&rt_waiter); |
2170 | rt_waiter.task = NULL; | 2185 | rt_waiter.task = NULL; |
2171 | 2186 | ||
2172 | q.pi_state = NULL; | ||
2173 | q.bitset = bitset; | ||
2174 | q.rt_waiter = &rt_waiter; | ||
2175 | |||
2176 | key2 = FUTEX_KEY_INIT; | 2187 | key2 = FUTEX_KEY_INIT; |
2177 | ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE); | 2188 | ret = get_futex_key(uaddr2, fshared, &key2, VERIFY_WRITE); |
2178 | if (unlikely(ret != 0)) | 2189 | if (unlikely(ret != 0)) |
2179 | goto out; | 2190 | goto out; |
2180 | 2191 | ||
2192 | q.pi_state = NULL; | ||
2193 | q.bitset = bitset; | ||
2194 | q.rt_waiter = &rt_waiter; | ||
2195 | q.requeue_pi_key = &key2; | ||
2196 | |||
2181 | /* Prepare to wait on uaddr. */ | 2197 | /* Prepare to wait on uaddr. */ |
2182 | ret = futex_wait_setup(uaddr, val, fshared, &q, &hb); | 2198 | ret = futex_wait_setup(uaddr, val, fshared, &q, &hb); |
2183 | if (ret) | 2199 | if (ret) |
@@ -2248,14 +2264,11 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared, | |||
2248 | rt_mutex_unlock(pi_mutex); | 2264 | rt_mutex_unlock(pi_mutex); |
2249 | } else if (ret == -EINTR) { | 2265 | } else if (ret == -EINTR) { |
2250 | /* | 2266 | /* |
2251 | * We've already been requeued, but we have no way to | 2267 | * We've already been requeued, but cannot restart by calling |
2252 | * restart by calling futex_lock_pi() directly. We | 2268 | * futex_lock_pi() directly. We could restart this syscall, but |
2253 | * could restart the syscall, but that will look at | 2269 | * it would detect that the user space "val" changed and return |
2254 | * the user space value and return right away. So we | 2270 | * -EWOULDBLOCK. Save the overhead of the restart and return |
2255 | * drop back with EWOULDBLOCK to tell user space that | 2271 | * -EWOULDBLOCK directly. |
2256 | * "val" has been changed. That's the same what the | ||
2257 | * restart of the syscall would do in | ||
2258 | * futex_wait_setup(). | ||
2259 | */ | 2272 | */ |
2260 | ret = -EWOULDBLOCK; | 2273 | ret = -EWOULDBLOCK; |
2261 | } | 2274 | } |