diff options
author | Peter Zijlstra <peterz@infradead.org> | 2013-10-31 13:11:53 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@kernel.org> | 2013-11-06 01:55:07 -0500 |
commit | 01768b42dc97a67b4fb33a2535c49fc1969880df (patch) | |
tree | 448a1aff2286e8e9752124964e725d7bd5d3dba8 /kernel/mutex.c | |
parent | c90423d1de12fbeaf0c898e1db0e962de347302b (diff) |
locking: Move the mutex code to kernel/locking/
Suggested-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/n/tip-1ditvncg30dgbpvrz2bxfmke@git.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'kernel/mutex.c')
-rw-r--r-- | kernel/mutex.c | 960 |
1 files changed, 0 insertions, 960 deletions
diff --git a/kernel/mutex.c b/kernel/mutex.c deleted file mode 100644 index d24105b1b794..000000000000 --- a/kernel/mutex.c +++ /dev/null | |||
@@ -1,960 +0,0 @@ | |||
1 | /* | ||
2 | * kernel/mutex.c | ||
3 | * | ||
4 | * Mutexes: blocking mutual exclusion locks | ||
5 | * | ||
6 | * Started by Ingo Molnar: | ||
7 | * | ||
8 | * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> | ||
9 | * | ||
10 | * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and | ||
11 | * David Howells for suggestions and improvements. | ||
12 | * | ||
13 | * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline | ||
14 | * from the -rt tree, where it was originally implemented for rtmutexes | ||
15 | * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale | ||
16 | * and Sven Dietrich. | ||
17 | * | ||
18 | * Also see Documentation/mutex-design.txt. | ||
19 | */ | ||
20 | #include <linux/mutex.h> | ||
21 | #include <linux/ww_mutex.h> | ||
22 | #include <linux/sched.h> | ||
23 | #include <linux/sched/rt.h> | ||
24 | #include <linux/export.h> | ||
25 | #include <linux/spinlock.h> | ||
26 | #include <linux/interrupt.h> | ||
27 | #include <linux/debug_locks.h> | ||
28 | |||
29 | /* | ||
30 | * In the DEBUG case we are using the "NULL fastpath" for mutexes, | ||
31 | * which forces all calls into the slowpath: | ||
32 | */ | ||
33 | #ifdef CONFIG_DEBUG_MUTEXES | ||
34 | # include "mutex-debug.h" | ||
35 | # include <asm-generic/mutex-null.h> | ||
36 | #else | ||
37 | # include "mutex.h" | ||
38 | # include <asm/mutex.h> | ||
39 | #endif | ||
40 | |||
41 | /* | ||
42 | * A negative mutex count indicates that waiters are sleeping waiting for the | ||
43 | * mutex. | ||
44 | */ | ||
45 | #define MUTEX_SHOW_NO_WAITER(mutex) (atomic_read(&(mutex)->count) >= 0) | ||
46 | |||
47 | void | ||
48 | __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) | ||
49 | { | ||
50 | atomic_set(&lock->count, 1); | ||
51 | spin_lock_init(&lock->wait_lock); | ||
52 | INIT_LIST_HEAD(&lock->wait_list); | ||
53 | mutex_clear_owner(lock); | ||
54 | #ifdef CONFIG_MUTEX_SPIN_ON_OWNER | ||
55 | lock->spin_mlock = NULL; | ||
56 | #endif | ||
57 | |||
58 | debug_mutex_init(lock, name, key); | ||
59 | } | ||
60 | |||
61 | EXPORT_SYMBOL(__mutex_init); | ||
62 | |||
63 | #ifndef CONFIG_DEBUG_LOCK_ALLOC | ||
64 | /* | ||
65 | * We split the mutex lock/unlock logic into separate fastpath and | ||
66 | * slowpath functions, to reduce the register pressure on the fastpath. | ||
67 | * We also put the fastpath first in the kernel image, to make sure the | ||
68 | * branch is predicted by the CPU as default-untaken. | ||
69 | */ | ||
70 | static __used noinline void __sched | ||
71 | __mutex_lock_slowpath(atomic_t *lock_count); | ||
72 | |||
73 | /** | ||
74 | * mutex_lock - acquire the mutex | ||
75 | * @lock: the mutex to be acquired | ||
76 | * | ||
77 | * Lock the mutex exclusively for this task. If the mutex is not | ||
78 | * available right now, it will sleep until it can get it. | ||
79 | * | ||
80 | * The mutex must later on be released by the same task that | ||
81 | * acquired it. Recursive locking is not allowed. The task | ||
82 | * may not exit without first unlocking the mutex. Also, kernel | ||
83 | * memory where the mutex resides mutex must not be freed with | ||
84 | * the mutex still locked. The mutex must first be initialized | ||
85 | * (or statically defined) before it can be locked. memset()-ing | ||
86 | * the mutex to 0 is not allowed. | ||
87 | * | ||
88 | * ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging | ||
89 | * checks that will enforce the restrictions and will also do | ||
90 | * deadlock debugging. ) | ||
91 | * | ||
92 | * This function is similar to (but not equivalent to) down(). | ||
93 | */ | ||
94 | void __sched mutex_lock(struct mutex *lock) | ||
95 | { | ||
96 | might_sleep(); | ||
97 | /* | ||
98 | * The locking fastpath is the 1->0 transition from | ||
99 | * 'unlocked' into 'locked' state. | ||
100 | */ | ||
101 | __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath); | ||
102 | mutex_set_owner(lock); | ||
103 | } | ||
104 | |||
105 | EXPORT_SYMBOL(mutex_lock); | ||
106 | #endif | ||
107 | |||
108 | #ifdef CONFIG_MUTEX_SPIN_ON_OWNER | ||
109 | /* | ||
110 | * In order to avoid a stampede of mutex spinners from acquiring the mutex | ||
111 | * more or less simultaneously, the spinners need to acquire a MCS lock | ||
112 | * first before spinning on the owner field. | ||
113 | * | ||
114 | * We don't inline mspin_lock() so that perf can correctly account for the | ||
115 | * time spent in this lock function. | ||
116 | */ | ||
117 | struct mspin_node { | ||
118 | struct mspin_node *next ; | ||
119 | int locked; /* 1 if lock acquired */ | ||
120 | }; | ||
121 | #define MLOCK(mutex) ((struct mspin_node **)&((mutex)->spin_mlock)) | ||
122 | |||
123 | static noinline | ||
124 | void mspin_lock(struct mspin_node **lock, struct mspin_node *node) | ||
125 | { | ||
126 | struct mspin_node *prev; | ||
127 | |||
128 | /* Init node */ | ||
129 | node->locked = 0; | ||
130 | node->next = NULL; | ||
131 | |||
132 | prev = xchg(lock, node); | ||
133 | if (likely(prev == NULL)) { | ||
134 | /* Lock acquired */ | ||
135 | node->locked = 1; | ||
136 | return; | ||
137 | } | ||
138 | ACCESS_ONCE(prev->next) = node; | ||
139 | smp_wmb(); | ||
140 | /* Wait until the lock holder passes the lock down */ | ||
141 | while (!ACCESS_ONCE(node->locked)) | ||
142 | arch_mutex_cpu_relax(); | ||
143 | } | ||
144 | |||
145 | static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node) | ||
146 | { | ||
147 | struct mspin_node *next = ACCESS_ONCE(node->next); | ||
148 | |||
149 | if (likely(!next)) { | ||
150 | /* | ||
151 | * Release the lock by setting it to NULL | ||
152 | */ | ||
153 | if (cmpxchg(lock, node, NULL) == node) | ||
154 | return; | ||
155 | /* Wait until the next pointer is set */ | ||
156 | while (!(next = ACCESS_ONCE(node->next))) | ||
157 | arch_mutex_cpu_relax(); | ||
158 | } | ||
159 | ACCESS_ONCE(next->locked) = 1; | ||
160 | smp_wmb(); | ||
161 | } | ||
162 | |||
163 | /* | ||
164 | * Mutex spinning code migrated from kernel/sched/core.c | ||
165 | */ | ||
166 | |||
167 | static inline bool owner_running(struct mutex *lock, struct task_struct *owner) | ||
168 | { | ||
169 | if (lock->owner != owner) | ||
170 | return false; | ||
171 | |||
172 | /* | ||
173 | * Ensure we emit the owner->on_cpu, dereference _after_ checking | ||
174 | * lock->owner still matches owner, if that fails, owner might | ||
175 | * point to free()d memory, if it still matches, the rcu_read_lock() | ||
176 | * ensures the memory stays valid. | ||
177 | */ | ||
178 | barrier(); | ||
179 | |||
180 | return owner->on_cpu; | ||
181 | } | ||
182 | |||
183 | /* | ||
184 | * Look out! "owner" is an entirely speculative pointer | ||
185 | * access and not reliable. | ||
186 | */ | ||
187 | static noinline | ||
188 | int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner) | ||
189 | { | ||
190 | rcu_read_lock(); | ||
191 | while (owner_running(lock, owner)) { | ||
192 | if (need_resched()) | ||
193 | break; | ||
194 | |||
195 | arch_mutex_cpu_relax(); | ||
196 | } | ||
197 | rcu_read_unlock(); | ||
198 | |||
199 | /* | ||
200 | * We break out the loop above on need_resched() and when the | ||
201 | * owner changed, which is a sign for heavy contention. Return | ||
202 | * success only when lock->owner is NULL. | ||
203 | */ | ||
204 | return lock->owner == NULL; | ||
205 | } | ||
206 | |||
207 | /* | ||
208 | * Initial check for entering the mutex spinning loop | ||
209 | */ | ||
210 | static inline int mutex_can_spin_on_owner(struct mutex *lock) | ||
211 | { | ||
212 | struct task_struct *owner; | ||
213 | int retval = 1; | ||
214 | |||
215 | rcu_read_lock(); | ||
216 | owner = ACCESS_ONCE(lock->owner); | ||
217 | if (owner) | ||
218 | retval = owner->on_cpu; | ||
219 | rcu_read_unlock(); | ||
220 | /* | ||
221 | * if lock->owner is not set, the mutex owner may have just acquired | ||
222 | * it and not set the owner yet or the mutex has been released. | ||
223 | */ | ||
224 | return retval; | ||
225 | } | ||
226 | #endif | ||
227 | |||
228 | static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count); | ||
229 | |||
230 | /** | ||
231 | * mutex_unlock - release the mutex | ||
232 | * @lock: the mutex to be released | ||
233 | * | ||
234 | * Unlock a mutex that has been locked by this task previously. | ||
235 | * | ||
236 | * This function must not be used in interrupt context. Unlocking | ||
237 | * of a not locked mutex is not allowed. | ||
238 | * | ||
239 | * This function is similar to (but not equivalent to) up(). | ||
240 | */ | ||
241 | void __sched mutex_unlock(struct mutex *lock) | ||
242 | { | ||
243 | /* | ||
244 | * The unlocking fastpath is the 0->1 transition from 'locked' | ||
245 | * into 'unlocked' state: | ||
246 | */ | ||
247 | #ifndef CONFIG_DEBUG_MUTEXES | ||
248 | /* | ||
249 | * When debugging is enabled we must not clear the owner before time, | ||
250 | * the slow path will always be taken, and that clears the owner field | ||
251 | * after verifying that it was indeed current. | ||
252 | */ | ||
253 | mutex_clear_owner(lock); | ||
254 | #endif | ||
255 | __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath); | ||
256 | } | ||
257 | |||
258 | EXPORT_SYMBOL(mutex_unlock); | ||
259 | |||
260 | /** | ||
261 | * ww_mutex_unlock - release the w/w mutex | ||
262 | * @lock: the mutex to be released | ||
263 | * | ||
264 | * Unlock a mutex that has been locked by this task previously with any of the | ||
265 | * ww_mutex_lock* functions (with or without an acquire context). It is | ||
266 | * forbidden to release the locks after releasing the acquire context. | ||
267 | * | ||
268 | * This function must not be used in interrupt context. Unlocking | ||
269 | * of a unlocked mutex is not allowed. | ||
270 | */ | ||
271 | void __sched ww_mutex_unlock(struct ww_mutex *lock) | ||
272 | { | ||
273 | /* | ||
274 | * The unlocking fastpath is the 0->1 transition from 'locked' | ||
275 | * into 'unlocked' state: | ||
276 | */ | ||
277 | if (lock->ctx) { | ||
278 | #ifdef CONFIG_DEBUG_MUTEXES | ||
279 | DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired); | ||
280 | #endif | ||
281 | if (lock->ctx->acquired > 0) | ||
282 | lock->ctx->acquired--; | ||
283 | lock->ctx = NULL; | ||
284 | } | ||
285 | |||
286 | #ifndef CONFIG_DEBUG_MUTEXES | ||
287 | /* | ||
288 | * When debugging is enabled we must not clear the owner before time, | ||
289 | * the slow path will always be taken, and that clears the owner field | ||
290 | * after verifying that it was indeed current. | ||
291 | */ | ||
292 | mutex_clear_owner(&lock->base); | ||
293 | #endif | ||
294 | __mutex_fastpath_unlock(&lock->base.count, __mutex_unlock_slowpath); | ||
295 | } | ||
296 | EXPORT_SYMBOL(ww_mutex_unlock); | ||
297 | |||
298 | static inline int __sched | ||
299 | __mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx) | ||
300 | { | ||
301 | struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); | ||
302 | struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx); | ||
303 | |||
304 | if (!hold_ctx) | ||
305 | return 0; | ||
306 | |||
307 | if (unlikely(ctx == hold_ctx)) | ||
308 | return -EALREADY; | ||
309 | |||
310 | if (ctx->stamp - hold_ctx->stamp <= LONG_MAX && | ||
311 | (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) { | ||
312 | #ifdef CONFIG_DEBUG_MUTEXES | ||
313 | DEBUG_LOCKS_WARN_ON(ctx->contending_lock); | ||
314 | ctx->contending_lock = ww; | ||
315 | #endif | ||
316 | return -EDEADLK; | ||
317 | } | ||
318 | |||
319 | return 0; | ||
320 | } | ||
321 | |||
322 | static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww, | ||
323 | struct ww_acquire_ctx *ww_ctx) | ||
324 | { | ||
325 | #ifdef CONFIG_DEBUG_MUTEXES | ||
326 | /* | ||
327 | * If this WARN_ON triggers, you used ww_mutex_lock to acquire, | ||
328 | * but released with a normal mutex_unlock in this call. | ||
329 | * | ||
330 | * This should never happen, always use ww_mutex_unlock. | ||
331 | */ | ||
332 | DEBUG_LOCKS_WARN_ON(ww->ctx); | ||
333 | |||
334 | /* | ||
335 | * Not quite done after calling ww_acquire_done() ? | ||
336 | */ | ||
337 | DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire); | ||
338 | |||
339 | if (ww_ctx->contending_lock) { | ||
340 | /* | ||
341 | * After -EDEADLK you tried to | ||
342 | * acquire a different ww_mutex? Bad! | ||
343 | */ | ||
344 | DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww); | ||
345 | |||
346 | /* | ||
347 | * You called ww_mutex_lock after receiving -EDEADLK, | ||
348 | * but 'forgot' to unlock everything else first? | ||
349 | */ | ||
350 | DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0); | ||
351 | ww_ctx->contending_lock = NULL; | ||
352 | } | ||
353 | |||
354 | /* | ||
355 | * Naughty, using a different class will lead to undefined behavior! | ||
356 | */ | ||
357 | DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class); | ||
358 | #endif | ||
359 | ww_ctx->acquired++; | ||
360 | } | ||
361 | |||
362 | /* | ||
363 | * after acquiring lock with fastpath or when we lost out in contested | ||
364 | * slowpath, set ctx and wake up any waiters so they can recheck. | ||
365 | * | ||
366 | * This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set, | ||
367 | * as the fastpath and opportunistic spinning are disabled in that case. | ||
368 | */ | ||
369 | static __always_inline void | ||
370 | ww_mutex_set_context_fastpath(struct ww_mutex *lock, | ||
371 | struct ww_acquire_ctx *ctx) | ||
372 | { | ||
373 | unsigned long flags; | ||
374 | struct mutex_waiter *cur; | ||
375 | |||
376 | ww_mutex_lock_acquired(lock, ctx); | ||
377 | |||
378 | lock->ctx = ctx; | ||
379 | |||
380 | /* | ||
381 | * The lock->ctx update should be visible on all cores before | ||
382 | * the atomic read is done, otherwise contended waiters might be | ||
383 | * missed. The contended waiters will either see ww_ctx == NULL | ||
384 | * and keep spinning, or it will acquire wait_lock, add itself | ||
385 | * to waiter list and sleep. | ||
386 | */ | ||
387 | smp_mb(); /* ^^^ */ | ||
388 | |||
389 | /* | ||
390 | * Check if lock is contended, if not there is nobody to wake up | ||
391 | */ | ||
392 | if (likely(atomic_read(&lock->base.count) == 0)) | ||
393 | return; | ||
394 | |||
395 | /* | ||
396 | * Uh oh, we raced in fastpath, wake up everyone in this case, | ||
397 | * so they can see the new lock->ctx. | ||
398 | */ | ||
399 | spin_lock_mutex(&lock->base.wait_lock, flags); | ||
400 | list_for_each_entry(cur, &lock->base.wait_list, list) { | ||
401 | debug_mutex_wake_waiter(&lock->base, cur); | ||
402 | wake_up_process(cur->task); | ||
403 | } | ||
404 | spin_unlock_mutex(&lock->base.wait_lock, flags); | ||
405 | } | ||
406 | |||
407 | /* | ||
408 | * Lock a mutex (possibly interruptible), slowpath: | ||
409 | */ | ||
410 | static __always_inline int __sched | ||
411 | __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, | ||
412 | struct lockdep_map *nest_lock, unsigned long ip, | ||
413 | struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx) | ||
414 | { | ||
415 | struct task_struct *task = current; | ||
416 | struct mutex_waiter waiter; | ||
417 | unsigned long flags; | ||
418 | int ret; | ||
419 | |||
420 | preempt_disable(); | ||
421 | mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); | ||
422 | |||
423 | #ifdef CONFIG_MUTEX_SPIN_ON_OWNER | ||
424 | /* | ||
425 | * Optimistic spinning. | ||
426 | * | ||
427 | * We try to spin for acquisition when we find that there are no | ||
428 | * pending waiters and the lock owner is currently running on a | ||
429 | * (different) CPU. | ||
430 | * | ||
431 | * The rationale is that if the lock owner is running, it is likely to | ||
432 | * release the lock soon. | ||
433 | * | ||
434 | * Since this needs the lock owner, and this mutex implementation | ||
435 | * doesn't track the owner atomically in the lock field, we need to | ||
436 | * track it non-atomically. | ||
437 | * | ||
438 | * We can't do this for DEBUG_MUTEXES because that relies on wait_lock | ||
439 | * to serialize everything. | ||
440 | * | ||
441 | * The mutex spinners are queued up using MCS lock so that only one | ||
442 | * spinner can compete for the mutex. However, if mutex spinning isn't | ||
443 | * going to happen, there is no point in going through the lock/unlock | ||
444 | * overhead. | ||
445 | */ | ||
446 | if (!mutex_can_spin_on_owner(lock)) | ||
447 | goto slowpath; | ||
448 | |||
449 | for (;;) { | ||
450 | struct task_struct *owner; | ||
451 | struct mspin_node node; | ||
452 | |||
453 | if (use_ww_ctx && ww_ctx->acquired > 0) { | ||
454 | struct ww_mutex *ww; | ||
455 | |||
456 | ww = container_of(lock, struct ww_mutex, base); | ||
457 | /* | ||
458 | * If ww->ctx is set the contents are undefined, only | ||
459 | * by acquiring wait_lock there is a guarantee that | ||
460 | * they are not invalid when reading. | ||
461 | * | ||
462 | * As such, when deadlock detection needs to be | ||
463 | * performed the optimistic spinning cannot be done. | ||
464 | */ | ||
465 | if (ACCESS_ONCE(ww->ctx)) | ||
466 | goto slowpath; | ||
467 | } | ||
468 | |||
469 | /* | ||
470 | * If there's an owner, wait for it to either | ||
471 | * release the lock or go to sleep. | ||
472 | */ | ||
473 | mspin_lock(MLOCK(lock), &node); | ||
474 | owner = ACCESS_ONCE(lock->owner); | ||
475 | if (owner && !mutex_spin_on_owner(lock, owner)) { | ||
476 | mspin_unlock(MLOCK(lock), &node); | ||
477 | goto slowpath; | ||
478 | } | ||
479 | |||
480 | if ((atomic_read(&lock->count) == 1) && | ||
481 | (atomic_cmpxchg(&lock->count, 1, 0) == 1)) { | ||
482 | lock_acquired(&lock->dep_map, ip); | ||
483 | if (use_ww_ctx) { | ||
484 | struct ww_mutex *ww; | ||
485 | ww = container_of(lock, struct ww_mutex, base); | ||
486 | |||
487 | ww_mutex_set_context_fastpath(ww, ww_ctx); | ||
488 | } | ||
489 | |||
490 | mutex_set_owner(lock); | ||
491 | mspin_unlock(MLOCK(lock), &node); | ||
492 | preempt_enable(); | ||
493 | return 0; | ||
494 | } | ||
495 | mspin_unlock(MLOCK(lock), &node); | ||
496 | |||
497 | /* | ||
498 | * When there's no owner, we might have preempted between the | ||
499 | * owner acquiring the lock and setting the owner field. If | ||
500 | * we're an RT task that will live-lock because we won't let | ||
501 | * the owner complete. | ||
502 | */ | ||
503 | if (!owner && (need_resched() || rt_task(task))) | ||
504 | goto slowpath; | ||
505 | |||
506 | /* | ||
507 | * The cpu_relax() call is a compiler barrier which forces | ||
508 | * everything in this loop to be re-loaded. We don't need | ||
509 | * memory barriers as we'll eventually observe the right | ||
510 | * values at the cost of a few extra spins. | ||
511 | */ | ||
512 | arch_mutex_cpu_relax(); | ||
513 | } | ||
514 | slowpath: | ||
515 | #endif | ||
516 | spin_lock_mutex(&lock->wait_lock, flags); | ||
517 | |||
518 | /* once more, can we acquire the lock? */ | ||
519 | if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1)) | ||
520 | goto skip_wait; | ||
521 | |||
522 | debug_mutex_lock_common(lock, &waiter); | ||
523 | debug_mutex_add_waiter(lock, &waiter, task_thread_info(task)); | ||
524 | |||
525 | /* add waiting tasks to the end of the waitqueue (FIFO): */ | ||
526 | list_add_tail(&waiter.list, &lock->wait_list); | ||
527 | waiter.task = task; | ||
528 | |||
529 | lock_contended(&lock->dep_map, ip); | ||
530 | |||
531 | for (;;) { | ||
532 | /* | ||
533 | * Lets try to take the lock again - this is needed even if | ||
534 | * we get here for the first time (shortly after failing to | ||
535 | * acquire the lock), to make sure that we get a wakeup once | ||
536 | * it's unlocked. Later on, if we sleep, this is the | ||
537 | * operation that gives us the lock. We xchg it to -1, so | ||
538 | * that when we release the lock, we properly wake up the | ||
539 | * other waiters: | ||
540 | */ | ||
541 | if (MUTEX_SHOW_NO_WAITER(lock) && | ||
542 | (atomic_xchg(&lock->count, -1) == 1)) | ||
543 | break; | ||
544 | |||
545 | /* | ||
546 | * got a signal? (This code gets eliminated in the | ||
547 | * TASK_UNINTERRUPTIBLE case.) | ||
548 | */ | ||
549 | if (unlikely(signal_pending_state(state, task))) { | ||
550 | ret = -EINTR; | ||
551 | goto err; | ||
552 | } | ||
553 | |||
554 | if (use_ww_ctx && ww_ctx->acquired > 0) { | ||
555 | ret = __mutex_lock_check_stamp(lock, ww_ctx); | ||
556 | if (ret) | ||
557 | goto err; | ||
558 | } | ||
559 | |||
560 | __set_task_state(task, state); | ||
561 | |||
562 | /* didn't get the lock, go to sleep: */ | ||
563 | spin_unlock_mutex(&lock->wait_lock, flags); | ||
564 | schedule_preempt_disabled(); | ||
565 | spin_lock_mutex(&lock->wait_lock, flags); | ||
566 | } | ||
567 | mutex_remove_waiter(lock, &waiter, current_thread_info()); | ||
568 | /* set it to 0 if there are no waiters left: */ | ||
569 | if (likely(list_empty(&lock->wait_list))) | ||
570 | atomic_set(&lock->count, 0); | ||
571 | debug_mutex_free_waiter(&waiter); | ||
572 | |||
573 | skip_wait: | ||
574 | /* got the lock - cleanup and rejoice! */ | ||
575 | lock_acquired(&lock->dep_map, ip); | ||
576 | mutex_set_owner(lock); | ||
577 | |||
578 | if (use_ww_ctx) { | ||
579 | struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); | ||
580 | struct mutex_waiter *cur; | ||
581 | |||
582 | /* | ||
583 | * This branch gets optimized out for the common case, | ||
584 | * and is only important for ww_mutex_lock. | ||
585 | */ | ||
586 | ww_mutex_lock_acquired(ww, ww_ctx); | ||
587 | ww->ctx = ww_ctx; | ||
588 | |||
589 | /* | ||
590 | * Give any possible sleeping processes the chance to wake up, | ||
591 | * so they can recheck if they have to back off. | ||
592 | */ | ||
593 | list_for_each_entry(cur, &lock->wait_list, list) { | ||
594 | debug_mutex_wake_waiter(lock, cur); | ||
595 | wake_up_process(cur->task); | ||
596 | } | ||
597 | } | ||
598 | |||
599 | spin_unlock_mutex(&lock->wait_lock, flags); | ||
600 | preempt_enable(); | ||
601 | return 0; | ||
602 | |||
603 | err: | ||
604 | mutex_remove_waiter(lock, &waiter, task_thread_info(task)); | ||
605 | spin_unlock_mutex(&lock->wait_lock, flags); | ||
606 | debug_mutex_free_waiter(&waiter); | ||
607 | mutex_release(&lock->dep_map, 1, ip); | ||
608 | preempt_enable(); | ||
609 | return ret; | ||
610 | } | ||
611 | |||
612 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
613 | void __sched | ||
614 | mutex_lock_nested(struct mutex *lock, unsigned int subclass) | ||
615 | { | ||
616 | might_sleep(); | ||
617 | __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, | ||
618 | subclass, NULL, _RET_IP_, NULL, 0); | ||
619 | } | ||
620 | |||
621 | EXPORT_SYMBOL_GPL(mutex_lock_nested); | ||
622 | |||
623 | void __sched | ||
624 | _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) | ||
625 | { | ||
626 | might_sleep(); | ||
627 | __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, | ||
628 | 0, nest, _RET_IP_, NULL, 0); | ||
629 | } | ||
630 | |||
631 | EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock); | ||
632 | |||
633 | int __sched | ||
634 | mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass) | ||
635 | { | ||
636 | might_sleep(); | ||
637 | return __mutex_lock_common(lock, TASK_KILLABLE, | ||
638 | subclass, NULL, _RET_IP_, NULL, 0); | ||
639 | } | ||
640 | EXPORT_SYMBOL_GPL(mutex_lock_killable_nested); | ||
641 | |||
642 | int __sched | ||
643 | mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) | ||
644 | { | ||
645 | might_sleep(); | ||
646 | return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, | ||
647 | subclass, NULL, _RET_IP_, NULL, 0); | ||
648 | } | ||
649 | |||
650 | EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested); | ||
651 | |||
652 | static inline int | ||
653 | ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) | ||
654 | { | ||
655 | #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH | ||
656 | unsigned tmp; | ||
657 | |||
658 | if (ctx->deadlock_inject_countdown-- == 0) { | ||
659 | tmp = ctx->deadlock_inject_interval; | ||
660 | if (tmp > UINT_MAX/4) | ||
661 | tmp = UINT_MAX; | ||
662 | else | ||
663 | tmp = tmp*2 + tmp + tmp/2; | ||
664 | |||
665 | ctx->deadlock_inject_interval = tmp; | ||
666 | ctx->deadlock_inject_countdown = tmp; | ||
667 | ctx->contending_lock = lock; | ||
668 | |||
669 | ww_mutex_unlock(lock); | ||
670 | |||
671 | return -EDEADLK; | ||
672 | } | ||
673 | #endif | ||
674 | |||
675 | return 0; | ||
676 | } | ||
677 | |||
678 | int __sched | ||
679 | __ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) | ||
680 | { | ||
681 | int ret; | ||
682 | |||
683 | might_sleep(); | ||
684 | ret = __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, | ||
685 | 0, &ctx->dep_map, _RET_IP_, ctx, 1); | ||
686 | if (!ret && ctx->acquired > 1) | ||
687 | return ww_mutex_deadlock_injection(lock, ctx); | ||
688 | |||
689 | return ret; | ||
690 | } | ||
691 | EXPORT_SYMBOL_GPL(__ww_mutex_lock); | ||
692 | |||
693 | int __sched | ||
694 | __ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) | ||
695 | { | ||
696 | int ret; | ||
697 | |||
698 | might_sleep(); | ||
699 | ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, | ||
700 | 0, &ctx->dep_map, _RET_IP_, ctx, 1); | ||
701 | |||
702 | if (!ret && ctx->acquired > 1) | ||
703 | return ww_mutex_deadlock_injection(lock, ctx); | ||
704 | |||
705 | return ret; | ||
706 | } | ||
707 | EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible); | ||
708 | |||
709 | #endif | ||
710 | |||
711 | /* | ||
712 | * Release the lock, slowpath: | ||
713 | */ | ||
714 | static inline void | ||
715 | __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested) | ||
716 | { | ||
717 | struct mutex *lock = container_of(lock_count, struct mutex, count); | ||
718 | unsigned long flags; | ||
719 | |||
720 | spin_lock_mutex(&lock->wait_lock, flags); | ||
721 | mutex_release(&lock->dep_map, nested, _RET_IP_); | ||
722 | debug_mutex_unlock(lock); | ||
723 | |||
724 | /* | ||
725 | * some architectures leave the lock unlocked in the fastpath failure | ||
726 | * case, others need to leave it locked. In the later case we have to | ||
727 | * unlock it here | ||
728 | */ | ||
729 | if (__mutex_slowpath_needs_to_unlock()) | ||
730 | atomic_set(&lock->count, 1); | ||
731 | |||
732 | if (!list_empty(&lock->wait_list)) { | ||
733 | /* get the first entry from the wait-list: */ | ||
734 | struct mutex_waiter *waiter = | ||
735 | list_entry(lock->wait_list.next, | ||
736 | struct mutex_waiter, list); | ||
737 | |||
738 | debug_mutex_wake_waiter(lock, waiter); | ||
739 | |||
740 | wake_up_process(waiter->task); | ||
741 | } | ||
742 | |||
743 | spin_unlock_mutex(&lock->wait_lock, flags); | ||
744 | } | ||
745 | |||
746 | /* | ||
747 | * Release the lock, slowpath: | ||
748 | */ | ||
749 | static __used noinline void | ||
750 | __mutex_unlock_slowpath(atomic_t *lock_count) | ||
751 | { | ||
752 | __mutex_unlock_common_slowpath(lock_count, 1); | ||
753 | } | ||
754 | |||
755 | #ifndef CONFIG_DEBUG_LOCK_ALLOC | ||
756 | /* | ||
757 | * Here come the less common (and hence less performance-critical) APIs: | ||
758 | * mutex_lock_interruptible() and mutex_trylock(). | ||
759 | */ | ||
760 | static noinline int __sched | ||
761 | __mutex_lock_killable_slowpath(struct mutex *lock); | ||
762 | |||
763 | static noinline int __sched | ||
764 | __mutex_lock_interruptible_slowpath(struct mutex *lock); | ||
765 | |||
766 | /** | ||
767 | * mutex_lock_interruptible - acquire the mutex, interruptible | ||
768 | * @lock: the mutex to be acquired | ||
769 | * | ||
770 | * Lock the mutex like mutex_lock(), and return 0 if the mutex has | ||
771 | * been acquired or sleep until the mutex becomes available. If a | ||
772 | * signal arrives while waiting for the lock then this function | ||
773 | * returns -EINTR. | ||
774 | * | ||
775 | * This function is similar to (but not equivalent to) down_interruptible(). | ||
776 | */ | ||
777 | int __sched mutex_lock_interruptible(struct mutex *lock) | ||
778 | { | ||
779 | int ret; | ||
780 | |||
781 | might_sleep(); | ||
782 | ret = __mutex_fastpath_lock_retval(&lock->count); | ||
783 | if (likely(!ret)) { | ||
784 | mutex_set_owner(lock); | ||
785 | return 0; | ||
786 | } else | ||
787 | return __mutex_lock_interruptible_slowpath(lock); | ||
788 | } | ||
789 | |||
790 | EXPORT_SYMBOL(mutex_lock_interruptible); | ||
791 | |||
792 | int __sched mutex_lock_killable(struct mutex *lock) | ||
793 | { | ||
794 | int ret; | ||
795 | |||
796 | might_sleep(); | ||
797 | ret = __mutex_fastpath_lock_retval(&lock->count); | ||
798 | if (likely(!ret)) { | ||
799 | mutex_set_owner(lock); | ||
800 | return 0; | ||
801 | } else | ||
802 | return __mutex_lock_killable_slowpath(lock); | ||
803 | } | ||
804 | EXPORT_SYMBOL(mutex_lock_killable); | ||
805 | |||
806 | static __used noinline void __sched | ||
807 | __mutex_lock_slowpath(atomic_t *lock_count) | ||
808 | { | ||
809 | struct mutex *lock = container_of(lock_count, struct mutex, count); | ||
810 | |||
811 | __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, | ||
812 | NULL, _RET_IP_, NULL, 0); | ||
813 | } | ||
814 | |||
815 | static noinline int __sched | ||
816 | __mutex_lock_killable_slowpath(struct mutex *lock) | ||
817 | { | ||
818 | return __mutex_lock_common(lock, TASK_KILLABLE, 0, | ||
819 | NULL, _RET_IP_, NULL, 0); | ||
820 | } | ||
821 | |||
822 | static noinline int __sched | ||
823 | __mutex_lock_interruptible_slowpath(struct mutex *lock) | ||
824 | { | ||
825 | return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, | ||
826 | NULL, _RET_IP_, NULL, 0); | ||
827 | } | ||
828 | |||
829 | static noinline int __sched | ||
830 | __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) | ||
831 | { | ||
832 | return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0, | ||
833 | NULL, _RET_IP_, ctx, 1); | ||
834 | } | ||
835 | |||
836 | static noinline int __sched | ||
837 | __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock, | ||
838 | struct ww_acquire_ctx *ctx) | ||
839 | { | ||
840 | return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0, | ||
841 | NULL, _RET_IP_, ctx, 1); | ||
842 | } | ||
843 | |||
844 | #endif | ||
845 | |||
846 | /* | ||
847 | * Spinlock based trylock, we take the spinlock and check whether we | ||
848 | * can get the lock: | ||
849 | */ | ||
850 | static inline int __mutex_trylock_slowpath(atomic_t *lock_count) | ||
851 | { | ||
852 | struct mutex *lock = container_of(lock_count, struct mutex, count); | ||
853 | unsigned long flags; | ||
854 | int prev; | ||
855 | |||
856 | spin_lock_mutex(&lock->wait_lock, flags); | ||
857 | |||
858 | prev = atomic_xchg(&lock->count, -1); | ||
859 | if (likely(prev == 1)) { | ||
860 | mutex_set_owner(lock); | ||
861 | mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); | ||
862 | } | ||
863 | |||
864 | /* Set it back to 0 if there are no waiters: */ | ||
865 | if (likely(list_empty(&lock->wait_list))) | ||
866 | atomic_set(&lock->count, 0); | ||
867 | |||
868 | spin_unlock_mutex(&lock->wait_lock, flags); | ||
869 | |||
870 | return prev == 1; | ||
871 | } | ||
872 | |||
873 | /** | ||
874 | * mutex_trylock - try to acquire the mutex, without waiting | ||
875 | * @lock: the mutex to be acquired | ||
876 | * | ||
877 | * Try to acquire the mutex atomically. Returns 1 if the mutex | ||
878 | * has been acquired successfully, and 0 on contention. | ||
879 | * | ||
880 | * NOTE: this function follows the spin_trylock() convention, so | ||
881 | * it is negated from the down_trylock() return values! Be careful | ||
882 | * about this when converting semaphore users to mutexes. | ||
883 | * | ||
884 | * This function must not be used in interrupt context. The | ||
885 | * mutex must be released by the same task that acquired it. | ||
886 | */ | ||
887 | int __sched mutex_trylock(struct mutex *lock) | ||
888 | { | ||
889 | int ret; | ||
890 | |||
891 | ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath); | ||
892 | if (ret) | ||
893 | mutex_set_owner(lock); | ||
894 | |||
895 | return ret; | ||
896 | } | ||
897 | EXPORT_SYMBOL(mutex_trylock); | ||
898 | |||
899 | #ifndef CONFIG_DEBUG_LOCK_ALLOC | ||
900 | int __sched | ||
901 | __ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) | ||
902 | { | ||
903 | int ret; | ||
904 | |||
905 | might_sleep(); | ||
906 | |||
907 | ret = __mutex_fastpath_lock_retval(&lock->base.count); | ||
908 | |||
909 | if (likely(!ret)) { | ||
910 | ww_mutex_set_context_fastpath(lock, ctx); | ||
911 | mutex_set_owner(&lock->base); | ||
912 | } else | ||
913 | ret = __ww_mutex_lock_slowpath(lock, ctx); | ||
914 | return ret; | ||
915 | } | ||
916 | EXPORT_SYMBOL(__ww_mutex_lock); | ||
917 | |||
918 | int __sched | ||
919 | __ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) | ||
920 | { | ||
921 | int ret; | ||
922 | |||
923 | might_sleep(); | ||
924 | |||
925 | ret = __mutex_fastpath_lock_retval(&lock->base.count); | ||
926 | |||
927 | if (likely(!ret)) { | ||
928 | ww_mutex_set_context_fastpath(lock, ctx); | ||
929 | mutex_set_owner(&lock->base); | ||
930 | } else | ||
931 | ret = __ww_mutex_lock_interruptible_slowpath(lock, ctx); | ||
932 | return ret; | ||
933 | } | ||
934 | EXPORT_SYMBOL(__ww_mutex_lock_interruptible); | ||
935 | |||
936 | #endif | ||
937 | |||
938 | /** | ||
939 | * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 | ||
940 | * @cnt: the atomic which we are to dec | ||
941 | * @lock: the mutex to return holding if we dec to 0 | ||
942 | * | ||
943 | * return true and hold lock if we dec to 0, return false otherwise | ||
944 | */ | ||
945 | int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) | ||
946 | { | ||
947 | /* dec if we can't possibly hit 0 */ | ||
948 | if (atomic_add_unless(cnt, -1, 1)) | ||
949 | return 0; | ||
950 | /* we might hit 0, so take the lock */ | ||
951 | mutex_lock(lock); | ||
952 | if (!atomic_dec_and_test(cnt)) { | ||
953 | /* when we actually did the dec, we didn't hit 0 */ | ||
954 | mutex_unlock(lock); | ||
955 | return 0; | ||
956 | } | ||
957 | /* we hit 0, and we hold the lock */ | ||
958 | return 1; | ||
959 | } | ||
960 | EXPORT_SYMBOL(atomic_dec_and_mutex_lock); | ||