aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/mutex.h358
-rw-r--r--include/linux/reservation.h2
-rw-r--r--include/linux/ww_mutex.h378
-rw-r--r--kernel/mutex.c1
-rw-r--r--lib/locking-selftest.c1
5 files changed, 381 insertions, 359 deletions
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index 3793ed7feeeb..ccd4260834c5 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -78,40 +78,6 @@ struct mutex_waiter {
78#endif 78#endif
79}; 79};
80 80
81struct ww_class {
82 atomic_long_t stamp;
83 struct lock_class_key acquire_key;
84 struct lock_class_key mutex_key;
85 const char *acquire_name;
86 const char *mutex_name;
87};
88
89struct ww_acquire_ctx {
90 struct task_struct *task;
91 unsigned long stamp;
92 unsigned acquired;
93#ifdef CONFIG_DEBUG_MUTEXES
94 unsigned done_acquire;
95 struct ww_class *ww_class;
96 struct ww_mutex *contending_lock;
97#endif
98#ifdef CONFIG_DEBUG_LOCK_ALLOC
99 struct lockdep_map dep_map;
100#endif
101#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
102 unsigned deadlock_inject_interval;
103 unsigned deadlock_inject_countdown;
104#endif
105};
106
107struct ww_mutex {
108 struct mutex base;
109 struct ww_acquire_ctx *ctx;
110#ifdef CONFIG_DEBUG_MUTEXES
111 struct ww_class *ww_class;
112#endif
113};
114
115#ifdef CONFIG_DEBUG_MUTEXES 81#ifdef CONFIG_DEBUG_MUTEXES
116# include <linux/mutex-debug.h> 82# include <linux/mutex-debug.h>
117#else 83#else
@@ -136,11 +102,8 @@ static inline void mutex_destroy(struct mutex *lock) {}
136#ifdef CONFIG_DEBUG_LOCK_ALLOC 102#ifdef CONFIG_DEBUG_LOCK_ALLOC
137# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ 103# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
138 , .dep_map = { .name = #lockname } 104 , .dep_map = { .name = #lockname }
139# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) \
140 , .ww_class = &ww_class
141#else 105#else
142# define __DEP_MAP_MUTEX_INITIALIZER(lockname) 106# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
143# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class)
144#endif 107#endif
145 108
146#define __MUTEX_INITIALIZER(lockname) \ 109#define __MUTEX_INITIALIZER(lockname) \
@@ -150,49 +113,13 @@ static inline void mutex_destroy(struct mutex *lock) {}
150 __DEBUG_MUTEX_INITIALIZER(lockname) \ 113 __DEBUG_MUTEX_INITIALIZER(lockname) \
151 __DEP_MAP_MUTEX_INITIALIZER(lockname) } 114 __DEP_MAP_MUTEX_INITIALIZER(lockname) }
152 115
153#define __WW_CLASS_INITIALIZER(ww_class) \
154 { .stamp = ATOMIC_LONG_INIT(0) \
155 , .acquire_name = #ww_class "_acquire" \
156 , .mutex_name = #ww_class "_mutex" }
157
158#define __WW_MUTEX_INITIALIZER(lockname, class) \
159 { .base = { \__MUTEX_INITIALIZER(lockname) } \
160 __WW_CLASS_MUTEX_INITIALIZER(lockname, class) }
161
162#define DEFINE_MUTEX(mutexname) \ 116#define DEFINE_MUTEX(mutexname) \
163 struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) 117 struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
164 118
165#define DEFINE_WW_CLASS(classname) \
166 struct ww_class classname = __WW_CLASS_INITIALIZER(classname)
167
168#define DEFINE_WW_MUTEX(mutexname, ww_class) \
169 struct ww_mutex mutexname = __WW_MUTEX_INITIALIZER(mutexname, ww_class)
170
171
172extern void __mutex_init(struct mutex *lock, const char *name, 119extern void __mutex_init(struct mutex *lock, const char *name,
173 struct lock_class_key *key); 120 struct lock_class_key *key);
174 121
175/** 122/**
176 * ww_mutex_init - initialize the w/w mutex
177 * @lock: the mutex to be initialized
178 * @ww_class: the w/w class the mutex should belong to
179 *
180 * Initialize the w/w mutex to unlocked state and associate it with the given
181 * class.
182 *
183 * It is not allowed to initialize an already locked mutex.
184 */
185static inline void ww_mutex_init(struct ww_mutex *lock,
186 struct ww_class *ww_class)
187{
188 __mutex_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key);
189 lock->ctx = NULL;
190#ifdef CONFIG_DEBUG_MUTEXES
191 lock->ww_class = ww_class;
192#endif
193}
194
195/**
196 * mutex_is_locked - is the mutex locked 123 * mutex_is_locked - is the mutex locked
197 * @lock: the mutex to be queried 124 * @lock: the mutex to be queried
198 * 125 *
@@ -246,291 +173,6 @@ extern int __must_check mutex_lock_killable(struct mutex *lock);
246extern int mutex_trylock(struct mutex *lock); 173extern int mutex_trylock(struct mutex *lock);
247extern void mutex_unlock(struct mutex *lock); 174extern void mutex_unlock(struct mutex *lock);
248 175
249/**
250 * ww_acquire_init - initialize a w/w acquire context
251 * @ctx: w/w acquire context to initialize
252 * @ww_class: w/w class of the context
253 *
254 * Initializes an context to acquire multiple mutexes of the given w/w class.
255 *
256 * Context-based w/w mutex acquiring can be done in any order whatsoever within
257 * a given lock class. Deadlocks will be detected and handled with the
258 * wait/wound logic.
259 *
260 * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can
261 * result in undetected deadlocks and is so forbidden. Mixing different contexts
262 * for the same w/w class when acquiring mutexes can also result in undetected
263 * deadlocks, and is hence also forbidden. Both types of abuse will be caught by
264 * enabling CONFIG_PROVE_LOCKING.
265 *
266 * Nesting of acquire contexts for _different_ w/w classes is possible, subject
267 * to the usual locking rules between different lock classes.
268 *
269 * An acquire context must be released with ww_acquire_fini by the same task
270 * before the memory is freed. It is recommended to allocate the context itself
271 * on the stack.
272 */
273static inline void ww_acquire_init(struct ww_acquire_ctx *ctx,
274 struct ww_class *ww_class)
275{
276 ctx->task = current;
277 ctx->stamp = atomic_long_inc_return(&ww_class->stamp);
278 ctx->acquired = 0;
279#ifdef CONFIG_DEBUG_MUTEXES
280 ctx->ww_class = ww_class;
281 ctx->done_acquire = 0;
282 ctx->contending_lock = NULL;
283#endif
284#ifdef CONFIG_DEBUG_LOCK_ALLOC
285 debug_check_no_locks_freed((void *)ctx, sizeof(*ctx));
286 lockdep_init_map(&ctx->dep_map, ww_class->acquire_name,
287 &ww_class->acquire_key, 0);
288 mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_);
289#endif
290#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
291 ctx->deadlock_inject_interval = 1;
292 ctx->deadlock_inject_countdown = ctx->stamp & 0xf;
293#endif
294}
295
296/**
297 * ww_acquire_done - marks the end of the acquire phase
298 * @ctx: the acquire context
299 *
300 * Marks the end of the acquire phase, any further w/w mutex lock calls using
301 * this context are forbidden.
302 *
303 * Calling this function is optional, it is just useful to document w/w mutex
304 * code and clearly designated the acquire phase from actually using the locked
305 * data structures.
306 */
307static inline void ww_acquire_done(struct ww_acquire_ctx *ctx)
308{
309#ifdef CONFIG_DEBUG_MUTEXES
310 lockdep_assert_held(ctx);
311
312 DEBUG_LOCKS_WARN_ON(ctx->done_acquire);
313 ctx->done_acquire = 1;
314#endif
315}
316
317/**
318 * ww_acquire_fini - releases a w/w acquire context
319 * @ctx: the acquire context to free
320 *
321 * Releases a w/w acquire context. This must be called _after_ all acquired w/w
322 * mutexes have been released with ww_mutex_unlock.
323 */
324static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx)
325{
326#ifdef CONFIG_DEBUG_MUTEXES
327 mutex_release(&ctx->dep_map, 0, _THIS_IP_);
328
329