diff options
| -rw-r--r-- | include/linux/mutex.h | 358 | ||||
| -rw-r--r-- | include/linux/reservation.h | 2 | ||||
| -rw-r--r-- | include/linux/ww_mutex.h | 378 | ||||
| -rw-r--r-- | kernel/mutex.c | 1 | ||||
| -rw-r--r-- | lib/locking-selftest.c | 1 |
5 files changed, 381 insertions, 359 deletions
diff --git a/include/linux/mutex.h b/include/linux/mutex.h index 3793ed7feeeb..ccd4260834c5 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h | |||
| @@ -78,40 +78,6 @@ struct mutex_waiter { | |||
| 78 | #endif | 78 | #endif |
| 79 | }; | 79 | }; |
| 80 | 80 | ||
| 81 | struct ww_class { | ||
| 82 | atomic_long_t stamp; | ||
| 83 | struct lock_class_key acquire_key; | ||
| 84 | struct lock_class_key mutex_key; | ||
| 85 | const char *acquire_name; | ||
| 86 | const char *mutex_name; | ||
| 87 | }; | ||
| 88 | |||
| 89 | struct ww_acquire_ctx { | ||
| 90 | struct task_struct *task; | ||
| 91 | unsigned long stamp; | ||
| 92 | unsigned acquired; | ||
| 93 | #ifdef CONFIG_DEBUG_MUTEXES | ||
| 94 | unsigned done_acquire; | ||
| 95 | struct ww_class *ww_class; | ||
| 96 | struct ww_mutex *contending_lock; | ||
| 97 | #endif | ||
| 98 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 99 | struct lockdep_map dep_map; | ||
| 100 | #endif | ||
| 101 | #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH | ||
| 102 | unsigned deadlock_inject_interval; | ||
| 103 | unsigned deadlock_inject_countdown; | ||
| 104 | #endif | ||
| 105 | }; | ||
| 106 | |||
| 107 | struct ww_mutex { | ||
| 108 | struct mutex base; | ||
| 109 | struct ww_acquire_ctx *ctx; | ||
| 110 | #ifdef CONFIG_DEBUG_MUTEXES | ||
| 111 | struct ww_class *ww_class; | ||
| 112 | #endif | ||
| 113 | }; | ||
| 114 | |||
| 115 | #ifdef CONFIG_DEBUG_MUTEXES | 81 | #ifdef CONFIG_DEBUG_MUTEXES |
| 116 | # include <linux/mutex-debug.h> | 82 | # include <linux/mutex-debug.h> |
| 117 | #else | 83 | #else |
| @@ -136,11 +102,8 @@ static inline void mutex_destroy(struct mutex *lock) {} | |||
| 136 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 102 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 137 | # define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ | 103 | # define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ |
| 138 | , .dep_map = { .name = #lockname } | 104 | , .dep_map = { .name = #lockname } |
| 139 | # define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) \ | ||
| 140 | , .ww_class = &ww_class | ||
| 141 | #else | 105 | #else |
| 142 | # define __DEP_MAP_MUTEX_INITIALIZER(lockname) | 106 | # define __DEP_MAP_MUTEX_INITIALIZER(lockname) |
| 143 | # define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) | ||
| 144 | #endif | 107 | #endif |
| 145 | 108 | ||
| 146 | #define __MUTEX_INITIALIZER(lockname) \ | 109 | #define __MUTEX_INITIALIZER(lockname) \ |
| @@ -150,49 +113,13 @@ static inline void mutex_destroy(struct mutex *lock) {} | |||
| 150 | __DEBUG_MUTEX_INITIALIZER(lockname) \ | 113 | __DEBUG_MUTEX_INITIALIZER(lockname) \ |
| 151 | __DEP_MAP_MUTEX_INITIALIZER(lockname) } | 114 | __DEP_MAP_MUTEX_INITIALIZER(lockname) } |
| 152 | 115 | ||
| 153 | #define __WW_CLASS_INITIALIZER(ww_class) \ | ||
| 154 | { .stamp = ATOMIC_LONG_INIT(0) \ | ||
| 155 | , .acquire_name = #ww_class "_acquire" \ | ||
| 156 | , .mutex_name = #ww_class "_mutex" } | ||
| 157 | |||
| 158 | #define __WW_MUTEX_INITIALIZER(lockname, class) \ | ||
| 159 | { .base = { \__MUTEX_INITIALIZER(lockname) } \ | ||
| 160 | __WW_CLASS_MUTEX_INITIALIZER(lockname, class) } | ||
| 161 | |||
| 162 | #define DEFINE_MUTEX(mutexname) \ | 116 | #define DEFINE_MUTEX(mutexname) \ |
| 163 | struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) | 117 | struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) |
| 164 | 118 | ||
| 165 | #define DEFINE_WW_CLASS(classname) \ | ||
| 166 | struct ww_class classname = __WW_CLASS_INITIALIZER(classname) | ||
| 167 | |||
| 168 | #define DEFINE_WW_MUTEX(mutexname, ww_class) \ | ||
| 169 | struct ww_mutex mutexname = __WW_MUTEX_INITIALIZER(mutexname, ww_class) | ||
| 170 | |||
| 171 | |||
| 172 | extern void __mutex_init(struct mutex *lock, const char *name, | 119 | extern void __mutex_init(struct mutex *lock, const char *name, |
| 173 | struct lock_class_key *key); | 120 | struct lock_class_key *key); |
| 174 | 121 | ||
| 175 | /** | 122 | /** |
| 176 | * ww_mutex_init - initialize the w/w mutex | ||
| 177 | * @lock: the mutex to be initialized | ||
| 178 | * @ww_class: the w/w class the mutex should belong to | ||
| 179 | * | ||
| 180 | * Initialize the w/w mutex to unlocked state and associate it with the given | ||
| 181 | * class. | ||
| 182 | * | ||
| 183 | * It is not allowed to initialize an already locked mutex. | ||
| 184 | */ | ||
| 185 | static inline void ww_mutex_init(struct ww_mutex *lock, | ||
| 186 | struct ww_class *ww_class) | ||
| 187 | { | ||
| 188 | __mutex_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); | ||
| 189 | lock->ctx = NULL; | ||
| 190 | #ifdef CONFIG_DEBUG_MUTEXES | ||
| 191 | lock->ww_class = ww_class; | ||
| 192 | #endif | ||
| 193 | } | ||
| 194 | |||
| 195 | /** | ||
| 196 | * mutex_is_locked - is the mutex locked | 123 | * mutex_is_locked - is the mutex locked |
| 197 | * @lock: the mutex to be queried | 124 | * @lock: the mutex to be queried |
| 198 | * | 125 | * |
| @@ -246,291 +173,6 @@ extern int __must_check mutex_lock_killable(struct mutex *lock); | |||
| 246 | extern int mutex_trylock(struct mutex *lock); | 173 | extern int mutex_trylock(struct mutex *lock); |
| 247 | extern void mutex_unlock(struct mutex *lock); | 174 | extern void mutex_unlock(struct mutex *lock); |
| 248 | 175 | ||
| 249 | /** | ||
| 250 | * ww_acquire_init - initialize a w/w acquire context | ||
| 251 | * @ctx: w/w acquire context to initialize | ||
| 252 | * @ww_class: w/w class of the context | ||
| 253 | * | ||
| 254 | * Initializes an context to acquire multiple mutexes of the given w/w class. | ||
| 255 | * | ||
| 256 | * Context-based w/w mutex acquiring can be done in any order whatsoever within | ||
| 257 | * a given lock class. Deadlocks will be detected and handled with the | ||
| 258 | * wait/wound logic. | ||
| 259 | * | ||
| 260 | * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can | ||
| 261 | * result in undetected deadlocks and is so forbidden. Mixing different contexts | ||
| 262 | * for the same w/w class when acquiring mutexes can also result in undetected | ||
| 263 | * deadlocks, and is hence also forbidden. Both types of abuse will be caught by | ||
| 264 | * enabling CONFIG_PROVE_LOCKING. | ||
| 265 | * | ||
| 266 | * Nesting of acquire contexts for _different_ w/w classes is possible, subject | ||
| 267 | * to the usual locking rules between different lock classes. | ||
| 268 | * | ||
| 269 | * An acquire context must be released with ww_acquire_fini by the same task | ||
| 270 | * before the memory is freed. It is recommended to allocate the context itself | ||
| 271 | * on the stack. | ||
| 272 | */ | ||
| 273 | static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, | ||
| 274 | struct ww_class *ww_class) | ||
| 275 | { | ||
| 276 | ctx->task = current; | ||
| 277 | ctx->stamp = atomic_long_inc_return(&ww_class->stamp); | ||
| 278 | ctx->acquired = 0; | ||
| 279 | #ifdef CONFIG_DEBUG_MUTEXES | ||
| 280 | ctx->ww_class = ww_class; | ||
| 281 | ctx->done_acquire = 0; | ||
| 282 | ctx->contending_lock = NULL; | ||
| 283 | #endif | ||
| 284 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 285 | debug_check_no_locks_freed((void *)ctx, sizeof(*ctx)); | ||
| 286 | lockdep_init_map(&ctx->dep_map, ww_class->acquire_name, | ||
| 287 | &ww_class->acquire_key, 0); | ||
| 288 | mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_); | ||
| 289 | #endif | ||
| 290 | #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH | ||
| 291 | ctx->deadlock_inject_interval = 1; | ||
| 292 | ctx->deadlock_inject_countdown = ctx->stamp & 0xf; | ||
| 293 | #endif | ||
| 294 | } | ||
| 295 | |||
| 296 | /** | ||
| 297 | * ww_acquire_done - marks the end of the acquire phase | ||
| 298 | * @ctx: the acquire context | ||
| 299 | * | ||
| 300 | * Marks the end of the acquire phase, any further w/w mutex lock calls using | ||
| 301 | * this context are forbidden. | ||
| 302 | * | ||
| 303 | * Calling this function is optional, it is just useful to document w/w mutex | ||
| 304 | * code and clearly designated the acquire phase from actually using the locked | ||
| 305 | * data structures. | ||
| 306 | */ | ||
| 307 | static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) | ||
| 308 | { | ||
| 309 | #ifdef CONFIG_DEBUG_MUTEXES | ||
| 310 | lockdep_assert_held(ctx); | ||
| 311 | |||
| 312 | DEBUG_LOCKS_WARN_ON(ctx->done_acquire); | ||
| 313 | ctx->done_acquire = 1; | ||
| 314 | #endif | ||
| 315 | } | ||
| 316 | |||
| 317 | /** | ||
| 318 | * ww_acquire_fini - releases a w/w acquire context | ||
| 319 | * @ctx: the acquire context to free | ||
| 320 | * | ||
| 321 | * Releases a w/w acquire context. This must be called _after_ all acquired w/w | ||
| 322 | * mutexes have been released with ww_mutex_unlock. | ||
| 323 | */ | ||
| 324 | static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) | ||
| 325 | { | ||
| 326 | #ifdef CONFIG_DEBUG_MUTEXES | ||
| 327 | mutex_release(&ctx->dep_map, 0, _THIS_IP_); | ||
| 328 | |||
| 329 | |||
