diff options
| author | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2010-04-28 17:39:09 -0400 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2010-08-19 20:17:59 -0400 |
| commit | ca5ecddfa8fcbd948c95530e7e817cee9fb43a3d (patch) | |
| tree | 0ad1c320c2e6612a9a230a58d8588850701c048c /include | |
| parent | d34a16661ed0fed433c9469d7cfa3ca4d30ca42e (diff) | |
rcu: define __rcu address space modifier for sparse
This commit provides definitions for the __rcu annotation defined earlier.
This annotation permits sparse to check for correct use of RCU-protected
pointers. If a pointer that is annotated with __rcu is accessed
directly (as opposed to via rcu_dereference(), rcu_assign_pointer(),
or one of their variants), sparse can be made to complain. To enable
such complaints, use the new default-disabled CONFIG_SPARSE_RCU_POINTER
kernel configuration option. Please note that these sparse complaints are
intended to be a debugging aid, -not- a code-style-enforcement mechanism.
There are special rcu_dereference_protected() and rcu_access_pointer()
accessors for use when RCU read-side protection is not required, for
example, when no other CPU has access to the data structure in question
or while the current CPU hold the update-side lock.
This patch also updates a number of docbook comments that were showing
their age.
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Christopher Li <sparse@chrisli.org>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/compiler.h | 4 | ||||
| -rw-r--r-- | include/linux/rcupdate.h | 352 | ||||
| -rw-r--r-- | include/linux/srcu.h | 27 |
3 files changed, 240 insertions, 143 deletions
diff --git a/include/linux/compiler.h b/include/linux/compiler.h index c1a62c56a660..320d6c94ff84 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h | |||
| @@ -16,7 +16,11 @@ | |||
| 16 | # define __release(x) __context__(x,-1) | 16 | # define __release(x) __context__(x,-1) |
| 17 | # define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) | 17 | # define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) |
| 18 | # define __percpu __attribute__((noderef, address_space(3))) | 18 | # define __percpu __attribute__((noderef, address_space(3))) |
| 19 | #ifdef CONFIG_SPARSE_RCU_POINTER | ||
| 20 | # define __rcu __attribute__((noderef, address_space(4))) | ||
| 21 | #else | ||
| 19 | # define __rcu | 22 | # define __rcu |
| 23 | #endif | ||
| 20 | extern void __chk_user_ptr(const volatile void __user *); | 24 | extern void __chk_user_ptr(const volatile void __user *); |
| 21 | extern void __chk_io_ptr(const volatile void __iomem *); | 25 | extern void __chk_io_ptr(const volatile void __iomem *); |
| 22 | #else | 26 | #else |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 9fbc54a2585d..b973dea2d6b0 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
| @@ -41,6 +41,7 @@ | |||
| 41 | #include <linux/lockdep.h> | 41 | #include <linux/lockdep.h> |
| 42 | #include <linux/completion.h> | 42 | #include <linux/completion.h> |
| 43 | #include <linux/debugobjects.h> | 43 | #include <linux/debugobjects.h> |
| 44 | #include <linux/compiler.h> | ||
| 44 | 45 | ||
| 45 | #ifdef CONFIG_RCU_TORTURE_TEST | 46 | #ifdef CONFIG_RCU_TORTURE_TEST |
| 46 | extern int rcutorture_runnable; /* for sysctl */ | 47 | extern int rcutorture_runnable; /* for sysctl */ |
| @@ -120,14 +121,15 @@ extern struct lockdep_map rcu_sched_lock_map; | |||
| 120 | extern int debug_lockdep_rcu_enabled(void); | 121 | extern int debug_lockdep_rcu_enabled(void); |
| 121 | 122 | ||
| 122 | /** | 123 | /** |
| 123 | * rcu_read_lock_held - might we be in RCU read-side critical section? | 124 | * rcu_read_lock_held() - might we be in RCU read-side critical section? |
| 124 | * | 125 | * |
| 125 | * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU | 126 | * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU |
| 126 | * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, | 127 | * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, |
| 127 | * this assumes we are in an RCU read-side critical section unless it can | 128 | * this assumes we are in an RCU read-side critical section unless it can |
| 128 | * prove otherwise. | 129 | * prove otherwise. This is useful for debug checks in functions that |
| 130 | * require that they be called within an RCU read-side critical section. | ||
| 129 | * | 131 | * |
| 130 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot | 132 | * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot |
| 131 | * and while lockdep is disabled. | 133 | * and while lockdep is disabled. |
| 132 | */ | 134 | */ |
| 133 | static inline int rcu_read_lock_held(void) | 135 | static inline int rcu_read_lock_held(void) |
| @@ -144,14 +146,16 @@ static inline int rcu_read_lock_held(void) | |||
| 144 | extern int rcu_read_lock_bh_held(void); | 146 | extern int rcu_read_lock_bh_held(void); |
| 145 | 147 | ||
| 146 | /** | 148 | /** |
| 147 | * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? | 149 | * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? |
| 148 | * | 150 | * |
| 149 | * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an | 151 | * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an |
| 150 | * RCU-sched read-side critical section. In absence of | 152 | * RCU-sched read-side critical section. In absence of |
| 151 | * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side | 153 | * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side |
| 152 | * critical section unless it can prove otherwise. Note that disabling | 154 | * critical section unless it can prove otherwise. Note that disabling |
| 153 | * of preemption (including disabling irqs) counts as an RCU-sched | 155 | * of preemption (including disabling irqs) counts as an RCU-sched |
| 154 | * read-side critical section. | 156 | * read-side critical section. This is useful for debug checks in functions |
| 157 | * that required that they be called within an RCU-sched read-side | ||
| 158 | * critical section. | ||
| 155 | * | 159 | * |
| 156 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot | 160 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot |
| 157 | * and while lockdep is disabled. | 161 | * and while lockdep is disabled. |
| @@ -220,41 +224,155 @@ extern int rcu_my_thread_group_empty(void); | |||
| 220 | } \ | 224 | } \ |
| 221 | } while (0) | 225 | } while (0) |
| 222 | 226 | ||
| 227 | #else /* #ifdef CONFIG_PROVE_RCU */ | ||
| 228 | |||
| 229 | #define __do_rcu_dereference_check(c) do { } while (0) | ||
| 230 | |||
| 231 | #endif /* #else #ifdef CONFIG_PROVE_RCU */ | ||
| 232 | |||
| 233 | /* | ||
| 234 | * Helper functions for rcu_dereference_check(), rcu_dereference_protected() | ||
| 235 | * and rcu_assign_pointer(). Some of these could be folded into their | ||
| 236 | * callers, but they are left separate in order to ease introduction of | ||
| 237 | * multiple flavors of pointers to match the multiple flavors of RCU | ||
| 238 | * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in | ||
| 239 | * the future. | ||
| 240 | */ | ||
| 241 | #define __rcu_access_pointer(p, space) \ | ||
| 242 | ({ \ | ||
| 243 | typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \ | ||
| 244 | (void) (((typeof (*p) space *)p) == p); \ | ||
| 245 | ((typeof(*p) __force __kernel *)(_________p1)); \ | ||
| 246 | }) | ||
| 247 | #define __rcu_dereference_check(p, c, space) \ | ||
| 248 | ({ \ | ||
| 249 | typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \ | ||
| 250 | __do_rcu_dereference_check(c); \ | ||
| 251 | (void) (((typeof (*p) space *)p) == p); \ | ||
| 252 | smp_read_barrier_depends(); \ | ||
| 253 | ((typeof(*p) __force __kernel *)(_________p1)); \ | ||
| 254 | }) | ||
| 255 | #define __rcu_dereference_protected(p, c, space) \ | ||
| 256 | ({ \ | ||
| 257 | __do_rcu_dereference_check(c); \ | ||
| 258 | (void) (((typeof (*p) space *)p) == p); \ | ||
| 259 | ((typeof(*p) __force __kernel *)(p)); \ | ||
| 260 | }) | ||
| 261 | |||
| 262 | #define __rcu_dereference_index_check(p, c) \ | ||
| 263 | ({ \ | ||
| 264 | typeof(p) _________p1 = ACCESS_ONCE(p); \ | ||
| 265 | __do_rcu_dereference_check(c); \ | ||
| 266 | smp_read_barrier_depends(); \ | ||
| 267 | (_________p1); \ | ||
| 268 | }) | ||
| 269 | #define __rcu_assign_pointer(p, v, space) \ | ||
| 270 | ({ \ | ||
| 271 | if (!__builtin_constant_p(v) || \ | ||
| 272 | ((v) != NULL)) \ | ||
| 273 | smp_wmb(); \ | ||
| 274 | (p) = (typeof(*v) __force space *)(v); \ | ||
| 275 | }) | ||
| 276 | |||
| 277 | |||
| 278 | /** | ||
| 279 | * rcu_access_pointer() - fetch RCU pointer with no dereferencing | ||
| 280 | * @p: The pointer to read | ||
| 281 | * | ||
| 282 | * Return the value of the specified RCU-protected pointer, but omit the | ||
| 283 | * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful | ||
| 284 | * when the value of this pointer is accessed, but the pointer is not | ||
| 285 | * dereferenced, for example, when testing an RCU-protected pointer against | ||
| 286 | * NULL. Although rcu_access_pointer() may also be used in cases where | ||
| 287 | * update-side locks prevent the value of the pointer from changing, you | ||
| 288 | * should instead use rcu_dereference_protected() for this use case. | ||
| 289 | */ | ||
| 290 | #define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu) | ||
| 291 | |||
| 223 | /** | 292 | /** |
| 224 | * rcu_dereference_check - rcu_dereference with debug checking | 293 | * rcu_dereference_check() - rcu_dereference with debug checking |
| 225 | * @p: The pointer to read, prior to dereferencing | 294 | * @p: The pointer to read, prior to dereferencing |
| 226 | * @c: The conditions under which the dereference will take place | 295 | * @c: The conditions under which the dereference will take place |
| 227 | * | 296 | * |
| 228 | * Do an rcu_dereference(), but check that the conditions under which the | 297 | * Do an rcu_dereference(), but check that the conditions under which the |
| 229 | * dereference will take place are correct. Typically the conditions indicate | 298 | * dereference will take place are correct. Typically the conditions |
| 230 | * the various locking conditions that should be held at that point. The check | 299 | * indicate the various locking conditions that should be held at that |
| 231 | * should return true if the conditions are satisfied. | 300 | * point. The check should return true if the conditions are satisfied. |
| 301 | * An implicit check for being in an RCU read-side critical section | ||
| 302 | * (rcu_read_lock()) is included. | ||
| 232 | * | 303 | * |
| 233 | * For example: | 304 | * For example: |
| 234 | * | 305 | * |
| 235 | * bar = rcu_dereference_check(foo->bar, rcu_read_lock_held() || | 306 | * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock)); |
| 236 | * lockdep_is_held(&foo->lock)); | ||
| 237 | * | 307 | * |
| 238 | * could be used to indicate to lockdep that foo->bar may only be dereferenced | 308 | * could be used to indicate to lockdep that foo->bar may only be dereferenced |
| 239 | * if either the RCU read lock is held, or that the lock required to replace | 309 | * if either rcu_read_lock() is held, or that the lock required to replace |
| 240 | * the bar struct at foo->bar is held. | 310 | * the bar struct at foo->bar is held. |
| 241 | * | 311 | * |
| 242 | * Note that the list of conditions may also include indications of when a lock | 312 | * Note that the list of conditions may also include indications of when a lock |
| 243 | * need not be held, for example during initialisation or destruction of the | 313 | * need not be held, for example during initialisation or destruction of the |
| 244 | * target struct: | 314 | * target struct: |
| 245 | * | 315 | * |
| 246 | * bar = rcu_dereference_check(foo->bar, rcu_read_lock_held() || | 316 | * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) || |
| 247 | * lockdep_is_held(&foo->lock) || | ||
| 248 | * atomic_read(&foo->usage) == 0); | 317 | * atomic_read(&foo->usage) == 0); |
| 318 | * | ||
| 319 | * Inserts memory barriers on architectures that require them | ||
| 320 | * (currently only the Alpha), prevents the compiler from refetching | ||
| 321 | * (and from merging fetches), and, more importantly, documents exactly | ||
| 322 | * which pointers are protected by RCU and checks that the pointer is | ||
| 323 | * annotated as __rcu. | ||
| 249 | */ | 324 | */ |
| 250 | #define rcu_dereference_check(p, c) \ | 325 | #define rcu_dereference_check(p, c) \ |
| 251 | ({ \ | 326 | __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu) |
| 252 | __do_rcu_dereference_check(c); \ | 327 | |
| 253 | rcu_dereference_raw(p); \ | 328 | /** |
| 254 | }) | 329 | * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking |
| 330 | * @p: The pointer to read, prior to dereferencing | ||
| 331 | * @c: The conditions under which the dereference will take place | ||
| 332 | * | ||
| 333 | * This is the RCU-bh counterpart to rcu_dereference_check(). | ||
| 334 | */ | ||
| 335 | #define rcu_dereference_bh_check(p, c) \ | ||
| 336 | __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu) | ||
| 255 | 337 | ||
| 256 | /** | 338 | /** |
| 257 | * rcu_dereference_protected - fetch RCU pointer when updates prevented | 339 | * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking |
| 340 | * @p: The pointer to read, prior to dereferencing | ||
| 341 | * @c: The conditions under which the dereference will take place | ||
| 342 | * | ||
| 343 | * This is the RCU-sched counterpart to rcu_dereference_check(). | ||
| 344 | */ | ||
| 345 | #define rcu_dereference_sched_check(p, c) \ | ||
| 346 | __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \ | ||
| 347 | __rcu) | ||
| 348 | |||
| 349 | #define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/ | ||
| 350 | |||
| 351 | /** | ||
| 352 | * rcu_dereference_index_check() - rcu_dereference for indices with debug checking | ||
| 353 | * @p: The pointer to read, prior to dereferencing | ||
| 354 | * @c: The conditions under which the dereference will take place | ||
| 355 | * | ||
| 356 | * Similar to rcu_dereference_check(), but omits the sparse checking. | ||
| 357 | * This allows rcu_dereference_index_check() to be used on integers, | ||
| 358 | * which can then be used as array indices. Attempting to use | ||
| 359 | * rcu_dereference_check() on an integer will give compiler warnings | ||
| 360 | * because the sparse address-space mechanism relies on dereferencing | ||
| 361 | * the RCU-protected pointer. Dereferencing integers is not something | ||
| 362 | * that even gcc will put up with. | ||
| 363 | * | ||
| 364 | * Note that this function does not implicitly check for RCU read-side | ||
| 365 | * critical sections. If this function gains lots of uses, it might | ||
| 366 | * make sense to provide versions for each flavor of RCU, but it does | ||
| 367 | * not make sense as of early 2010. | ||
| 368 | */ | ||
| 369 | #define rcu_dereference_index_check(p, c) \ | ||
| 370 | __rcu_dereference_index_check((p), (c)) | ||
| 371 | |||
| 372 | /** | ||
| 373 | * rcu_dereference_protected() - fetch RCU pointer when updates prevented | ||
| 374 | * @p: The pointer to read, prior to dereferencing | ||
| 375 | * @c: The conditions under which the dereference will take place | ||
| 258 | * | 376 | * |
| 259 | * Return the value of the specified RCU-protected pointer, but omit | 377 | * Return the value of the specified RCU-protected pointer, but omit |
| 260 | * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This | 378 | * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This |
| @@ -263,35 +381,61 @@ extern int rcu_my_thread_group_empty(void); | |||
| 263 | * prevent the compiler from repeating this reference or combining it | 381 | * prevent the compiler from repeating this reference or combining it |
| 264 | * with other references, so it should not be used without protection | 382 | * with other references, so it should not be used without protection |
| 265 | * of appropriate locks. | 383 | * of appropriate locks. |
| 384 | * | ||
| 385 | * This function is only for update-side use. Using this function | ||
| 386 | * when protected only by rcu_read_lock() will result in infrequent | ||
| 387 | * but very ugly failures. | ||
| 266 | */ | 388 | */ |
| 267 | #define rcu_dereference_protected(p, c) \ | 389 | #define rcu_dereference_protected(p, c) \ |
| 268 | ({ \ | 390 | __rcu_dereference_protected((p), (c), __rcu) |
| 269 | __do_rcu_dereference_check(c); \ | ||
| 270 | (p); \ | ||
| 271 | }) | ||
| 272 | 391 | ||
| 273 | #else /* #ifdef CONFIG_PROVE_RCU */ | 392 | /** |
| 393 | * rcu_dereference_bh_protected() - fetch RCU-bh pointer when updates prevented | ||
| 394 | * @p: The pointer to read, prior to dereferencing | ||
| 395 | * @c: The conditions under which the dereference will take place | ||
| 396 | * | ||
| 397 | * This is the RCU-bh counterpart to rcu_dereference_protected(). | ||
| 398 | */ | ||
| 399 | #define rcu_dereference_bh_protected(p, c) \ | ||
| 400 | __rcu_dereference_protected((p), (c), __rcu) | ||
| 274 | 401 | ||
| 275 | #define rcu_dereference_check(p, c) rcu_dereference_raw(p) | 402 | /** |
| 276 | #define rcu_dereference_protected(p, c) (p) | 403 | * rcu_dereference_sched_protected() - fetch RCU-sched pointer when updates prevented |
| 404 | * @p: The pointer to read, prior to dereferencing | ||
| 405 | * @c: The conditions under which the dereference will take place | ||
| 406 | * | ||
| 407 | * This is the RCU-sched counterpart to rcu_dereference_protected(). | ||
| 408 | */ | ||
| 409 | #define rcu_dereference_sched_protected(p, c) \ | ||
| 410 | __rcu_dereference_protected((p), (c), __rcu) | ||
| 277 | 411 | ||
| 278 | #endif /* #else #ifdef CONFIG_PROVE_RCU */ | ||
| 279 | 412 | ||
| 280 | /** | 413 | /** |
| 281 | * rcu_access_pointer - fetch RCU pointer with no dereferencing | 414 | * rcu_dereference() - fetch RCU-protected pointer for dereferencing |
| 415 | * @p: The pointer to read, prior to dereferencing | ||
| 282 | * | 416 | * |
| 283 | * Return the value of the specified RCU-protected pointer, but omit the | 417 | * This is a simple wrapper around rcu_dereference_check(). |
| 284 | * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful | ||
| 285 | * when the value of this pointer is accessed, but the pointer is not | ||
| 286 | * dereferenced, for example, when testing an RCU-protected pointer against | ||
| 287 | * NULL. This may also be used in cases where update-side locks prevent | ||
| 288 | * the value of the pointer from changing, but rcu_dereference_protected() | ||
| 289 | * is a lighter-weight primitive for this use case. | ||
| 290 | */ | 418 | */ |
| 291 | #define rcu_access_pointer(p) ACCESS_ONCE(p) | 419 | #define rcu_dereference(p) rcu_dereference_check(p, 0) |
| 292 | 420 | ||
| 293 | /** | 421 | /** |
| 294 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. | 422 | * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing |
| 423 | * @p: The pointer to read, prior to dereferencing | ||
| 424 | * | ||
| 425 | * Makes rcu_dereference_check() do the dirty work. | ||
| 426 | */ | ||
| 427 | #define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0) | ||
| 428 | |||
| 429 | /** | ||
| 430 | * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing | ||
| 431 | * @p: The pointer to read, prior to dereferencing | ||
| 432 | * | ||
| 433 | * Makes rcu_dereference_check() do the dirty work. | ||
| 434 | */ | ||
| 435 | #define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0) | ||
| 436 | |||
| 437 | /** | ||
| 438 | * rcu_read_lock() - mark the beginning of an RCU read-side critical section | ||
| 295 | * | 439 | * |
| 296 | * When synchronize_rcu() is invoked on one CPU while other CPUs | 440 | * When synchronize_rcu() is invoked on one CPU while other CPUs |
| 297 | * are within RCU read-side critical sections, then the | 441 | * are within RCU read-side critical sections, then the |
| @@ -337,7 +481,7 @@ static inline void rcu_read_lock(void) | |||
| 337 | */ | 481 | */ |
| 338 | 482 | ||
| 339 | /** | 483 | /** |
| 340 | * rcu_read_unlock - marks the end of an RCU read-side critical section. | 484 | * rcu_read_unlock() - marks the end of an RCU read-side critical section. |
| 341 | * | 485 | * |
| 342 | * See rcu_read_lock() for more information. | 486 | * See rcu_read_lock() for more information. |
| 343 | */ | 487 | */ |
| @@ -349,15 +493,16 @@ static inline void rcu_read_unlock(void) | |||
| 349 | } | 493 | } |
| 350 | 494 | ||
| 351 | /** | 495 | /** |
| 352 | * rcu_read_lock_bh - mark the beginning of a softirq-only RCU critical section | 496 | * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section |
| 353 | * | 497 | * |
| 354 | * This is equivalent of rcu_read_lock(), but to be used when updates | 498 | * This is equivalent of rcu_read_lock(), but to be used when updates |
| 355 | * are being done using call_rcu_bh(). Since call_rcu_bh() callbacks | 499 | * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since |
| 356 | * consider completion of a softirq handler to be a quiescent state, | 500 | * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a |
| 357 | * a process in RCU read-side critical section must be protected by | 501 | * softirq handler to be a quiescent state, a process in RCU read-side |
| 358 | * disabling softirqs. Read-side critical sections in interrupt context | 502 | * critical section must be protected by disabling softirqs. Read-side |
| 359 | * can use just rcu_read_lock(). | 503 | * critical sections in interrupt context can use just rcu_read_lock(), |
| 360 | * | 504 | * though this should at least be commented to avoid confusing people |
| 505 | * reading the code. | ||
| 361 | */ | 506 | */ |
| 362 | static inline void rcu_read_lock_bh(void) | 507 | static inline void rcu_read_lock_bh(void) |
| 363 | { | 508 | { |
| @@ -379,13 +524,12 @@ static inline void rcu_read_unlock_bh(void) | |||
| 379 | } | 524 | } |
| 380 | 525 | ||
| 381 | /** | 526 | /** |
| 382 | * rcu_read_lock_sched - mark the beginning of a RCU-classic critical section | 527 | * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section |
| 383 | * | 528 | * |
| 384 | * Should be used with either | 529 | * This is equivalent of rcu_read_lock(), but to be used when updates |
| 385 | * - synchronize_sched() | 530 | * are being done using call_rcu_sched() or synchronize_rcu_sched(). |
| 386 | * or | 531 | * Read-side critical sections can also be introduced by anything that |
| 387 | * - call_rcu_sched() and rcu_barrier_sched() | 532 | * disables preemption, including local_irq_disable() and friends. |
| 388 | * on the write-side to insure proper synchronization. | ||
| 389 | */ | 533 | */ |
| 390 | static inline void rcu_read_lock_sched(void) | 534 | static inline void rcu_read_lock_sched(void) |
| 391 | { | 535 | { |
| @@ -420,54 +564,14 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) | |||
| 420 | preempt_enable_notrace(); | 564 | preempt_enable_notrace(); |
| 421 | } | 565 | } |
| 422 | 566 | ||
| 423 | |||
| 424 | /** | 567 | /** |
| 425 | * rcu_dereference_raw - fetch an RCU-protected pointer | 568 | * rcu_assign_pointer() - assign to RCU-protected pointer |
| 569 | * @p: pointer to assign to | ||
| 570 | * @v: value to assign (publish) | ||
| 426 | * | 571 | * |
| 427 | * The caller must be within some flavor of RCU read-side critical | 572 | * Assigns the specified value to the specified RCU-protected |
| 428 | * section, or must be otherwise preventing the pointer from changing, | 573 | * pointer, ensuring that any concurrent RCU readers will see |
| 429 | * for example, by holding an appropriate lock. This pointer may later | 574 | * any prior initialization. Returns the value assigned. |
| 430 | * be safely dereferenced. It is the caller's responsibility to have | ||
| 431 | * done the right thing, as this primitive does no checking of any kind. | ||
| 432 | * | ||
| 433 | * Inserts memory barriers on architectures that require them | ||
| 434 | * (currently only the Alpha), and, more importantly, documents | ||
| 435 | * exactly which pointers are protected by RCU. | ||
| 436 | */ | ||
| 437 | #define rcu_dereference_raw(p) ({ \ | ||
| 438 | typeof(p) _________p1 = ACCESS_ONCE(p); \ | ||
| 439 | smp_read_barrier_depends(); \ | ||
| 440 | (_________p1); \ | ||
| 441 | }) | ||
| 442 | |||
| 443 | /** | ||
| 444 | * rcu_dereference - fetch an RCU-protected pointer, checking for RCU | ||
| 445 | * | ||
| 446 | * Makes rcu_dereference_check() do the dirty work. | ||
| 447 | */ | ||
| 448 | #define rcu_dereference(p) \ | ||
| 449 | rcu_dereference_check(p, rcu_read_lock_held()) | ||
| 450 | |||
| 451 | /** | ||
| 452 | * rcu_dereference_bh - fetch an RCU-protected pointer, checking for RCU-bh | ||
| 453 | * | ||
| 454 | * Makes rcu_dereference_check() do the dirty work. | ||
| 455 | */ | ||
| 456 | #define rcu_dereference_bh(p) \ | ||
| 457 | rcu_dereference_check(p, rcu_read_lock_bh_held()) | ||
| 458 | |||
| 459 | /** | ||
| 460 | * rcu_dereference_sched - fetch RCU-protected pointer, checking for RCU-sched | ||
| 461 | * | ||
| 462 | * Makes rcu_dereference_check() do the dirty work. | ||
| 463 | */ | ||
| 464 | #define rcu_dereference_sched(p) \ | ||
| 465 | rcu_dereference_check(p, rcu_read_lock_sched_held()) | ||
| 466 | |||
| 467 | /** | ||
| 468 | * rcu_assign_pointer - assign (publicize) a pointer to a newly | ||
| 469 | * initialized structure that will be dereferenced by RCU read-side | ||
| 470 | * critical sections. Returns the value assigned. | ||
| 471 | * | 575 | * |
| 472 | * Inserts memory barriers on architectures that require them | 576 | * Inserts memory barriers on architectures that require them |
| 473 | * (pretty much all of them other than x86), and also prevents | 577 | * (pretty much all of them other than x86), and also prevents |
| @@ -476,14 +580,17 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) | |||
| 476 | * call documents which pointers will be dereferenced by RCU read-side | 580 | * call documents which pointers will be dereferenced by RCU read-side |
| 477 | * code. | 581 | * code. |
| 478 | */ | 582 | */ |
| 479 | |||
| 480 | #define rcu_assign_pointer(p, v) \ | 583 | #define rcu_assign_pointer(p, v) \ |
| 481 | ({ \ | 584 | __rcu_assign_pointer((p), (v), __rcu) |
| 482 | if (!__builtin_constant_p(v) || \ | 585 | |
| 483 | ((v) != NULL)) \ | 586 | /** |
| 484 | smp_wmb(); \ | 587 | * RCU_INIT_POINTER() - initialize an RCU protected pointer |
| 485 | (p) = (v); \ | 588 | * |
| 486 | }) | 589 | * Initialize an RCU-protected pointer in such a way to avoid RCU-lockdep |
| 590 | * splats. | ||
| 591 | */ | ||
| 592 | #define RCU_INIT_POINTER(p, v) \ | ||
| 593 | p = (typeof(*v) __force __rcu *)(v) | ||
| 487 | 594 | ||
| 488 | /* Infrastructure to implement the synchronize_() primitives. */ | 595 | /* Infrastructure to implement the synchronize_() primitives. */ |
| 489 | 596 | ||
| @@ -495,7 +602,7 @@ struct rcu_synchronize { | |||
| 495 | extern void wakeme_after_rcu(struct rcu_head *head); | 602 | extern void wakeme_after_rcu(struct rcu_head *head); |
| 496 | 603 | ||
| 497 | /** | 604 | /** |
| 498 | * call_rcu - Queue an RCU callback for invocation after a grace period. | 605 | * call_rcu() - Queue an RCU callback for invocation after a grace period. |
| 499 | * @head: structure to be used for queueing the RCU updates. | 606 | * @head: structure to be used for queueing the RCU updates. |
| 500 | * @func: actual update function to be invoked after the grace period | 607 | * @func: actual update function to be invoked after the grace period |
| 501 | * | 608 | * |
| @@ -509,7 +616,7 @@ extern void call_rcu(struct rcu_head *head, | |||
| 509 | void (*func)(struct rcu_head *head)); | 616 | void (*func)(struct rcu_head *head)); |
| 510 | 617 | ||
| 511 | /** | 618 | /** |
| 512 | * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. | 619 | * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. |
| 513 | * @head: structure to be used for queueing the RCU updates. | 620 | * @head: structure to be used for queueing the RCU updates. |
| 514 | * @func: actual update function to be invoked after the grace period | 621 | * @func: actual update function to be invoked after the grace period |
| 515 | * | 622 | * |
| @@ -566,37 +673,4 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) | |||
| 566 | } | 673 | } |
| 567 | #endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ | 674 | #endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ |
| 568 | 675 | ||
| 569 | #ifndef CONFIG_PROVE_RCU | ||
| 570 | #define __do_rcu_dereference_check(c) do { } while (0) | ||
| 571 | #endif /* #ifdef CONFIG_PROVE_RCU */ | ||
| 572 | |||
| 573 | #define __rcu_dereference_index_check(p, c) \ | ||
| 574 | ({ \ | ||
| 575 | typeof(p) _________p1 = ACCESS_ONCE(p); \ | ||
| 576 | __do_rcu_dereference_check(c); \ | ||
| 577 | smp_read_barrier_depends(); \ | ||
| 578 | (_________p1); \ | ||
| 579 | }) | ||
| 580 | |||
| 581 | /** | ||
| 582 | * rcu_dereference_index_check() - rcu_dereference for indices with debug checking | ||
| 583 | * @p: The pointer to read, prior to dereferencing | ||
| 584 | * @c: The conditions under which the dereference will take place | ||
| 585 | * | ||
| 586 | * Similar to rcu_dereference_check(), but omits the sparse checking. | ||
| 587 | * This allows rcu_dereference_index_check() to be used on integers, | ||
| 588 | * which can then be used as array indices. Attempting to use | ||
| 589 | * rcu_dereference_check() on an integer will give compiler warnings | ||
| 590 | * because the sparse address-space mechanism relies on dereferencing | ||
| 591 | * the RCU-protected pointer. Dereferencing integers is not something | ||
| 592 | * that even gcc will put up with. | ||
| 593 | * | ||
| 594 | * Note that this function does not implicitly check for RCU read-side | ||
| 595 | * critical sections. If this function gains lots of uses, it might | ||
| 596 | * make sense to provide versions for each flavor of RCU, but it does | ||
| 597 | * not make sense as of early 2010. | ||
| 598 | */ | ||
| 599 | #define rcu_dereference_index_check(p, c) \ | ||
| 600 | __rcu_dereference_index_check((p), (c)) | ||
| 601 | |||
| 602 | #endif /* __LINUX_RCUPDATE_H */ | 676 | #endif /* __LINUX_RCUPDATE_H */ |
diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 4d5d2f546dbf..6f456a720ff0 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h | |||
| @@ -108,12 +108,31 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp) | |||
| 108 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 108 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
| 109 | 109 | ||
| 110 | /** | 110 | /** |
| 111 | * srcu_dereference - fetch SRCU-protected pointer with checking | 111 | * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing |
| 112 | * @p: the pointer to fetch and protect for later dereferencing | ||
| 113 | * @sp: pointer to the srcu_struct, which is used to check that we | ||
| 114 | * really are in an SRCU read-side critical section. | ||
| 115 | * @c: condition to check for update-side use | ||
| 112 | * | 116 | * |
| 113 | * Makes rcu_dereference_check() do the dirty work. | 117 | * If PROVE_RCU is enabled, invoking this outside of an RCU read-side |
| 118 | * critical section will result in an RCU-lockdep splat, unless @c evaluates | ||
| 119 | * to 1. The @c argument will normally be a logical expression containing | ||
| 120 | * lockdep_is_held() calls. | ||
| 114 | */ | 121 | */ |
| 115 | #define srcu_dereference(p, sp) \ | 122 | #define srcu_dereference_check(p, sp, c) \ |
| 116 | rcu_dereference_check(p, srcu_read_lock_held(sp)) | 123 | __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu) |
| 124 | |||
| 125 | /** | ||
| 126 | * srcu_dereference - fetch SRCU-protected pointer for later dereferencing | ||
| 127 | * @p: the pointer to fetch and protect for later dereferencing | ||
| 128 | * @sp: pointer to the srcu_struct, which is used to check that we | ||
| 129 | * really are in an SRCU read-side critical section. | ||
| 130 | * | ||
| 131 | * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU | ||
| 132 | * is enabled, invoking this outside of an RCU read-side critical | ||
| 133 | * section will result in an RCU-lockdep splat. | ||
| 134 | */ | ||
| 135 | #define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0) | ||
| 117 | 136 | ||
| 118 | /** | 137 | /** |
| 119 | * srcu_read_lock - register a new reader for an SRCU-protected structure. | 138 | * srcu_read_lock - register a new reader for an SRCU-protected structure. |
