aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2010-04-28 17:39:09 -0400
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2010-08-19 20:17:59 -0400
commitca5ecddfa8fcbd948c95530e7e817cee9fb43a3d (patch)
tree0ad1c320c2e6612a9a230a58d8588850701c048c /include/linux
parentd34a16661ed0fed433c9469d7cfa3ca4d30ca42e (diff)
rcu: define __rcu address space modifier for sparse
This commit provides definitions for the __rcu annotation defined earlier. This annotation permits sparse to check for correct use of RCU-protected pointers. If a pointer that is annotated with __rcu is accessed directly (as opposed to via rcu_dereference(), rcu_assign_pointer(), or one of their variants), sparse can be made to complain. To enable such complaints, use the new default-disabled CONFIG_SPARSE_RCU_POINTER kernel configuration option. Please note that these sparse complaints are intended to be a debugging aid, -not- a code-style-enforcement mechanism. There are special rcu_dereference_protected() and rcu_access_pointer() accessors for use when RCU read-side protection is not required, for example, when no other CPU has access to the data structure in question or while the current CPU hold the update-side lock. This patch also updates a number of docbook comments that were showing their age. Signed-off-by: Arnd Bergmann <arnd@arndb.de> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Christopher Li <sparse@chrisli.org> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/compiler.h4
-rw-r--r--include/linux/rcupdate.h352
-rw-r--r--include/linux/srcu.h27
3 files changed, 240 insertions, 143 deletions
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index c1a62c56a660..320d6c94ff84 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -16,7 +16,11 @@
16# define __release(x) __context__(x,-1) 16# define __release(x) __context__(x,-1)
17# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) 17# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0)
18# define __percpu __attribute__((noderef, address_space(3))) 18# define __percpu __attribute__((noderef, address_space(3)))
19#ifdef CONFIG_SPARSE_RCU_POINTER
20# define __rcu __attribute__((noderef, address_space(4)))
21#else
19# define __rcu 22# define __rcu
23#endif
20extern void __chk_user_ptr(const volatile void __user *); 24extern void __chk_user_ptr(const volatile void __user *);
21extern void __chk_io_ptr(const volatile void __iomem *); 25extern void __chk_io_ptr(const volatile void __iomem *);
22#else 26#else
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 9fbc54a2585d..b973dea2d6b0 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -41,6 +41,7 @@
41#include <linux/lockdep.h> 41#include <linux/lockdep.h>
42#include <linux/completion.h> 42#include <linux/completion.h>
43#include <linux/debugobjects.h> 43#include <linux/debugobjects.h>
44#include <linux/compiler.h>
44 45
45#ifdef CONFIG_RCU_TORTURE_TEST 46#ifdef CONFIG_RCU_TORTURE_TEST
46extern int rcutorture_runnable; /* for sysctl */ 47extern int rcutorture_runnable; /* for sysctl */
@@ -120,14 +121,15 @@ extern struct lockdep_map rcu_sched_lock_map;
120extern int debug_lockdep_rcu_enabled(void); 121extern int debug_lockdep_rcu_enabled(void);
121 122
122/** 123/**
123 * rcu_read_lock_held - might we be in RCU read-side critical section? 124 * rcu_read_lock_held() - might we be in RCU read-side critical section?
124 * 125 *
125 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU 126 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
126 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, 127 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
127 * this assumes we are in an RCU read-side critical section unless it can 128 * this assumes we are in an RCU read-side critical section unless it can
128 * prove otherwise. 129 * prove otherwise. This is useful for debug checks in functions that
130 * require that they be called within an RCU read-side critical section.
129 * 131 *
130 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot 132 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
131 * and while lockdep is disabled. 133 * and while lockdep is disabled.
132 */ 134 */
133static inline int rcu_read_lock_held(void) 135static inline int rcu_read_lock_held(void)
@@ -144,14 +146,16 @@ static inline int rcu_read_lock_held(void)
144extern int rcu_read_lock_bh_held(void); 146extern int rcu_read_lock_bh_held(void);
145 147
146/** 148/**
147 * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? 149 * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
148 * 150 *
149 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an 151 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
150 * RCU-sched read-side critical section. In absence of 152 * RCU-sched read-side critical section. In absence of
151 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side 153 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
152 * critical section unless it can prove otherwise. Note that disabling 154 * critical section unless it can prove otherwise. Note that disabling
153 * of preemption (including disabling irqs) counts as an RCU-sched 155 * of preemption (including disabling irqs) counts as an RCU-sched
154 * read-side critical section. 156 * read-side critical section. This is useful for debug checks in functions
157 * that required that they be called within an RCU-sched read-side
158 * critical section.
155 * 159 *
156 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot 160 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
157 * and while lockdep is disabled. 161 * and while lockdep is disabled.
@@ -220,41 +224,155 @@ extern int rcu_my_thread_group_empty(void);
220 } \ 224 } \
221 } while (0) 225 } while (0)
222 226
227#else /* #ifdef CONFIG_PROVE_RCU */
228
229#define __do_rcu_dereference_check(c) do { } while (0)
230
231#endif /* #else #ifdef CONFIG_PROVE_RCU */
232
233/*
234 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
235 * and rcu_assign_pointer(). Some of these could be folded into their
236 * callers, but they are left separate in order to ease introduction of
237 * multiple flavors of pointers to match the multiple flavors of RCU
238 * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
239 * the future.
240 */
241#define __rcu_access_pointer(p, space) \
242 ({ \
243 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
244 (void) (((typeof (*p) space *)p) == p); \
245 ((typeof(*p) __force __kernel *)(_________p1)); \
246 })
247#define __rcu_dereference_check(p, c, space) \
248 ({ \
249 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
250 __do_rcu_dereference_check(c); \
251 (void) (((typeof (*p) space *)p) == p); \
252 smp_read_barrier_depends(); \
253 ((typeof(*p) __force __kernel *)(_________p1)); \
254 })
255#define __rcu_dereference_protected(p, c, space) \
256 ({ \
257 __do_rcu_dereference_check(c); \
258 (void) (((typeof (*p) space *)p) == p); \
259 ((typeof(*p) __force __kernel *)(p)); \
260 })
261
262#define __rcu_dereference_index_check(p, c) \
263 ({ \
264 typeof(p) _________p1 = ACCESS_ONCE(p); \
265 __do_rcu_dereference_check(c); \
266 smp_read_barrier_depends(); \
267 (_________p1); \
268 })
269#define __rcu_assign_pointer(p, v, space) \
270 ({ \
271 if (!__builtin_constant_p(v) || \
272 ((v) != NULL)) \
273 smp_wmb(); \
274 (p) = (typeof(*v) __force space *)(v); \
275 })
276
277
278/**
279 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
280 * @p: The pointer to read
281 *
282 * Return the value of the specified RCU-protected pointer, but omit the
283 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
284 * when the value of this pointer is accessed, but the pointer is not
285 * dereferenced, for example, when testing an RCU-protected pointer against
286 * NULL. Although rcu_access_pointer() may also be used in cases where
287 * update-side locks prevent the value of the pointer from changing, you
288 * should instead use rcu_dereference_protected() for this use case.
289 */
290#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
291
223/** 292/**
224 * rcu_dereference_check - rcu_dereference with debug checking 293 * rcu_dereference_check() - rcu_dereference with debug checking
225 * @p: The pointer to read, prior to dereferencing 294 * @p: The pointer to read, prior to dereferencing
226 * @c: The conditions under which the dereference will take place 295 * @c: The conditions under which the dereference will take place
227 * 296 *
228 * Do an rcu_dereference(), but check that the conditions under which the 297 * Do an rcu_dereference(), but check that the conditions under which the
229 * dereference will take place are correct. Typically the conditions indicate 298 * dereference will take place are correct. Typically the conditions
230 * the various locking conditions that should be held at that point. The check 299 * indicate the various locking conditions that should be held at that
231 * should return true if the conditions are satisfied. 300 * point. The check should return true if the conditions are satisfied.
301 * An implicit check for being in an RCU read-side critical section
302 * (rcu_read_lock()) is included.
232 * 303 *
233 * For example: 304 * For example:
234 * 305 *
235 * bar = rcu_dereference_check(foo->bar, rcu_read_lock_held() || 306 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
236 * lockdep_is_held(&foo->lock));
237 * 307 *
238 * could be used to indicate to lockdep that foo->bar may only be dereferenced 308 * could be used to indicate to lockdep that foo->bar may only be dereferenced
239 * if either the RCU read lock is held, or that the lock required to replace 309 * if either rcu_read_lock() is held, or that the lock required to replace
240 * the bar struct at foo->bar is held. 310 * the bar struct at foo->bar is held.
241 * 311 *
242 * Note that the list of conditions may also include indications of when a lock 312 * Note that the list of conditions may also include indications of when a lock
243 * need not be held, for example during initialisation or destruction of the 313 * need not be held, for example during initialisation or destruction of the
244 * target struct: 314 * target struct:
245 * 315 *
246 * bar = rcu_dereference_check(foo->bar, rcu_read_lock_held() || 316 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
247 * lockdep_is_held(&foo->lock) ||
248 * atomic_read(&foo->usage) == 0); 317 * atomic_read(&foo->usage) == 0);
318 *
319 * Inserts memory barriers on architectures that require them
320 * (currently only the Alpha), prevents the compiler from refetching
321 * (and from merging fetches), and, more importantly, documents exactly
322 * which pointers are protected by RCU and checks that the pointer is
323 * annotated as __rcu.
249 */ 324 */
250#define rcu_dereference_check(p, c) \ 325#define rcu_dereference_check(p, c) \
251 ({ \ 326 __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
252 __do_rcu_dereference_check(c); \ 327
253 rcu_dereference_raw(p); \ 328/**
254 }) 329 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
330 * @p: The pointer to read, prior to dereferencing
331 * @c: The conditions under which the dereference will take place
332 *
333 * This is the RCU-bh counterpart to rcu_dereference_check().
334 */
335#define rcu_dereference_bh_check(p, c) \
336 __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
255 337
256/** 338/**
257 * rcu_dereference_protected - fetch RCU pointer when updates prevented 339 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
340 * @p: The pointer to read, prior to dereferencing
341 * @c: The conditions under which the dereference will take place
342 *
343 * This is the RCU-sched counterpart to rcu_dereference_check().
344 */
345#define rcu_dereference_sched_check(p, c) \
346 __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
347 __rcu)
348
349#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
350
351/**
352 * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
353 * @p: The pointer to read, prior to dereferencing
354 * @c: The conditions under which the dereference will take place
355 *
356 * Similar to rcu_dereference_check(), but omits the sparse checking.
357 * This allows rcu_dereference_index_check() to be used on integers,
358 * which can then be used as array indices. Attempting to use
359 * rcu_dereference_check() on an integer will give compiler warnings
360 * because the sparse address-space mechanism relies on dereferencing
361 * the RCU-protected pointer. Dereferencing integers is not something
362 * that even gcc will put up with.
363 *
364 * Note that this function does not implicitly check for RCU read-side
365 * critical sections. If this function gains lots of uses, it might
366 * make sense to provide versions for each flavor of RCU, but it does
367 * not make sense as of early 2010.
368 */
369#define rcu_dereference_index_check(p, c) \
370 __rcu_dereference_index_check((p), (c))
371
372/**
373 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
374 * @p: The pointer to read, prior to dereferencing
375 * @c: The conditions under which the dereference will take place
258 * 376 *
259 * Return the value of the specified RCU-protected pointer, but omit 377 * Return the value of the specified RCU-protected pointer, but omit
260 * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This 378 * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
@@ -263,35 +381,61 @@ extern int rcu_my_thread_group_empty(void);
263 * prevent the compiler from repeating this reference or combining it 381 * prevent the compiler from repeating this reference or combining it
264 * with other references, so it should not be used without protection 382 * with other references, so it should not be used without protection
265 * of appropriate locks. 383 * of appropriate locks.
384 *
385 * This function is only for update-side use. Using this function
386 * when protected only by rcu_read_lock() will result in infrequent
387 * but very ugly failures.
266 */ 388 */
267#define rcu_dereference_protected(p, c) \ 389#define rcu_dereference_protected(p, c) \
268 ({ \ 390 __rcu_dereference_protected((p), (c), __rcu)
269 __do_rcu_dereference_check(c); \
270 (p); \
271 })
272 391
273#else /* #ifdef CONFIG_PROVE_RCU */ 392/**
393 * rcu_dereference_bh_protected() - fetch RCU-bh pointer when updates prevented
394 * @p: The pointer to read, prior to dereferencing
395 * @c: The conditions under which the dereference will take place
396 *
397 * This is the RCU-bh counterpart to rcu_dereference_protected().
398 */
399#define rcu_dereference_bh_protected(p, c) \
400 __rcu_dereference_protected((p), (c), __rcu)
274 401
275#define rcu_dereference_check(p, c) rcu_dereference_raw(p) 402/**
276#define rcu_dereference_protected(p, c) (p) 403 * rcu_dereference_sched_protected() - fetch RCU-sched pointer when updates prevented
404 * @p: The pointer to read, prior to dereferencing
405 * @c: The conditions under which the dereference will take place
406 *
407 * This is the RCU-sched counterpart to rcu_dereference_protected().
408 */
409#define rcu_dereference_sched_protected(p, c) \
410 __rcu_dereference_protected((p), (c), __rcu)
277 411
278#endif /* #else #ifdef CONFIG_PROVE_RCU */
279 412
280/** 413/**
281 * rcu_access_pointer - fetch RCU pointer with no dereferencing 414 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
415 * @p: The pointer to read, prior to dereferencing
282 * 416 *
283 * Return the value of the specified RCU-protected pointer, but omit the 417 * This is a simple wrapper around rcu_dereference_check().
284 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
285 * when the value of this pointer is accessed, but the pointer is not
286 * dereferenced, for example, when testing an RCU-protected pointer against
287 * NULL. This may also be used in cases where update-side locks prevent
288 * the value of the pointer from changing, but rcu_dereference_protected()
289 * is a lighter-weight primitive for this use case.
290 */ 418 */
291#define rcu_access_pointer(p) ACCESS_ONCE(p) 419#define rcu_dereference(p) rcu_dereference_check(p, 0)
292 420
293/** 421/**
294 * rcu_read_lock - mark the beginning of an RCU read-side critical section. 422 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
423 * @p: The pointer to read, prior to dereferencing
424 *
425 * Makes rcu_dereference_check() do the dirty work.
426 */
427#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
428
429/**
430 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
431 * @p: The pointer to read, prior to dereferencing
432 *
433 * Makes rcu_dereference_check() do the dirty work.
434 */
435#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
436
437/**
438 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
295 * 439 *
296 * When synchronize_rcu() is invoked on one CPU while other CPUs 440 * When synchronize_rcu() is invoked on one CPU while other CPUs
297 * are within RCU read-side critical sections, then the 441 * are within RCU read-side critical sections, then the
@@ -337,7 +481,7 @@ static inline void rcu_read_lock(void)
337 */ 481 */
338 482
339/** 483/**
340 * rcu_read_unlock - marks the end of an RCU read-side critical section. 484 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
341 * 485 *
342 * See rcu_read_lock() for more information. 486 * See rcu_read_lock() for more information.
343 */ 487 */
@@ -349,15 +493,16 @@ static inline void rcu_read_unlock(void)
349} 493}
350 494
351/** 495/**
352 * rcu_read_lock_bh - mark the beginning of a softirq-only RCU critical section 496 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
353 * 497 *
354 * This is equivalent of rcu_read_lock(), but to be used when updates 498 * This is equivalent of rcu_read_lock(), but to be used when updates
355 * are being done using call_rcu_bh(). Since call_rcu_bh() callbacks 499 * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
356 * consider completion of a softirq handler to be a quiescent state, 500 * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
357 * a process in RCU read-side critical section must be protected by 501 * softirq handler to be a quiescent state, a process in RCU read-side
358 * disabling softirqs. Read-side critical sections in interrupt context 502 * critical section must be protected by disabling softirqs. Read-side
359 * can use just rcu_read_lock(). 503 * critical sections in interrupt context can use just rcu_read_lock(),
360 * 504 * though this should at least be commented to avoid confusing people
505 * reading the code.
361 */ 506 */
362static inline void rcu_read_lock_bh(void) 507static inline void rcu_read_lock_bh(void)
363{ 508{
@@ -379,13 +524,12 @@ static inline void rcu_read_unlock_bh(void)
379} 524}
380 525
381/** 526/**
382 * rcu_read_lock_sched - mark the beginning of a RCU-classic critical section 527 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
383 * 528 *
384 * Should be used with either 529 * This is equivalent of rcu_read_lock(), but to be used when updates
385 * - synchronize_sched() 530 * are being done using call_rcu_sched() or synchronize_rcu_sched().
386 * or 531 * Read-side critical sections can also be introduced by anything that
387 * - call_rcu_sched() and rcu_barrier_sched() 532 * disables preemption, including local_irq_disable() and friends.
388 * on the write-side to insure proper synchronization.
389 */ 533 */
390static inline void rcu_read_lock_sched(void) 534static inline void rcu_read_lock_sched(void)
391{ 535{
@@ -420,54 +564,14 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
420 preempt_enable_notrace(); 564 preempt_enable_notrace();
421} 565}
422 566
423
424/** 567/**
425 * rcu_dereference_raw - fetch an RCU-protected pointer 568 * rcu_assign_pointer() - assign to RCU-protected pointer
569 * @p: pointer to assign to
570 * @v: value to assign (publish)
426 * 571 *
427 * The caller must be within some flavor of RCU read-side critical 572 * Assigns the specified value to the specified RCU-protected
428 * section, or must be otherwise preventing the pointer from changing, 573 * pointer, ensuring that any concurrent RCU readers will see
429 * for example, by holding an appropriate lock. This pointer may later 574 * any prior initialization. Returns the value assigned.
430 * be safely dereferenced. It is the caller's responsibility to have
431 * done the right thing, as this primitive does no checking of any kind.
432 *
433 * Inserts memory barriers on architectures that require them
434 * (currently only the Alpha), and, more importantly, documents
435 * exactly which pointers are protected by RCU.
436 */
437#define rcu_dereference_raw(p) ({ \
438 typeof(p) _________p1 = ACCESS_ONCE(p); \
439 smp_read_barrier_depends(); \
440 (_________p1); \
441 })
442
443/**
444 * rcu_dereference - fetch an RCU-protected pointer, checking for RCU
445 *
446 * Makes rcu_dereference_check() do the dirty work.
447 */
448#define rcu_dereference(p) \
449 rcu_dereference_check(p, rcu_read_lock_held())
450
451/**
452 * rcu_dereference_bh - fetch an RCU-protected pointer, checking for RCU-bh
453 *
454 * Makes rcu_dereference_check() do the dirty work.
455 */
456#define rcu_dereference_bh(p) \
457 rcu_dereference_check(p, rcu_read_lock_bh_held())
458
459/**
460 * rcu_dereference_sched - fetch RCU-protected pointer, checking for RCU-sched
461 *
462 * Makes rcu_dereference_check() do the dirty work.
463 */
464#define rcu_dereference_sched(p) \
465 rcu_dereference_check(p, rcu_read_lock_sched_held())
466
467/**
468 * rcu_assign_pointer - assign (publicize) a pointer to a newly
469 * initialized structure that will be dereferenced by RCU read-side
470 * critical sections. Returns the value assigned.
471 * 575 *
472 * Inserts memory barriers on architectures that require them 576 * Inserts memory barriers on architectures that require them
473 * (pretty much all of them other than x86), and also prevents 577 * (pretty much all of them other than x86), and also prevents
@@ -476,14 +580,17 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
476 * call documents which pointers will be dereferenced by RCU read-side 580 * call documents which pointers will be dereferenced by RCU read-side
477 * code. 581 * code.
478 */ 582 */
479
480#define rcu_assign_pointer(p, v) \ 583#define rcu_assign_pointer(p, v) \
481 ({ \ 584 __rcu_assign_pointer((p), (v), __rcu)
482 if (!__builtin_constant_p(v) || \ 585
483 ((v) != NULL)) \ 586/**
484 smp_wmb(); \ 587 * RCU_INIT_POINTER() - initialize an RCU protected pointer
485 (p) = (v); \ 588 *
486 }) 589 * Initialize an RCU-protected pointer in such a way to avoid RCU-lockdep
590 * splats.
591 */
592#define RCU_INIT_POINTER(p, v) \
593 p = (typeof(*v) __force __rcu *)(v)
487 594
488/* Infrastructure to implement the synchronize_() primitives. */ 595/* Infrastructure to implement the synchronize_() primitives. */
489 596
@@ -495,7 +602,7 @@ struct rcu_synchronize {
495extern void wakeme_after_rcu(struct rcu_head *head); 602extern void wakeme_after_rcu(struct rcu_head *head);
496 603
497/** 604/**
498 * call_rcu - Queue an RCU callback for invocation after a grace period. 605 * call_rcu() - Queue an RCU callback for invocation after a grace period.
499 * @head: structure to be used for queueing the RCU updates. 606 * @head: structure to be used for queueing the RCU updates.
500 * @func: actual update function to be invoked after the grace period 607 * @func: actual update function to be invoked after the grace period
501 * 608 *
@@ -509,7 +616,7 @@ extern void call_rcu(struct rcu_head *head,
509 void (*func)(struct rcu_head *head)); 616 void (*func)(struct rcu_head *head));
510 617
511/** 618/**
512 * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. 619 * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
513 * @head: structure to be used for queueing the RCU updates. 620 * @head: structure to be used for queueing the RCU updates.
514 * @func: actual update function to be invoked after the grace period 621 * @func: actual update function to be invoked after the grace period
515 * 622 *
@@ -566,37 +673,4 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
566} 673}
567#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 674#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
568 675
569#ifndef CONFIG_PROVE_RCU
570#define __do_rcu_dereference_check(c) do { } while (0)
571#endif /* #ifdef CONFIG_PROVE_RCU */
572
573#define __rcu_dereference_index_check(p, c) \
574 ({ \
575 typeof(p) _________p1 = ACCESS_ONCE(p); \
576 __do_rcu_dereference_check(c); \
577 smp_read_barrier_depends(); \
578 (_________p1); \
579 })
580
581/**
582 * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
583 * @p: The pointer to read, prior to dereferencing
584 * @c: The conditions under which the dereference will take place
585 *
586 * Similar to rcu_dereference_check(), but omits the sparse checking.
587 * This allows rcu_dereference_index_check() to be used on integers,
588 * which can then be used as array indices. Attempting to use
589 * rcu_dereference_check() on an integer will give compiler warnings
590 * because the sparse address-space mechanism relies on dereferencing
591 * the RCU-protected pointer. Dereferencing integers is not something
592 * that even gcc will put up with.
593 *
594 * Note that this function does not implicitly check for RCU read-side
595 * critical sections. If this function gains lots of uses, it might
596 * make sense to provide versions for each flavor of RCU, but it does
597 * not make sense as of early 2010.
598 */
599#define rcu_dereference_index_check(p, c) \
600 __rcu_dereference_index_check((p), (c))
601
602#endif /* __LINUX_RCUPDATE_H */ 676#endif /* __LINUX_RCUPDATE_H */
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 4d5d2f546dbf..6f456a720ff0 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -108,12 +108,31 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp)
108#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 108#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
109 109
110/** 110/**
111 * srcu_dereference - fetch SRCU-protected pointer with checking 111 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
112 * @p: the pointer to fetch and protect for later dereferencing
113 * @sp: pointer to the srcu_struct, which is used to check that we
114 * really are in an SRCU read-side critical section.
115 * @c: condition to check for update-side use
112 * 116 *
113 * Makes rcu_dereference_check() do the dirty work. 117 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
118 * critical section will result in an RCU-lockdep splat, unless @c evaluates
119 * to 1. The @c argument will normally be a logical expression containing
120 * lockdep_is_held() calls.
114 */ 121 */
115#define srcu_dereference(p, sp) \ 122#define srcu_dereference_check(p, sp, c) \
116 rcu_dereference_check(p, srcu_read_lock_held(sp)) 123 __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu)
124
125/**
126 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
127 * @p: the pointer to fetch and protect for later dereferencing
128 * @sp: pointer to the srcu_struct, which is used to check that we
129 * really are in an SRCU read-side critical section.
130 *
131 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
132 * is enabled, invoking this outside of an RCU read-side critical
133 * section will result in an RCU-lockdep splat.
134 */
135#define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0)
117 136
118/** 137/**
119 * srcu_read_lock - register a new reader for an SRCU-protected structure. 138 * srcu_read_lock - register a new reader for an SRCU-protected structure.