diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2010-02-28 13:13:16 -0500 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2010-02-28 13:13:16 -0500 |
| commit | 642c4c75a765d7a3244ab39c8e6fb09be21eca5b (patch) | |
| tree | ce0be9b476f362835d3a3d6e4fd32801cd15c9fe /include | |
| parent | f91b22c35f6b0ae06ec5b67922eca1999c3b6e0a (diff) | |
| parent | 71da81324c83ef65bb196c7f874ac1c6996d8287 (diff) | |
Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (44 commits)
rcu: Fix accelerated GPs for last non-dynticked CPU
rcu: Make non-RCU_PROVE_LOCKING rcu_read_lock_sched_held() understand boot
rcu: Fix accelerated grace periods for last non-dynticked CPU
rcu: Export rcu_scheduler_active
rcu: Make rcu_read_lock_sched_held() take boot time into account
rcu: Make lockdep_rcu_dereference() message less alarmist
sched, cgroups: Fix module export
rcu: Add RCU_CPU_STALL_VERBOSE to dump detailed per-task information
rcu: Fix rcutorture mod_timer argument to delay one jiffy
rcu: Fix deadlock in TREE_PREEMPT_RCU CPU stall detection
rcu: Convert to raw_spinlocks
rcu: Stop overflowing signed integers
rcu: Use canonical URL for Mathieu's dissertation
rcu: Accelerate grace period if last non-dynticked CPU
rcu: Fix citation of Mathieu's dissertation
rcu: Documentation update for CONFIG_PROVE_RCU
security: Apply lockdep-based checking to rcu_dereference() uses
idr: Apply lockdep-based diagnostics to rcu_dereference() uses
radix-tree: Disable RCU lockdep checking in radix tree
vfs: Abstract rcu_dereference_check for files-fdtable use
...
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/cgroup.h | 5 | ||||
| -rw-r--r-- | include/linux/cpumask.h | 14 | ||||
| -rw-r--r-- | include/linux/cred.h | 2 | ||||
| -rw-r--r-- | include/linux/fdtable.h | 11 | ||||
| -rw-r--r-- | include/linux/lockdep.h | 4 | ||||
| -rw-r--r-- | include/linux/rculist.h | 14 | ||||
| -rw-r--r-- | include/linux/rculist_nulls.h | 4 | ||||
| -rw-r--r-- | include/linux/rcupdate.h | 165 | ||||
| -rw-r--r-- | include/linux/rcutiny.h | 16 | ||||
| -rw-r--r-- | include/linux/rcutree.h | 4 | ||||
| -rw-r--r-- | include/linux/rtnetlink.h | 3 | ||||
| -rw-r--r-- | include/linux/srcu.h | 95 | ||||
| -rw-r--r-- | include/net/addrconf.h | 4 |
13 files changed, 305 insertions, 36 deletions
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 0008dee66514..c9bbcb2a75ae 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h | |||
| @@ -28,6 +28,7 @@ struct css_id; | |||
| 28 | extern int cgroup_init_early(void); | 28 | extern int cgroup_init_early(void); |
| 29 | extern int cgroup_init(void); | 29 | extern int cgroup_init(void); |
| 30 | extern void cgroup_lock(void); | 30 | extern void cgroup_lock(void); |
| 31 | extern int cgroup_lock_is_held(void); | ||
| 31 | extern bool cgroup_lock_live_group(struct cgroup *cgrp); | 32 | extern bool cgroup_lock_live_group(struct cgroup *cgrp); |
| 32 | extern void cgroup_unlock(void); | 33 | extern void cgroup_unlock(void); |
| 33 | extern void cgroup_fork(struct task_struct *p); | 34 | extern void cgroup_fork(struct task_struct *p); |
| @@ -486,7 +487,9 @@ static inline struct cgroup_subsys_state *cgroup_subsys_state( | |||
| 486 | static inline struct cgroup_subsys_state *task_subsys_state( | 487 | static inline struct cgroup_subsys_state *task_subsys_state( |
| 487 | struct task_struct *task, int subsys_id) | 488 | struct task_struct *task, int subsys_id) |
| 488 | { | 489 | { |
| 489 | return rcu_dereference(task->cgroups->subsys[subsys_id]); | 490 | return rcu_dereference_check(task->cgroups->subsys[subsys_id], |
| 491 | rcu_read_lock_held() || | ||
| 492 | cgroup_lock_is_held()); | ||
| 490 | } | 493 | } |
| 491 | 494 | ||
| 492 | static inline struct cgroup* task_cgroup(struct task_struct *task, | 495 | static inline struct cgroup* task_cgroup(struct task_struct *task, |
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index d77b54733c5b..dbcee7647d9a 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h | |||
| @@ -143,6 +143,8 @@ static inline unsigned int cpumask_any_but(const struct cpumask *mask, | |||
| 143 | 143 | ||
| 144 | #define for_each_cpu(cpu, mask) \ | 144 | #define for_each_cpu(cpu, mask) \ |
| 145 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) | 145 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) |
| 146 | #define for_each_cpu_not(cpu, mask) \ | ||
| 147 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) | ||
| 146 | #define for_each_cpu_and(cpu, mask, and) \ | 148 | #define for_each_cpu_and(cpu, mask, and) \ |
| 147 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and) | 149 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and) |
| 148 | #else | 150 | #else |
| @@ -203,6 +205,18 @@ int cpumask_any_but(const struct cpumask *mask, unsigned int cpu); | |||
| 203 | (cpu) < nr_cpu_ids;) | 205 | (cpu) < nr_cpu_ids;) |
| 204 | 206 | ||
| 205 | /** | 207 | /** |
| 208 | * for_each_cpu_not - iterate over every cpu in a complemented mask | ||
| 209 | * @cpu: the (optionally unsigned) integer iterator | ||
| 210 | * @mask: the cpumask pointer | ||
| 211 | * | ||
| 212 | * After the loop, cpu is >= nr_cpu_ids. | ||
| 213 | */ | ||
| 214 | #define for_each_cpu_not(cpu, mask) \ | ||
| 215 | for ((cpu) = -1; \ | ||
| 216 | (cpu) = cpumask_next_zero((cpu), (mask)), \ | ||
| 217 | (cpu) < nr_cpu_ids;) | ||
| 218 | |||
| 219 | /** | ||
| 206 | * for_each_cpu_and - iterate over every cpu in both masks | 220 | * for_each_cpu_and - iterate over every cpu in both masks |
| 207 | * @cpu: the (optionally unsigned) integer iterator | 221 | * @cpu: the (optionally unsigned) integer iterator |
| 208 | * @mask: the first cpumask pointer | 222 | * @mask: the first cpumask pointer |
diff --git a/include/linux/cred.h b/include/linux/cred.h index 4e3387a89cb9..4db09f89b637 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h | |||
| @@ -280,7 +280,7 @@ static inline void put_cred(const struct cred *_cred) | |||
| 280 | * task or by holding tasklist_lock to prevent it from being unlinked. | 280 | * task or by holding tasklist_lock to prevent it from being unlinked. |
| 281 | */ | 281 | */ |
| 282 | #define __task_cred(task) \ | 282 | #define __task_cred(task) \ |
| 283 | ((const struct cred *)(rcu_dereference((task)->real_cred))) | 283 | ((const struct cred *)(rcu_dereference_check((task)->real_cred, rcu_read_lock_held() || lockdep_is_held(&tasklist_lock)))) |
| 284 | 284 | ||
| 285 | /** | 285 | /** |
| 286 | * get_task_cred - Get another task's objective credentials | 286 | * get_task_cred - Get another task's objective credentials |
diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index a2ec74bc4812..013dc529e95f 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h | |||
| @@ -57,7 +57,14 @@ struct files_struct { | |||
| 57 | struct file * fd_array[NR_OPEN_DEFAULT]; | 57 | struct file * fd_array[NR_OPEN_DEFAULT]; |
| 58 | }; | 58 | }; |
| 59 | 59 | ||
| 60 | #define files_fdtable(files) (rcu_dereference((files)->fdt)) | 60 | #define rcu_dereference_check_fdtable(files, fdtfd) \ |
| 61 | (rcu_dereference_check((fdtfd), \ | ||
| 62 | rcu_read_lock_held() || \ | ||
| 63 | lockdep_is_held(&(files)->file_lock) || \ | ||
| 64 | atomic_read(&(files)->count) == 1)) | ||
| 65 | |||
| 66 | #define files_fdtable(files) \ | ||
| 67 | (rcu_dereference_check_fdtable((files), (files)->fdt)) | ||
| 61 | 68 | ||
| 62 | struct file_operations; | 69 | struct file_operations; |
| 63 | struct vfsmount; | 70 | struct vfsmount; |
| @@ -78,7 +85,7 @@ static inline struct file * fcheck_files(struct files_struct *files, unsigned in | |||
| 78 | struct fdtable *fdt = files_fdtable(files); | 85 | struct fdtable *fdt = files_fdtable(files); |
| 79 | 86 | ||
| 80 | if (fd < fdt->max_fds) | 87 | if (fd < fdt->max_fds) |
| 81 | file = rcu_dereference(fdt->fd[fd]); | 88 | file = rcu_dereference_check_fdtable(files, fdt->fd[fd]); |
| 82 | return file; | 89 | return file; |
| 83 | } | 90 | } |
| 84 | 91 | ||
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 9ccf0e286b2a..10206a87da19 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h | |||
| @@ -534,4 +534,8 @@ do { \ | |||
| 534 | # define might_lock_read(lock) do { } while (0) | 534 | # define might_lock_read(lock) do { } while (0) |
| 535 | #endif | 535 | #endif |
| 536 | 536 | ||
| 537 | #ifdef CONFIG_PROVE_RCU | ||
| 538 | extern void lockdep_rcu_dereference(const char *file, const int line); | ||
| 539 | #endif | ||
| 540 | |||
| 537 | #endif /* __LINUX_LOCKDEP_H */ | 541 | #endif /* __LINUX_LOCKDEP_H */ |
diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 1bf0f708c4fc..779d70749beb 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h | |||
| @@ -208,7 +208,7 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
| 208 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). | 208 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
| 209 | */ | 209 | */ |
| 210 | #define list_entry_rcu(ptr, type, member) \ | 210 | #define list_entry_rcu(ptr, type, member) \ |
| 211 | container_of(rcu_dereference(ptr), type, member) | 211 | container_of(rcu_dereference_raw(ptr), type, member) |
| 212 | 212 | ||
| 213 | /** | 213 | /** |
| 214 | * list_first_entry_rcu - get the first element from a list | 214 | * list_first_entry_rcu - get the first element from a list |
| @@ -225,9 +225,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
| 225 | list_entry_rcu((ptr)->next, type, member) | 225 | list_entry_rcu((ptr)->next, type, member) |
| 226 | 226 | ||
| 227 | #define __list_for_each_rcu(pos, head) \ | 227 | #define __list_for_each_rcu(pos, head) \ |
| 228 | for (pos = rcu_dereference((head)->next); \ | 228 | for (pos = rcu_dereference_raw((head)->next); \ |
| 229 | pos != (head); \ | 229 | pos != (head); \ |
| 230 | pos = rcu_dereference(pos->next)) | 230 | pos = rcu_dereference_raw(pos->next)) |
| 231 | 231 | ||
| 232 | /** | 232 | /** |
| 233 | * list_for_each_entry_rcu - iterate over rcu list of given type | 233 | * list_for_each_entry_rcu - iterate over rcu list of given type |
| @@ -257,9 +257,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
| 257 | * as long as the traversal is guarded by rcu_read_lock(). | 257 | * as long as the traversal is guarded by rcu_read_lock(). |
| 258 | */ | 258 | */ |
| 259 | #define list_for_each_continue_rcu(pos, head) \ | 259 | #define list_for_each_continue_rcu(pos, head) \ |
| 260 | for ((pos) = rcu_dereference((pos)->next); \ | 260 | for ((pos) = rcu_dereference_raw((pos)->next); \ |
| 261 | prefetch((pos)->next), (pos) != (head); \ | 261 | prefetch((pos)->next), (pos) != (head); \ |
| 262 | (pos) = rcu_dereference((pos)->next)) | 262 | (pos) = rcu_dereference_raw((pos)->next)) |
| 263 | 263 | ||
| 264 | /** | 264 | /** |
| 265 | * list_for_each_entry_continue_rcu - continue iteration over list of given type | 265 | * list_for_each_entry_continue_rcu - continue iteration over list of given type |
| @@ -418,10 +418,10 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, | |||
| 418 | * as long as the traversal is guarded by rcu_read_lock(). | 418 | * as long as the traversal is guarded by rcu_read_lock(). |
| 419 | */ | 419 | */ |
| 420 | #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ | 420 | #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ |
| 421 | for (pos = rcu_dereference((head)->first); \ | 421 | for (pos = rcu_dereference_raw((head)->first); \ |
| 422 | pos && ({ prefetch(pos->next); 1; }) && \ | 422 | pos && ({ prefetch(pos->next); 1; }) && \ |
| 423 | ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ | 423 | ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ |
| 424 | pos = rcu_dereference(pos->next)) | 424 | pos = rcu_dereference_raw(pos->next)) |
| 425 | 425 | ||
| 426 | #endif /* __KERNEL__ */ | 426 | #endif /* __KERNEL__ */ |
| 427 | #endif | 427 | #endif |
diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index 589a40919f01..b70ffe53cb9f 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h | |||
| @@ -101,10 +101,10 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, | |||
| 101 | * | 101 | * |
| 102 | */ | 102 | */ |
| 103 | #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ | 103 | #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ |
| 104 | for (pos = rcu_dereference((head)->first); \ | 104 | for (pos = rcu_dereference_raw((head)->first); \ |
| 105 | (!is_a_nulls(pos)) && \ | 105 | (!is_a_nulls(pos)) && \ |
| 106 | ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ | 106 | ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ |
| 107 | pos = rcu_dereference(pos->next)) | 107 | pos = rcu_dereference_raw(pos->next)) |
| 108 | 108 | ||
| 109 | #endif | 109 | #endif |
| 110 | #endif | 110 | #endif |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 24440f4bf476..c84373626336 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
| @@ -62,6 +62,8 @@ extern int sched_expedited_torture_stats(char *page); | |||
| 62 | 62 | ||
| 63 | /* Internal to kernel */ | 63 | /* Internal to kernel */ |
| 64 | extern void rcu_init(void); | 64 | extern void rcu_init(void); |
| 65 | extern int rcu_scheduler_active; | ||
| 66 | extern void rcu_scheduler_starting(void); | ||
| 65 | 67 | ||
| 66 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) | 68 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) |
| 67 | #include <linux/rcutree.h> | 69 | #include <linux/rcutree.h> |
| @@ -78,14 +80,120 @@ extern void rcu_init(void); | |||
| 78 | } while (0) | 80 | } while (0) |
| 79 | 81 | ||
| 80 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 82 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 83 | |||
| 81 | extern struct lockdep_map rcu_lock_map; | 84 | extern struct lockdep_map rcu_lock_map; |
| 82 | # define rcu_read_acquire() \ | 85 | # define rcu_read_acquire() \ |
| 83 | lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | 86 | lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) |
| 84 | # define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_) | 87 | # define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_) |
| 85 | #else | 88 | |
| 86 | # define rcu_read_acquire() do { } while (0) | 89 | extern struct lockdep_map rcu_bh_lock_map; |
| 87 | # define rcu_read_release() do { } while (0) | 90 | # define rcu_read_acquire_bh() \ |
| 88 | #endif | 91 | lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) |
| 92 | # define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_) | ||
| 93 | |||
| 94 | extern struct lockdep_map rcu_sched_lock_map; | ||
| 95 | # define rcu_read_acquire_sched() \ | ||
| 96 | lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | ||
| 97 | # define rcu_read_release_sched() \ | ||
| 98 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) | ||
| 99 | |||
| 100 | /** | ||
| 101 | * rcu_read_lock_held - might we be in RCU read-side critical section? | ||
| 102 | * | ||
| 103 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
| 104 | * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
| 105 | * this assumes we are in an RCU read-side critical section unless it can | ||
| 106 | * prove otherwise. | ||
| 107 | */ | ||
| 108 | static inline int rcu_read_lock_held(void) | ||
| 109 | { | ||
| 110 | if (debug_locks) | ||
| 111 | return lock_is_held(&rcu_lock_map); | ||
| 112 | return 1; | ||
| 113 | } | ||
| 114 | |||
| 115 | /** | ||
| 116 | * rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section? | ||
| 117 | * | ||
| 118 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
| 119 | * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
| 120 | * this assumes we are in an RCU-bh read-side critical section unless it can | ||
| 121 | * prove otherwise. | ||
| 122 | */ | ||
| 123 | static inline int rcu_read_lock_bh_held(void) | ||
| 124 | { | ||
| 125 | if (debug_locks) | ||
| 126 | return lock_is_held(&rcu_bh_lock_map); | ||
| 127 | return 1; | ||
| 128 | } | ||
| 129 | |||
| 130 | /** | ||
| 131 | * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? | ||
| 132 | * | ||
| 133 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in an | ||
| 134 | * RCU-sched read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
| 135 | * this assumes we are in an RCU-sched read-side critical section unless it | ||
| 136 | * can prove otherwise. Note that disabling of preemption (including | ||
| 137 | * disabling irqs) counts as an RCU-sched read-side critical section. | ||
| 138 | */ | ||
| 139 | static inline int rcu_read_lock_sched_held(void) | ||
| 140 | { | ||
| 141 | int lockdep_opinion = 0; | ||
| 142 | |||
| 143 | if (debug_locks) | ||
| 144 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | ||
| 145 | return lockdep_opinion || preempt_count() != 0 || !rcu_scheduler_active; | ||
| 146 | } | ||
| 147 | |||
| 148 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 149 | |||
| 150 | # define rcu_read_acquire() do { } while (0) | ||
| 151 | # define rcu_read_release() do { } while (0) | ||
| 152 | # define rcu_read_acquire_bh() do { } while (0) | ||
| 153 | # define rcu_read_release_bh() do { } while (0) | ||
| 154 | # define rcu_read_acquire_sched() do { } while (0) | ||
| 155 | # define rcu_read_release_sched() do { } while (0) | ||
| 156 | |||
| 157 | static inline int rcu_read_lock_held(void) | ||
| 158 | { | ||
| 159 | return 1; | ||
| 160 | } | ||
| 161 | |||
| 162 | static inline int rcu_read_lock_bh_held(void) | ||
| 163 | { | ||
| 164 | return 1; | ||
| 165 | } | ||
| 166 | |||
| 167 | static inline int rcu_read_lock_sched_held(void) | ||
| 168 | { | ||
| 169 | return preempt_count() != 0 || !rcu_scheduler_active; | ||
| 170 | } | ||
| 171 | |||
| 172 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 173 | |||
| 174 | #ifdef CONFIG_PROVE_RCU | ||
| 175 | |||
| 176 | /** | ||
| 177 | * rcu_dereference_check - rcu_dereference with debug checking | ||
| 178 | * | ||
| 179 | * Do an rcu_dereference(), but check that the context is correct. | ||
| 180 | * For example, rcu_dereference_check(gp, rcu_read_lock_held()) to | ||
| 181 | * ensure that the rcu_dereference_check() executes within an RCU | ||
| 182 | * read-side critical section. It is also possible to check for | ||
| 183 | * locks being held, for example, by using lockdep_is_held(). | ||
| 184 | */ | ||
| 185 | #define rcu_dereference_check(p, c) \ | ||
| 186 | ({ \ | ||
| 187 | if (debug_locks && !(c)) \ | ||
| 188 | lockdep_rcu_dereference(__FILE__, __LINE__); \ | ||
| 189 | rcu_dereference_raw(p); \ | ||
| 190 | }) | ||
| 191 | |||
| 192 | #else /* #ifdef CONFIG_PROVE_RCU */ | ||
| 193 | |||
| 194 | #define rcu_dereference_check(p, c) rcu_dereference_raw(p) | ||
| 195 | |||
| 196 | #endif /* #else #ifdef CONFIG_PROVE_RCU */ | ||
| 89 | 197 | ||
| 90 | /** | 198 | /** |
| 91 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. | 199 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. |
| @@ -160,7 +268,7 @@ static inline void rcu_read_lock_bh(void) | |||
| 160 | { | 268 | { |
| 161 | __rcu_read_lock_bh(); | 269 | __rcu_read_lock_bh(); |
| 162 | __acquire(RCU_BH); | 270 | __acquire(RCU_BH); |
| 163 | rcu_read_acquire(); | 271 | rcu_read_acquire_bh(); |
| 164 | } | 272 | } |
| 165 | 273 | ||
| 166 | /* | 274 | /* |
| @@ -170,7 +278,7 @@ static inline void rcu_read_lock_bh(void) | |||
| 170 | */ | 278 | */ |
| 171 | static inline void rcu_read_unlock_bh(void) | 279 | static inline void rcu_read_unlock_bh(void) |
| 172 | { | 280 | { |
| 173 | rcu_read_release(); | 281 | rcu_read_release_bh(); |
| 174 | __release(RCU_BH); | 282 | __release(RCU_BH); |
| 175 | __rcu_read_unlock_bh(); | 283 | __rcu_read_unlock_bh(); |
| 176 | } | 284 | } |
| @@ -188,7 +296,7 @@ static inline void rcu_read_lock_sched(void) | |||
| 188 | { | 296 | { |
| 189 | preempt_disable(); | 297 | preempt_disable(); |
| 190 | __acquire(RCU_SCHED); | 298 | __acquire(RCU_SCHED); |
| 191 | rcu_read_acquire(); | 299 | rcu_read_acquire_sched(); |
| 192 | } | 300 | } |
| 193 | 301 | ||
| 194 | /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ | 302 | /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ |
| @@ -205,7 +313,7 @@ static inline notrace void rcu_read_lock_sched_notrace(void) | |||
| 205 | */ | 313 | */ |
| 206 | static inline void rcu_read_unlock_sched(void) | 314 | static inline void rcu_read_unlock_sched(void) |
| 207 | { | 315 | { |
| 208 | rcu_read_release(); | 316 | rcu_read_release_sched(); |
| 209 | __release(RCU_SCHED); | 317 | __release(RCU_SCHED); |
| 210 | preempt_enable(); | 318 | preempt_enable(); |
| 211 | } | 319 | } |
| @@ -219,22 +327,49 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) | |||
| 219 | 327 | ||
| 220 | 328 | ||
| 221 | /** | 329 | /** |
| 222 | * rcu_dereference - fetch an RCU-protected pointer in an | 330 | * rcu_dereference_raw - fetch an RCU-protected pointer |
| 223 | * RCU read-side critical section. This pointer may later | 331 | * |
| 224 | * be safely dereferenced. | 332 | * The caller must be within some flavor of RCU read-side critical |
| 333 | * section, or must be otherwise preventing the pointer from changing, | ||
| 334 | * for example, by holding an appropriate lock. This pointer may later | ||
| 335 | * be safely dereferenced. It is the caller's responsibility to have | ||
| 336 | * done the right thing, as this primitive does no checking of any kind. | ||
| 225 | * | 337 | * |
| 226 | * Inserts memory barriers on architectures that require them | 338 | * Inserts memory barriers on architectures that require them |
| 227 | * (currently only the Alpha), and, more importantly, documents | 339 | * (currently only the Alpha), and, more importantly, documents |
| 228 | * exactly which pointers are protected by RCU. | 340 | * exactly which pointers are protected by RCU. |
| 229 | */ | 341 | */ |
| 230 | 342 | #define rcu_dereference_raw(p) ({ \ | |
| 231 | #define rcu_dereference(p) ({ \ | ||
| 232 | typeof(p) _________p1 = ACCESS_ONCE(p); \ | 343 | typeof(p) _________p1 = ACCESS_ONCE(p); \ |
| 233 | smp_read_barrier_depends(); \ | 344 | smp_read_barrier_depends(); \ |
| 234 | (_________p1); \ | 345 | (_________p1); \ |
| 235 | }) | 346 | }) |
| 236 | 347 | ||
| 237 | /** | 348 | /** |
| 349 | * rcu_dereference - fetch an RCU-protected pointer, checking for RCU | ||
| 350 | * | ||
| 351 | * Makes rcu_dereference_check() do the dirty work. | ||
| 352 | */ | ||
| 353 | #define rcu_dereference(p) \ | ||
| 354 | rcu_dereference_check(p, rcu_read_lock_held()) | ||
| 355 | |||
| 356 | /** | ||
| 357 | * rcu_dereference_bh - fetch an RCU-protected pointer, checking for RCU-bh | ||
| 358 | * | ||
| 359 | * Makes rcu_dereference_check() do the dirty work. | ||
| 360 | */ | ||
| 361 | #define rcu_dereference_bh(p) \ | ||
| 362 | rcu_dereference_check(p, rcu_read_lock_bh_held()) | ||
| 363 | |||
| 364 | /** | ||
| 365 | * rcu_dereference_sched - fetch RCU-protected pointer, checking for RCU-sched | ||
| 366 | * | ||
| 367 | * Makes rcu_dereference_check() do the dirty work. | ||
| 368 | */ | ||
| 369 | #define rcu_dereference_sched(p) \ | ||
| 370 | rcu_dereference_check(p, rcu_read_lock_sched_held()) | ||
| 371 | |||
| 372 | /** | ||
| 238 | * rcu_assign_pointer - assign (publicize) a pointer to a newly | 373 | * rcu_assign_pointer - assign (publicize) a pointer to a newly |
| 239 | * initialized structure that will be dereferenced by RCU read-side | 374 | * initialized structure that will be dereferenced by RCU read-side |
| 240 | * critical sections. Returns the value assigned. | 375 | * critical sections. Returns the value assigned. |
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 96cc307ed9f4..a5195875480a 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h | |||
| @@ -62,6 +62,18 @@ static inline long rcu_batches_completed_bh(void) | |||
| 62 | 62 | ||
| 63 | extern int rcu_expedited_torture_stats(char *page); | 63 | extern int rcu_expedited_torture_stats(char *page); |
| 64 | 64 | ||
| 65 | static inline void rcu_force_quiescent_state(void) | ||
| 66 | { | ||
| 67 | } | ||
| 68 | |||
| 69 | static inline void rcu_bh_force_quiescent_state(void) | ||
| 70 | { | ||
| 71 | } | ||
| 72 | |||
| 73 | static inline void rcu_sched_force_quiescent_state(void) | ||
| 74 | { | ||
| 75 | } | ||
| 76 | |||
| 65 | #define synchronize_rcu synchronize_sched | 77 | #define synchronize_rcu synchronize_sched |
| 66 | 78 | ||
| 67 | static inline void synchronize_rcu_expedited(void) | 79 | static inline void synchronize_rcu_expedited(void) |
| @@ -93,10 +105,6 @@ static inline void rcu_exit_nohz(void) | |||
| 93 | 105 | ||
| 94 | #endif /* #else #ifdef CONFIG_NO_HZ */ | 106 | #endif /* #else #ifdef CONFIG_NO_HZ */ |
| 95 | 107 | ||
| 96 | static inline void rcu_scheduler_starting(void) | ||
| 97 | { | ||
| 98 | } | ||
| 99 | |||
| 100 | static inline void exit_rcu(void) | 108 | static inline void exit_rcu(void) |
| 101 | { | 109 | { |
| 102 | } | 110 | } |
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 8044b1b94333..42cc3a04779e 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h | |||
| @@ -35,7 +35,6 @@ struct notifier_block; | |||
| 35 | extern void rcu_sched_qs(int cpu); | 35 | extern void rcu_sched_qs(int cpu); |
| 36 | extern void rcu_bh_qs(int cpu); | 36 | extern void rcu_bh_qs(int cpu); |
| 37 | extern int rcu_needs_cpu(int cpu); | 37 | extern int rcu_needs_cpu(int cpu); |
| 38 | extern void rcu_scheduler_starting(void); | ||
| 39 | extern int rcu_expedited_torture_stats(char *page); | 38 | extern int rcu_expedited_torture_stats(char *page); |
| 40 | 39 | ||
| 41 | #ifdef CONFIG_TREE_PREEMPT_RCU | 40 | #ifdef CONFIG_TREE_PREEMPT_RCU |
| @@ -99,6 +98,9 @@ extern void rcu_check_callbacks(int cpu, int user); | |||
| 99 | extern long rcu_batches_completed(void); | 98 | extern long rcu_batches_completed(void); |
| 100 | extern long rcu_batches_completed_bh(void); | 99 | extern long rcu_batches_completed_bh(void); |
| 101 | extern long rcu_batches_completed_sched(void); | 100 | extern long rcu_batches_completed_sched(void); |
| 101 | extern void rcu_force_quiescent_state(void); | ||
| 102 | extern void rcu_bh_force_quiescent_state(void); | ||
| 103 | extern void rcu_sched_force_quiescent_state(void); | ||
| 102 | 104 | ||
| 103 | #ifdef CONFIG_NO_HZ | 105 | #ifdef CONFIG_NO_HZ |
| 104 | void rcu_enter_nohz(void); | 106 | void rcu_enter_nohz(void); |
diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 05330fc5b436..5c52fa43785c 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h | |||
| @@ -735,6 +735,9 @@ extern void rtnl_lock(void); | |||
| 735 | extern void rtnl_unlock(void); | 735 | extern void rtnl_unlock(void); |
| 736 | extern int rtnl_trylock(void); | 736 | extern int rtnl_trylock(void); |
| 737 | extern int rtnl_is_locked(void); | 737 | extern int rtnl_is_locked(void); |
| 738 | #ifdef CONFIG_PROVE_LOCKING | ||
| 739 | extern int lockdep_rtnl_is_held(void); | ||
| 740 | #endif /* #ifdef CONFIG_PROVE_LOCKING */ | ||
| 738 | 741 | ||
| 739 | extern void rtnetlink_init(void); | 742 | extern void rtnetlink_init(void); |
| 740 | extern void __rtnl_unlock(void); | 743 | extern void __rtnl_unlock(void); |
diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 4765d97dcafb..3084f80909cd 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h | |||
| @@ -35,6 +35,9 @@ struct srcu_struct { | |||
| 35 | int completed; | 35 | int completed; |
| 36 | struct srcu_struct_array *per_cpu_ref; | 36 | struct srcu_struct_array *per_cpu_ref; |
| 37 | struct mutex mutex; | 37 | struct mutex mutex; |
| 38 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 39 | struct lockdep_map dep_map; | ||
| 40 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 38 | }; | 41 | }; |
| 39 | 42 | ||
| 40 | #ifndef CONFIG_PREEMPT | 43 | #ifndef CONFIG_PREEMPT |
| @@ -43,12 +46,100 @@ struct srcu_struct { | |||
| 43 | #define srcu_barrier() | 46 | #define srcu_barrier() |
| 44 | #endif /* #else #ifndef CONFIG_PREEMPT */ | 47 | #endif /* #else #ifndef CONFIG_PREEMPT */ |
| 45 | 48 | ||
| 49 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 50 | |||
| 51 | int __init_srcu_struct(struct srcu_struct *sp, const char *name, | ||
| 52 | struct lock_class_key *key); | ||
| 53 | |||
| 54 | #define init_srcu_struct(sp) \ | ||
| 55 | ({ \ | ||
| 56 | static struct lock_class_key __srcu_key; \ | ||
| 57 | \ | ||
| 58 | __init_srcu_struct((sp), #sp, &__srcu_key); \ | ||
| 59 | }) | ||
| 60 | |||
| 61 | # define srcu_read_acquire(sp) \ | ||
| 62 | lock_acquire(&(sp)->dep_map, 0, 0, 2, 1, NULL, _THIS_IP_) | ||
| 63 | # define srcu_read_release(sp) \ | ||
| 64 | lock_release(&(sp)->dep_map, 1, _THIS_IP_) | ||
| 65 | |||
| 66 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 67 | |||
| 46 | int init_srcu_struct(struct srcu_struct *sp); | 68 | int init_srcu_struct(struct srcu_struct *sp); |
| 69 | |||
| 70 | # define srcu_read_acquire(sp) do { } while (0) | ||
| 71 | # define srcu_read_release(sp) do { } while (0) | ||
| 72 | |||
| 73 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 74 | |||
| 47 | void cleanup_srcu_struct(struct srcu_struct *sp); | 75 | void cleanup_srcu_struct(struct srcu_struct *sp); |
| 48 | int srcu_read_lock(struct srcu_struct *sp) __acquires(sp); | 76 | int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp); |
| 49 | void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); | 77 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); |
| 50 | void synchronize_srcu(struct srcu_struct *sp); | 78 | void synchronize_srcu(struct srcu_struct *sp); |
| 51 | void synchronize_srcu_expedited(struct srcu_struct *sp); | 79 | void synchronize_srcu_expedited(struct srcu_struct *sp); |
| 52 | long srcu_batches_completed(struct srcu_struct *sp); | 80 | long srcu_batches_completed(struct srcu_struct *sp); |
| 53 | 81 | ||
| 82 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
| 83 | |||
| 84 | /** | ||
| 85 | * srcu_read_lock_held - might we be in SRCU read-side critical section? | ||
| 86 | * | ||
| 87 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
| 88 | * an SRCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
| 89 | * this assumes we are in an SRCU read-side critical section unless it can | ||
| 90 | * prove otherwise. | ||
| 91 | */ | ||
| 92 | static inline int srcu_read_lock_held(struct srcu_struct *sp) | ||
| 93 | { | ||
| 94 | if (debug_locks) | ||
| 95 | return lock_is_held(&sp->dep_map); | ||
| 96 | return 1; | ||
| 97 | } | ||
| 98 | |||
| 99 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 100 | |||
| 101 | static inline int srcu_read_lock_held(struct srcu_struct *sp) | ||
| 102 | { | ||
| 103 | return 1; | ||
| 104 | } | ||
| 105 | |||
| 106 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
| 107 | |||
| 108 | /** | ||
| 109 | * srcu_dereference - fetch SRCU-protected pointer with checking | ||
| 110 | * | ||
| 111 | * Makes rcu_dereference_check() do the dirty work. | ||
| 112 | */ | ||
| 113 | #define srcu_dereference(p, sp) \ | ||
| 114 | rcu_dereference_check(p, srcu_read_lock_held(sp)) | ||
| 115 | |||
| 116 | /** | ||
| 117 | * srcu_read_lock - register a new reader for an SRCU-protected structure. | ||
| 118 | * @sp: srcu_struct in which to register the new reader. | ||
| 119 | * | ||
| 120 | * Enter an SRCU read-side critical section. Note that SRCU read-side | ||
| 121 | * critical sections may be nested. | ||
| 122 | */ | ||
| 123 | static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) | ||
| 124 | { | ||
| 125 | int retval = __srcu_read_lock(sp); | ||
| 126 | |||
| 127 | srcu_read_acquire(sp); | ||
| 128 | return retval; | ||
| 129 | } | ||
| 130 | |||
| 131 | /** | ||
| 132 | * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. | ||
| 133 | * @sp: srcu_struct in which to unregister the old reader. | ||
| 134 | * @idx: return value from corresponding srcu_read_lock(). | ||
| 135 | * | ||
| 136 | * Exit an SRCU read-side critical section. | ||
| 137 | */ | ||
| 138 | static inline void srcu_read_unlock(struct srcu_struct *sp, int idx) | ||
| 139 | __releases(sp) | ||
| 140 | { | ||
| 141 | srcu_read_release(sp); | ||
| 142 | __srcu_read_unlock(sp, idx); | ||
| 143 | } | ||
| 144 | |||
| 54 | #endif | 145 | #endif |
diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 0f7c37825fc1..45375b41a2a0 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h | |||
| @@ -177,7 +177,9 @@ extern int unregister_inet6addr_notifier(struct notifier_block *nb); | |||
| 177 | static inline struct inet6_dev * | 177 | static inline struct inet6_dev * |
| 178 | __in6_dev_get(struct net_device *dev) | 178 | __in6_dev_get(struct net_device *dev) |
| 179 | { | 179 | { |
| 180 | return rcu_dereference(dev->ip6_ptr); | 180 | return rcu_dereference_check(dev->ip6_ptr, |
| 181 | rcu_read_lock_held() || | ||
| 182 | lockdep_rtnl_is_held()); | ||
| 181 | } | 183 | } |
| 182 | 184 | ||
| 183 | static inline struct inet6_dev * | 185 | static inline struct inet6_dev * |
