diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2010-02-28 13:13:16 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2010-02-28 13:13:16 -0500 |
commit | 642c4c75a765d7a3244ab39c8e6fb09be21eca5b (patch) | |
tree | ce0be9b476f362835d3a3d6e4fd32801cd15c9fe /include | |
parent | f91b22c35f6b0ae06ec5b67922eca1999c3b6e0a (diff) | |
parent | 71da81324c83ef65bb196c7f874ac1c6996d8287 (diff) |
Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (44 commits)
rcu: Fix accelerated GPs for last non-dynticked CPU
rcu: Make non-RCU_PROVE_LOCKING rcu_read_lock_sched_held() understand boot
rcu: Fix accelerated grace periods for last non-dynticked CPU
rcu: Export rcu_scheduler_active
rcu: Make rcu_read_lock_sched_held() take boot time into account
rcu: Make lockdep_rcu_dereference() message less alarmist
sched, cgroups: Fix module export
rcu: Add RCU_CPU_STALL_VERBOSE to dump detailed per-task information
rcu: Fix rcutorture mod_timer argument to delay one jiffy
rcu: Fix deadlock in TREE_PREEMPT_RCU CPU stall detection
rcu: Convert to raw_spinlocks
rcu: Stop overflowing signed integers
rcu: Use canonical URL for Mathieu's dissertation
rcu: Accelerate grace period if last non-dynticked CPU
rcu: Fix citation of Mathieu's dissertation
rcu: Documentation update for CONFIG_PROVE_RCU
security: Apply lockdep-based checking to rcu_dereference() uses
idr: Apply lockdep-based diagnostics to rcu_dereference() uses
radix-tree: Disable RCU lockdep checking in radix tree
vfs: Abstract rcu_dereference_check for files-fdtable use
...
Diffstat (limited to 'include')
-rw-r--r-- | include/linux/cgroup.h | 5 | ||||
-rw-r--r-- | include/linux/cpumask.h | 14 | ||||
-rw-r--r-- | include/linux/cred.h | 2 | ||||
-rw-r--r-- | include/linux/fdtable.h | 11 | ||||
-rw-r--r-- | include/linux/lockdep.h | 4 | ||||
-rw-r--r-- | include/linux/rculist.h | 14 | ||||
-rw-r--r-- | include/linux/rculist_nulls.h | 4 | ||||
-rw-r--r-- | include/linux/rcupdate.h | 165 | ||||
-rw-r--r-- | include/linux/rcutiny.h | 16 | ||||
-rw-r--r-- | include/linux/rcutree.h | 4 | ||||
-rw-r--r-- | include/linux/rtnetlink.h | 3 | ||||
-rw-r--r-- | include/linux/srcu.h | 95 | ||||
-rw-r--r-- | include/net/addrconf.h | 4 |
13 files changed, 305 insertions, 36 deletions
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 0008dee66514..c9bbcb2a75ae 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h | |||
@@ -28,6 +28,7 @@ struct css_id; | |||
28 | extern int cgroup_init_early(void); | 28 | extern int cgroup_init_early(void); |
29 | extern int cgroup_init(void); | 29 | extern int cgroup_init(void); |
30 | extern void cgroup_lock(void); | 30 | extern void cgroup_lock(void); |
31 | extern int cgroup_lock_is_held(void); | ||
31 | extern bool cgroup_lock_live_group(struct cgroup *cgrp); | 32 | extern bool cgroup_lock_live_group(struct cgroup *cgrp); |
32 | extern void cgroup_unlock(void); | 33 | extern void cgroup_unlock(void); |
33 | extern void cgroup_fork(struct task_struct *p); | 34 | extern void cgroup_fork(struct task_struct *p); |
@@ -486,7 +487,9 @@ static inline struct cgroup_subsys_state *cgroup_subsys_state( | |||
486 | static inline struct cgroup_subsys_state *task_subsys_state( | 487 | static inline struct cgroup_subsys_state *task_subsys_state( |
487 | struct task_struct *task, int subsys_id) | 488 | struct task_struct *task, int subsys_id) |
488 | { | 489 | { |
489 | return rcu_dereference(task->cgroups->subsys[subsys_id]); | 490 | return rcu_dereference_check(task->cgroups->subsys[subsys_id], |
491 | rcu_read_lock_held() || | ||
492 | cgroup_lock_is_held()); | ||
490 | } | 493 | } |
491 | 494 | ||
492 | static inline struct cgroup* task_cgroup(struct task_struct *task, | 495 | static inline struct cgroup* task_cgroup(struct task_struct *task, |
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index d77b54733c5b..dbcee7647d9a 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h | |||
@@ -143,6 +143,8 @@ static inline unsigned int cpumask_any_but(const struct cpumask *mask, | |||
143 | 143 | ||
144 | #define for_each_cpu(cpu, mask) \ | 144 | #define for_each_cpu(cpu, mask) \ |
145 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) | 145 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) |
146 | #define for_each_cpu_not(cpu, mask) \ | ||
147 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) | ||
146 | #define for_each_cpu_and(cpu, mask, and) \ | 148 | #define for_each_cpu_and(cpu, mask, and) \ |
147 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and) | 149 | for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and) |
148 | #else | 150 | #else |
@@ -203,6 +205,18 @@ int cpumask_any_but(const struct cpumask *mask, unsigned int cpu); | |||
203 | (cpu) < nr_cpu_ids;) | 205 | (cpu) < nr_cpu_ids;) |
204 | 206 | ||
205 | /** | 207 | /** |
208 | * for_each_cpu_not - iterate over every cpu in a complemented mask | ||
209 | * @cpu: the (optionally unsigned) integer iterator | ||
210 | * @mask: the cpumask pointer | ||
211 | * | ||
212 | * After the loop, cpu is >= nr_cpu_ids. | ||
213 | */ | ||
214 | #define for_each_cpu_not(cpu, mask) \ | ||
215 | for ((cpu) = -1; \ | ||
216 | (cpu) = cpumask_next_zero((cpu), (mask)), \ | ||
217 | (cpu) < nr_cpu_ids;) | ||
218 | |||
219 | /** | ||
206 | * for_each_cpu_and - iterate over every cpu in both masks | 220 | * for_each_cpu_and - iterate over every cpu in both masks |
207 | * @cpu: the (optionally unsigned) integer iterator | 221 | * @cpu: the (optionally unsigned) integer iterator |
208 | * @mask: the first cpumask pointer | 222 | * @mask: the first cpumask pointer |
diff --git a/include/linux/cred.h b/include/linux/cred.h index 4e3387a89cb9..4db09f89b637 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h | |||
@@ -280,7 +280,7 @@ static inline void put_cred(const struct cred *_cred) | |||
280 | * task or by holding tasklist_lock to prevent it from being unlinked. | 280 | * task or by holding tasklist_lock to prevent it from being unlinked. |
281 | */ | 281 | */ |
282 | #define __task_cred(task) \ | 282 | #define __task_cred(task) \ |
283 | ((const struct cred *)(rcu_dereference((task)->real_cred))) | 283 | ((const struct cred *)(rcu_dereference_check((task)->real_cred, rcu_read_lock_held() || lockdep_is_held(&tasklist_lock)))) |
284 | 284 | ||
285 | /** | 285 | /** |
286 | * get_task_cred - Get another task's objective credentials | 286 | * get_task_cred - Get another task's objective credentials |
diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index a2ec74bc4812..013dc529e95f 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h | |||
@@ -57,7 +57,14 @@ struct files_struct { | |||
57 | struct file * fd_array[NR_OPEN_DEFAULT]; | 57 | struct file * fd_array[NR_OPEN_DEFAULT]; |
58 | }; | 58 | }; |
59 | 59 | ||
60 | #define files_fdtable(files) (rcu_dereference((files)->fdt)) | 60 | #define rcu_dereference_check_fdtable(files, fdtfd) \ |
61 | (rcu_dereference_check((fdtfd), \ | ||
62 | rcu_read_lock_held() || \ | ||
63 | lockdep_is_held(&(files)->file_lock) || \ | ||
64 | atomic_read(&(files)->count) == 1)) | ||
65 | |||
66 | #define files_fdtable(files) \ | ||
67 | (rcu_dereference_check_fdtable((files), (files)->fdt)) | ||
61 | 68 | ||
62 | struct file_operations; | 69 | struct file_operations; |
63 | struct vfsmount; | 70 | struct vfsmount; |
@@ -78,7 +85,7 @@ static inline struct file * fcheck_files(struct files_struct *files, unsigned in | |||
78 | struct fdtable *fdt = files_fdtable(files); | 85 | struct fdtable *fdt = files_fdtable(files); |
79 | 86 | ||
80 | if (fd < fdt->max_fds) | 87 | if (fd < fdt->max_fds) |
81 | file = rcu_dereference(fdt->fd[fd]); | 88 | file = rcu_dereference_check_fdtable(files, fdt->fd[fd]); |
82 | return file; | 89 | return file; |
83 | } | 90 | } |
84 | 91 | ||
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 9ccf0e286b2a..10206a87da19 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h | |||
@@ -534,4 +534,8 @@ do { \ | |||
534 | # define might_lock_read(lock) do { } while (0) | 534 | # define might_lock_read(lock) do { } while (0) |
535 | #endif | 535 | #endif |
536 | 536 | ||
537 | #ifdef CONFIG_PROVE_RCU | ||
538 | extern void lockdep_rcu_dereference(const char *file, const int line); | ||
539 | #endif | ||
540 | |||
537 | #endif /* __LINUX_LOCKDEP_H */ | 541 | #endif /* __LINUX_LOCKDEP_H */ |
diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 1bf0f708c4fc..779d70749beb 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h | |||
@@ -208,7 +208,7 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
208 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). | 208 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
209 | */ | 209 | */ |
210 | #define list_entry_rcu(ptr, type, member) \ | 210 | #define list_entry_rcu(ptr, type, member) \ |
211 | container_of(rcu_dereference(ptr), type, member) | 211 | container_of(rcu_dereference_raw(ptr), type, member) |
212 | 212 | ||
213 | /** | 213 | /** |
214 | * list_first_entry_rcu - get the first element from a list | 214 | * list_first_entry_rcu - get the first element from a list |
@@ -225,9 +225,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
225 | list_entry_rcu((ptr)->next, type, member) | 225 | list_entry_rcu((ptr)->next, type, member) |
226 | 226 | ||
227 | #define __list_for_each_rcu(pos, head) \ | 227 | #define __list_for_each_rcu(pos, head) \ |
228 | for (pos = rcu_dereference((head)->next); \ | 228 | for (pos = rcu_dereference_raw((head)->next); \ |
229 | pos != (head); \ | 229 | pos != (head); \ |
230 | pos = rcu_dereference(pos->next)) | 230 | pos = rcu_dereference_raw(pos->next)) |
231 | 231 | ||
232 | /** | 232 | /** |
233 | * list_for_each_entry_rcu - iterate over rcu list of given type | 233 | * list_for_each_entry_rcu - iterate over rcu list of given type |
@@ -257,9 +257,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
257 | * as long as the traversal is guarded by rcu_read_lock(). | 257 | * as long as the traversal is guarded by rcu_read_lock(). |
258 | */ | 258 | */ |
259 | #define list_for_each_continue_rcu(pos, head) \ | 259 | #define list_for_each_continue_rcu(pos, head) \ |
260 | for ((pos) = rcu_dereference((pos)->next); \ | 260 | for ((pos) = rcu_dereference_raw((pos)->next); \ |
261 | prefetch((pos)->next), (pos) != (head); \ | 261 | prefetch((pos)->next), (pos) != (head); \ |
262 | (pos) = rcu_dereference((pos)->next)) | 262 | (pos) = rcu_dereference_raw((pos)->next)) |
263 | 263 | ||
264 | /** | 264 | /** |
265 | * list_for_each_entry_continue_rcu - continue iteration over list of given type | 265 | * list_for_each_entry_continue_rcu - continue iteration over list of given type |
@@ -418,10 +418,10 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, | |||
418 | * as long as the traversal is guarded by rcu_read_lock(). | 418 | * as long as the traversal is guarded by rcu_read_lock(). |
419 | */ | 419 | */ |
420 | #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ | 420 | #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ |
421 | for (pos = rcu_dereference((head)->first); \ | 421 | for (pos = rcu_dereference_raw((head)->first); \ |
422 | pos && ({ prefetch(pos->next); 1; }) && \ | 422 | pos && ({ prefetch(pos->next); 1; }) && \ |
423 | ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ | 423 | ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ |
424 | pos = rcu_dereference(pos->next)) | 424 | pos = rcu_dereference_raw(pos->next)) |
425 | 425 | ||
426 | #endif /* __KERNEL__ */ | 426 | #endif /* __KERNEL__ */ |
427 | #endif | 427 | #endif |
diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index 589a40919f01..b70ffe53cb9f 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h | |||
@@ -101,10 +101,10 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, | |||
101 | * | 101 | * |
102 | */ | 102 | */ |
103 | #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ | 103 | #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ |
104 | for (pos = rcu_dereference((head)->first); \ | 104 | for (pos = rcu_dereference_raw((head)->first); \ |
105 | (!is_a_nulls(pos)) && \ | 105 | (!is_a_nulls(pos)) && \ |
106 | ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ | 106 | ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ |
107 | pos = rcu_dereference(pos->next)) | 107 | pos = rcu_dereference_raw(pos->next)) |
108 | 108 | ||
109 | #endif | 109 | #endif |
110 | #endif | 110 | #endif |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 24440f4bf476..c84373626336 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -62,6 +62,8 @@ extern int sched_expedited_torture_stats(char *page); | |||
62 | 62 | ||
63 | /* Internal to kernel */ | 63 | /* Internal to kernel */ |
64 | extern void rcu_init(void); | 64 | extern void rcu_init(void); |
65 | extern int rcu_scheduler_active; | ||
66 | extern void rcu_scheduler_starting(void); | ||
65 | 67 | ||
66 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) | 68 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) |
67 | #include <linux/rcutree.h> | 69 | #include <linux/rcutree.h> |
@@ -78,14 +80,120 @@ extern void rcu_init(void); | |||
78 | } while (0) | 80 | } while (0) |
79 | 81 | ||
80 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 82 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
83 | |||
81 | extern struct lockdep_map rcu_lock_map; | 84 | extern struct lockdep_map rcu_lock_map; |
82 | # define rcu_read_acquire() \ | 85 | # define rcu_read_acquire() \ |
83 | lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | 86 | lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) |
84 | # define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_) | 87 | # define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_) |
85 | #else | 88 | |
86 | # define rcu_read_acquire() do { } while (0) | 89 | extern struct lockdep_map rcu_bh_lock_map; |
87 | # define rcu_read_release() do { } while (0) | 90 | # define rcu_read_acquire_bh() \ |
88 | #endif | 91 | lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) |
92 | # define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_) | ||
93 | |||
94 | extern struct lockdep_map rcu_sched_lock_map; | ||
95 | # define rcu_read_acquire_sched() \ | ||
96 | lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | ||
97 | # define rcu_read_release_sched() \ | ||
98 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) | ||
99 | |||
100 | /** | ||
101 | * rcu_read_lock_held - might we be in RCU read-side critical section? | ||
102 | * | ||
103 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
104 | * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
105 | * this assumes we are in an RCU read-side critical section unless it can | ||
106 | * prove otherwise. | ||
107 | */ | ||
108 | static inline int rcu_read_lock_held(void) | ||
109 | { | ||
110 | if (debug_locks) | ||
111 | return lock_is_held(&rcu_lock_map); | ||
112 | return 1; | ||
113 | } | ||
114 | |||
115 | /** | ||
116 | * rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section? | ||
117 | * | ||
118 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
119 | * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
120 | * this assumes we are in an RCU-bh read-side critical section unless it can | ||
121 | * prove otherwise. | ||
122 | */ | ||
123 | static inline int rcu_read_lock_bh_held(void) | ||
124 | { | ||
125 | if (debug_locks) | ||
126 | return lock_is_held(&rcu_bh_lock_map); | ||
127 | return 1; | ||
128 | } | ||
129 | |||
130 | /** | ||
131 | * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? | ||
132 | * | ||
133 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in an | ||
134 | * RCU-sched read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
135 | * this assumes we are in an RCU-sched read-side critical section unless it | ||
136 | * can prove otherwise. Note that disabling of preemption (including | ||
137 | * disabling irqs) counts as an RCU-sched read-side critical section. | ||
138 | */ | ||
139 | static inline int rcu_read_lock_sched_held(void) | ||
140 | { | ||
141 | int lockdep_opinion = 0; | ||
142 | |||
143 | if (debug_locks) | ||
144 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | ||
145 | return lockdep_opinion || preempt_count() != 0 || !rcu_scheduler_active; | ||
146 | } | ||
147 | |||
148 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
149 | |||
150 | # define rcu_read_acquire() do { } while (0) | ||
151 | # define rcu_read_release() do { } while (0) | ||
152 | # define rcu_read_acquire_bh() do { } while (0) | ||
153 | # define rcu_read_release_bh() do { } while (0) | ||
154 | # define rcu_read_acquire_sched() do { } while (0) | ||
155 | # define rcu_read_release_sched() do { } while (0) | ||
156 | |||
157 | static inline int rcu_read_lock_held(void) | ||
158 | { | ||
159 | return 1; | ||
160 | } | ||
161 | |||
162 | static inline int rcu_read_lock_bh_held(void) | ||
163 | { | ||
164 | return 1; | ||
165 | } | ||
166 | |||
167 | static inline int rcu_read_lock_sched_held(void) | ||
168 | { | ||
169 | return preempt_count() != 0 || !rcu_scheduler_active; | ||
170 | } | ||
171 | |||
172 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
173 | |||
174 | #ifdef CONFIG_PROVE_RCU | ||
175 | |||
176 | /** | ||
177 | * rcu_dereference_check - rcu_dereference with debug checking | ||
178 | * | ||
179 | * Do an rcu_dereference(), but check that the context is correct. | ||
180 | * For example, rcu_dereference_check(gp, rcu_read_lock_held()) to | ||
181 | * ensure that the rcu_dereference_check() executes within an RCU | ||
182 | * read-side critical section. It is also possible to check for | ||
183 | * locks being held, for example, by using lockdep_is_held(). | ||
184 | */ | ||
185 | #define rcu_dereference_check(p, c) \ | ||
186 | ({ \ | ||
187 | if (debug_locks && !(c)) \ | ||
188 | lockdep_rcu_dereference(__FILE__, __LINE__); \ | ||
189 | rcu_dereference_raw(p); \ | ||
190 | }) | ||
191 | |||
192 | #else /* #ifdef CONFIG_PROVE_RCU */ | ||
193 | |||
194 | #define rcu_dereference_check(p, c) rcu_dereference_raw(p) | ||
195 | |||
196 | #endif /* #else #ifdef CONFIG_PROVE_RCU */ | ||
89 | 197 | ||
90 | /** | 198 | /** |
91 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. | 199 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. |
@@ -160,7 +268,7 @@ static inline void rcu_read_lock_bh(void) | |||
160 | { | 268 | { |
161 | __rcu_read_lock_bh(); | 269 | __rcu_read_lock_bh(); |
162 | __acquire(RCU_BH); | 270 | __acquire(RCU_BH); |
163 | rcu_read_acquire(); | 271 | rcu_read_acquire_bh(); |
164 | } | 272 | } |
165 | 273 | ||
166 | /* | 274 | /* |
@@ -170,7 +278,7 @@ static inline void rcu_read_lock_bh(void) | |||
170 | */ | 278 | */ |
171 | static inline void rcu_read_unlock_bh(void) | 279 | static inline void rcu_read_unlock_bh(void) |
172 | { | 280 | { |
173 | rcu_read_release(); | 281 | rcu_read_release_bh(); |
174 | __release(RCU_BH); | 282 | __release(RCU_BH); |
175 | __rcu_read_unlock_bh(); | 283 | __rcu_read_unlock_bh(); |
176 | } | 284 | } |
@@ -188,7 +296,7 @@ static inline void rcu_read_lock_sched(void) | |||
188 | { | 296 | { |
189 | preempt_disable(); | 297 | preempt_disable(); |
190 | __acquire(RCU_SCHED); | 298 | __acquire(RCU_SCHED); |
191 | rcu_read_acquire(); | 299 | rcu_read_acquire_sched(); |
192 | } | 300 | } |
193 | 301 | ||
194 | /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ | 302 | /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ |
@@ -205,7 +313,7 @@ static inline notrace void rcu_read_lock_sched_notrace(void) | |||
205 | */ | 313 | */ |
206 | static inline void rcu_read_unlock_sched(void) | 314 | static inline void rcu_read_unlock_sched(void) |
207 | { | 315 | { |
208 | rcu_read_release(); | 316 | rcu_read_release_sched(); |
209 | __release(RCU_SCHED); | 317 | __release(RCU_SCHED); |
210 | preempt_enable(); | 318 | preempt_enable(); |
211 | } | 319 | } |
@@ -219,22 +327,49 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) | |||
219 | 327 | ||
220 | 328 | ||
221 | /** | 329 | /** |
222 | * rcu_dereference - fetch an RCU-protected pointer in an | 330 | * rcu_dereference_raw - fetch an RCU-protected pointer |
223 | * RCU read-side critical section. This pointer may later | 331 | * |
224 | * be safely dereferenced. | 332 | * The caller must be within some flavor of RCU read-side critical |
333 | * section, or must be otherwise preventing the pointer from changing, | ||
334 | * for example, by holding an appropriate lock. This pointer may later | ||
335 | * be safely dereferenced. It is the caller's responsibility to have | ||
336 | * done the right thing, as this primitive does no checking of any kind. | ||
225 | * | 337 | * |
226 | * Inserts memory barriers on architectures that require them | 338 | * Inserts memory barriers on architectures that require them |
227 | * (currently only the Alpha), and, more importantly, documents | 339 | * (currently only the Alpha), and, more importantly, documents |
228 | * exactly which pointers are protected by RCU. | 340 | * exactly which pointers are protected by RCU. |
229 | */ | 341 | */ |
230 | 342 | #define rcu_dereference_raw(p) ({ \ | |
231 | #define rcu_dereference(p) ({ \ | ||
232 | typeof(p) _________p1 = ACCESS_ONCE(p); \ | 343 | typeof(p) _________p1 = ACCESS_ONCE(p); \ |
233 | smp_read_barrier_depends(); \ | 344 | smp_read_barrier_depends(); \ |
234 | (_________p1); \ | 345 | (_________p1); \ |
235 | }) | 346 | }) |
236 | 347 | ||
237 | /** | 348 | /** |
349 | * rcu_dereference - fetch an RCU-protected pointer, checking for RCU | ||
350 | * | ||
351 | * Makes rcu_dereference_check() do the dirty work. | ||
352 | */ | ||
353 | #define rcu_dereference(p) \ | ||
354 | rcu_dereference_check(p, rcu_read_lock_held()) | ||
355 | |||
356 | /** | ||
357 | * rcu_dereference_bh - fetch an RCU-protected pointer, checking for RCU-bh | ||
358 | * | ||
359 | * Makes rcu_dereference_check() do the dirty work. | ||
360 | */ | ||
361 | #define rcu_dereference_bh(p) \ | ||
362 | rcu_dereference_check(p, rcu_read_lock_bh_held()) | ||
363 | |||
364 | /** | ||
365 | * rcu_dereference_sched - fetch RCU-protected pointer, checking for RCU-sched | ||
366 | * | ||
367 | * Makes rcu_dereference_check() do the dirty work. | ||
368 | */ | ||
369 | #define rcu_dereference_sched(p) \ | ||
370 | rcu_dereference_check(p, rcu_read_lock_sched_held()) | ||
371 | |||
372 | /** | ||
238 | * rcu_assign_pointer - assign (publicize) a pointer to a newly | 373 | * rcu_assign_pointer - assign (publicize) a pointer to a newly |
239 | * initialized structure that will be dereferenced by RCU read-side | 374 | * initialized structure that will be dereferenced by RCU read-side |
240 | * critical sections. Returns the value assigned. | 375 | * critical sections. Returns the value assigned. |
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 96cc307ed9f4..a5195875480a 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h | |||
@@ -62,6 +62,18 @@ static inline long rcu_batches_completed_bh(void) | |||
62 | 62 | ||
63 | extern int rcu_expedited_torture_stats(char *page); | 63 | extern int rcu_expedited_torture_stats(char *page); |
64 | 64 | ||
65 | static inline void rcu_force_quiescent_state(void) | ||
66 | { | ||
67 | } | ||
68 | |||
69 | static inline void rcu_bh_force_quiescent_state(void) | ||
70 | { | ||
71 | } | ||
72 | |||
73 | static inline void rcu_sched_force_quiescent_state(void) | ||
74 | { | ||
75 | } | ||
76 | |||
65 | #define synchronize_rcu synchronize_sched | 77 | #define synchronize_rcu synchronize_sched |
66 | 78 | ||
67 | static inline void synchronize_rcu_expedited(void) | 79 | static inline void synchronize_rcu_expedited(void) |
@@ -93,10 +105,6 @@ static inline void rcu_exit_nohz(void) | |||
93 | 105 | ||
94 | #endif /* #else #ifdef CONFIG_NO_HZ */ | 106 | #endif /* #else #ifdef CONFIG_NO_HZ */ |
95 | 107 | ||
96 | static inline void rcu_scheduler_starting(void) | ||
97 | { | ||
98 | } | ||
99 | |||
100 | static inline void exit_rcu(void) | 108 | static inline void exit_rcu(void) |
101 | { | 109 | { |
102 | } | 110 | } |
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 8044b1b94333..42cc3a04779e 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h | |||
@@ -35,7 +35,6 @@ struct notifier_block; | |||
35 | extern void rcu_sched_qs(int cpu); | 35 | extern void rcu_sched_qs(int cpu); |
36 | extern void rcu_bh_qs(int cpu); | 36 | extern void rcu_bh_qs(int cpu); |
37 | extern int rcu_needs_cpu(int cpu); | 37 | extern int rcu_needs_cpu(int cpu); |
38 | extern void rcu_scheduler_starting(void); | ||
39 | extern int rcu_expedited_torture_stats(char *page); | 38 | extern int rcu_expedited_torture_stats(char *page); |
40 | 39 | ||
41 | #ifdef CONFIG_TREE_PREEMPT_RCU | 40 | #ifdef CONFIG_TREE_PREEMPT_RCU |
@@ -99,6 +98,9 @@ extern void rcu_check_callbacks(int cpu, int user); | |||
99 | extern long rcu_batches_completed(void); | 98 | extern long rcu_batches_completed(void); |
100 | extern long rcu_batches_completed_bh(void); | 99 | extern long rcu_batches_completed_bh(void); |
101 | extern long rcu_batches_completed_sched(void); | 100 | extern long rcu_batches_completed_sched(void); |
101 | extern void rcu_force_quiescent_state(void); | ||
102 | extern void rcu_bh_force_quiescent_state(void); | ||
103 | extern void rcu_sched_force_quiescent_state(void); | ||
102 | 104 | ||
103 | #ifdef CONFIG_NO_HZ | 105 | #ifdef CONFIG_NO_HZ |
104 | void rcu_enter_nohz(void); | 106 | void rcu_enter_nohz(void); |
diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 05330fc5b436..5c52fa43785c 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h | |||
@@ -735,6 +735,9 @@ extern void rtnl_lock(void); | |||
735 | extern void rtnl_unlock(void); | 735 | extern void rtnl_unlock(void); |
736 | extern int rtnl_trylock(void); | 736 | extern int rtnl_trylock(void); |
737 | extern int rtnl_is_locked(void); | 737 | extern int rtnl_is_locked(void); |
738 | #ifdef CONFIG_PROVE_LOCKING | ||
739 | extern int lockdep_rtnl_is_held(void); | ||
740 | #endif /* #ifdef CONFIG_PROVE_LOCKING */ | ||
738 | 741 | ||
739 | extern void rtnetlink_init(void); | 742 | extern void rtnetlink_init(void); |
740 | extern void __rtnl_unlock(void); | 743 | extern void __rtnl_unlock(void); |
diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 4765d97dcafb..3084f80909cd 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h | |||
@@ -35,6 +35,9 @@ struct srcu_struct { | |||
35 | int completed; | 35 | int completed; |
36 | struct srcu_struct_array *per_cpu_ref; | 36 | struct srcu_struct_array *per_cpu_ref; |
37 | struct mutex mutex; | 37 | struct mutex mutex; |
38 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
39 | struct lockdep_map dep_map; | ||
40 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
38 | }; | 41 | }; |
39 | 42 | ||
40 | #ifndef CONFIG_PREEMPT | 43 | #ifndef CONFIG_PREEMPT |
@@ -43,12 +46,100 @@ struct srcu_struct { | |||
43 | #define srcu_barrier() | 46 | #define srcu_barrier() |
44 | #endif /* #else #ifndef CONFIG_PREEMPT */ | 47 | #endif /* #else #ifndef CONFIG_PREEMPT */ |
45 | 48 | ||
49 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
50 | |||
51 | int __init_srcu_struct(struct srcu_struct *sp, const char *name, | ||
52 | struct lock_class_key *key); | ||
53 | |||
54 | #define init_srcu_struct(sp) \ | ||
55 | ({ \ | ||
56 | static struct lock_class_key __srcu_key; \ | ||
57 | \ | ||
58 | __init_srcu_struct((sp), #sp, &__srcu_key); \ | ||
59 | }) | ||
60 | |||
61 | # define srcu_read_acquire(sp) \ | ||
62 | lock_acquire(&(sp)->dep_map, 0, 0, 2, 1, NULL, _THIS_IP_) | ||
63 | # define srcu_read_release(sp) \ | ||
64 | lock_release(&(sp)->dep_map, 1, _THIS_IP_) | ||
65 | |||
66 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
67 | |||
46 | int init_srcu_struct(struct srcu_struct *sp); | 68 | int init_srcu_struct(struct srcu_struct *sp); |
69 | |||
70 | # define srcu_read_acquire(sp) do { } while (0) | ||
71 | # define srcu_read_release(sp) do { } while (0) | ||
72 | |||
73 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
74 | |||
47 | void cleanup_srcu_struct(struct srcu_struct *sp); | 75 | void cleanup_srcu_struct(struct srcu_struct *sp); |
48 | int srcu_read_lock(struct srcu_struct *sp) __acquires(sp); | 76 | int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp); |
49 | void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); | 77 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); |
50 | void synchronize_srcu(struct srcu_struct *sp); | 78 | void synchronize_srcu(struct srcu_struct *sp); |
51 | void synchronize_srcu_expedited(struct srcu_struct *sp); | 79 | void synchronize_srcu_expedited(struct srcu_struct *sp); |
52 | long srcu_batches_completed(struct srcu_struct *sp); | 80 | long srcu_batches_completed(struct srcu_struct *sp); |
53 | 81 | ||
82 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
83 | |||
84 | /** | ||
85 | * srcu_read_lock_held - might we be in SRCU read-side critical section? | ||
86 | * | ||
87 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
88 | * an SRCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
89 | * this assumes we are in an SRCU read-side critical section unless it can | ||
90 | * prove otherwise. | ||
91 | */ | ||
92 | static inline int srcu_read_lock_held(struct srcu_struct *sp) | ||
93 | { | ||
94 | if (debug_locks) | ||
95 | return lock_is_held(&sp->dep_map); | ||
96 | return 1; | ||
97 | } | ||
98 | |||
99 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
100 | |||
101 | static inline int srcu_read_lock_held(struct srcu_struct *sp) | ||
102 | { | ||
103 | return 1; | ||
104 | } | ||
105 | |||
106 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | ||
107 | |||
108 | /** | ||
109 | * srcu_dereference - fetch SRCU-protected pointer with checking | ||
110 | * | ||
111 | * Makes rcu_dereference_check() do the dirty work. | ||
112 | */ | ||
113 | #define srcu_dereference(p, sp) \ | ||
114 | rcu_dereference_check(p, srcu_read_lock_held(sp)) | ||
115 | |||
116 | /** | ||
117 | * srcu_read_lock - register a new reader for an SRCU-protected structure. | ||
118 | * @sp: srcu_struct in which to register the new reader. | ||
119 | * | ||
120 | * Enter an SRCU read-side critical section. Note that SRCU read-side | ||
121 | * critical sections may be nested. | ||
122 | */ | ||
123 | static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) | ||
124 | { | ||
125 | int retval = __srcu_read_lock(sp); | ||
126 | |||
127 | srcu_read_acquire(sp); | ||
128 | return retval; | ||
129 | } | ||
130 | |||
131 | /** | ||
132 | * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. | ||
133 | * @sp: srcu_struct in which to unregister the old reader. | ||
134 | * @idx: return value from corresponding srcu_read_lock(). | ||
135 | * | ||
136 | * Exit an SRCU read-side critical section. | ||
137 | */ | ||
138 | static inline void srcu_read_unlock(struct srcu_struct *sp, int idx) | ||
139 | __releases(sp) | ||
140 | { | ||
141 | srcu_read_release(sp); | ||
142 | __srcu_read_unlock(sp, idx); | ||
143 | } | ||
144 | |||
54 | #endif | 145 | #endif |
diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 0f7c37825fc1..45375b41a2a0 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h | |||
@@ -177,7 +177,9 @@ extern int unregister_inet6addr_notifier(struct notifier_block *nb); | |||
177 | static inline struct inet6_dev * | 177 | static inline struct inet6_dev * |
178 | __in6_dev_get(struct net_device *dev) | 178 | __in6_dev_get(struct net_device *dev) |
179 | { | 179 | { |
180 | return rcu_dereference(dev->ip6_ptr); | 180 | return rcu_dereference_check(dev->ip6_ptr, |
181 | rcu_read_lock_held() || | ||
182 | lockdep_rtnl_is_held()); | ||
181 | } | 183 | } |
182 | 184 | ||
183 | static inline struct inet6_dev * | 185 | static inline struct inet6_dev * |