diff options
author | Arnd Bergmann <arnd@arndb.de> | 2010-02-25 10:55:13 -0500 |
---|---|---|
committer | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2010-08-19 20:18:00 -0400 |
commit | 67bdbffd696f29a0b68aa8daa285783a06651583 (patch) | |
tree | 1b3304e2fcc51e1aa8946d808b8f6f1d0cb3dbe1 | |
parent | ca5ecddfa8fcbd948c95530e7e817cee9fb43a3d (diff) |
rculist: avoid __rcu annotations
This avoids warnings from missing __rcu annotations
in the rculist implementation, making it possible to
use the same lists in both RCU and non-RCU cases.
We can add rculist annotations later, together with
lockdep support for rculist, which is missing as well,
but that may involve changing all the users.
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@us.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
-rw-r--r-- | include/linux/rculist.h | 53 | ||||
-rw-r--r-- | include/linux/rculist_nulls.h | 16 | ||||
-rw-r--r-- | kernel/pid.c | 2 |
3 files changed, 46 insertions, 25 deletions
diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 4ec3b38ce9c5..c10b1050dbe6 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h | |||
@@ -10,6 +10,12 @@ | |||
10 | #include <linux/rcupdate.h> | 10 | #include <linux/rcupdate.h> |
11 | 11 | ||
12 | /* | 12 | /* |
13 | * return the ->next pointer of a list_head in an rcu safe | ||
14 | * way, we must not access it directly | ||
15 | */ | ||
16 | #define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next))) | ||
17 | |||
18 | /* | ||
13 | * Insert a new entry between two known consecutive entries. | 19 | * Insert a new entry between two known consecutive entries. |
14 | * | 20 | * |
15 | * This is only for internal list manipulation where we know | 21 | * This is only for internal list manipulation where we know |
@@ -20,7 +26,7 @@ static inline void __list_add_rcu(struct list_head *new, | |||
20 | { | 26 | { |
21 | new->next = next; | 27 | new->next = next; |
22 | new->prev = prev; | 28 | new->prev = prev; |
23 | rcu_assign_pointer(prev->next, new); | 29 | rcu_assign_pointer(list_next_rcu(prev), new); |
24 | next->prev = new; | 30 | next->prev = new; |
25 | } | 31 | } |
26 | 32 | ||
@@ -138,7 +144,7 @@ static inline void list_replace_rcu(struct list_head *old, | |||
138 | { | 144 | { |
139 | new->next = old->next; | 145 | new->next = old->next; |
140 | new->prev = old->prev; | 146 | new->prev = old->prev; |
141 | rcu_assign_pointer(new->prev->next, new); | 147 | rcu_assign_pointer(list_next_rcu(new->prev), new); |
142 | new->next->prev = new; | 148 | new->next->prev = new; |
143 | old->prev = LIST_POISON2; | 149 | old->prev = LIST_POISON2; |
144 | } | 150 | } |
@@ -193,7 +199,7 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
193 | */ | 199 | */ |
194 | 200 | ||
195 | last->next = at; | 201 | last->next = at; |
196 | rcu_assign_pointer(head->next, first); | 202 | rcu_assign_pointer(list_next_rcu(head), first); |
197 | first->prev = head; | 203 | first->prev = head; |
198 | at->prev = last; | 204 | at->prev = last; |
199 | } | 205 | } |
@@ -208,7 +214,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
208 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). | 214 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
209 | */ | 215 | */ |
210 | #define list_entry_rcu(ptr, type, member) \ | 216 | #define list_entry_rcu(ptr, type, member) \ |
211 | container_of(rcu_dereference_raw(ptr), type, member) | 217 | ({typeof (*ptr) __rcu *__ptr = (typeof (*ptr) __rcu __force *)ptr; \ |
218 | container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \ | ||
219 | }) | ||
212 | 220 | ||
213 | /** | 221 | /** |
214 | * list_first_entry_rcu - get the first element from a list | 222 | * list_first_entry_rcu - get the first element from a list |
@@ -225,9 +233,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
225 | list_entry_rcu((ptr)->next, type, member) | 233 | list_entry_rcu((ptr)->next, type, member) |
226 | 234 | ||
227 | #define __list_for_each_rcu(pos, head) \ | 235 | #define __list_for_each_rcu(pos, head) \ |
228 | for (pos = rcu_dereference_raw((head)->next); \ | 236 | for (pos = rcu_dereference_raw(list_next_rcu(head)); \ |
229 | pos != (head); \ | 237 | pos != (head); \ |
230 | pos = rcu_dereference_raw(pos->next)) | 238 | pos = rcu_dereference_raw(list_next_rcu((pos))) |
231 | 239 | ||
232 | /** | 240 | /** |
233 | * list_for_each_entry_rcu - iterate over rcu list of given type | 241 | * list_for_each_entry_rcu - iterate over rcu list of given type |
@@ -257,9 +265,9 @@ static inline void list_splice_init_rcu(struct list_head *list, | |||
257 | * as long as the traversal is guarded by rcu_read_lock(). | 265 | * as long as the traversal is guarded by rcu_read_lock(). |
258 | */ | 266 | */ |
259 | #define list_for_each_continue_rcu(pos, head) \ | 267 | #define list_for_each_continue_rcu(pos, head) \ |
260 | for ((pos) = rcu_dereference_raw((pos)->next); \ | 268 | for ((pos) = rcu_dereference_raw(list_next_rcu(pos)); \ |
261 | prefetch((pos)->next), (pos) != (head); \ | 269 | prefetch((pos)->next), (pos) != (head); \ |
262 | (pos) = rcu_dereference_raw((pos)->next)) | 270 | (pos) = rcu_dereference_raw(list_next_rcu(pos))) |
263 | 271 | ||
264 | /** | 272 | /** |
265 | * list_for_each_entry_continue_rcu - continue iteration over list of given type | 273 | * list_for_each_entry_continue_rcu - continue iteration over list of given type |
@@ -314,12 +322,19 @@ static inline void hlist_replace_rcu(struct hlist_node *old, | |||
314 | 322 | ||
315 | new->next = next; | 323 | new->next = next; |
316 | new->pprev = old->pprev; | 324 | new->pprev = old->pprev; |
317 | rcu_assign_pointer(*new->pprev, new); | 325 | rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new); |
318 | if (next) | 326 | if (next) |
319 | new->next->pprev = &new->next; | 327 | new->next->pprev = &new->next; |
320 | old->pprev = LIST_POISON2; | 328 | old->pprev = LIST_POISON2; |
321 | } | 329 | } |
322 | 330 | ||
331 | /* | ||
332 | * return the first or the next element in an RCU protected hlist | ||
333 | */ | ||
334 | #define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first))) | ||
335 | #define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next))) | ||
336 | #define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev))) | ||
337 | |||
323 | /** | 338 | /** |
324 | * hlist_add_head_rcu | 339 | * hlist_add_head_rcu |
325 | * @n: the element to add to the hash list. | 340 | * @n: the element to add to the hash list. |
@@ -346,7 +361,7 @@ static inline void hlist_add_head_rcu(struct hlist_node *n, | |||
346 | 361 | ||
347 | n->next = first; | 362 | n->next = first; |
348 | n->pprev = &h->first; | 363 | n->pprev = &h->first; |
349 | rcu_assign_pointer(h->first, n); | 364 | rcu_assign_pointer(hlist_first_rcu(h), n); |
350 | if (first) | 365 | if (first) |
351 | first->pprev = &n->next; | 366 | first->pprev = &n->next; |
352 | } | 367 | } |
@@ -374,7 +389,7 @@ static inline void hlist_add_before_rcu(struct hlist_node *n, | |||
374 | { | 389 | { |
375 | n->pprev = next->pprev; | 390 | n->pprev = next->pprev; |
376 | n->next = next; | 391 | n->next = next; |
377 | rcu_assign_pointer(*(n->pprev), n); | 392 | rcu_assign_pointer(hlist_pprev_rcu(n), n); |
378 | next->pprev = &n->next; | 393 | next->pprev = &n->next; |
379 | } | 394 | } |
380 | 395 | ||
@@ -401,15 +416,15 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, | |||
401 | { | 416 | { |
402 | n->next = prev->next; | 417 | n->next = prev->next; |
403 | n->pprev = &prev->next; | 418 | n->pprev = &prev->next; |
404 | rcu_assign_pointer(prev->next, n); | 419 | rcu_assign_pointer(hlist_next_rcu(prev), n); |
405 | if (n->next) | 420 | if (n->next) |
406 | n->next->pprev = &n->next; | 421 | n->next->pprev = &n->next; |
407 | } | 422 | } |
408 | 423 | ||
409 | #define __hlist_for_each_rcu(pos, head) \ | 424 | #define __hlist_for_each_rcu(pos, head) \ |
410 | for (pos = rcu_dereference((head)->first); \ | 425 | for (pos = rcu_dereference(hlist_first_rcu(head)); \ |
411 | pos && ({ prefetch(pos->next); 1; }); \ | 426 | pos && ({ prefetch(pos->next); 1; }); \ |
412 | pos = rcu_dereference(pos->next)) | 427 | pos = rcu_dereference(hlist_next_rcu(pos))) |
413 | 428 | ||
414 | /** | 429 | /** |
415 | * hlist_for_each_entry_rcu - iterate over rcu list of given type | 430 | * hlist_for_each_entry_rcu - iterate over rcu list of given type |
@@ -422,11 +437,11 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, | |||
422 | * the _rcu list-mutation primitives such as hlist_add_head_rcu() | 437 | * the _rcu list-mutation primitives such as hlist_add_head_rcu() |
423 | * as long as the traversal is guarded by rcu_read_lock(). | 438 | * as long as the traversal is guarded by rcu_read_lock(). |
424 | */ | 439 | */ |
425 | #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ | 440 | #define hlist_for_each_entry_rcu(tpos, pos, head, member) \ |
426 | for (pos = rcu_dereference_raw((head)->first); \ | 441 | for (pos = rcu_dereference_raw(hlist_first_rcu(head)); \ |
427 | pos && ({ prefetch(pos->next); 1; }) && \ | 442 | pos && ({ prefetch(pos->next); 1; }) && \ |
428 | ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ | 443 | ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ |
429 | pos = rcu_dereference_raw(pos->next)) | 444 | pos = rcu_dereference_raw(hlist_next_rcu(pos))) |
430 | 445 | ||
431 | /** | 446 | /** |
432 | * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type | 447 | * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type |
diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index b70ffe53cb9f..2ae13714828b 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h | |||
@@ -37,6 +37,12 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) | |||
37 | } | 37 | } |
38 | } | 38 | } |
39 | 39 | ||
40 | #define hlist_nulls_first_rcu(head) \ | ||
41 | (*((struct hlist_nulls_node __rcu __force **)&(head)->first)) | ||
42 | |||
43 | #define hlist_nulls_next_rcu(node) \ | ||
44 | (*((struct hlist_nulls_node __rcu __force **)&(node)->next)) | ||
45 | |||
40 | /** | 46 | /** |
41 | * hlist_nulls_del_rcu - deletes entry from hash list without re-initialization | 47 | * hlist_nulls_del_rcu - deletes entry from hash list without re-initialization |
42 | * @n: the element to delete from the hash list. | 48 | * @n: the element to delete from the hash list. |
@@ -88,7 +94,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, | |||
88 | 94 | ||
89 | n->next = first; | 95 | n->next = first; |
90 | n->pprev = &h->first; | 96 | n->pprev = &h->first; |
91 | rcu_assign_pointer(h->first, n); | 97 | rcu_assign_pointer(hlist_nulls_first_rcu(h), n); |
92 | if (!is_a_nulls(first)) | 98 | if (!is_a_nulls(first)) |
93 | first->pprev = &n->next; | 99 | first->pprev = &n->next; |
94 | } | 100 | } |
@@ -100,11 +106,11 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, | |||
100 | * @member: the name of the hlist_nulls_node within the struct. | 106 | * @member: the name of the hlist_nulls_node within the struct. |
101 | * | 107 | * |
102 | */ | 108 | */ |
103 | #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ | 109 | #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ |
104 | for (pos = rcu_dereference_raw((head)->first); \ | 110 | for (pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \ |
105 | (!is_a_nulls(pos)) && \ | 111 | (!is_a_nulls(pos)) && \ |
106 | ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ | 112 | ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ |
107 | pos = rcu_dereference_raw(pos->next)) | 113 | pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos))) |
108 | 114 | ||
109 | #endif | 115 | #endif |
110 | #endif | 116 | #endif |
diff --git a/kernel/pid.c b/kernel/pid.c index d55c6fb8d087..0f90c2f713f1 100644 --- a/kernel/pid.c +++ b/kernel/pid.c | |||
@@ -401,7 +401,7 @@ struct task_struct *pid_task(struct pid *pid, enum pid_type type) | |||
401 | struct task_struct *result = NULL; | 401 | struct task_struct *result = NULL; |
402 | if (pid) { | 402 | if (pid) { |
403 | struct hlist_node *first; | 403 | struct hlist_node *first; |
404 | first = rcu_dereference_check(pid->tasks[type].first, | 404 | first = rcu_dereference_check(hlist_first_rcu(&pid->tasks[type]), |
405 | rcu_read_lock_held() || | 405 | rcu_read_lock_held() || |
406 | lockdep_tasklist_lock_is_held()); | 406 | lockdep_tasklist_lock_is_held()); |
407 | if (first) | 407 | if (first) |