aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArnd Bergmann <arnd@arndb.de>2010-02-25 10:55:13 -0500
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2010-08-19 20:18:00 -0400
commit67bdbffd696f29a0b68aa8daa285783a06651583 (patch)
tree1b3304e2fcc51e1aa8946d808b8f6f1d0cb3dbe1
parentca5ecddfa8fcbd948c95530e7e817cee9fb43a3d (diff)
rculist: avoid __rcu annotations
This avoids warnings from missing __rcu annotations in the rculist implementation, making it possible to use the same lists in both RCU and non-RCU cases. We can add rculist annotations later, together with lockdep support for rculist, which is missing as well, but that may involve changing all the users. Signed-off-by: Arnd Bergmann <arnd@arndb.de> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Pavel Emelyanov <xemul@openvz.org> Cc: Sukadev Bhattiprolu <sukadev@us.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
-rw-r--r--include/linux/rculist.h53
-rw-r--r--include/linux/rculist_nulls.h16
-rw-r--r--kernel/pid.c2
3 files changed, 46 insertions, 25 deletions
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index 4ec3b38ce9c5..c10b1050dbe6 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -10,6 +10,12 @@
10#include <linux/rcupdate.h> 10#include <linux/rcupdate.h>
11 11
12/* 12/*
13 * return the ->next pointer of a list_head in an rcu safe
14 * way, we must not access it directly
15 */
16#define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
17
18/*
13 * Insert a new entry between two known consecutive entries. 19 * Insert a new entry between two known consecutive entries.
14 * 20 *
15 * This is only for internal list manipulation where we know 21 * This is only for internal list manipulation where we know
@@ -20,7 +26,7 @@ static inline void __list_add_rcu(struct list_head *new,
20{ 26{
21 new->next = next; 27 new->next = next;
22 new->prev = prev; 28 new->prev = prev;
23 rcu_assign_pointer(prev->next, new); 29 rcu_assign_pointer(list_next_rcu(prev), new);
24 next->prev = new; 30 next->prev = new;
25} 31}
26 32
@@ -138,7 +144,7 @@ static inline void list_replace_rcu(struct list_head *old,
138{ 144{
139 new->next = old->next; 145 new->next = old->next;
140 new->prev = old->prev; 146 new->prev = old->prev;
141 rcu_assign_pointer(new->prev->next, new); 147 rcu_assign_pointer(list_next_rcu(new->prev), new);
142 new->next->prev = new; 148 new->next->prev = new;
143 old->prev = LIST_POISON2; 149 old->prev = LIST_POISON2;
144} 150}
@@ -193,7 +199,7 @@ static inline void list_splice_init_rcu(struct list_head *list,
193 */ 199 */
194 200
195 last->next = at; 201 last->next = at;
196 rcu_assign_pointer(head->next, first); 202 rcu_assign_pointer(list_next_rcu(head), first);
197 first->prev = head; 203 first->prev = head;
198 at->prev = last; 204 at->prev = last;
199} 205}
@@ -208,7 +214,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
208 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). 214 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
209 */ 215 */
210#define list_entry_rcu(ptr, type, member) \ 216#define list_entry_rcu(ptr, type, member) \
211 container_of(rcu_dereference_raw(ptr), type, member) 217 ({typeof (*ptr) __rcu *__ptr = (typeof (*ptr) __rcu __force *)ptr; \
218 container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \
219 })
212 220
213/** 221/**
214 * list_first_entry_rcu - get the first element from a list 222 * list_first_entry_rcu - get the first element from a list
@@ -225,9 +233,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
225 list_entry_rcu((ptr)->next, type, member) 233 list_entry_rcu((ptr)->next, type, member)
226 234
227#define __list_for_each_rcu(pos, head) \ 235#define __list_for_each_rcu(pos, head) \
228 for (pos = rcu_dereference_raw((head)->next); \ 236 for (pos = rcu_dereference_raw(list_next_rcu(head)); \
229 pos != (head); \ 237 pos != (head); \
230 pos = rcu_dereference_raw(pos->next)) 238 pos = rcu_dereference_raw(list_next_rcu((pos)))
231 239
232/** 240/**
233 * list_for_each_entry_rcu - iterate over rcu list of given type 241 * list_for_each_entry_rcu - iterate over rcu list of given type
@@ -257,9 +265,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
257 * as long as the traversal is guarded by rcu_read_lock(). 265 * as long as the traversal is guarded by rcu_read_lock().
258 */ 266 */
259#define list_for_each_continue_rcu(pos, head) \ 267#define list_for_each_continue_rcu(pos, head) \
260 for ((pos) = rcu_dereference_raw((pos)->next); \ 268 for ((pos) = rcu_dereference_raw(list_next_rcu(pos)); \
261 prefetch((pos)->next), (pos) != (head); \ 269 prefetch((pos)->next), (pos) != (head); \
262 (pos) = rcu_dereference_raw((pos)->next)) 270 (pos) = rcu_dereference_raw(list_next_rcu(pos)))
263 271
264/** 272/**
265 * list_for_each_entry_continue_rcu - continue iteration over list of given type 273 * list_for_each_entry_continue_rcu - continue iteration over list of given type
@@ -314,12 +322,19 @@ static inline void hlist_replace_rcu(struct hlist_node *old,
314 322
315 new->next = next; 323 new->next = next;
316 new->pprev = old->pprev; 324 new->pprev = old->pprev;
317 rcu_assign_pointer(*new->pprev, new); 325 rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
318 if (next) 326 if (next)
319 new->next->pprev = &new->next; 327 new->next->pprev = &new->next;
320 old->pprev = LIST_POISON2; 328 old->pprev = LIST_POISON2;
321} 329}
322 330
331/*
332 * return the first or the next element in an RCU protected hlist
333 */
334#define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first)))
335#define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next)))
336#define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
337
323/** 338/**
324 * hlist_add_head_rcu 339 * hlist_add_head_rcu
325 * @n: the element to add to the hash list. 340 * @n: the element to add to the hash list.
@@ -346,7 +361,7 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,
346 361
347 n->next = first; 362 n->next = first;
348 n->pprev = &h->first; 363 n->pprev = &h->first;
349 rcu_assign_pointer(h->first, n); 364 rcu_assign_pointer(hlist_first_rcu(h), n);
350 if (first) 365 if (first)
351 first->pprev = &n->next; 366 first->pprev = &n->next;
352} 367}
@@ -374,7 +389,7 @@ static inline void hlist_add_before_rcu(struct hlist_node *n,
374{ 389{
375 n->pprev = next->pprev; 390 n->pprev = next->pprev;
376 n->next = next; 391 n->next = next;
377 rcu_assign_pointer(*(n->pprev), n); 392 rcu_assign_pointer(hlist_pprev_rcu(n), n);
378 next->pprev = &n->next; 393 next->pprev = &n->next;
379} 394}
380 395
@@ -401,15 +416,15 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
401{ 416{
402 n->next = prev->next; 417 n->next = prev->next;
403 n->pprev = &prev->next; 418 n->pprev = &prev->next;
404 rcu_assign_pointer(prev->next, n); 419 rcu_assign_pointer(hlist_next_rcu(prev), n);
405 if (n->next) 420 if (n->next)
406 n->next->pprev = &n->next; 421 n->next->pprev = &n->next;
407} 422}
408 423
409#define __hlist_for_each_rcu(pos, head) \ 424#define __hlist_for_each_rcu(pos, head) \
410 for (pos = rcu_dereference((head)->first); \ 425 for (pos = rcu_dereference(hlist_first_rcu(head)); \
411 pos && ({ prefetch(pos->next); 1; }); \ 426 pos && ({ prefetch(pos->next); 1; }); \
412 pos = rcu_dereference(pos->next)) 427 pos = rcu_dereference(hlist_next_rcu(pos)))
413 428
414/** 429/**
415 * hlist_for_each_entry_rcu - iterate over rcu list of given type 430 * hlist_for_each_entry_rcu - iterate over rcu list of given type
@@ -422,11 +437,11 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
422 * the _rcu list-mutation primitives such as hlist_add_head_rcu() 437 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
423 * as long as the traversal is guarded by rcu_read_lock(). 438 * as long as the traversal is guarded by rcu_read_lock().
424 */ 439 */
425#define hlist_for_each_entry_rcu(tpos, pos, head, member) \ 440#define hlist_for_each_entry_rcu(tpos, pos, head, member) \
426 for (pos = rcu_dereference_raw((head)->first); \ 441 for (pos = rcu_dereference_raw(hlist_first_rcu(head)); \
427 pos && ({ prefetch(pos->next); 1; }) && \ 442 pos && ({ prefetch(pos->next); 1; }) && \
428 ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ 443 ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
429 pos = rcu_dereference_raw(pos->next)) 444 pos = rcu_dereference_raw(hlist_next_rcu(pos)))
430 445
431/** 446/**
432 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type 447 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h
index b70ffe53cb9f..2ae13714828b 100644
--- a/include/linux/rculist_nulls.h
+++ b/include/linux/rculist_nulls.h
@@ -37,6 +37,12 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n)
37 } 37 }
38} 38}
39 39
40#define hlist_nulls_first_rcu(head) \
41 (*((struct hlist_nulls_node __rcu __force **)&(head)->first))
42
43#define hlist_nulls_next_rcu(node) \
44 (*((struct hlist_nulls_node __rcu __force **)&(node)->next))
45
40/** 46/**
41 * hlist_nulls_del_rcu - deletes entry from hash list without re-initialization 47 * hlist_nulls_del_rcu - deletes entry from hash list without re-initialization
42 * @n: the element to delete from the hash list. 48 * @n: the element to delete from the hash list.
@@ -88,7 +94,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,
88 94
89 n->next = first; 95 n->next = first;
90 n->pprev = &h->first; 96 n->pprev = &h->first;
91 rcu_assign_pointer(h->first, n); 97 rcu_assign_pointer(hlist_nulls_first_rcu(h), n);
92 if (!is_a_nulls(first)) 98 if (!is_a_nulls(first))
93 first->pprev = &n->next; 99 first->pprev = &n->next;
94} 100}
@@ -100,11 +106,11 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,
100 * @member: the name of the hlist_nulls_node within the struct. 106 * @member: the name of the hlist_nulls_node within the struct.
101 * 107 *
102 */ 108 */
103#define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ 109#define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \
104 for (pos = rcu_dereference_raw((head)->first); \ 110 for (pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \
105 (!is_a_nulls(pos)) && \ 111 (!is_a_nulls(pos)) && \
106 ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ 112 ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \
107 pos = rcu_dereference_raw(pos->next)) 113 pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos)))
108 114
109#endif 115#endif
110#endif 116#endif
diff --git a/kernel/pid.c b/kernel/pid.c
index d55c6fb8d087..0f90c2f713f1 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -401,7 +401,7 @@ struct task_struct *pid_task(struct pid *pid, enum pid_type type)
401 struct task_struct *result = NULL; 401 struct task_struct *result = NULL;
402 if (pid) { 402 if (pid) {
403 struct hlist_node *first; 403 struct hlist_node *first;
404 first = rcu_dereference_check(pid->tasks[type].first, 404 first = rcu_dereference_check(hlist_first_rcu(&pid->tasks[type]),
405 rcu_read_lock_held() || 405 rcu_read_lock_held() ||
406 lockdep_tasklist_lock_is_held()); 406 lockdep_tasklist_lock_is_held());
407 if (first) 407 if (first)