aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--fs/file.c2
-rw-r--r--include/linux/rculist.h17
-rw-r--r--include/linux/rcupdate.h94
-rw-r--r--include/linux/rcutiny.h10
-rw-r--r--include/linux/rcutree.h6
-rw-r--r--include/linux/srcu.h4
-rw-r--r--kernel/ksysfs.c2
-rw-r--r--kernel/notifier.c2
-rw-r--r--kernel/rcu/rcu.h7
-rw-r--r--kernel/rcu/srcu.c11
-rw-r--r--kernel/rcu/tiny.c8
-rw-r--r--kernel/rcu/tiny_plugin.h4
-rw-r--r--kernel/rcu/torture.c8
-rw-r--r--kernel/rcu/tree.c18
-rw-r--r--kernel/rcu/tree.h4
-rw-r--r--kernel/rcu/tree_plugin.h19
-rw-r--r--kernel/rcu/tree_trace.c6
-rw-r--r--kernel/rcu/update.c5
18 files changed, 114 insertions, 113 deletions
diff --git a/fs/file.c b/fs/file.c
index db25c2bdfe46..18f7d27855c4 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -497,7 +497,7 @@ repeat:
497 error = fd; 497 error = fd;
498#if 1 498#if 1
499 /* Sanity check */ 499 /* Sanity check */
500 if (rcu_dereference_raw(fdt->fd[fd]) != NULL) { 500 if (rcu_access_pointer(fdt->fd[fd]) != NULL) {
501 printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd); 501 printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd);
502 rcu_assign_pointer(fdt->fd[fd], NULL); 502 rcu_assign_pointer(fdt->fd[fd], NULL);
503 } 503 }
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index dbaf99084112..8183b46fbaa2 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -247,9 +247,10 @@ static inline void list_splice_init_rcu(struct list_head *list,
247 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). 247 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
248 */ 248 */
249#define list_entry_rcu(ptr, type, member) \ 249#define list_entry_rcu(ptr, type, member) \
250 ({typeof (*ptr) __rcu *__ptr = (typeof (*ptr) __rcu __force *)ptr; \ 250({ \
251 container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \ 251 typeof(*ptr) __rcu *__ptr = (typeof(*ptr) __rcu __force *)ptr; \
252 }) 252 container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \
253})
253 254
254/** 255/**
255 * Where are list_empty_rcu() and list_first_entry_rcu()? 256 * Where are list_empty_rcu() and list_first_entry_rcu()?
@@ -285,11 +286,11 @@ static inline void list_splice_init_rcu(struct list_head *list,
285 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). 286 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
286 */ 287 */
287#define list_first_or_null_rcu(ptr, type, member) \ 288#define list_first_or_null_rcu(ptr, type, member) \
288 ({struct list_head *__ptr = (ptr); \ 289({ \
289 struct list_head *__next = ACCESS_ONCE(__ptr->next); \ 290 struct list_head *__ptr = (ptr); \
290 likely(__ptr != __next) ? \ 291 struct list_head *__next = ACCESS_ONCE(__ptr->next); \
291 list_entry_rcu(__next, type, member) : NULL; \ 292 likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
292 }) 293})
293 294
294/** 295/**
295 * list_for_each_entry_rcu - iterate over rcu list of given type 296 * list_for_each_entry_rcu - iterate over rcu list of given type
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 72bf3a01a4ee..cda2583d67e9 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2001 18 * Copyright IBM Corporation, 2001
19 * 19 *
@@ -44,7 +44,9 @@
44#include <linux/debugobjects.h> 44#include <linux/debugobjects.h>
45#include <linux/bug.h> 45#include <linux/bug.h>
46#include <linux/compiler.h> 46#include <linux/compiler.h>
47#include <asm/barrier.h>
47 48
49extern int rcu_expedited; /* for sysctl */
48#ifdef CONFIG_RCU_TORTURE_TEST 50#ifdef CONFIG_RCU_TORTURE_TEST
49extern int rcutorture_runnable; /* for sysctl */ 51extern int rcutorture_runnable; /* for sysctl */
50#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */ 52#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
@@ -479,11 +481,9 @@ static inline void rcu_preempt_sleep_check(void)
479 do { \ 481 do { \
480 rcu_preempt_sleep_check(); \ 482 rcu_preempt_sleep_check(); \
481 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \ 483 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \
482 "Illegal context switch in RCU-bh" \ 484 "Illegal context switch in RCU-bh read-side critical section"); \
483 " read-side critical section"); \
484 rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \ 485 rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \
485 "Illegal context switch in RCU-sched"\ 486 "Illegal context switch in RCU-sched read-side critical section"); \
486 " read-side critical section"); \
487 } while (0) 487 } while (0)
488 488
489#else /* #ifdef CONFIG_PROVE_RCU */ 489#else /* #ifdef CONFIG_PROVE_RCU */
@@ -510,43 +510,40 @@ static inline void rcu_preempt_sleep_check(void)
510#endif /* #else #ifdef __CHECKER__ */ 510#endif /* #else #ifdef __CHECKER__ */
511 511
512#define __rcu_access_pointer(p, space) \ 512#define __rcu_access_pointer(p, space) \
513 ({ \ 513({ \
514 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \ 514 typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \
515 rcu_dereference_sparse(p, space); \ 515 rcu_dereference_sparse(p, space); \
516 ((typeof(*p) __force __kernel *)(_________p1)); \ 516 ((typeof(*p) __force __kernel *)(_________p1)); \
517 }) 517})
518#define __rcu_dereference_check(p, c, space) \ 518#define __rcu_dereference_check(p, c, space) \
519 ({ \ 519({ \
520 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \ 520 typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \
521 rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \ 521 rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \
522 " usage"); \ 522 rcu_dereference_sparse(p, space); \
523 rcu_dereference_sparse(p, space); \ 523 smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
524 smp_read_barrier_depends(); \ 524 ((typeof(*p) __force __kernel *)(_________p1)); \
525 ((typeof(*p) __force __kernel *)(_________p1)); \ 525})
526 })
527#define __rcu_dereference_protected(p, c, space) \ 526#define __rcu_dereference_protected(p, c, space) \
528 ({ \ 527({ \
529 rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \ 528 rcu_lockdep_assert(c, "suspicious rcu_dereference_protected() usage"); \
530 " usage"); \ 529 rcu_dereference_sparse(p, space); \
531 rcu_dereference_sparse(p, space); \ 530 ((typeof(*p) __force __kernel *)(p)); \
532 ((typeof(*p) __force __kernel *)(p)); \ 531})
533 })
534 532
535#define __rcu_access_index(p, space) \ 533#define __rcu_access_index(p, space) \
536 ({ \ 534({ \
537 typeof(p) _________p1 = ACCESS_ONCE(p); \ 535 typeof(p) _________p1 = ACCESS_ONCE(p); \
538 rcu_dereference_sparse(p, space); \ 536 rcu_dereference_sparse(p, space); \
539 (_________p1); \ 537 (_________p1); \
540 }) 538})
541#define __rcu_dereference_index_check(p, c) \ 539#define __rcu_dereference_index_check(p, c) \
542 ({ \ 540({ \
543 typeof(p) _________p1 = ACCESS_ONCE(p); \ 541 typeof(p) _________p1 = ACCESS_ONCE(p); \
544 rcu_lockdep_assert(c, \ 542 rcu_lockdep_assert(c, \
545 "suspicious rcu_dereference_index_check()" \ 543 "suspicious rcu_dereference_index_check() usage"); \
546 " usage"); \ 544 smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
547 smp_read_barrier_depends(); \ 545 (_________p1); \
548 (_________p1); \ 546})
549 })
550 547
551/** 548/**
552 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable 549 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
@@ -585,12 +582,7 @@ static inline void rcu_preempt_sleep_check(void)
585 * please be careful when making changes to rcu_assign_pointer() and the 582 * please be careful when making changes to rcu_assign_pointer() and the
586 * other macros that it invokes. 583 * other macros that it invokes.
587 */ 584 */
588#define rcu_assign_pointer(p, v) \ 585#define rcu_assign_pointer(p, v) smp_store_release(&p, RCU_INITIALIZER(v))
589 do { \
590 smp_wmb(); \
591 ACCESS_ONCE(p) = RCU_INITIALIZER(v); \
592 } while (0)
593
594 586
595/** 587/**
596 * rcu_access_pointer() - fetch RCU pointer with no dereferencing 588 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
@@ -1015,11 +1007,21 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
1015#define kfree_rcu(ptr, rcu_head) \ 1007#define kfree_rcu(ptr, rcu_head) \
1016 __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) 1008 __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
1017 1009
1018#ifdef CONFIG_RCU_NOCB_CPU 1010#if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL)
1011static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
1012{
1013 *delta_jiffies = ULONG_MAX;
1014 return 0;
1015}
1016#endif /* #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) */
1017
1018#if defined(CONFIG_RCU_NOCB_CPU_ALL)
1019static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
1020#elif defined(CONFIG_RCU_NOCB_CPU)
1019bool rcu_is_nocb_cpu(int cpu); 1021bool rcu_is_nocb_cpu(int cpu);
1020#else 1022#else
1021static inline bool rcu_is_nocb_cpu(int cpu) { return false; } 1023static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
1022#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ 1024#endif
1023 1025
1024 1026
1025/* Only for use by adaptive-ticks code. */ 1027/* Only for use by adaptive-ticks code. */
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 6f01771b571c..e8cb6e3b52a7 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2008 18 * Copyright IBM Corporation, 2008
19 * 19 *
@@ -68,12 +68,6 @@ static inline void kfree_call_rcu(struct rcu_head *head,
68 call_rcu(head, func); 68 call_rcu(head, func);
69} 69}
70 70
71static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
72{
73 *delta_jiffies = ULONG_MAX;
74 return 0;
75}
76
77static inline void rcu_note_context_switch(int cpu) 71static inline void rcu_note_context_switch(int cpu)
78{ 72{
79 rcu_sched_qs(cpu); 73 rcu_sched_qs(cpu);
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 72137ee8c603..e9c63884df0a 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2008 18 * Copyright IBM Corporation, 2008
19 * 19 *
@@ -31,7 +31,9 @@
31#define __LINUX_RCUTREE_H 31#define __LINUX_RCUTREE_H
32 32
33void rcu_note_context_switch(int cpu); 33void rcu_note_context_switch(int cpu);
34#ifndef CONFIG_RCU_NOCB_CPU_ALL
34int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies); 35int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies);
36#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
35void rcu_cpu_stall_reset(void); 37void rcu_cpu_stall_reset(void);
36 38
37/* 39/*
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 9b058eecd403..a2783cb5d275 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright (C) IBM Corporation, 2006 18 * Copyright (C) IBM Corporation, 2006
19 * Copyright (C) Fujitsu, 2012 19 * Copyright (C) Fujitsu, 2012
diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
index d945a949760f..e660964086e2 100644
--- a/kernel/ksysfs.c
+++ b/kernel/ksysfs.c
@@ -19,6 +19,8 @@
19#include <linux/sched.h> 19#include <linux/sched.h>
20#include <linux/capability.h> 20#include <linux/capability.h>
21 21
22#include <linux/rcupdate.h> /* rcu_expedited */
23
22#define KERNEL_ATTR_RO(_name) \ 24#define KERNEL_ATTR_RO(_name) \
23static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 25static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
24 26
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 2d5cc4ccff7f..db4c8b08a50c 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -309,7 +309,7 @@ int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
309 * racy then it does not matter what the result of the test 309 * racy then it does not matter what the result of the test
310 * is, we re-check the list after having taken the lock anyway: 310 * is, we re-check the list after having taken the lock anyway:
311 */ 311 */
312 if (rcu_dereference_raw(nh->head)) { 312 if (rcu_access_pointer(nh->head)) {
313 down_read(&nh->rwsem); 313 down_read(&nh->rwsem);
314 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, 314 ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
315 nr_calls); 315 nr_calls);
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 79c3877e9c5b..bfda2726ca45 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2011 18 * Copyright IBM Corporation, 2011
19 * 19 *
@@ -23,6 +23,7 @@
23#ifndef __LINUX_RCU_H 23#ifndef __LINUX_RCU_H
24#define __LINUX_RCU_H 24#define __LINUX_RCU_H
25 25
26#include <trace/events/rcu.h>
26#ifdef CONFIG_RCU_TRACE 27#ifdef CONFIG_RCU_TRACE
27#define RCU_TRACE(stmt) stmt 28#define RCU_TRACE(stmt) stmt
28#else /* #ifdef CONFIG_RCU_TRACE */ 29#else /* #ifdef CONFIG_RCU_TRACE */
@@ -116,8 +117,6 @@ static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)
116 } 117 }
117} 118}
118 119
119extern int rcu_expedited;
120
121#ifdef CONFIG_RCU_STALL_COMMON 120#ifdef CONFIG_RCU_STALL_COMMON
122 121
123extern int rcu_cpu_stall_suppress; 122extern int rcu_cpu_stall_suppress;
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index 3318d8284384..c639556f3fa0 100644
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright (C) IBM Corporation, 2006 18 * Copyright (C) IBM Corporation, 2006
19 * Copyright (C) Fujitsu, 2012 19 * Copyright (C) Fujitsu, 2012
@@ -36,8 +36,6 @@
36#include <linux/delay.h> 36#include <linux/delay.h>
37#include <linux/srcu.h> 37#include <linux/srcu.h>
38 38
39#include <trace/events/rcu.h>
40
41#include "rcu.h" 39#include "rcu.h"
42 40
43/* 41/*
@@ -398,7 +396,7 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *head,
398 rcu_batch_queue(&sp->batch_queue, head); 396 rcu_batch_queue(&sp->batch_queue, head);
399 if (!sp->running) { 397 if (!sp->running) {
400 sp->running = true; 398 sp->running = true;
401 schedule_delayed_work(&sp->work, 0); 399 queue_delayed_work(system_power_efficient_wq, &sp->work, 0);
402 } 400 }
403 spin_unlock_irqrestore(&sp->queue_lock, flags); 401 spin_unlock_irqrestore(&sp->queue_lock, flags);
404} 402}
@@ -674,7 +672,8 @@ static void srcu_reschedule(struct srcu_struct *sp)
674 } 672 }
675 673
676 if (pending) 674 if (pending)
677 schedule_delayed_work(&sp->work, SRCU_INTERVAL); 675 queue_delayed_work(system_power_efficient_wq,
676 &sp->work, SRCU_INTERVAL);
678} 677}
679 678
680/* 679/*
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index 1254f312d024..d9efcc13008c 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2008 18 * Copyright IBM Corporation, 2008
19 * 19 *
@@ -37,10 +37,6 @@
37#include <linux/prefetch.h> 37#include <linux/prefetch.h>
38#include <linux/ftrace_event.h> 38#include <linux/ftrace_event.h>
39 39
40#ifdef CONFIG_RCU_TRACE
41#include <trace/events/rcu.h>
42#endif /* #else #ifdef CONFIG_RCU_TRACE */
43
44#include "rcu.h" 40#include "rcu.h"
45 41
46/* Forward declarations for tiny_plugin.h. */ 42/* Forward declarations for tiny_plugin.h. */
diff --git a/kernel/rcu/tiny_plugin.h b/kernel/rcu/tiny_plugin.h
index 280d06cae352..431528520562 100644
--- a/kernel/rcu/tiny_plugin.h
+++ b/kernel/rcu/tiny_plugin.h
@@ -14,8 +14,8 @@
14 * GNU General Public License for more details. 14 * GNU General Public License for more details.
15 * 15 *
16 * You should have received a copy of the GNU General Public License 16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software 17 * along with this program; if not, you can access it online at
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 * 19 *
20 * Copyright (c) 2010 Linaro 20 * Copyright (c) 2010 Linaro
21 * 21 *
diff --git a/kernel/rcu/torture.c b/kernel/rcu/torture.c
index 732f8ae3086a..022c5312b725 100644
--- a/kernel/rcu/torture.c
+++ b/kernel/rcu/torture.c
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright (C) IBM Corporation, 2005, 2006 18 * Copyright (C) IBM Corporation, 2005, 2006
19 * 19 *
@@ -1352,7 +1352,7 @@ rcu_torture_shutdown(void *arg)
1352 unsigned long jiffies_snap; 1352 unsigned long jiffies_snap;
1353 1353
1354 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task started"); 1354 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task started");
1355 jiffies_snap = ACCESS_ONCE(jiffies); 1355 jiffies_snap = jiffies;
1356 while (ULONG_CMP_LT(jiffies_snap, shutdown_time) && 1356 while (ULONG_CMP_LT(jiffies_snap, shutdown_time) &&
1357 !kthread_should_stop()) { 1357 !kthread_should_stop()) {
1358 delta = shutdown_time - jiffies_snap; 1358 delta = shutdown_time - jiffies_snap;
@@ -1361,7 +1361,7 @@ rcu_torture_shutdown(void *arg)
1361 "rcu_torture_shutdown task: %lu jiffies remaining\n", 1361 "rcu_torture_shutdown task: %lu jiffies remaining\n",
1362 torture_type, delta); 1362 torture_type, delta);
1363 schedule_timeout_interruptible(delta); 1363 schedule_timeout_interruptible(delta);
1364 jiffies_snap = ACCESS_ONCE(jiffies); 1364 jiffies_snap = jiffies;
1365 } 1365 }
1366 if (kthread_should_stop()) { 1366 if (kthread_should_stop()) {
1367 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task stopping"); 1367 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task stopping");
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index b3d116cd072d..351faba48b91 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2008 18 * Copyright IBM Corporation, 2008
19 * 19 *
@@ -58,8 +58,6 @@
58#include <linux/suspend.h> 58#include <linux/suspend.h>
59 59
60#include "tree.h" 60#include "tree.h"
61#include <trace/events/rcu.h>
62
63#include "rcu.h" 61#include "rcu.h"
64 62
65MODULE_ALIAS("rcutree"); 63MODULE_ALIAS("rcutree");
@@ -837,7 +835,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
837 * to the next. Only do this for the primary flavor of RCU. 835 * to the next. Only do this for the primary flavor of RCU.
838 */ 836 */
839 if (rdp->rsp == rcu_state && 837 if (rdp->rsp == rcu_state &&
840 ULONG_CMP_GE(ACCESS_ONCE(jiffies), rdp->rsp->jiffies_resched)) { 838 ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
841 rdp->rsp->jiffies_resched += 5; 839 rdp->rsp->jiffies_resched += 5;
842 resched_cpu(rdp->cpu); 840 resched_cpu(rdp->cpu);
843 } 841 }
@@ -847,7 +845,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
847 845
848static void record_gp_stall_check_time(struct rcu_state *rsp) 846static void record_gp_stall_check_time(struct rcu_state *rsp)
849{ 847{
850 unsigned long j = ACCESS_ONCE(jiffies); 848 unsigned long j = jiffies;
851 unsigned long j1; 849 unsigned long j1;
852 850
853 rsp->gp_start = j; 851 rsp->gp_start = j;
@@ -1005,7 +1003,7 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
1005 1003
1006 if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp)) 1004 if (rcu_cpu_stall_suppress || !rcu_gp_in_progress(rsp))
1007 return; 1005 return;
1008 j = ACCESS_ONCE(jiffies); 1006 j = jiffies;
1009 1007
1010 /* 1008 /*
1011 * Lots of memory barriers to reject false positives. 1009 * Lots of memory barriers to reject false positives.
@@ -2304,7 +2302,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
2304 if (rnp_old != NULL) 2302 if (rnp_old != NULL)
2305 raw_spin_unlock(&rnp_old->fqslock); 2303 raw_spin_unlock(&rnp_old->fqslock);
2306 if (ret) { 2304 if (ret) {
2307 rsp->n_force_qs_lh++; 2305 ACCESS_ONCE(rsp->n_force_qs_lh)++;
2308 return; 2306 return;
2309 } 2307 }
2310 rnp_old = rnp; 2308 rnp_old = rnp;
@@ -2316,7 +2314,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
2316 smp_mb__after_unlock_lock(); 2314 smp_mb__after_unlock_lock();
2317 raw_spin_unlock(&rnp_old->fqslock); 2315 raw_spin_unlock(&rnp_old->fqslock);
2318 if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { 2316 if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
2319 rsp->n_force_qs_lh++; 2317 ACCESS_ONCE(rsp->n_force_qs_lh)++;
2320 raw_spin_unlock_irqrestore(&rnp_old->lock, flags); 2318 raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
2321 return; /* Someone beat us to it. */ 2319 return; /* Someone beat us to it. */
2322 } 2320 }
@@ -2880,7 +2878,7 @@ static int rcu_pending(int cpu)
2880 * non-NULL, store an indication of whether all callbacks are lazy. 2878 * non-NULL, store an indication of whether all callbacks are lazy.
2881 * (If there are no callbacks, all of them are deemed to be lazy.) 2879 * (If there are no callbacks, all of them are deemed to be lazy.)
2882 */ 2880 */
2883static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy) 2881static int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
2884{ 2882{
2885 bool al = true; 2883 bool al = true;
2886 bool hc = false; 2884 bool hc = false;
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 8c19873f1ac9..75dc3c39a02a 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -13,8 +13,8 @@
13 * GNU General Public License for more details. 13 * GNU General Public License for more details.
14 * 14 *
15 * You should have received a copy of the GNU General Public License 15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software 16 * along with this program; if not, you can access it online at
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * 18 *
19 * Copyright IBM Corporation, 2008 19 * Copyright IBM Corporation, 2008
20 * 20 *
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 6e2ef4b2b920..962d1d589929 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -14,8 +14,8 @@
14 * GNU General Public License for more details. 14 * GNU General Public License for more details.
15 * 15 *
16 * You should have received a copy of the GNU General Public License 16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software 17 * along with this program; if not, you can access it online at
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 * 19 *
20 * Copyright Red Hat, 2009 20 * Copyright Red Hat, 2009
21 * Copyright IBM Corporation, 2009 21 * Copyright IBM Corporation, 2009
@@ -1586,11 +1586,13 @@ static void rcu_prepare_kthreads(int cpu)
1586 * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs 1586 * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1587 * any flavor of RCU. 1587 * any flavor of RCU.
1588 */ 1588 */
1589#ifndef CONFIG_RCU_NOCB_CPU_ALL
1589int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) 1590int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
1590{ 1591{
1591 *delta_jiffies = ULONG_MAX; 1592 *delta_jiffies = ULONG_MAX;
1592 return rcu_cpu_has_callbacks(cpu, NULL); 1593 return rcu_cpu_has_callbacks(cpu, NULL);
1593} 1594}
1595#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1594 1596
1595/* 1597/*
1596 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up 1598 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
@@ -1656,7 +1658,7 @@ extern int tick_nohz_active;
1656 * only if it has been awhile since the last time we did so. Afterwards, 1658 * only if it has been awhile since the last time we did so. Afterwards,
1657 * if there are any callbacks ready for immediate invocation, return true. 1659 * if there are any callbacks ready for immediate invocation, return true.
1658 */ 1660 */
1659static bool rcu_try_advance_all_cbs(void) 1661static bool __maybe_unused rcu_try_advance_all_cbs(void)
1660{ 1662{
1661 bool cbs_ready = false; 1663 bool cbs_ready = false;
1662 struct rcu_data *rdp; 1664 struct rcu_data *rdp;
@@ -1696,6 +1698,7 @@ static bool rcu_try_advance_all_cbs(void)
1696 * 1698 *
1697 * The caller must have disabled interrupts. 1699 * The caller must have disabled interrupts.
1698 */ 1700 */
1701#ifndef CONFIG_RCU_NOCB_CPU_ALL
1699int rcu_needs_cpu(int cpu, unsigned long *dj) 1702int rcu_needs_cpu(int cpu, unsigned long *dj)
1700{ 1703{
1701 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); 1704 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
@@ -1726,6 +1729,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)
1726 } 1729 }
1727 return 0; 1730 return 0;
1728} 1731}
1732#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1729 1733
1730/* 1734/*
1731 * Prepare a CPU for idle from an RCU perspective. The first major task 1735 * Prepare a CPU for idle from an RCU perspective. The first major task
@@ -1739,6 +1743,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)
1739 */ 1743 */
1740static void rcu_prepare_for_idle(int cpu) 1744static void rcu_prepare_for_idle(int cpu)
1741{ 1745{
1746#ifndef CONFIG_RCU_NOCB_CPU_ALL
1742 struct rcu_data *rdp; 1747 struct rcu_data *rdp;
1743 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); 1748 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1744 struct rcu_node *rnp; 1749 struct rcu_node *rnp;
@@ -1790,6 +1795,7 @@ static void rcu_prepare_for_idle(int cpu)
1790 rcu_accelerate_cbs(rsp, rnp, rdp); 1795 rcu_accelerate_cbs(rsp, rnp, rdp);
1791 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ 1796 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
1792 } 1797 }
1798#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1793} 1799}
1794 1800
1795/* 1801/*
@@ -1799,11 +1805,12 @@ static void rcu_prepare_for_idle(int cpu)
1799 */ 1805 */
1800static void rcu_cleanup_after_idle(int cpu) 1806static void rcu_cleanup_after_idle(int cpu)
1801{ 1807{
1802 1808#ifndef CONFIG_RCU_NOCB_CPU_ALL
1803 if (rcu_is_nocb_cpu(cpu)) 1809 if (rcu_is_nocb_cpu(cpu))
1804 return; 1810 return;
1805 if (rcu_try_advance_all_cbs()) 1811 if (rcu_try_advance_all_cbs())
1806 invoke_rcu_core(); 1812 invoke_rcu_core();
1813#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1807} 1814}
1808 1815
1809/* 1816/*
@@ -2101,6 +2108,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
2101 init_waitqueue_head(&rnp->nocb_gp_wq[1]); 2108 init_waitqueue_head(&rnp->nocb_gp_wq[1]);
2102} 2109}
2103 2110
2111#ifndef CONFIG_RCU_NOCB_CPU_ALL
2104/* Is the specified CPU a no-CPUs CPU? */ 2112/* Is the specified CPU a no-CPUs CPU? */
2105bool rcu_is_nocb_cpu(int cpu) 2113bool rcu_is_nocb_cpu(int cpu)
2106{ 2114{
@@ -2108,6 +2116,7 @@ bool rcu_is_nocb_cpu(int cpu)
2108 return cpumask_test_cpu(cpu, rcu_nocb_mask); 2116 return cpumask_test_cpu(cpu, rcu_nocb_mask);
2109 return false; 2117 return false;
2110} 2118}
2119#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
2111 2120
2112/* 2121/*
2113 * Enqueue the specified string of rcu_head structures onto the specified 2122 * Enqueue the specified string of rcu_head structures onto the specified
@@ -2893,7 +2902,7 @@ static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2893 * CPU unless the grace period has extended for too long. 2902 * CPU unless the grace period has extended for too long.
2894 * 2903 *
2895 * This code relies on the fact that all NO_HZ_FULL CPUs are also 2904 * This code relies on the fact that all NO_HZ_FULL CPUs are also
2896 * CONFIG_RCU_NOCB_CPUs. 2905 * CONFIG_RCU_NOCB_CPU CPUs.
2897 */ 2906 */
2898static bool rcu_nohz_full_cpu(struct rcu_state *rsp) 2907static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
2899{ 2908{
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index 4def475336d4..5cdc62e1beeb 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2008 18 * Copyright IBM Corporation, 2008
19 * 19 *
@@ -273,7 +273,7 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
273 seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n", 273 seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n",
274 rsp->n_force_qs, rsp->n_force_qs_ngp, 274 rsp->n_force_qs, rsp->n_force_qs_ngp,
275 rsp->n_force_qs - rsp->n_force_qs_ngp, 275 rsp->n_force_qs - rsp->n_force_qs_ngp,
276 rsp->n_force_qs_lh, rsp->qlen_lazy, rsp->qlen); 276 ACCESS_ONCE(rsp->n_force_qs_lh), rsp->qlen_lazy, rsp->qlen);
277 for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < rcu_num_nodes; rnp++) { 277 for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < rcu_num_nodes; rnp++) {
278 if (rnp->level != level) { 278 if (rnp->level != level) {
279 seq_puts(m, "\n"); 279 seq_puts(m, "\n");
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index c54609faf233..4c0a9b0af469 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -12,8 +12,8 @@
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, you can access it online at
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 * 17 *
18 * Copyright IBM Corporation, 2001 18 * Copyright IBM Corporation, 2001
19 * 19 *
@@ -49,7 +49,6 @@
49#include <linux/module.h> 49#include <linux/module.h>
50 50
51#define CREATE_TRACE_POINTS 51#define CREATE_TRACE_POINTS
52#include <trace/events/rcu.h>
53 52
54#include "rcu.h" 53#include "rcu.h"
55 54