diff options
-rw-r--r-- | include/linux/rcupdate.h | 4 | ||||
-rw-r--r-- | include/linux/rcutiny.h | 4 | ||||
-rw-r--r-- | include/linux/rcutree.h | 1 | ||||
-rw-r--r-- | kernel/rcupdate.c | 18 | ||||
-rw-r--r-- | kernel/rcutree.c | 19 |
5 files changed, 21 insertions, 25 deletions
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a4de31bd7b4..fcea332a8424 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -62,6 +62,8 @@ extern int sched_expedited_torture_stats(char *page); | |||
62 | 62 | ||
63 | /* Internal to kernel */ | 63 | /* Internal to kernel */ |
64 | extern void rcu_init(void); | 64 | extern void rcu_init(void); |
65 | extern int rcu_scheduler_active; | ||
66 | extern void rcu_scheduler_starting(void); | ||
65 | 67 | ||
66 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) | 68 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) |
67 | #include <linux/rcutree.h> | 69 | #include <linux/rcutree.h> |
@@ -140,7 +142,7 @@ static inline int rcu_read_lock_sched_held(void) | |||
140 | 142 | ||
141 | if (debug_locks) | 143 | if (debug_locks) |
142 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | 144 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); |
143 | return lockdep_opinion || preempt_count() != 0; | 145 | return lockdep_opinion || preempt_count() != 0 || !rcu_scheduler_active; |
144 | } | 146 | } |
145 | 147 | ||
146 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 148 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 2b70d4e37383..a5195875480a 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h | |||
@@ -105,10 +105,6 @@ static inline void rcu_exit_nohz(void) | |||
105 | 105 | ||
106 | #endif /* #else #ifdef CONFIG_NO_HZ */ | 106 | #endif /* #else #ifdef CONFIG_NO_HZ */ |
107 | 107 | ||
108 | static inline void rcu_scheduler_starting(void) | ||
109 | { | ||
110 | } | ||
111 | |||
112 | static inline void exit_rcu(void) | 108 | static inline void exit_rcu(void) |
113 | { | 109 | { |
114 | } | 110 | } |
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 704a010f686c..42cc3a04779e 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h | |||
@@ -35,7 +35,6 @@ struct notifier_block; | |||
35 | extern void rcu_sched_qs(int cpu); | 35 | extern void rcu_sched_qs(int cpu); |
36 | extern void rcu_bh_qs(int cpu); | 36 | extern void rcu_bh_qs(int cpu); |
37 | extern int rcu_needs_cpu(int cpu); | 37 | extern int rcu_needs_cpu(int cpu); |
38 | extern void rcu_scheduler_starting(void); | ||
39 | extern int rcu_expedited_torture_stats(char *page); | 38 | extern int rcu_expedited_torture_stats(char *page); |
40 | 39 | ||
41 | #ifdef CONFIG_TREE_PREEMPT_RCU | 40 | #ifdef CONFIG_TREE_PREEMPT_RCU |
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index 033cb55c26df..7bfa004572b1 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c | |||
@@ -44,6 +44,7 @@ | |||
44 | #include <linux/cpu.h> | 44 | #include <linux/cpu.h> |
45 | #include <linux/mutex.h> | 45 | #include <linux/mutex.h> |
46 | #include <linux/module.h> | 46 | #include <linux/module.h> |
47 | #include <linux/kernel_stat.h> | ||
47 | 48 | ||
48 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 49 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
49 | static struct lock_class_key rcu_lock_key; | 50 | static struct lock_class_key rcu_lock_key; |
@@ -62,6 +63,23 @@ struct lockdep_map rcu_sched_lock_map = | |||
62 | EXPORT_SYMBOL_GPL(rcu_sched_lock_map); | 63 | EXPORT_SYMBOL_GPL(rcu_sched_lock_map); |
63 | #endif | 64 | #endif |
64 | 65 | ||
66 | int rcu_scheduler_active __read_mostly; | ||
67 | |||
68 | /* | ||
69 | * This function is invoked towards the end of the scheduler's initialization | ||
70 | * process. Before this is called, the idle task might contain | ||
71 | * RCU read-side critical sections (during which time, this idle | ||
72 | * task is booting the system). After this function is called, the | ||
73 | * idle tasks are prohibited from containing RCU read-side critical | ||
74 | * sections. | ||
75 | */ | ||
76 | void rcu_scheduler_starting(void) | ||
77 | { | ||
78 | WARN_ON(num_online_cpus() != 1); | ||
79 | WARN_ON(nr_context_switches() > 0); | ||
80 | rcu_scheduler_active = 1; | ||
81 | } | ||
82 | |||
65 | /* | 83 | /* |
66 | * Awaken the corresponding synchronize_rcu() instance now that a | 84 | * Awaken the corresponding synchronize_rcu() instance now that a |
67 | * grace period has elapsed. | 85 | * grace period has elapsed. |
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 525d39810616..335bfe4f0076 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
@@ -46,7 +46,6 @@ | |||
46 | #include <linux/cpu.h> | 46 | #include <linux/cpu.h> |
47 | #include <linux/mutex.h> | 47 | #include <linux/mutex.h> |
48 | #include <linux/time.h> | 48 | #include <linux/time.h> |
49 | #include <linux/kernel_stat.h> | ||
50 | 49 | ||
51 | #include "rcutree.h" | 50 | #include "rcutree.h" |
52 | 51 | ||
@@ -81,9 +80,6 @@ DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); | |||
81 | struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state); | 80 | struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state); |
82 | DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); | 81 | DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); |
83 | 82 | ||
84 | static int rcu_scheduler_active __read_mostly; | ||
85 | |||
86 | |||
87 | /* | 83 | /* |
88 | * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s | 84 | * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s |
89 | * permit this function to be invoked without holding the root rcu_node | 85 | * permit this function to be invoked without holding the root rcu_node |
@@ -1565,21 +1561,6 @@ static int rcu_needs_cpu_quick_check(int cpu) | |||
1565 | rcu_preempt_needs_cpu(cpu); | 1561 | rcu_preempt_needs_cpu(cpu); |
1566 | } | 1562 | } |
1567 | 1563 | ||
1568 | /* | ||
1569 | * This function is invoked towards the end of the scheduler's initialization | ||
1570 | * process. Before this is called, the idle task might contain | ||
1571 | * RCU read-side critical sections (during which time, this idle | ||
1572 | * task is booting the system). After this function is called, the | ||
1573 | * idle tasks are prohibited from containing RCU read-side critical | ||
1574 | * sections. | ||
1575 | */ | ||
1576 | void rcu_scheduler_starting(void) | ||
1577 | { | ||
1578 | WARN_ON(num_online_cpus() != 1); | ||
1579 | WARN_ON(nr_context_switches() > 0); | ||
1580 | rcu_scheduler_active = 1; | ||
1581 | } | ||
1582 | |||
1583 | static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; | 1564 | static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; |
1584 | static atomic_t rcu_barrier_cpu_count; | 1565 | static atomic_t rcu_barrier_cpu_count; |
1585 | static DEFINE_MUTEX(rcu_barrier_mutex); | 1566 | static DEFINE_MUTEX(rcu_barrier_mutex); |