diff options
Diffstat (limited to 'include/linux/rcupdate.h')
-rw-r--r-- | include/linux/rcupdate.h | 58 |
1 files changed, 39 insertions, 19 deletions
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index c84373626336..872a98e13d6a 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -41,6 +41,10 @@ | |||
41 | #include <linux/lockdep.h> | 41 | #include <linux/lockdep.h> |
42 | #include <linux/completion.h> | 42 | #include <linux/completion.h> |
43 | 43 | ||
44 | #ifdef CONFIG_RCU_TORTURE_TEST | ||
45 | extern int rcutorture_runnable; /* for sysctl */ | ||
46 | #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */ | ||
47 | |||
44 | /** | 48 | /** |
45 | * struct rcu_head - callback structure for use with RCU | 49 | * struct rcu_head - callback structure for use with RCU |
46 | * @next: next update requests in a list | 50 | * @next: next update requests in a list |
@@ -97,6 +101,11 @@ extern struct lockdep_map rcu_sched_lock_map; | |||
97 | # define rcu_read_release_sched() \ | 101 | # define rcu_read_release_sched() \ |
98 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) | 102 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) |
99 | 103 | ||
104 | static inline int debug_lockdep_rcu_enabled(void) | ||
105 | { | ||
106 | return likely(rcu_scheduler_active && debug_locks); | ||
107 | } | ||
108 | |||
100 | /** | 109 | /** |
101 | * rcu_read_lock_held - might we be in RCU read-side critical section? | 110 | * rcu_read_lock_held - might we be in RCU read-side critical section? |
102 | * | 111 | * |
@@ -104,28 +113,21 @@ extern struct lockdep_map rcu_sched_lock_map; | |||
104 | * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, | 113 | * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, |
105 | * this assumes we are in an RCU read-side critical section unless it can | 114 | * this assumes we are in an RCU read-side critical section unless it can |
106 | * prove otherwise. | 115 | * prove otherwise. |
116 | * | ||
117 | * Check rcu_scheduler_active to prevent false positives during boot. | ||
107 | */ | 118 | */ |
108 | static inline int rcu_read_lock_held(void) | 119 | static inline int rcu_read_lock_held(void) |
109 | { | 120 | { |
110 | if (debug_locks) | 121 | if (!debug_lockdep_rcu_enabled()) |
111 | return lock_is_held(&rcu_lock_map); | 122 | return 1; |
112 | return 1; | 123 | return lock_is_held(&rcu_lock_map); |
113 | } | 124 | } |
114 | 125 | ||
115 | /** | 126 | /* |
116 | * rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section? | 127 | * rcu_read_lock_bh_held() is defined out of line to avoid #include-file |
117 | * | 128 | * hell. |
118 | * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in | ||
119 | * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING, | ||
120 | * this assumes we are in an RCU-bh read-side critical section unless it can | ||
121 | * prove otherwise. | ||
122 | */ | 129 | */ |
123 | static inline int rcu_read_lock_bh_held(void) | 130 | extern int rcu_read_lock_bh_held(void); |
124 | { | ||
125 | if (debug_locks) | ||
126 | return lock_is_held(&rcu_bh_lock_map); | ||
127 | return 1; | ||
128 | } | ||
129 | 131 | ||
130 | /** | 132 | /** |
131 | * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? | 133 | * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? |
@@ -135,15 +137,26 @@ static inline int rcu_read_lock_bh_held(void) | |||
135 | * this assumes we are in an RCU-sched read-side critical section unless it | 137 | * this assumes we are in an RCU-sched read-side critical section unless it |
136 | * can prove otherwise. Note that disabling of preemption (including | 138 | * can prove otherwise. Note that disabling of preemption (including |
137 | * disabling irqs) counts as an RCU-sched read-side critical section. | 139 | * disabling irqs) counts as an RCU-sched read-side critical section. |
140 | * | ||
141 | * Check rcu_scheduler_active to prevent false positives during boot. | ||
138 | */ | 142 | */ |
143 | #ifdef CONFIG_PREEMPT | ||
139 | static inline int rcu_read_lock_sched_held(void) | 144 | static inline int rcu_read_lock_sched_held(void) |
140 | { | 145 | { |
141 | int lockdep_opinion = 0; | 146 | int lockdep_opinion = 0; |
142 | 147 | ||
148 | if (!debug_lockdep_rcu_enabled()) | ||
149 | return 1; | ||
143 | if (debug_locks) | 150 | if (debug_locks) |
144 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | 151 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); |
145 | return lockdep_opinion || preempt_count() != 0 || !rcu_scheduler_active; | 152 | return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); |
146 | } | 153 | } |
154 | #else /* #ifdef CONFIG_PREEMPT */ | ||
155 | static inline int rcu_read_lock_sched_held(void) | ||
156 | { | ||
157 | return 1; | ||
158 | } | ||
159 | #endif /* #else #ifdef CONFIG_PREEMPT */ | ||
147 | 160 | ||
148 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 161 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
149 | 162 | ||
@@ -164,10 +177,17 @@ static inline int rcu_read_lock_bh_held(void) | |||
164 | return 1; | 177 | return 1; |
165 | } | 178 | } |
166 | 179 | ||
180 | #ifdef CONFIG_PREEMPT | ||
181 | static inline int rcu_read_lock_sched_held(void) | ||
182 | { | ||
183 | return !rcu_scheduler_active || preempt_count() != 0 || irqs_disabled(); | ||
184 | } | ||
185 | #else /* #ifdef CONFIG_PREEMPT */ | ||
167 | static inline int rcu_read_lock_sched_held(void) | 186 | static inline int rcu_read_lock_sched_held(void) |
168 | { | 187 | { |
169 | return preempt_count() != 0 || !rcu_scheduler_active; | 188 | return 1; |
170 | } | 189 | } |
190 | #endif /* #else #ifdef CONFIG_PREEMPT */ | ||
171 | 191 | ||
172 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 192 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
173 | 193 | ||
@@ -184,7 +204,7 @@ static inline int rcu_read_lock_sched_held(void) | |||
184 | */ | 204 | */ |
185 | #define rcu_dereference_check(p, c) \ | 205 | #define rcu_dereference_check(p, c) \ |
186 | ({ \ | 206 | ({ \ |
187 | if (debug_locks && !(c)) \ | 207 | if (debug_lockdep_rcu_enabled() && !(c)) \ |
188 | lockdep_rcu_dereference(__FILE__, __LINE__); \ | 208 | lockdep_rcu_dereference(__FILE__, __LINE__); \ |
189 | rcu_dereference_raw(p); \ | 209 | rcu_dereference_raw(p); \ |
190 | }) | 210 | }) |