diff options
Diffstat (limited to 'kernel/rcutiny.c')
-rw-r--r-- | kernel/rcutiny.c | 21 |
1 files changed, 11 insertions, 10 deletions
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c index a0714a51b6d7..aa344111de3e 100644 --- a/kernel/rcutiny.c +++ b/kernel/rcutiny.c | |||
@@ -44,7 +44,6 @@ | |||
44 | 44 | ||
45 | /* Forward declarations for rcutiny_plugin.h. */ | 45 | /* Forward declarations for rcutiny_plugin.h. */ |
46 | struct rcu_ctrlblk; | 46 | struct rcu_ctrlblk; |
47 | static void invoke_rcu_callbacks(void); | ||
48 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp); | 47 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp); |
49 | static void rcu_process_callbacks(struct softirq_action *unused); | 48 | static void rcu_process_callbacks(struct softirq_action *unused); |
50 | static void __call_rcu(struct rcu_head *head, | 49 | static void __call_rcu(struct rcu_head *head, |
@@ -205,7 +204,7 @@ static int rcu_is_cpu_rrupt_from_idle(void) | |||
205 | */ | 204 | */ |
206 | static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) | 205 | static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) |
207 | { | 206 | { |
208 | reset_cpu_stall_ticks(rcp); | 207 | RCU_TRACE(reset_cpu_stall_ticks(rcp)); |
209 | if (rcp->rcucblist != NULL && | 208 | if (rcp->rcucblist != NULL && |
210 | rcp->donetail != rcp->curtail) { | 209 | rcp->donetail != rcp->curtail) { |
211 | rcp->donetail = rcp->curtail; | 210 | rcp->donetail = rcp->curtail; |
@@ -227,7 +226,7 @@ void rcu_sched_qs(int cpu) | |||
227 | local_irq_save(flags); | 226 | local_irq_save(flags); |
228 | if (rcu_qsctr_help(&rcu_sched_ctrlblk) + | 227 | if (rcu_qsctr_help(&rcu_sched_ctrlblk) + |
229 | rcu_qsctr_help(&rcu_bh_ctrlblk)) | 228 | rcu_qsctr_help(&rcu_bh_ctrlblk)) |
230 | invoke_rcu_callbacks(); | 229 | raise_softirq(RCU_SOFTIRQ); |
231 | local_irq_restore(flags); | 230 | local_irq_restore(flags); |
232 | } | 231 | } |
233 | 232 | ||
@@ -240,7 +239,7 @@ void rcu_bh_qs(int cpu) | |||
240 | 239 | ||
241 | local_irq_save(flags); | 240 | local_irq_save(flags); |
242 | if (rcu_qsctr_help(&rcu_bh_ctrlblk)) | 241 | if (rcu_qsctr_help(&rcu_bh_ctrlblk)) |
243 | invoke_rcu_callbacks(); | 242 | raise_softirq(RCU_SOFTIRQ); |
244 | local_irq_restore(flags); | 243 | local_irq_restore(flags); |
245 | } | 244 | } |
246 | 245 | ||
@@ -252,12 +251,11 @@ void rcu_bh_qs(int cpu) | |||
252 | */ | 251 | */ |
253 | void rcu_check_callbacks(int cpu, int user) | 252 | void rcu_check_callbacks(int cpu, int user) |
254 | { | 253 | { |
255 | check_cpu_stalls(); | 254 | RCU_TRACE(check_cpu_stalls()); |
256 | if (user || rcu_is_cpu_rrupt_from_idle()) | 255 | if (user || rcu_is_cpu_rrupt_from_idle()) |
257 | rcu_sched_qs(cpu); | 256 | rcu_sched_qs(cpu); |
258 | else if (!in_softirq()) | 257 | else if (!in_softirq()) |
259 | rcu_bh_qs(cpu); | 258 | rcu_bh_qs(cpu); |
260 | rcu_preempt_check_callbacks(); | ||
261 | } | 259 | } |
262 | 260 | ||
263 | /* | 261 | /* |
@@ -278,7 +276,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) | |||
278 | ACCESS_ONCE(rcp->rcucblist), | 276 | ACCESS_ONCE(rcp->rcucblist), |
279 | need_resched(), | 277 | need_resched(), |
280 | is_idle_task(current), | 278 | is_idle_task(current), |
281 | rcu_is_callbacks_kthread())); | 279 | false)); |
282 | return; | 280 | return; |
283 | } | 281 | } |
284 | 282 | ||
@@ -290,7 +288,6 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) | |||
290 | *rcp->donetail = NULL; | 288 | *rcp->donetail = NULL; |
291 | if (rcp->curtail == rcp->donetail) | 289 | if (rcp->curtail == rcp->donetail) |
292 | rcp->curtail = &rcp->rcucblist; | 290 | rcp->curtail = &rcp->rcucblist; |
293 | rcu_preempt_remove_callbacks(rcp); | ||
294 | rcp->donetail = &rcp->rcucblist; | 291 | rcp->donetail = &rcp->rcucblist; |
295 | local_irq_restore(flags); | 292 | local_irq_restore(flags); |
296 | 293 | ||
@@ -309,14 +306,13 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) | |||
309 | RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count)); | 306 | RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count)); |
310 | RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count, 0, need_resched(), | 307 | RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count, 0, need_resched(), |
311 | is_idle_task(current), | 308 | is_idle_task(current), |
312 | rcu_is_callbacks_kthread())); | 309 | false)); |
313 | } | 310 | } |
314 | 311 | ||
315 | static void rcu_process_callbacks(struct softirq_action *unused) | 312 | static void rcu_process_callbacks(struct softirq_action *unused) |
316 | { | 313 | { |
317 | __rcu_process_callbacks(&rcu_sched_ctrlblk); | 314 | __rcu_process_callbacks(&rcu_sched_ctrlblk); |
318 | __rcu_process_callbacks(&rcu_bh_ctrlblk); | 315 | __rcu_process_callbacks(&rcu_bh_ctrlblk); |
319 | rcu_preempt_process_callbacks(); | ||
320 | } | 316 | } |
321 | 317 | ||
322 | /* | 318 | /* |
@@ -382,3 +378,8 @@ void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) | |||
382 | __call_rcu(head, func, &rcu_bh_ctrlblk); | 378 | __call_rcu(head, func, &rcu_bh_ctrlblk); |
383 | } | 379 | } |
384 | EXPORT_SYMBOL_GPL(call_rcu_bh); | 380 | EXPORT_SYMBOL_GPL(call_rcu_bh); |
381 | |||
382 | void rcu_init(void) | ||
383 | { | ||
384 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | ||
385 | } | ||