diff options
Diffstat (limited to 'kernel/rcuclassic.c')
-rw-r--r-- | kernel/rcuclassic.c | 52 |
1 files changed, 49 insertions, 3 deletions
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c index f4ffbd0f306f..16eeeaa9d618 100644 --- a/kernel/rcuclassic.c +++ b/kernel/rcuclassic.c | |||
@@ -89,8 +89,22 @@ static void force_quiescent_state(struct rcu_data *rdp, | |||
89 | /* | 89 | /* |
90 | * Don't send IPI to itself. With irqs disabled, | 90 | * Don't send IPI to itself. With irqs disabled, |
91 | * rdp->cpu is the current cpu. | 91 | * rdp->cpu is the current cpu. |
92 | * | ||
93 | * cpu_online_map is updated by the _cpu_down() | ||
94 | * using stop_machine_run(). Since we're in irqs disabled | ||
95 | * section, stop_machine_run() is not exectuting, hence | ||
96 | * the cpu_online_map is stable. | ||
97 | * | ||
98 | * However, a cpu might have been offlined _just_ before | ||
99 | * we disabled irqs while entering here. | ||
100 | * And rcu subsystem might not yet have handled the CPU_DEAD | ||
101 | * notification, leading to the offlined cpu's bit | ||
102 | * being set in the rcp->cpumask. | ||
103 | * | ||
104 | * Hence cpumask = (rcp->cpumask & cpu_online_map) to prevent | ||
105 | * sending smp_reschedule() to an offlined CPU. | ||
92 | */ | 106 | */ |
93 | cpumask = rcp->cpumask; | 107 | cpus_and(cpumask, rcp->cpumask, cpu_online_map); |
94 | cpu_clear(rdp->cpu, cpumask); | 108 | cpu_clear(rdp->cpu, cpumask); |
95 | for_each_cpu_mask(cpu, cpumask) | 109 | for_each_cpu_mask(cpu, cpumask) |
96 | smp_send_reschedule(cpu); | 110 | smp_send_reschedule(cpu); |
@@ -373,6 +387,10 @@ static void __rcu_offline_cpu(struct rcu_data *this_rdp, | |||
373 | rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); | 387 | rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); |
374 | rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); | 388 | rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); |
375 | rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); | 389 | rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); |
390 | |||
391 | local_irq_disable(); | ||
392 | this_rdp->qlen += rdp->qlen; | ||
393 | local_irq_enable(); | ||
376 | } | 394 | } |
377 | 395 | ||
378 | static void rcu_offline_cpu(int cpu) | 396 | static void rcu_offline_cpu(int cpu) |
@@ -502,10 +520,38 @@ void rcu_check_callbacks(int cpu, int user) | |||
502 | if (user || | 520 | if (user || |
503 | (idle_cpu(cpu) && !in_softirq() && | 521 | (idle_cpu(cpu) && !in_softirq() && |
504 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) { | 522 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) { |
523 | |||
524 | /* | ||
525 | * Get here if this CPU took its interrupt from user | ||
526 | * mode or from the idle loop, and if this is not a | ||
527 | * nested interrupt. In this case, the CPU is in | ||
528 | * a quiescent state, so count it. | ||
529 | * | ||
530 | * Also do a memory barrier. This is needed to handle | ||
531 | * the case where writes from a preempt-disable section | ||
532 | * of code get reordered into schedule() by this CPU's | ||
533 | * write buffer. The memory barrier makes sure that | ||
534 | * the rcu_qsctr_inc() and rcu_bh_qsctr_inc() are see | ||
535 | * by other CPUs to happen after any such write. | ||
536 | */ | ||
537 | |||
538 | smp_mb(); /* See above block comment. */ | ||
505 | rcu_qsctr_inc(cpu); | 539 | rcu_qsctr_inc(cpu); |
506 | rcu_bh_qsctr_inc(cpu); | 540 | rcu_bh_qsctr_inc(cpu); |
507 | } else if (!in_softirq()) | 541 | |
542 | } else if (!in_softirq()) { | ||
543 | |||
544 | /* | ||
545 | * Get here if this CPU did not take its interrupt from | ||
546 | * softirq, in other words, if it is not interrupting | ||
547 | * a rcu_bh read-side critical section. This is an _bh | ||
548 | * critical section, so count it. The memory barrier | ||
549 | * is needed for the same reason as is the above one. | ||
550 | */ | ||
551 | |||
552 | smp_mb(); /* See above block comment. */ | ||
508 | rcu_bh_qsctr_inc(cpu); | 553 | rcu_bh_qsctr_inc(cpu); |
554 | } | ||
509 | raise_rcu_softirq(); | 555 | raise_rcu_softirq(); |
510 | } | 556 | } |
511 | 557 | ||
@@ -529,7 +575,7 @@ static void __cpuinit rcu_online_cpu(int cpu) | |||
529 | 575 | ||
530 | rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); | 576 | rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); |
531 | rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); | 577 | rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); |
532 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); | 578 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
533 | } | 579 | } |
534 | 580 | ||
535 | static int __cpuinit rcu_cpu_notify(struct notifier_block *self, | 581 | static int __cpuinit rcu_cpu_notify(struct notifier_block *self, |