aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/rcutree.c27
-rw-r--r--kernel/rcutree_plugin.h10
2 files changed, 11 insertions, 26 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index e8624ebf2320..ae4a553e37ce 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -767,10 +767,10 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
767 767
768/* 768/*
769 * Record a quiescent state for the specified CPU, which must either be 769 * Record a quiescent state for the specified CPU, which must either be
770 * the current CPU or an offline CPU. The lastcomp argument is used to 770 * the current CPU. The lastcomp argument is used to make sure we are
771 * make sure we are still in the grace period of interest. We don't want 771 * still in the grace period of interest. We don't want to end the current
772 * to end the current grace period based on quiescent states detected in 772 * grace period based on quiescent states detected in an earlier grace
773 * an earlier grace period! 773 * period!
774 */ 774 */
775static void 775static void
776cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) 776cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
@@ -805,7 +805,6 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
805 * This GP can't end until cpu checks in, so all of our 805 * This GP can't end until cpu checks in, so all of our
806 * callbacks can be processed during the next GP. 806 * callbacks can be processed during the next GP.
807 */ 807 */
808 rdp = rsp->rda[smp_processor_id()];
809 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; 808 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
810 809
811 cpu_quiet_msk(mask, rsp, rnp, flags); /* releases rnp->lock */ 810 cpu_quiet_msk(mask, rsp, rnp, flags); /* releases rnp->lock */
@@ -881,9 +880,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
881 880
882 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */ 881 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
883 882
884 /* Being offline is a quiescent state, so go record it. */
885 cpu_quiet(cpu, rsp, rdp, lastcomp);
886
887 /* 883 /*
888 * Move callbacks from the outgoing CPU to the running CPU. 884 * Move callbacks from the outgoing CPU to the running CPU.
889 * Note that the outgoing CPU is now quiscent, so it is now 885 * Note that the outgoing CPU is now quiscent, so it is now
@@ -1448,20 +1444,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable)
1448 rnp = rnp->parent; 1444 rnp = rnp->parent;
1449 } while (rnp != NULL && !(rnp->qsmaskinit & mask)); 1445 } while (rnp != NULL && !(rnp->qsmaskinit & mask));
1450 1446
1451 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */ 1447 spin_unlock_irqrestore(&rsp->onofflock, flags);
1452
1453 /*
1454 * A new grace period might start here. If so, we will be part of
1455 * it, and its gpnum will be greater than ours, so we will
1456 * participate. It is also possible for the gpnum to have been
1457 * incremented before this function was called, and the bitmasks
1458 * to not be filled out until now, in which case we will also
1459 * participate due to our gpnum being behind.
1460 */
1461
1462 /* Since it is coming online, the CPU is in a quiescent state. */
1463 cpu_quiet(cpu, rsp, rdp, lastcomp);
1464 local_irq_restore(flags);
1465} 1448}
1466 1449
1467static void __cpuinit rcu_online_cpu(int cpu) 1450static void __cpuinit rcu_online_cpu(int cpu)
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 5f94619450af..cd6047cc7fc2 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -117,9 +117,9 @@ static void rcu_preempt_note_context_switch(int cpu)
117 * on line! 117 * on line!
118 */ 118 */
119 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0); 119 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
120 phase = !(rnp->qsmask & rdp->grpmask) ^ (rnp->gpnum & 0x1); 120 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
121 phase = (rnp->gpnum + !(rnp->qsmask & rdp->grpmask)) & 0x1;
121 list_add(&t->rcu_node_entry, &rnp->blocked_tasks[phase]); 122 list_add(&t->rcu_node_entry, &rnp->blocked_tasks[phase]);
122 smp_mb(); /* Ensure later ctxt swtch seen after above. */
123 spin_unlock_irqrestore(&rnp->lock, flags); 123 spin_unlock_irqrestore(&rnp->lock, flags);
124 } 124 }
125 125
@@ -133,7 +133,9 @@ static void rcu_preempt_note_context_switch(int cpu)
133 * means that we continue to block the current grace period. 133 * means that we continue to block the current grace period.
134 */ 134 */
135 rcu_preempt_qs(cpu); 135 rcu_preempt_qs(cpu);
136 local_irq_save(flags);
136 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS; 137 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
138 local_irq_restore(flags);
137} 139}
138 140
139/* 141/*
@@ -189,10 +191,10 @@ static void rcu_read_unlock_special(struct task_struct *t)
189 */ 191 */
190 for (;;) { 192 for (;;) {
191 rnp = t->rcu_blocked_node; 193 rnp = t->rcu_blocked_node;
192 spin_lock(&rnp->lock); 194 spin_lock(&rnp->lock); /* irqs already disabled. */
193 if (rnp == t->rcu_blocked_node) 195 if (rnp == t->rcu_blocked_node)
194 break; 196 break;
195 spin_unlock(&rnp->lock); 197 spin_unlock(&rnp->lock); /* irqs remain disabled. */
196 } 198 }
197 empty = list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]); 199 empty = list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]);
198 list_del_init(&t->rcu_node_entry); 200 list_del_init(&t->rcu_node_entry);