aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcutree.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcutree.c')
-rw-r--r--kernel/rcutree.c31
1 files changed, 25 insertions, 6 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 055f1a941a9e..69f4efec3449 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -61,7 +61,7 @@ static struct lock_class_key rcu_root_class;
61 NUM_RCU_LVL_2, \ 61 NUM_RCU_LVL_2, \
62 NUM_RCU_LVL_3, /* == MAX_RCU_LVLS */ \ 62 NUM_RCU_LVL_3, /* == MAX_RCU_LVLS */ \
63 }, \ 63 }, \
64 .signaled = RCU_SIGNAL_INIT, \ 64 .signaled = RCU_GP_IDLE, \
65 .gpnum = -300, \ 65 .gpnum = -300, \
66 .completed = -300, \ 66 .completed = -300, \
67 .onofflock = __SPIN_LOCK_UNLOCKED(&name.onofflock), \ 67 .onofflock = __SPIN_LOCK_UNLOCKED(&name.onofflock), \
@@ -659,14 +659,17 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
659 * irqs disabled. 659 * irqs disabled.
660 */ 660 */
661 rcu_for_each_node_breadth_first(rsp, rnp) { 661 rcu_for_each_node_breadth_first(rsp, rnp) {
662 spin_lock(&rnp->lock); /* irqs already disabled. */ 662 spin_lock(&rnp->lock); /* irqs already disabled. */
663 rcu_preempt_check_blocked_tasks(rnp); 663 rcu_preempt_check_blocked_tasks(rnp);
664 rnp->qsmask = rnp->qsmaskinit; 664 rnp->qsmask = rnp->qsmaskinit;
665 rnp->gpnum = rsp->gpnum; 665 rnp->gpnum = rsp->gpnum;
666 spin_unlock(&rnp->lock); /* irqs already disabled. */ 666 spin_unlock(&rnp->lock); /* irqs remain disabled. */
667 } 667 }
668 668
669 rnp = rcu_get_root(rsp);
670 spin_lock(&rnp->lock); /* irqs already disabled. */
669 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state now OK. */ 671 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state now OK. */
672 spin_unlock(&rnp->lock); /* irqs remain disabled. */
670 spin_unlock_irqrestore(&rsp->onofflock, flags); 673 spin_unlock_irqrestore(&rsp->onofflock, flags);
671} 674}
672 675
@@ -708,6 +711,7 @@ static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags)
708{ 711{
709 WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); 712 WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
710 rsp->completed = rsp->gpnum; 713 rsp->completed = rsp->gpnum;
714 rsp->signaled = RCU_GP_IDLE;
711 rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]); 715 rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
712 rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */ 716 rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */
713} 717}
@@ -915,7 +919,20 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
915 spin_unlock(&rnp->lock); /* irqs remain disabled. */ 919 spin_unlock(&rnp->lock); /* irqs remain disabled. */
916 break; 920 break;
917 } 921 }
918 rcu_preempt_offline_tasks(rsp, rnp, rdp); 922
923 /*
924 * If there was a task blocking the current grace period,
925 * and if all CPUs have checked in, we need to propagate
926 * the quiescent state up the rcu_node hierarchy. But that
927 * is inconvenient at the moment due to deadlock issues if
928 * this should end the current grace period. So set the
929 * offlined CPU's bit in ->qsmask in order to force the
930 * next force_quiescent_state() invocation to clean up this
931 * mess in a deadlock-free manner.
932 */
933 if (rcu_preempt_offline_tasks(rsp, rnp, rdp) && !rnp->qsmask)
934 rnp->qsmask |= mask;
935
919 mask = rnp->grpmask; 936 mask = rnp->grpmask;
920 spin_unlock(&rnp->lock); /* irqs remain disabled. */ 937 spin_unlock(&rnp->lock); /* irqs remain disabled. */
921 rnp = rnp->parent; 938 rnp = rnp->parent;
@@ -1151,9 +1168,10 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1151 } 1168 }
1152 spin_unlock(&rnp->lock); 1169 spin_unlock(&rnp->lock);
1153 switch (signaled) { 1170 switch (signaled) {
1171 case RCU_GP_IDLE:
1154 case RCU_GP_INIT: 1172 case RCU_GP_INIT:
1155 1173
1156 break; /* grace period still initializing, ignore. */ 1174 break; /* grace period idle or initializing, ignore. */
1157 1175
1158 case RCU_SAVE_DYNTICK: 1176 case RCU_SAVE_DYNTICK:
1159 1177
@@ -1167,7 +1185,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1167 1185
1168 /* Update state, record completion counter. */ 1186 /* Update state, record completion counter. */
1169 spin_lock(&rnp->lock); 1187 spin_lock(&rnp->lock);
1170 if (lastcomp == rsp->completed) { 1188 if (lastcomp == rsp->completed &&
1189 rsp->signaled == RCU_SAVE_DYNTICK) {
1171 rsp->signaled = RCU_FORCE_QS; 1190 rsp->signaled = RCU_FORCE_QS;
1172 dyntick_record_completed(rsp, lastcomp); 1191 dyntick_record_completed(rsp, lastcomp);
1173 } 1192 }