aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcutree.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcutree.c')
-rw-r--r--kernel/rcutree.c13
1 files changed, 9 insertions, 4 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 6c99553e9f15..e8624ebf2320 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -628,8 +628,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
628 628
629 /* Special-case the common single-level case. */ 629 /* Special-case the common single-level case. */
630 if (NUM_RCU_NODES == 1) { 630 if (NUM_RCU_NODES == 1) {
631 rnp->qsmask = rnp->qsmaskinit;
632 rcu_preempt_check_blocked_tasks(rnp); 631 rcu_preempt_check_blocked_tasks(rnp);
632 rnp->qsmask = rnp->qsmaskinit;
633 rnp->gpnum = rsp->gpnum; 633 rnp->gpnum = rsp->gpnum;
634 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */ 634 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */
635 spin_unlock_irqrestore(&rnp->lock, flags); 635 spin_unlock_irqrestore(&rnp->lock, flags);
@@ -662,8 +662,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
662 rnp_end = &rsp->node[NUM_RCU_NODES]; 662 rnp_end = &rsp->node[NUM_RCU_NODES];
663 for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++) { 663 for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++) {
664 spin_lock(&rnp_cur->lock); /* irqs already disabled. */ 664 spin_lock(&rnp_cur->lock); /* irqs already disabled. */
665 rnp_cur->qsmask = rnp_cur->qsmaskinit;
666 rcu_preempt_check_blocked_tasks(rnp); 665 rcu_preempt_check_blocked_tasks(rnp);
666 rnp_cur->qsmask = rnp_cur->qsmaskinit;
667 rnp->gpnum = rsp->gpnum; 667 rnp->gpnum = rsp->gpnum;
668 spin_unlock(&rnp_cur->lock); /* irqs already disabled. */ 668 spin_unlock(&rnp_cur->lock); /* irqs already disabled. */
669 } 669 }
@@ -708,6 +708,7 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
708static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags) 708static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags)
709 __releases(rnp->lock) 709 __releases(rnp->lock)
710{ 710{
711 WARN_ON_ONCE(rsp->completed == rsp->gpnum);
711 rsp->completed = rsp->gpnum; 712 rsp->completed = rsp->gpnum;
712 rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]); 713 rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
713 rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */ 714 rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */
@@ -725,6 +726,8 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
725 unsigned long flags) 726 unsigned long flags)
726 __releases(rnp->lock) 727 __releases(rnp->lock)
727{ 728{
729 struct rcu_node *rnp_c;
730
728 /* Walk up the rcu_node hierarchy. */ 731 /* Walk up the rcu_node hierarchy. */
729 for (;;) { 732 for (;;) {
730 if (!(rnp->qsmask & mask)) { 733 if (!(rnp->qsmask & mask)) {
@@ -748,8 +751,10 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
748 break; 751 break;
749 } 752 }
750 spin_unlock_irqrestore(&rnp->lock, flags); 753 spin_unlock_irqrestore(&rnp->lock, flags);
754 rnp_c = rnp;
751 rnp = rnp->parent; 755 rnp = rnp->parent;
752 spin_lock_irqsave(&rnp->lock, flags); 756 spin_lock_irqsave(&rnp->lock, flags);
757 WARN_ON_ONCE(rnp_c->qsmask);
753 } 758 }
754 759
755 /* 760 /*
@@ -858,7 +863,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
858 spin_lock_irqsave(&rsp->onofflock, flags); 863 spin_lock_irqsave(&rsp->onofflock, flags);
859 864
860 /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */ 865 /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */
861 rnp = rdp->mynode; 866 rnp = rdp->mynode; /* this is the outgoing CPU's rnp. */
862 mask = rdp->grpmask; /* rnp->grplo is constant. */ 867 mask = rdp->grpmask; /* rnp->grplo is constant. */
863 do { 868 do {
864 spin_lock(&rnp->lock); /* irqs already disabled. */ 869 spin_lock(&rnp->lock); /* irqs already disabled. */
@@ -867,7 +872,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
867 spin_unlock(&rnp->lock); /* irqs remain disabled. */ 872 spin_unlock(&rnp->lock); /* irqs remain disabled. */
868 break; 873 break;
869 } 874 }
870 rcu_preempt_offline_tasks(rsp, rnp); 875 rcu_preempt_offline_tasks(rsp, rnp, rdp);
871 mask = rnp->grpmask; 876 mask = rnp->grpmask;
872 spin_unlock(&rnp->lock); /* irqs remain disabled. */ 877 spin_unlock(&rnp->lock); /* irqs remain disabled. */
873 rnp = rnp->parent; 878 rnp = rnp->parent;