diff options
Diffstat (limited to 'kernel/rcutree.c')
-rw-r--r-- | kernel/rcutree.c | 13 |
1 files changed, 9 insertions, 4 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 6c99553e9f15..e8624ebf2320 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
@@ -628,8 +628,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) | |||
628 | 628 | ||
629 | /* Special-case the common single-level case. */ | 629 | /* Special-case the common single-level case. */ |
630 | if (NUM_RCU_NODES == 1) { | 630 | if (NUM_RCU_NODES == 1) { |
631 | rnp->qsmask = rnp->qsmaskinit; | ||
632 | rcu_preempt_check_blocked_tasks(rnp); | 631 | rcu_preempt_check_blocked_tasks(rnp); |
632 | rnp->qsmask = rnp->qsmaskinit; | ||
633 | rnp->gpnum = rsp->gpnum; | 633 | rnp->gpnum = rsp->gpnum; |
634 | rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */ | 634 | rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */ |
635 | spin_unlock_irqrestore(&rnp->lock, flags); | 635 | spin_unlock_irqrestore(&rnp->lock, flags); |
@@ -662,8 +662,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) | |||
662 | rnp_end = &rsp->node[NUM_RCU_NODES]; | 662 | rnp_end = &rsp->node[NUM_RCU_NODES]; |
663 | for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++) { | 663 | for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++) { |
664 | spin_lock(&rnp_cur->lock); /* irqs already disabled. */ | 664 | spin_lock(&rnp_cur->lock); /* irqs already disabled. */ |
665 | rnp_cur->qsmask = rnp_cur->qsmaskinit; | ||
666 | rcu_preempt_check_blocked_tasks(rnp); | 665 | rcu_preempt_check_blocked_tasks(rnp); |
666 | rnp_cur->qsmask = rnp_cur->qsmaskinit; | ||
667 | rnp->gpnum = rsp->gpnum; | 667 | rnp->gpnum = rsp->gpnum; |
668 | spin_unlock(&rnp_cur->lock); /* irqs already disabled. */ | 668 | spin_unlock(&rnp_cur->lock); /* irqs already disabled. */ |
669 | } | 669 | } |
@@ -708,6 +708,7 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp) | |||
708 | static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags) | 708 | static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags) |
709 | __releases(rnp->lock) | 709 | __releases(rnp->lock) |
710 | { | 710 | { |
711 | WARN_ON_ONCE(rsp->completed == rsp->gpnum); | ||
711 | rsp->completed = rsp->gpnum; | 712 | rsp->completed = rsp->gpnum; |
712 | rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]); | 713 | rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]); |
713 | rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */ | 714 | rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */ |
@@ -725,6 +726,8 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp, | |||
725 | unsigned long flags) | 726 | unsigned long flags) |
726 | __releases(rnp->lock) | 727 | __releases(rnp->lock) |
727 | { | 728 | { |
729 | struct rcu_node *rnp_c; | ||
730 | |||
728 | /* Walk up the rcu_node hierarchy. */ | 731 | /* Walk up the rcu_node hierarchy. */ |
729 | for (;;) { | 732 | for (;;) { |
730 | if (!(rnp->qsmask & mask)) { | 733 | if (!(rnp->qsmask & mask)) { |
@@ -748,8 +751,10 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp, | |||
748 | break; | 751 | break; |
749 | } | 752 | } |
750 | spin_unlock_irqrestore(&rnp->lock, flags); | 753 | spin_unlock_irqrestore(&rnp->lock, flags); |
754 | rnp_c = rnp; | ||
751 | rnp = rnp->parent; | 755 | rnp = rnp->parent; |
752 | spin_lock_irqsave(&rnp->lock, flags); | 756 | spin_lock_irqsave(&rnp->lock, flags); |
757 | WARN_ON_ONCE(rnp_c->qsmask); | ||
753 | } | 758 | } |
754 | 759 | ||
755 | /* | 760 | /* |
@@ -858,7 +863,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) | |||
858 | spin_lock_irqsave(&rsp->onofflock, flags); | 863 | spin_lock_irqsave(&rsp->onofflock, flags); |
859 | 864 | ||
860 | /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */ | 865 | /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */ |
861 | rnp = rdp->mynode; | 866 | rnp = rdp->mynode; /* this is the outgoing CPU's rnp. */ |
862 | mask = rdp->grpmask; /* rnp->grplo is constant. */ | 867 | mask = rdp->grpmask; /* rnp->grplo is constant. */ |
863 | do { | 868 | do { |
864 | spin_lock(&rnp->lock); /* irqs already disabled. */ | 869 | spin_lock(&rnp->lock); /* irqs already disabled. */ |
@@ -867,7 +872,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) | |||
867 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | 872 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ |
868 | break; | 873 | break; |
869 | } | 874 | } |
870 | rcu_preempt_offline_tasks(rsp, rnp); | 875 | rcu_preempt_offline_tasks(rsp, rnp, rdp); |
871 | mask = rnp->grpmask; | 876 | mask = rnp->grpmask; |
872 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | 877 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ |
873 | rnp = rnp->parent; | 878 | rnp = rnp->parent; |