aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/rcu/tree.c6
-rw-r--r--kernel/rcu/tree.h4
2 files changed, 10 insertions, 0 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 7f872721c54e..50e4f7ebf8cf 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -101,6 +101,7 @@ struct rcu_state sname##_state = { \
101 .abbr = sabbr, \ 101 .abbr = sabbr, \
102 .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \ 102 .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \
103 .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \ 103 .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \
104 .ofl_lock = __SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \
104} 105}
105 106
106RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); 107RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
@@ -1900,11 +1901,13 @@ static bool rcu_gp_init(struct rcu_state *rsp)
1900 */ 1901 */
1901 rcu_for_each_leaf_node(rsp, rnp) { 1902 rcu_for_each_leaf_node(rsp, rnp) {
1902 rcu_gp_slow(rsp, gp_preinit_delay); 1903 rcu_gp_slow(rsp, gp_preinit_delay);
1904 spin_lock(&rsp->ofl_lock);
1903 raw_spin_lock_irq_rcu_node(rnp); 1905 raw_spin_lock_irq_rcu_node(rnp);
1904 if (rnp->qsmaskinit == rnp->qsmaskinitnext && 1906 if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
1905 !rnp->wait_blkd_tasks) { 1907 !rnp->wait_blkd_tasks) {
1906 /* Nothing to do on this leaf rcu_node structure. */ 1908 /* Nothing to do on this leaf rcu_node structure. */
1907 raw_spin_unlock_irq_rcu_node(rnp); 1909 raw_spin_unlock_irq_rcu_node(rnp);
1910 spin_unlock(&rsp->ofl_lock);
1908 continue; 1911 continue;
1909 } 1912 }
1910 1913
@@ -1940,6 +1943,7 @@ static bool rcu_gp_init(struct rcu_state *rsp)
1940 } 1943 }
1941 1944
1942 raw_spin_unlock_irq_rcu_node(rnp); 1945 raw_spin_unlock_irq_rcu_node(rnp);
1946 spin_unlock(&rsp->ofl_lock);
1943 } 1947 }
1944 1948
1945 /* 1949 /*
@@ -3749,6 +3753,7 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
3749 3753
3750 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */ 3754 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
3751 mask = rdp->grpmask; 3755 mask = rdp->grpmask;
3756 spin_lock(&rsp->ofl_lock);
3752 raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */ 3757 raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
3753 if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */ 3758 if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
3754 /* Report quiescent state -before- changing ->qsmaskinitnext! */ 3759 /* Report quiescent state -before- changing ->qsmaskinitnext! */
@@ -3757,6 +3762,7 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
3757 } 3762 }
3758 rnp->qsmaskinitnext &= ~mask; 3763 rnp->qsmaskinitnext &= ~mask;
3759 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 3764 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3765 spin_unlock(&rsp->ofl_lock);
3760} 3766}
3761 3767
3762/* 3768/*
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 3def94fc9c74..6683da6e4ecc 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -363,6 +363,10 @@ struct rcu_state {
363 const char *name; /* Name of structure. */ 363 const char *name; /* Name of structure. */
364 char abbr; /* Abbreviated name. */ 364 char abbr; /* Abbreviated name. */
365 struct list_head flavors; /* List of RCU flavors. */ 365 struct list_head flavors; /* List of RCU flavors. */
366
367 spinlock_t ofl_lock ____cacheline_internodealigned_in_smp;
368 /* Synchronize offline with */
369 /* GP pre-initialization. */
366}; 370};
367 371
368/* Values for rcu_state structure's gp_flags field. */ 372/* Values for rcu_state structure's gp_flags field. */