diff options
| -rw-r--r-- | kernel/rcu/tree.c | 6 | ||||
| -rw-r--r-- | kernel/rcu/tree.h | 4 |
2 files changed, 10 insertions, 0 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 7f872721c54e..50e4f7ebf8cf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
| @@ -101,6 +101,7 @@ struct rcu_state sname##_state = { \ | |||
| 101 | .abbr = sabbr, \ | 101 | .abbr = sabbr, \ |
| 102 | .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \ | 102 | .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \ |
| 103 | .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \ | 103 | .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \ |
| 104 | .ofl_lock = __SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \ | ||
| 104 | } | 105 | } |
| 105 | 106 | ||
| 106 | RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); | 107 | RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); |
| @@ -1900,11 +1901,13 @@ static bool rcu_gp_init(struct rcu_state *rsp) | |||
| 1900 | */ | 1901 | */ |
| 1901 | rcu_for_each_leaf_node(rsp, rnp) { | 1902 | rcu_for_each_leaf_node(rsp, rnp) { |
| 1902 | rcu_gp_slow(rsp, gp_preinit_delay); | 1903 | rcu_gp_slow(rsp, gp_preinit_delay); |
| 1904 | spin_lock(&rsp->ofl_lock); | ||
| 1903 | raw_spin_lock_irq_rcu_node(rnp); | 1905 | raw_spin_lock_irq_rcu_node(rnp); |
| 1904 | if (rnp->qsmaskinit == rnp->qsmaskinitnext && | 1906 | if (rnp->qsmaskinit == rnp->qsmaskinitnext && |
| 1905 | !rnp->wait_blkd_tasks) { | 1907 | !rnp->wait_blkd_tasks) { |
| 1906 | /* Nothing to do on this leaf rcu_node structure. */ | 1908 | /* Nothing to do on this leaf rcu_node structure. */ |
| 1907 | raw_spin_unlock_irq_rcu_node(rnp); | 1909 | raw_spin_unlock_irq_rcu_node(rnp); |
| 1910 | spin_unlock(&rsp->ofl_lock); | ||
| 1908 | continue; | 1911 | continue; |
| 1909 | } | 1912 | } |
| 1910 | 1913 | ||
| @@ -1940,6 +1943,7 @@ static bool rcu_gp_init(struct rcu_state *rsp) | |||
| 1940 | } | 1943 | } |
| 1941 | 1944 | ||
| 1942 | raw_spin_unlock_irq_rcu_node(rnp); | 1945 | raw_spin_unlock_irq_rcu_node(rnp); |
| 1946 | spin_unlock(&rsp->ofl_lock); | ||
| 1943 | } | 1947 | } |
| 1944 | 1948 | ||
| 1945 | /* | 1949 | /* |
| @@ -3749,6 +3753,7 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) | |||
| 3749 | 3753 | ||
| 3750 | /* Remove outgoing CPU from mask in the leaf rcu_node structure. */ | 3754 | /* Remove outgoing CPU from mask in the leaf rcu_node structure. */ |
| 3751 | mask = rdp->grpmask; | 3755 | mask = rdp->grpmask; |
| 3756 | spin_lock(&rsp->ofl_lock); | ||
| 3752 | raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */ | 3757 | raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */ |
| 3753 | if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */ | 3758 | if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */ |
| 3754 | /* Report quiescent state -before- changing ->qsmaskinitnext! */ | 3759 | /* Report quiescent state -before- changing ->qsmaskinitnext! */ |
| @@ -3757,6 +3762,7 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) | |||
| 3757 | } | 3762 | } |
| 3758 | rnp->qsmaskinitnext &= ~mask; | 3763 | rnp->qsmaskinitnext &= ~mask; |
| 3759 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | 3764 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 3765 | spin_unlock(&rsp->ofl_lock); | ||
| 3760 | } | 3766 | } |
| 3761 | 3767 | ||
| 3762 | /* | 3768 | /* |
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 3def94fc9c74..6683da6e4ecc 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h | |||
| @@ -363,6 +363,10 @@ struct rcu_state { | |||
| 363 | const char *name; /* Name of structure. */ | 363 | const char *name; /* Name of structure. */ |
| 364 | char abbr; /* Abbreviated name. */ | 364 | char abbr; /* Abbreviated name. */ |
| 365 | struct list_head flavors; /* List of RCU flavors. */ | 365 | struct list_head flavors; /* List of RCU flavors. */ |
| 366 | |||
| 367 | spinlock_t ofl_lock ____cacheline_internodealigned_in_smp; | ||
| 368 | /* Synchronize offline with */ | ||
| 369 | /* GP pre-initialization. */ | ||
| 366 | }; | 370 | }; |
| 367 | 371 | ||
| 368 | /* Values for rcu_state structure's gp_flags field. */ | 372 | /* Values for rcu_state structure's gp_flags field. */ |
