aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcu/tree.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcu/tree.c')
-rw-r--r--kernel/rcu/tree.c15
1 files changed, 8 insertions, 7 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 094ed8ff82b4..338ea61929bd 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3309,9 +3309,9 @@ static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s)
3309 return rcu_seq_done(&rsp->expedited_sequence, s); 3309 return rcu_seq_done(&rsp->expedited_sequence, s);
3310} 3310}
3311 3311
3312/* Common code for synchronize_sched_expedited() work-done checking. */ 3312/* Common code for synchronize_{rcu,sched}_expedited() work-done checking. */
3313static bool sync_sched_exp_wd(struct rcu_state *rsp, struct rcu_node *rnp, 3313static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp,
3314 atomic_long_t *stat, unsigned long s) 3314 atomic_long_t *stat, unsigned long s)
3315{ 3315{
3316 if (rcu_exp_gp_seq_done(rsp, s)) { 3316 if (rcu_exp_gp_seq_done(rsp, s)) {
3317 if (rnp) 3317 if (rnp)
@@ -3319,7 +3319,6 @@ static bool sync_sched_exp_wd(struct rcu_state *rsp, struct rcu_node *rnp,
3319 /* Ensure test happens before caller kfree(). */ 3319 /* Ensure test happens before caller kfree(). */
3320 smp_mb__before_atomic(); /* ^^^ */ 3320 smp_mb__before_atomic(); /* ^^^ */
3321 atomic_long_inc(stat); 3321 atomic_long_inc(stat);
3322 put_online_cpus();
3323 return true; 3322 return true;
3324 } 3323 }
3325 return false; 3324 return false;
@@ -3345,14 +3344,14 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
3345 */ 3344 */
3346 rnp0 = per_cpu_ptr(rsp->rda, raw_smp_processor_id())->mynode; 3345 rnp0 = per_cpu_ptr(rsp->rda, raw_smp_processor_id())->mynode;
3347 for (; rnp0 != NULL; rnp0 = rnp0->parent) { 3346 for (; rnp0 != NULL; rnp0 = rnp0->parent) {
3348 if (sync_sched_exp_wd(rsp, rnp1, &rsp->expedited_workdone1, s)) 3347 if (sync_exp_work_done(rsp, rnp1, &rsp->expedited_workdone1, s))
3349 return NULL; 3348 return NULL;
3350 mutex_lock(&rnp0->exp_funnel_mutex); 3349 mutex_lock(&rnp0->exp_funnel_mutex);
3351 if (rnp1) 3350 if (rnp1)
3352 mutex_unlock(&rnp1->exp_funnel_mutex); 3351 mutex_unlock(&rnp1->exp_funnel_mutex);
3353 rnp1 = rnp0; 3352 rnp1 = rnp0;
3354 } 3353 }
3355 if (sync_sched_exp_wd(rsp, rnp1, &rsp->expedited_workdone2, s)) 3354 if (sync_exp_work_done(rsp, rnp1, &rsp->expedited_workdone2, s))
3356 return NULL; 3355 return NULL;
3357 return rnp1; 3356 return rnp1;
3358} 3357}
@@ -3402,8 +3401,10 @@ void synchronize_sched_expedited(void)
3402 WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id())); 3401 WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
3403 3402
3404 rnp = exp_funnel_lock(rsp, s); 3403 rnp = exp_funnel_lock(rsp, s);
3405 if (rnp == NULL) 3404 if (rnp == NULL) {
3405 put_online_cpus();
3406 return; /* Someone else did our work for us. */ 3406 return; /* Someone else did our work for us. */
3407 }
3407 3408
3408 rcu_exp_gp_seq_start(rsp); 3409 rcu_exp_gp_seq_start(rsp);
3409 3410