diff options
Diffstat (limited to 'kernel/rcu/tree.c')
-rw-r--r-- | kernel/rcu/tree.c | 15 |
1 files changed, 8 insertions, 7 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 094ed8ff82b4..338ea61929bd 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
@@ -3309,9 +3309,9 @@ static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s) | |||
3309 | return rcu_seq_done(&rsp->expedited_sequence, s); | 3309 | return rcu_seq_done(&rsp->expedited_sequence, s); |
3310 | } | 3310 | } |
3311 | 3311 | ||
3312 | /* Common code for synchronize_sched_expedited() work-done checking. */ | 3312 | /* Common code for synchronize_{rcu,sched}_expedited() work-done checking. */ |
3313 | static bool sync_sched_exp_wd(struct rcu_state *rsp, struct rcu_node *rnp, | 3313 | static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp, |
3314 | atomic_long_t *stat, unsigned long s) | 3314 | atomic_long_t *stat, unsigned long s) |
3315 | { | 3315 | { |
3316 | if (rcu_exp_gp_seq_done(rsp, s)) { | 3316 | if (rcu_exp_gp_seq_done(rsp, s)) { |
3317 | if (rnp) | 3317 | if (rnp) |
@@ -3319,7 +3319,6 @@ static bool sync_sched_exp_wd(struct rcu_state *rsp, struct rcu_node *rnp, | |||
3319 | /* Ensure test happens before caller kfree(). */ | 3319 | /* Ensure test happens before caller kfree(). */ |
3320 | smp_mb__before_atomic(); /* ^^^ */ | 3320 | smp_mb__before_atomic(); /* ^^^ */ |
3321 | atomic_long_inc(stat); | 3321 | atomic_long_inc(stat); |
3322 | put_online_cpus(); | ||
3323 | return true; | 3322 | return true; |
3324 | } | 3323 | } |
3325 | return false; | 3324 | return false; |
@@ -3345,14 +3344,14 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s) | |||
3345 | */ | 3344 | */ |
3346 | rnp0 = per_cpu_ptr(rsp->rda, raw_smp_processor_id())->mynode; | 3345 | rnp0 = per_cpu_ptr(rsp->rda, raw_smp_processor_id())->mynode; |
3347 | for (; rnp0 != NULL; rnp0 = rnp0->parent) { | 3346 | for (; rnp0 != NULL; rnp0 = rnp0->parent) { |
3348 | if (sync_sched_exp_wd(rsp, rnp1, &rsp->expedited_workdone1, s)) | 3347 | if (sync_exp_work_done(rsp, rnp1, &rsp->expedited_workdone1, s)) |
3349 | return NULL; | 3348 | return NULL; |
3350 | mutex_lock(&rnp0->exp_funnel_mutex); | 3349 | mutex_lock(&rnp0->exp_funnel_mutex); |
3351 | if (rnp1) | 3350 | if (rnp1) |
3352 | mutex_unlock(&rnp1->exp_funnel_mutex); | 3351 | mutex_unlock(&rnp1->exp_funnel_mutex); |
3353 | rnp1 = rnp0; | 3352 | rnp1 = rnp0; |
3354 | } | 3353 | } |
3355 | if (sync_sched_exp_wd(rsp, rnp1, &rsp->expedited_workdone2, s)) | 3354 | if (sync_exp_work_done(rsp, rnp1, &rsp->expedited_workdone2, s)) |
3356 | return NULL; | 3355 | return NULL; |
3357 | return rnp1; | 3356 | return rnp1; |
3358 | } | 3357 | } |
@@ -3402,8 +3401,10 @@ void synchronize_sched_expedited(void) | |||
3402 | WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id())); | 3401 | WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id())); |
3403 | 3402 | ||
3404 | rnp = exp_funnel_lock(rsp, s); | 3403 | rnp = exp_funnel_lock(rsp, s); |
3405 | if (rnp == NULL) | 3404 | if (rnp == NULL) { |
3405 | put_online_cpus(); | ||
3406 | return; /* Someone else did our work for us. */ | 3406 | return; /* Someone else did our work for us. */ |
3407 | } | ||
3407 | 3408 | ||
3408 | rcu_exp_gp_seq_start(rsp); | 3409 | rcu_exp_gp_seq_start(rsp); |
3409 | 3410 | ||