diff options
Diffstat (limited to 'kernel/rcu/tree_plugin.h')
-rw-r--r-- | kernel/rcu/tree_plugin.h | 22 |
1 files changed, 11 insertions, 11 deletions
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 00dc411e9676..a7997e272564 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
@@ -2074,9 +2074,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force) | |||
2074 | 2074 | ||
2075 | if (!ACCESS_ONCE(rdp_leader->nocb_kthread)) | 2075 | if (!ACCESS_ONCE(rdp_leader->nocb_kthread)) |
2076 | return; | 2076 | return; |
2077 | if (!ACCESS_ONCE(rdp_leader->nocb_leader_wake) || force) { | 2077 | if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { |
2078 | /* Prior xchg orders against prior callback enqueue. */ | 2078 | /* Prior xchg orders against prior callback enqueue. */ |
2079 | ACCESS_ONCE(rdp_leader->nocb_leader_wake) = true; | 2079 | ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; |
2080 | wake_up(&rdp_leader->nocb_wq); | 2080 | wake_up(&rdp_leader->nocb_wq); |
2081 | } | 2081 | } |
2082 | } | 2082 | } |
@@ -2253,7 +2253,7 @@ wait_again: | |||
2253 | if (!rcu_nocb_poll) { | 2253 | if (!rcu_nocb_poll) { |
2254 | trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); | 2254 | trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); |
2255 | wait_event_interruptible(my_rdp->nocb_wq, | 2255 | wait_event_interruptible(my_rdp->nocb_wq, |
2256 | ACCESS_ONCE(my_rdp->nocb_leader_wake)); | 2256 | !ACCESS_ONCE(my_rdp->nocb_leader_sleep)); |
2257 | /* Memory barrier handled by smp_mb() calls below and repoll. */ | 2257 | /* Memory barrier handled by smp_mb() calls below and repoll. */ |
2258 | } else if (firsttime) { | 2258 | } else if (firsttime) { |
2259 | firsttime = false; /* Don't drown trace log with "Poll"! */ | 2259 | firsttime = false; /* Don't drown trace log with "Poll"! */ |
@@ -2292,12 +2292,12 @@ wait_again: | |||
2292 | schedule_timeout_interruptible(1); | 2292 | schedule_timeout_interruptible(1); |
2293 | 2293 | ||
2294 | /* Rescan in case we were a victim of memory ordering. */ | 2294 | /* Rescan in case we were a victim of memory ordering. */ |
2295 | my_rdp->nocb_leader_wake = false; | 2295 | my_rdp->nocb_leader_sleep = true; |
2296 | smp_mb(); /* Ensure _wake false before scan. */ | 2296 | smp_mb(); /* Ensure _sleep true before scan. */ |
2297 | for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) | 2297 | for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) |
2298 | if (ACCESS_ONCE(rdp->nocb_head)) { | 2298 | if (ACCESS_ONCE(rdp->nocb_head)) { |
2299 | /* Found CB, so short-circuit next wait. */ | 2299 | /* Found CB, so short-circuit next wait. */ |
2300 | my_rdp->nocb_leader_wake = true; | 2300 | my_rdp->nocb_leader_sleep = false; |
2301 | break; | 2301 | break; |
2302 | } | 2302 | } |
2303 | goto wait_again; | 2303 | goto wait_again; |
@@ -2307,17 +2307,17 @@ wait_again: | |||
2307 | rcu_nocb_wait_gp(my_rdp); | 2307 | rcu_nocb_wait_gp(my_rdp); |
2308 | 2308 | ||
2309 | /* | 2309 | /* |
2310 | * We left ->nocb_leader_wake set to reduce cache thrashing. | 2310 | * We left ->nocb_leader_sleep unset to reduce cache thrashing. |
2311 | * We clear it now, but recheck for new callbacks while | 2311 | * We set it now, but recheck for new callbacks while |
2312 | * traversing our follower list. | 2312 | * traversing our follower list. |
2313 | */ | 2313 | */ |
2314 | my_rdp->nocb_leader_wake = false; | 2314 | my_rdp->nocb_leader_sleep = true; |
2315 | smp_mb(); /* Ensure _wake false before scan of ->nocb_head. */ | 2315 | smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */ |
2316 | 2316 | ||
2317 | /* Each pass through the following loop wakes a follower, if needed. */ | 2317 | /* Each pass through the following loop wakes a follower, if needed. */ |
2318 | for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { | 2318 | for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { |
2319 | if (ACCESS_ONCE(rdp->nocb_head)) | 2319 | if (ACCESS_ONCE(rdp->nocb_head)) |
2320 | my_rdp->nocb_leader_wake = true; /* No need to wait. */ | 2320 | my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/ |
2321 | if (!rdp->nocb_gp_head) | 2321 | if (!rdp->nocb_gp_head) |
2322 | continue; /* No CBs, so no need to wake follower. */ | 2322 | continue; /* No CBs, so no need to wake follower. */ |
2323 | 2323 | ||