diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/rcu/tree.c | 22 | ||||
-rw-r--r-- | kernel/rcu/tree.h | 2 | ||||
-rw-r--r-- | kernel/rcu/tree_plugin.h | 6 | ||||
-rw-r--r-- | kernel/rcu/tree_trace.c | 4 |
4 files changed, 17 insertions, 17 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 7c158ffc7769..31e7021ced4d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
@@ -245,21 +245,21 @@ static int rcu_gp_in_progress(struct rcu_state *rsp) | |||
245 | */ | 245 | */ |
246 | void rcu_sched_qs(void) | 246 | void rcu_sched_qs(void) |
247 | { | 247 | { |
248 | if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) { | 248 | if (__this_cpu_read(rcu_sched_data.cpu_no_qs)) { |
249 | trace_rcu_grace_period(TPS("rcu_sched"), | 249 | trace_rcu_grace_period(TPS("rcu_sched"), |
250 | __this_cpu_read(rcu_sched_data.gpnum), | 250 | __this_cpu_read(rcu_sched_data.gpnum), |
251 | TPS("cpuqs")); | 251 | TPS("cpuqs")); |
252 | __this_cpu_write(rcu_sched_data.passed_quiesce, 1); | 252 | __this_cpu_write(rcu_sched_data.cpu_no_qs, false); |
253 | } | 253 | } |
254 | } | 254 | } |
255 | 255 | ||
256 | void rcu_bh_qs(void) | 256 | void rcu_bh_qs(void) |
257 | { | 257 | { |
258 | if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { | 258 | if (__this_cpu_read(rcu_bh_data.cpu_no_qs)) { |
259 | trace_rcu_grace_period(TPS("rcu_bh"), | 259 | trace_rcu_grace_period(TPS("rcu_bh"), |
260 | __this_cpu_read(rcu_bh_data.gpnum), | 260 | __this_cpu_read(rcu_bh_data.gpnum), |
261 | TPS("cpuqs")); | 261 | TPS("cpuqs")); |
262 | __this_cpu_write(rcu_bh_data.passed_quiesce, 1); | 262 | __this_cpu_write(rcu_bh_data.cpu_no_qs, false); |
263 | } | 263 | } |
264 | } | 264 | } |
265 | 265 | ||
@@ -1744,7 +1744,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, | |||
1744 | */ | 1744 | */ |
1745 | rdp->gpnum = rnp->gpnum; | 1745 | rdp->gpnum = rnp->gpnum; |
1746 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); | 1746 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); |
1747 | rdp->passed_quiesce = 0; | 1747 | rdp->cpu_no_qs = true; |
1748 | rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); | 1748 | rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); |
1749 | rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); | 1749 | rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); |
1750 | zero_cpu_stall_ticks(rdp); | 1750 | zero_cpu_stall_ticks(rdp); |
@@ -2337,7 +2337,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) | |||
2337 | rnp = rdp->mynode; | 2337 | rnp = rdp->mynode; |
2338 | raw_spin_lock_irqsave(&rnp->lock, flags); | 2338 | raw_spin_lock_irqsave(&rnp->lock, flags); |
2339 | smp_mb__after_unlock_lock(); | 2339 | smp_mb__after_unlock_lock(); |
2340 | if ((rdp->passed_quiesce == 0 && | 2340 | if ((rdp->cpu_no_qs && |
2341 | rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || | 2341 | rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || |
2342 | rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum || | 2342 | rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum || |
2343 | rdp->gpwrap) { | 2343 | rdp->gpwrap) { |
@@ -2348,7 +2348,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) | |||
2348 | * We will instead need a new quiescent state that lies | 2348 | * We will instead need a new quiescent state that lies |
2349 | * within the current grace period. | 2349 | * within the current grace period. |
2350 | */ | 2350 | */ |
2351 | rdp->passed_quiesce = 0; /* need qs for new gp. */ | 2351 | rdp->cpu_no_qs = true; /* need qs for new gp. */ |
2352 | rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); | 2352 | rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); |
2353 | raw_spin_unlock_irqrestore(&rnp->lock, flags); | 2353 | raw_spin_unlock_irqrestore(&rnp->lock, flags); |
2354 | return; | 2354 | return; |
@@ -2395,7 +2395,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) | |||
2395 | * Was there a quiescent state since the beginning of the grace | 2395 | * Was there a quiescent state since the beginning of the grace |
2396 | * period? If no, then exit and wait for the next call. | 2396 | * period? If no, then exit and wait for the next call. |
2397 | */ | 2397 | */ |
2398 | if (!rdp->passed_quiesce && | 2398 | if (rdp->cpu_no_qs && |
2399 | rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) | 2399 | rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) |
2400 | return; | 2400 | return; |
2401 | 2401 | ||
@@ -3828,11 +3828,11 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) | |||
3828 | 3828 | ||
3829 | /* Is the RCU core waiting for a quiescent state from this CPU? */ | 3829 | /* Is the RCU core waiting for a quiescent state from this CPU? */ |
3830 | if (rcu_scheduler_fully_active && | 3830 | if (rcu_scheduler_fully_active && |
3831 | rdp->core_needs_qs && !rdp->passed_quiesce && | 3831 | rdp->core_needs_qs && rdp->cpu_no_qs && |
3832 | rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) { | 3832 | rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) { |
3833 | rdp->n_rp_core_needs_qs++; | 3833 | rdp->n_rp_core_needs_qs++; |
3834 | } else if (rdp->core_needs_qs && | 3834 | } else if (rdp->core_needs_qs && |
3835 | (rdp->passed_quiesce || | 3835 | (!rdp->cpu_no_qs || |
3836 | rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) { | 3836 | rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) { |
3837 | rdp->n_rp_report_qs++; | 3837 | rdp->n_rp_report_qs++; |
3838 | return 1; | 3838 | return 1; |
@@ -4155,7 +4155,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) | |||
4155 | rdp->beenonline = true; /* We have now been online. */ | 4155 | rdp->beenonline = true; /* We have now been online. */ |
4156 | rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */ | 4156 | rdp->gpnum = rnp->completed; /* Make CPU later note any new GP. */ |
4157 | rdp->completed = rnp->completed; | 4157 | rdp->completed = rnp->completed; |
4158 | rdp->passed_quiesce = false; | 4158 | rdp->cpu_no_qs = true; |
4159 | rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu); | 4159 | rdp->rcu_qs_ctr_snap = per_cpu(rcu_qs_ctr, cpu); |
4160 | rdp->core_needs_qs = false; | 4160 | rdp->core_needs_qs = false; |
4161 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); | 4161 | trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); |
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 4a0f30676ba8..ded4ceebed76 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h | |||
@@ -302,7 +302,7 @@ struct rcu_data { | |||
302 | /* is aware of having started. */ | 302 | /* is aware of having started. */ |
303 | unsigned long rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */ | 303 | unsigned long rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */ |
304 | /* for rcu_all_qs() invocations. */ | 304 | /* for rcu_all_qs() invocations. */ |
305 | bool passed_quiesce; /* User-mode/idle loop etc. */ | 305 | bool cpu_no_qs; /* No QS yet for this CPU. */ |
306 | bool core_needs_qs; /* Core waits for quiesc state. */ | 306 | bool core_needs_qs; /* Core waits for quiesc state. */ |
307 | bool beenonline; /* CPU online at least once. */ | 307 | bool beenonline; /* CPU online at least once. */ |
308 | bool gpwrap; /* Possible gpnum/completed wrap. */ | 308 | bool gpwrap; /* Possible gpnum/completed wrap. */ |
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index e33b4f3b8e0a..6977ff0dccb9 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
@@ -265,11 +265,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp, | |||
265 | */ | 265 | */ |
266 | static void rcu_preempt_qs(void) | 266 | static void rcu_preempt_qs(void) |
267 | { | 267 | { |
268 | if (!__this_cpu_read(rcu_data_p->passed_quiesce)) { | 268 | if (__this_cpu_read(rcu_data_p->cpu_no_qs)) { |
269 | trace_rcu_grace_period(TPS("rcu_preempt"), | 269 | trace_rcu_grace_period(TPS("rcu_preempt"), |
270 | __this_cpu_read(rcu_data_p->gpnum), | 270 | __this_cpu_read(rcu_data_p->gpnum), |
271 | TPS("cpuqs")); | 271 | TPS("cpuqs")); |
272 | __this_cpu_write(rcu_data_p->passed_quiesce, 1); | 272 | __this_cpu_write(rcu_data_p->cpu_no_qs, false); |
273 | barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */ | 273 | barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */ |
274 | current->rcu_read_unlock_special.b.need_qs = false; | 274 | current->rcu_read_unlock_special.b.need_qs = false; |
275 | } | 275 | } |
@@ -620,7 +620,7 @@ static void rcu_preempt_check_callbacks(void) | |||
620 | } | 620 | } |
621 | if (t->rcu_read_lock_nesting > 0 && | 621 | if (t->rcu_read_lock_nesting > 0 && |
622 | __this_cpu_read(rcu_data_p->core_needs_qs) && | 622 | __this_cpu_read(rcu_data_p->core_needs_qs) && |
623 | !__this_cpu_read(rcu_data_p->passed_quiesce)) | 623 | __this_cpu_read(rcu_data_p->cpu_no_qs)) |
624 | t->rcu_read_unlock_special.b.need_qs = true; | 624 | t->rcu_read_unlock_special.b.need_qs = true; |
625 | } | 625 | } |
626 | 626 | ||
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c index 4ac25f8520d6..d373e57109b8 100644 --- a/kernel/rcu/tree_trace.c +++ b/kernel/rcu/tree_trace.c | |||
@@ -117,11 +117,11 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) | |||
117 | 117 | ||
118 | if (!rdp->beenonline) | 118 | if (!rdp->beenonline) |
119 | return; | 119 | return; |
120 | seq_printf(m, "%3d%cc=%ld g=%ld pq=%d/%d qp=%d", | 120 | seq_printf(m, "%3d%cc=%ld g=%ld cnq=%d/%d:%d", |
121 | rdp->cpu, | 121 | rdp->cpu, |
122 | cpu_is_offline(rdp->cpu) ? '!' : ' ', | 122 | cpu_is_offline(rdp->cpu) ? '!' : ' ', |
123 | ulong2long(rdp->completed), ulong2long(rdp->gpnum), | 123 | ulong2long(rdp->completed), ulong2long(rdp->gpnum), |
124 | rdp->passed_quiesce, | 124 | rdp->cpu_no_qs, |
125 | rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu), | 125 | rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu), |
126 | rdp->core_needs_qs); | 126 | rdp->core_needs_qs); |
127 | seq_printf(m, " dt=%d/%llx/%d df=%lu", | 127 | seq_printf(m, " dt=%d/%llx/%d df=%lu", |