aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/rcutree.c27
-rw-r--r--kernel/rcutree.h2
-rw-r--r--kernel/rcutree_plugin.h2
-rw-r--r--kernel/rcutree_trace.c12
4 files changed, 16 insertions, 27 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index a2eadd04fb29..6194402ec853 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -176,8 +176,6 @@ void rcu_sched_qs(int cpu)
176{ 176{
177 struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); 177 struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu);
178 178
179 rdp->passed_quiesce_gpnum = rdp->gpnum;
180 barrier();
181 if (rdp->passed_quiesce == 0) 179 if (rdp->passed_quiesce == 0)
182 trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs"); 180 trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");
183 rdp->passed_quiesce = 1; 181 rdp->passed_quiesce = 1;
@@ -187,8 +185,6 @@ void rcu_bh_qs(int cpu)
187{ 185{
188 struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); 186 struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
189 187
190 rdp->passed_quiesce_gpnum = rdp->gpnum;
191 barrier();
192 if (rdp->passed_quiesce == 0) 188 if (rdp->passed_quiesce == 0)
193 trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs"); 189 trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");
194 rdp->passed_quiesce = 1; 190 rdp->passed_quiesce = 1;
@@ -899,12 +895,8 @@ static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct
899 */ 895 */
900 rdp->gpnum = rnp->gpnum; 896 rdp->gpnum = rnp->gpnum;
901 trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart"); 897 trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
902 if (rnp->qsmask & rdp->grpmask) { 898 rdp->passed_quiesce = 0;
903 rdp->qs_pending = 1; 899 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
904 rdp->passed_quiesce = 0;
905 } else {
906 rdp->qs_pending = 0;
907 }
908 zero_cpu_stall_ticks(rdp); 900 zero_cpu_stall_ticks(rdp);
909 } 901 }
910} 902}
@@ -984,10 +976,13 @@ __rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
984 * our behalf. Catch up with this state to avoid noting 976 * our behalf. Catch up with this state to avoid noting
985 * spurious new grace periods. If another grace period 977 * spurious new grace periods. If another grace period
986 * has started, then rnp->gpnum will have advanced, so 978 * has started, then rnp->gpnum will have advanced, so
987 * we will detect this later on. 979 * we will detect this later on. Of course, any quiescent
980 * states we found for the old GP are now invalid.
988 */ 981 */
989 if (ULONG_CMP_LT(rdp->gpnum, rdp->completed)) 982 if (ULONG_CMP_LT(rdp->gpnum, rdp->completed)) {
990 rdp->gpnum = rdp->completed; 983 rdp->gpnum = rdp->completed;
984 rdp->passed_quiesce = 0;
985 }
991 986
992 /* 987 /*
993 * If RCU does not need a quiescent state from this CPU, 988 * If RCU does not need a quiescent state from this CPU,
@@ -1358,7 +1353,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
1358 * based on quiescent states detected in an earlier grace period! 1353 * based on quiescent states detected in an earlier grace period!
1359 */ 1354 */
1360static void 1355static void
1361rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastgp) 1356rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
1362{ 1357{
1363 unsigned long flags; 1358 unsigned long flags;
1364 unsigned long mask; 1359 unsigned long mask;
@@ -1366,7 +1361,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long las
1366 1361
1367 rnp = rdp->mynode; 1362 rnp = rdp->mynode;
1368 raw_spin_lock_irqsave(&rnp->lock, flags); 1363 raw_spin_lock_irqsave(&rnp->lock, flags);
1369 if (lastgp != rnp->gpnum || rnp->completed == rnp->gpnum) { 1364 if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum ||
1365 rnp->completed == rnp->gpnum) {
1370 1366
1371 /* 1367 /*
1372 * The grace period in which this quiescent state was 1368 * The grace period in which this quiescent state was
@@ -1425,7 +1421,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
1425 * Tell RCU we are done (but rcu_report_qs_rdp() will be the 1421 * Tell RCU we are done (but rcu_report_qs_rdp() will be the
1426 * judge of that). 1422 * judge of that).
1427 */ 1423 */
1428 rcu_report_qs_rdp(rdp->cpu, rsp, rdp, rdp->passed_quiesce_gpnum); 1424 rcu_report_qs_rdp(rdp->cpu, rsp, rdp);
1429} 1425}
1430 1426
1431#ifdef CONFIG_HOTPLUG_CPU 1427#ifdef CONFIG_HOTPLUG_CPU
@@ -2600,7 +2596,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
2600 rdp->completed = rnp->completed; 2596 rdp->completed = rnp->completed;
2601 rdp->passed_quiesce = 0; 2597 rdp->passed_quiesce = 0;
2602 rdp->qs_pending = 0; 2598 rdp->qs_pending = 0;
2603 rdp->passed_quiesce_gpnum = rnp->gpnum - 1;
2604 trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl"); 2599 trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl");
2605 } 2600 }
2606 raw_spin_unlock(&rnp->lock); /* irqs already disabled. */ 2601 raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 8f0293ce1517..935dd4ca6816 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -246,8 +246,6 @@ struct rcu_data {
246 /* in order to detect GP end. */ 246 /* in order to detect GP end. */
247 unsigned long gpnum; /* Highest gp number that this CPU */ 247 unsigned long gpnum; /* Highest gp number that this CPU */
248 /* is aware of having started. */ 248 /* is aware of having started. */
249 unsigned long passed_quiesce_gpnum;
250 /* gpnum at time of quiescent state. */
251 bool passed_quiesce; /* User-mode/idle loop etc. */ 249 bool passed_quiesce; /* User-mode/idle loop etc. */
252 bool qs_pending; /* Core waits for quiesc state. */ 250 bool qs_pending; /* Core waits for quiesc state. */
253 bool beenonline; /* CPU online at least once. */ 251 bool beenonline; /* CPU online at least once. */
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index b4e8eb24a5f1..4734afbea73a 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -137,8 +137,6 @@ static void rcu_preempt_qs(int cpu)
137{ 137{
138 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu); 138 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
139 139
140 rdp->passed_quiesce_gpnum = rdp->gpnum;
141 barrier();
142 if (rdp->passed_quiesce == 0) 140 if (rdp->passed_quiesce == 0)
143 trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs"); 141 trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
144 rdp->passed_quiesce = 1; 142 rdp->passed_quiesce = 1;
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index f54f0ceda0cf..bd4df13d4afb 100644
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -86,12 +86,11 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
86{ 86{
87 if (!rdp->beenonline) 87 if (!rdp->beenonline)
88 return; 88 return;
89 seq_printf(m, "%3d%cc=%lu g=%lu pq=%d pgp=%lu qp=%d", 89 seq_printf(m, "%3d%cc=%lu g=%lu pq=%d qp=%d",
90 rdp->cpu, 90 rdp->cpu,
91 cpu_is_offline(rdp->cpu) ? '!' : ' ', 91 cpu_is_offline(rdp->cpu) ? '!' : ' ',
92 rdp->completed, rdp->gpnum, 92 rdp->completed, rdp->gpnum,
93 rdp->passed_quiesce, rdp->passed_quiesce_gpnum, 93 rdp->passed_quiesce, rdp->qs_pending);
94 rdp->qs_pending);
95 seq_printf(m, " dt=%d/%llx/%d df=%lu", 94 seq_printf(m, " dt=%d/%llx/%d df=%lu",
96 atomic_read(&rdp->dynticks->dynticks), 95 atomic_read(&rdp->dynticks->dynticks),
97 rdp->dynticks->dynticks_nesting, 96 rdp->dynticks->dynticks_nesting,
@@ -150,12 +149,11 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
150{ 149{
151 if (!rdp->beenonline) 150 if (!rdp->beenonline)
152 return; 151 return;
153 seq_printf(m, "%d,%s,%lu,%lu,%d,%lu,%d", 152 seq_printf(m, "%d,%s,%lu,%lu,%d,%d",
154 rdp->cpu, 153 rdp->cpu,
155 cpu_is_offline(rdp->cpu) ? "\"N\"" : "\"Y\"", 154 cpu_is_offline(rdp->cpu) ? "\"N\"" : "\"Y\"",
156 rdp->completed, rdp->gpnum, 155 rdp->completed, rdp->gpnum,
157 rdp->passed_quiesce, rdp->passed_quiesce_gpnum, 156 rdp->passed_quiesce, rdp->qs_pending);
158 rdp->qs_pending);
159 seq_printf(m, ",%d,%llx,%d,%lu", 157 seq_printf(m, ",%d,%llx,%d,%lu",
160 atomic_read(&rdp->dynticks->dynticks), 158 atomic_read(&rdp->dynticks->dynticks),
161 rdp->dynticks->dynticks_nesting, 159 rdp->dynticks->dynticks_nesting,
@@ -186,7 +184,7 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
186 int cpu; 184 int cpu;
187 struct rcu_state *rsp; 185 struct rcu_state *rsp;
188 186
189 seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pgp\",\"pq\","); 187 seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pq\",");
190 seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\","); 188 seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\",");
191 seq_puts(m, "\"of\",\"qll\",\"ql\",\"qs\""); 189 seq_puts(m, "\"of\",\"qll\",\"ql\",\"qs\"");
192#ifdef CONFIG_RCU_BOOST 190#ifdef CONFIG_RCU_BOOST