aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/rcutree.c22
-rw-r--r--kernel/rcutree.h5
2 files changed, 7 insertions, 20 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index e4971192fa9c..6268f37adfc4 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1144,6 +1144,7 @@ void rcu_check_callbacks(int cpu, int user)
1144/* 1144/*
1145 * Scan the leaf rcu_node structures, processing dyntick state for any that 1145 * Scan the leaf rcu_node structures, processing dyntick state for any that
1146 * have not yet encountered a quiescent state, using the function specified. 1146 * have not yet encountered a quiescent state, using the function specified.
1147 * The caller must have suppressed start of new grace periods.
1147 */ 1148 */
1148static void rcu_process_dyntick(struct rcu_state *rsp, 1149static void rcu_process_dyntick(struct rcu_state *rsp,
1149 int (*f)(struct rcu_data *)) 1150 int (*f)(struct rcu_data *))
@@ -1157,7 +1158,7 @@ static void rcu_process_dyntick(struct rcu_state *rsp,
1157 rcu_for_each_leaf_node(rsp, rnp) { 1158 rcu_for_each_leaf_node(rsp, rnp) {
1158 mask = 0; 1159 mask = 0;
1159 spin_lock_irqsave(&rnp->lock, flags); 1160 spin_lock_irqsave(&rnp->lock, flags);
1160 if (rnp->completed != rsp->gpnum - 1) { 1161 if (!rcu_gp_in_progress(rsp)) {
1161 spin_unlock_irqrestore(&rnp->lock, flags); 1162 spin_unlock_irqrestore(&rnp->lock, flags);
1162 return; 1163 return;
1163 } 1164 }
@@ -1171,7 +1172,7 @@ static void rcu_process_dyntick(struct rcu_state *rsp,
1171 if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu])) 1172 if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
1172 mask |= bit; 1173 mask |= bit;
1173 } 1174 }
1174 if (mask != 0 && rnp->completed == rsp->gpnum - 1) { 1175 if (mask != 0 && rcu_gp_in_progress(rsp)) {
1175 1176
1176 /* rcu_report_qs_rnp() releases rnp->lock. */ 1177 /* rcu_report_qs_rnp() releases rnp->lock. */
1177 rcu_report_qs_rnp(mask, rsp, rnp, flags); 1178 rcu_report_qs_rnp(mask, rsp, rnp, flags);
@@ -1189,7 +1190,6 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1189{ 1190{
1190 unsigned long flags; 1191 unsigned long flags;
1191 struct rcu_node *rnp = rcu_get_root(rsp); 1192 struct rcu_node *rnp = rcu_get_root(rsp);
1192 u8 forcenow;
1193 1193
1194 if (!rcu_gp_in_progress(rsp)) 1194 if (!rcu_gp_in_progress(rsp))
1195 return; /* No grace period in progress, nothing to force. */ 1195 return; /* No grace period in progress, nothing to force. */
@@ -1224,21 +1224,9 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1224 /* Record dyntick-idle state. */ 1224 /* Record dyntick-idle state. */
1225 rcu_process_dyntick(rsp, dyntick_save_progress_counter); 1225 rcu_process_dyntick(rsp, dyntick_save_progress_counter);
1226 spin_lock(&rnp->lock); /* irqs already disabled */ 1226 spin_lock(&rnp->lock); /* irqs already disabled */
1227 if (!rcu_gp_in_progress(rsp)) 1227 if (rcu_gp_in_progress(rsp))
1228 break;
1229 /* fall into next case. */
1230
1231 case RCU_SAVE_COMPLETED:
1232
1233 /* Update state, record completion counter. */
1234 forcenow = 0;
1235 if (rsp->gpnum - 1 == rsp->completed) {
1236 forcenow = rsp->signaled == RCU_SAVE_COMPLETED;
1237 rsp->signaled = RCU_FORCE_QS; 1228 rsp->signaled = RCU_FORCE_QS;
1238 } 1229 break;
1239 if (!forcenow)
1240 break;
1241 /* fall into next case. */
1242 1230
1243 case RCU_FORCE_QS: 1231 case RCU_FORCE_QS:
1244 1232
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 534856121b06..edb6fae0fa94 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -237,12 +237,11 @@ struct rcu_data {
237#define RCU_GP_IDLE 0 /* No grace period in progress. */ 237#define RCU_GP_IDLE 0 /* No grace period in progress. */
238#define RCU_GP_INIT 1 /* Grace period being initialized. */ 238#define RCU_GP_INIT 1 /* Grace period being initialized. */
239#define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */ 239#define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */
240#define RCU_SAVE_COMPLETED 3 /* Need to save rsp->completed. */ 240#define RCU_FORCE_QS 3 /* Need to force quiescent state. */
241#define RCU_FORCE_QS 4 /* Need to force quiescent state. */
242#ifdef CONFIG_NO_HZ 241#ifdef CONFIG_NO_HZ
243#define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK 242#define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK
244#else /* #ifdef CONFIG_NO_HZ */ 243#else /* #ifdef CONFIG_NO_HZ */
245#define RCU_SIGNAL_INIT RCU_SAVE_COMPLETED 244#define RCU_SIGNAL_INIT RCU_FORCE_QS
246#endif /* #else #ifdef CONFIG_NO_HZ */ 245#endif /* #else #ifdef CONFIG_NO_HZ */
247 246
248#define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ 247#define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */