aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcutree.c
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2013-03-19 13:08:37 -0400
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2013-06-10 16:39:42 -0400
commit6eaef633d77f50f031dd355ff5f91aaa1aaf9885 (patch)
tree53f5ee20265cb8da7d67e6ce73165ed7bcbec039 /kernel/rcutree.c
parentefc151c33b971148894789dc7c5589dec46d4348 (diff)
rcu: Move code to apply callback-numbering simplifications
The addition of callback numbering allows combining the detection of the ends of old grace periods and the beginnings of new grace periods. This commit moves code to set the stage for this combining. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Diffstat (limited to 'kernel/rcutree.c')
-rw-r--r--kernel/rcutree.c118
1 files changed, 59 insertions, 59 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 1009c0ccd4b1..c36e52dc091d 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -985,65 +985,6 @@ void rcu_cpu_stall_reset(void)
985} 985}
986 986
987/* 987/*
988 * Update CPU-local rcu_data state to record the newly noticed grace period.
989 * This is used both when we started the grace period and when we notice
990 * that someone else started the grace period. The caller must hold the
991 * ->lock of the leaf rcu_node structure corresponding to the current CPU,
992 * and must have irqs disabled.
993 */
994static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
995{
996 if (rdp->gpnum != rnp->gpnum) {
997 /*
998 * If the current grace period is waiting for this CPU,
999 * set up to detect a quiescent state, otherwise don't
1000 * go looking for one.
1001 */
1002 rdp->gpnum = rnp->gpnum;
1003 trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
1004 rdp->passed_quiesce = 0;
1005 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
1006 zero_cpu_stall_ticks(rdp);
1007 }
1008}
1009
1010static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
1011{
1012 unsigned long flags;
1013 struct rcu_node *rnp;
1014
1015 local_irq_save(flags);
1016 rnp = rdp->mynode;
1017 if (rdp->gpnum == ACCESS_ONCE(rnp->gpnum) || /* outside lock. */
1018 !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */
1019 local_irq_restore(flags);
1020 return;
1021 }
1022 __note_new_gpnum(rsp, rnp, rdp);
1023 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1024}
1025
1026/*
1027 * Did someone else start a new RCU grace period start since we last
1028 * checked? Update local state appropriately if so. Must be called
1029 * on the CPU corresponding to rdp.
1030 */
1031static int
1032check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
1033{
1034 unsigned long flags;
1035 int ret = 0;
1036
1037 local_irq_save(flags);
1038 if (rdp->gpnum != rsp->gpnum) {
1039 note_new_gpnum(rsp, rdp);
1040 ret = 1;
1041 }
1042 local_irq_restore(flags);
1043 return ret;
1044}
1045
1046/*
1047 * Initialize the specified rcu_data structure's callback list to empty. 988 * Initialize the specified rcu_data structure's callback list to empty.
1048 */ 989 */
1049static void init_callback_list(struct rcu_data *rdp) 990static void init_callback_list(struct rcu_data *rdp)
@@ -1360,6 +1301,45 @@ __rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
1360} 1301}
1361 1302
1362/* 1303/*
1304 * Update CPU-local rcu_data state to record the newly noticed grace period.
1305 * This is used both when we started the grace period and when we notice
1306 * that someone else started the grace period. The caller must hold the
1307 * ->lock of the leaf rcu_node structure corresponding to the current CPU,
1308 * and must have irqs disabled.
1309 */
1310static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
1311{
1312 if (rdp->gpnum != rnp->gpnum) {
1313 /*
1314 * If the current grace period is waiting for this CPU,
1315 * set up to detect a quiescent state, otherwise don't
1316 * go looking for one.
1317 */
1318 rdp->gpnum = rnp->gpnum;
1319 trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
1320 rdp->passed_quiesce = 0;
1321 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
1322 zero_cpu_stall_ticks(rdp);
1323 }
1324}
1325
1326static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
1327{
1328 unsigned long flags;
1329 struct rcu_node *rnp;
1330
1331 local_irq_save(flags);
1332 rnp = rdp->mynode;
1333 if (rdp->gpnum == ACCESS_ONCE(rnp->gpnum) || /* outside lock. */
1334 !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */
1335 local_irq_restore(flags);
1336 return;
1337 }
1338 __note_new_gpnum(rsp, rnp, rdp);
1339 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1340}
1341
1342/*
1363 * Advance this CPU's callbacks, but only if the current grace period 1343 * Advance this CPU's callbacks, but only if the current grace period
1364 * has ended. This may be called only from the CPU to whom the rdp 1344 * has ended. This may be called only from the CPU to whom the rdp
1365 * belongs. 1345 * belongs.
@@ -1382,6 +1362,26 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
1382} 1362}
1383 1363
1384/* 1364/*
1365 * Did someone else start a new RCU grace period start since we last
1366 * checked? Update local state appropriately if so. Must be called
1367 * on the CPU corresponding to rdp.
1368 */
1369static int
1370check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
1371{
1372 unsigned long flags;
1373 int ret = 0;
1374
1375 local_irq_save(flags);
1376 if (rdp->gpnum != rsp->gpnum) {
1377 note_new_gpnum(rsp, rdp);
1378 ret = 1;
1379 }
1380 local_irq_restore(flags);
1381 return ret;
1382}
1383
1384/*
1385 * Do per-CPU grace-period initialization for running CPU. The caller 1385 * Do per-CPU grace-period initialization for running CPU. The caller
1386 * must hold the lock of the leaf rcu_node structure corresponding to 1386 * must hold the lock of the leaf rcu_node structure corresponding to
1387 * this CPU. 1387 * this CPU.