aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcutree.c
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2009-11-02 16:52:29 -0500
committerIngo Molnar <mingo@elte.hu>2009-11-09 22:12:11 -0500
commit9160306e6f5b68bb64630c9031c517ca1cf463db (patch)
treed8f78f99d417f34121c21e45a4675b2b1d5c2674 /kernel/rcutree.c
parentd09b62dfa336447c52a5ec9bb88adbc479b0f3b8 (diff)
rcu: Fix note_new_gpnum() uses of ->gpnum
Impose a clear locking design on the note_new_gpnum() function's use of the ->gpnum counter. This is done by updating rdp->gpnum only from the corresponding leaf rcu_node structure's rnp->gpnum field, and even then only under the protection of that same rcu_node structure's ->lock field. Performance and scalability are maintained using a form of double-checked locking, and excessive spinning is avoided by use of the spin_trylock() function. The use of spin_trylock() is safe due to the fact that CPUs who fail to acquire this lock will try again later. The hierarchical nature of the rcu_node data structure limits contention (which could be limited further if need be using the RCU_FANOUT kernel parameter). Without this patch, obscure but quite possible races could result in a quiescent state that occurred during one grace period to be accounted to the following grace period, causing this following grace period to end prematurely. Not good! Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: laijs@cn.fujitsu.com Cc: dipankar@in.ibm.com Cc: mathieu.desnoyers@polymtl.ca Cc: josh@joshtriplett.org Cc: dvhltc@us.ibm.com Cc: niv@us.ibm.com Cc: peterz@infradead.org Cc: rostedt@goodmis.org Cc: Valdis.Kletnieks@vt.edu Cc: dhowells@redhat.com Cc: <stable@kernel.org> # .32.x LKML-Reference: <12571987492350-git-send-email-> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel/rcutree.c')
-rw-r--r--kernel/rcutree.c32
1 files changed, 27 insertions, 5 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 9e068d112153..ec007dd22632 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -540,13 +540,33 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
540/* 540/*
541 * Update CPU-local rcu_data state to record the newly noticed grace period. 541 * Update CPU-local rcu_data state to record the newly noticed grace period.
542 * This is used both when we started the grace period and when we notice 542 * This is used both when we started the grace period and when we notice
543 * that someone else started the grace period. 543 * that someone else started the grace period. The caller must hold the
544 * ->lock of the leaf rcu_node structure corresponding to the current CPU,
545 * and must have irqs disabled.
544 */ 546 */
547static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
548{
549 if (rdp->gpnum != rnp->gpnum) {
550 rdp->qs_pending = 1;
551 rdp->passed_quiesc = 0;
552 rdp->gpnum = rnp->gpnum;
553 }
554}
555
545static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp) 556static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
546{ 557{
547 rdp->qs_pending = 1; 558 unsigned long flags;
548 rdp->passed_quiesc = 0; 559 struct rcu_node *rnp;
549 rdp->gpnum = rsp->gpnum; 560
561 local_irq_save(flags);
562 rnp = rdp->mynode;
563 if (rdp->gpnum == ACCESS_ONCE(rnp->gpnum) || /* outside lock. */
564 !spin_trylock(&rnp->lock)) { /* irqs already off, retry later. */
565 local_irq_restore(flags);
566 return;
567 }
568 __note_new_gpnum(rsp, rnp, rdp);
569 spin_unlock_irqrestore(&rnp->lock, flags);
550} 570}
551 571
552/* 572/*
@@ -637,6 +657,9 @@ rcu_start_gp_per_cpu(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
637 */ 657 */
638 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; 658 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
639 rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; 659 rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
660
661 /* Set state so that this CPU will detect the next quiescent state. */
662 __note_new_gpnum(rsp, rnp, rdp);
640} 663}
641 664
642/* 665/*
@@ -664,7 +687,6 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
664 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS; 687 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
665 record_gp_stall_check_time(rsp); 688 record_gp_stall_check_time(rsp);
666 dyntick_record_completed(rsp, rsp->completed - 1); 689 dyntick_record_completed(rsp, rsp->completed - 1);
667 note_new_gpnum(rsp, rdp);
668 690
669 /* Special-case the common single-level case. */ 691 /* Special-case the common single-level case. */
670 if (NUM_RCU_NODES == 1) { 692 if (NUM_RCU_NODES == 1) {