aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/rcutree.c13
-rw-r--r--kernel/rcutree.h4
2 files changed, 8 insertions, 9 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 8154a4a3491c..5d96d68d20f8 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -36,7 +36,7 @@
36#include <linux/interrupt.h> 36#include <linux/interrupt.h>
37#include <linux/sched.h> 37#include <linux/sched.h>
38#include <linux/nmi.h> 38#include <linux/nmi.h>
39#include <asm/atomic.h> 39#include <linux/atomic.h>
40#include <linux/bitops.h> 40#include <linux/bitops.h>
41#include <linux/module.h> 41#include <linux/module.h>
42#include <linux/completion.h> 42#include <linux/completion.h>
@@ -1526,13 +1526,10 @@ static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
1526 */ 1526 */
1527static void rcu_cpu_kthread_timer(unsigned long arg) 1527static void rcu_cpu_kthread_timer(unsigned long arg)
1528{ 1528{
1529 unsigned long flags;
1530 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg); 1529 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg);
1531 struct rcu_node *rnp = rdp->mynode; 1530 struct rcu_node *rnp = rdp->mynode;
1532 1531
1533 raw_spin_lock_irqsave(&rnp->lock, flags); 1532 atomic_or(rdp->grpmask, &rnp->wakemask);
1534 rnp->wakemask |= rdp->grpmask;
1535 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1536 invoke_rcu_node_kthread(rnp); 1533 invoke_rcu_node_kthread(rnp);
1537} 1534}
1538 1535
@@ -1680,11 +1677,11 @@ static int rcu_node_kthread(void *arg)
1680 1677
1681 for (;;) { 1678 for (;;) {
1682 rnp->node_kthread_status = RCU_KTHREAD_WAITING; 1679 rnp->node_kthread_status = RCU_KTHREAD_WAITING;
1683 wait_event_interruptible(rnp->node_wq, rnp->wakemask != 0); 1680 wait_event_interruptible(rnp->node_wq,
1681 atomic_read(&rnp->wakemask) != 0);
1684 rnp->node_kthread_status = RCU_KTHREAD_RUNNING; 1682 rnp->node_kthread_status = RCU_KTHREAD_RUNNING;
1685 raw_spin_lock_irqsave(&rnp->lock, flags); 1683 raw_spin_lock_irqsave(&rnp->lock, flags);
1686 mask = rnp->wakemask; 1684 mask = atomic_xchg(&rnp->wakemask, 0);
1687 rnp->wakemask = 0;
1688 rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */ 1685 rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */
1689 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) { 1686 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) {
1690 if ((mask & 0x1) == 0) 1687 if ((mask & 0x1) == 0)
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 93d4a1c2e88b..561dcb9a8d2c 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -119,7 +119,9 @@ struct rcu_node {
119 /* elements that need to drain to allow the */ 119 /* elements that need to drain to allow the */
120 /* current expedited grace period to */ 120 /* current expedited grace period to */
121 /* complete (only for TREE_PREEMPT_RCU). */ 121 /* complete (only for TREE_PREEMPT_RCU). */
122 unsigned long wakemask; /* CPUs whose kthread needs to be awakened. */ 122 atomic_t wakemask; /* CPUs whose kthread needs to be awakened. */
123 /* Since this has meaning only for leaf */
124 /* rcu_node structures, 32 bits suffices. */
123 unsigned long qsmaskinit; 125 unsigned long qsmaskinit;
124 /* Per-GP initial value for qsmask & expmask. */ 126 /* Per-GP initial value for qsmask & expmask. */
125 unsigned long grpmask; /* Mask to apply to parent qsmask. */ 127 unsigned long grpmask; /* Mask to apply to parent qsmask. */