diff options
-rw-r--r-- | kernel/rcutree.c | 13 | ||||
-rw-r--r-- | kernel/rcutree.h | 4 |
2 files changed, 8 insertions, 9 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 8154a4a3491c..5d96d68d20f8 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
@@ -36,7 +36,7 @@ | |||
36 | #include <linux/interrupt.h> | 36 | #include <linux/interrupt.h> |
37 | #include <linux/sched.h> | 37 | #include <linux/sched.h> |
38 | #include <linux/nmi.h> | 38 | #include <linux/nmi.h> |
39 | #include <asm/atomic.h> | 39 | #include <linux/atomic.h> |
40 | #include <linux/bitops.h> | 40 | #include <linux/bitops.h> |
41 | #include <linux/module.h> | 41 | #include <linux/module.h> |
42 | #include <linux/completion.h> | 42 | #include <linux/completion.h> |
@@ -1526,13 +1526,10 @@ static void rcu_cpu_kthread_setrt(int cpu, int to_rt) | |||
1526 | */ | 1526 | */ |
1527 | static void rcu_cpu_kthread_timer(unsigned long arg) | 1527 | static void rcu_cpu_kthread_timer(unsigned long arg) |
1528 | { | 1528 | { |
1529 | unsigned long flags; | ||
1530 | struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg); | 1529 | struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg); |
1531 | struct rcu_node *rnp = rdp->mynode; | 1530 | struct rcu_node *rnp = rdp->mynode; |
1532 | 1531 | ||
1533 | raw_spin_lock_irqsave(&rnp->lock, flags); | 1532 | atomic_or(rdp->grpmask, &rnp->wakemask); |
1534 | rnp->wakemask |= rdp->grpmask; | ||
1535 | raw_spin_unlock_irqrestore(&rnp->lock, flags); | ||
1536 | invoke_rcu_node_kthread(rnp); | 1533 | invoke_rcu_node_kthread(rnp); |
1537 | } | 1534 | } |
1538 | 1535 | ||
@@ -1680,11 +1677,11 @@ static int rcu_node_kthread(void *arg) | |||
1680 | 1677 | ||
1681 | for (;;) { | 1678 | for (;;) { |
1682 | rnp->node_kthread_status = RCU_KTHREAD_WAITING; | 1679 | rnp->node_kthread_status = RCU_KTHREAD_WAITING; |
1683 | wait_event_interruptible(rnp->node_wq, rnp->wakemask != 0); | 1680 | wait_event_interruptible(rnp->node_wq, |
1681 | atomic_read(&rnp->wakemask) != 0); | ||
1684 | rnp->node_kthread_status = RCU_KTHREAD_RUNNING; | 1682 | rnp->node_kthread_status = RCU_KTHREAD_RUNNING; |
1685 | raw_spin_lock_irqsave(&rnp->lock, flags); | 1683 | raw_spin_lock_irqsave(&rnp->lock, flags); |
1686 | mask = rnp->wakemask; | 1684 | mask = atomic_xchg(&rnp->wakemask, 0); |
1687 | rnp->wakemask = 0; | ||
1688 | rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */ | 1685 | rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */ |
1689 | for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) { | 1686 | for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) { |
1690 | if ((mask & 0x1) == 0) | 1687 | if ((mask & 0x1) == 0) |
diff --git a/kernel/rcutree.h b/kernel/rcutree.h index 93d4a1c2e88b..561dcb9a8d2c 100644 --- a/kernel/rcutree.h +++ b/kernel/rcutree.h | |||
@@ -119,7 +119,9 @@ struct rcu_node { | |||
119 | /* elements that need to drain to allow the */ | 119 | /* elements that need to drain to allow the */ |
120 | /* current expedited grace period to */ | 120 | /* current expedited grace period to */ |
121 | /* complete (only for TREE_PREEMPT_RCU). */ | 121 | /* complete (only for TREE_PREEMPT_RCU). */ |
122 | unsigned long wakemask; /* CPUs whose kthread needs to be awakened. */ | 122 | atomic_t wakemask; /* CPUs whose kthread needs to be awakened. */ |
123 | /* Since this has meaning only for leaf */ | ||
124 | /* rcu_node structures, 32 bits suffices. */ | ||
123 | unsigned long qsmaskinit; | 125 | unsigned long qsmaskinit; |
124 | /* Per-GP initial value for qsmask & expmask. */ | 126 | /* Per-GP initial value for qsmask & expmask. */ |
125 | unsigned long grpmask; /* Mask to apply to parent qsmask. */ | 127 | unsigned long grpmask; /* Mask to apply to parent qsmask. */ |