diff options
Diffstat (limited to 'kernel/rcutree.c')
-rw-r--r-- | kernel/rcutree.c | 23 |
1 files changed, 19 insertions, 4 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 0a8ec5b2e208..ae5c9ea68662 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
@@ -100,6 +100,7 @@ static char rcu_kthreads_spawnable; | |||
100 | 100 | ||
101 | static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); | 101 | static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); |
102 | static void invoke_rcu_cpu_kthread(void); | 102 | static void invoke_rcu_cpu_kthread(void); |
103 | static void __invoke_rcu_cpu_kthread(void); | ||
103 | 104 | ||
104 | #define RCU_KTHREAD_PRIO 1 /* RT priority for per-CPU kthreads. */ | 105 | #define RCU_KTHREAD_PRIO 1 /* RT priority for per-CPU kthreads. */ |
105 | 106 | ||
@@ -1442,13 +1443,21 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) | |||
1442 | } | 1443 | } |
1443 | 1444 | ||
1444 | /* If there are callbacks ready, invoke them. */ | 1445 | /* If there are callbacks ready, invoke them. */ |
1445 | rcu_do_batch(rsp, rdp); | 1446 | if (cpu_has_callbacks_ready_to_invoke(rdp)) |
1447 | __invoke_rcu_cpu_kthread(); | ||
1448 | } | ||
1449 | |||
1450 | static void rcu_kthread_do_work(void) | ||
1451 | { | ||
1452 | rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data)); | ||
1453 | rcu_do_batch(&rcu_bh_state, &__get_cpu_var(rcu_bh_data)); | ||
1454 | rcu_preempt_do_callbacks(); | ||
1446 | } | 1455 | } |
1447 | 1456 | ||
1448 | /* | 1457 | /* |
1449 | * Do softirq processing for the current CPU. | 1458 | * Do softirq processing for the current CPU. |
1450 | */ | 1459 | */ |
1451 | static void rcu_process_callbacks(void) | 1460 | static void rcu_process_callbacks(struct softirq_action *unused) |
1452 | { | 1461 | { |
1453 | __rcu_process_callbacks(&rcu_sched_state, | 1462 | __rcu_process_callbacks(&rcu_sched_state, |
1454 | &__get_cpu_var(rcu_sched_data)); | 1463 | &__get_cpu_var(rcu_sched_data)); |
@@ -1465,7 +1474,7 @@ static void rcu_process_callbacks(void) | |||
1465 | * the current CPU with interrupts disabled, the rcu_cpu_kthread_task | 1474 | * the current CPU with interrupts disabled, the rcu_cpu_kthread_task |
1466 | * cannot disappear out from under us. | 1475 | * cannot disappear out from under us. |
1467 | */ | 1476 | */ |
1468 | static void invoke_rcu_cpu_kthread(void) | 1477 | static void __invoke_rcu_cpu_kthread(void) |
1469 | { | 1478 | { |
1470 | unsigned long flags; | 1479 | unsigned long flags; |
1471 | 1480 | ||
@@ -1479,6 +1488,11 @@ static void invoke_rcu_cpu_kthread(void) | |||
1479 | local_irq_restore(flags); | 1488 | local_irq_restore(flags); |
1480 | } | 1489 | } |
1481 | 1490 | ||
1491 | static void invoke_rcu_cpu_kthread(void) | ||
1492 | { | ||
1493 | raise_softirq(RCU_SOFTIRQ); | ||
1494 | } | ||
1495 | |||
1482 | /* | 1496 | /* |
1483 | * Wake up the specified per-rcu_node-structure kthread. | 1497 | * Wake up the specified per-rcu_node-structure kthread. |
1484 | * Because the per-rcu_node kthreads are immortal, we don't need | 1498 | * Because the per-rcu_node kthreads are immortal, we don't need |
@@ -1613,7 +1627,7 @@ static int rcu_cpu_kthread(void *arg) | |||
1613 | *workp = 0; | 1627 | *workp = 0; |
1614 | local_irq_restore(flags); | 1628 | local_irq_restore(flags); |
1615 | if (work) | 1629 | if (work) |
1616 | rcu_process_callbacks(); | 1630 | rcu_kthread_do_work(); |
1617 | local_bh_enable(); | 1631 | local_bh_enable(); |
1618 | if (*workp != 0) | 1632 | if (*workp != 0) |
1619 | spincnt++; | 1633 | spincnt++; |
@@ -2387,6 +2401,7 @@ void __init rcu_init(void) | |||
2387 | rcu_init_one(&rcu_sched_state, &rcu_sched_data); | 2401 | rcu_init_one(&rcu_sched_state, &rcu_sched_data); |
2388 | rcu_init_one(&rcu_bh_state, &rcu_bh_data); | 2402 | rcu_init_one(&rcu_bh_state, &rcu_bh_data); |
2389 | __rcu_init_preempt(); | 2403 | __rcu_init_preempt(); |
2404 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | ||
2390 | 2405 | ||
2391 | /* | 2406 | /* |
2392 | * We don't need protection against CPU-hotplug here because | 2407 | * We don't need protection against CPU-hotplug here because |