diff options
-rw-r--r-- | kernel/rcu/tree.h | 4 | ||||
-rw-r--r-- | kernel/rcu/tree_plugin.h | 17 |
2 files changed, 8 insertions, 13 deletions
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 5d447ceba769..69bd6bec05bb 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h | |||
@@ -49,10 +49,6 @@ struct rcu_dynticks { | |||
49 | /* # times non-lazy CBs posted to CPU. */ | 49 | /* # times non-lazy CBs posted to CPU. */ |
50 | unsigned long nonlazy_posted_snap; | 50 | unsigned long nonlazy_posted_snap; |
51 | /* idle-period nonlazy_posted snapshot. */ | 51 | /* idle-period nonlazy_posted snapshot. */ |
52 | unsigned long last_accelerate; | ||
53 | /* Last jiffy CBs were accelerated. */ | ||
54 | unsigned long last_advance_all; | ||
55 | /* Last jiffy CBs were all advanced. */ | ||
56 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ | 52 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ |
57 | }; | 53 | }; |
58 | 54 | ||
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 6511032371c1..45708164ddf9 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
@@ -1597,16 +1597,14 @@ module_param(rcu_idle_lazy_gp_delay, int, 0644); | |||
1597 | static bool __maybe_unused rcu_try_advance_all_cbs(void) | 1597 | static bool __maybe_unused rcu_try_advance_all_cbs(void) |
1598 | { | 1598 | { |
1599 | bool cbs_ready = false; | 1599 | bool cbs_ready = false; |
1600 | struct rcu_data *rdp; | 1600 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
1601 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); | ||
1602 | struct rcu_node *rnp; | 1601 | struct rcu_node *rnp; |
1603 | 1602 | ||
1604 | /* Exit early if we advanced recently. */ | 1603 | /* Exit early if we advanced recently. */ |
1605 | if (jiffies == rdtp->last_advance_all) | 1604 | if (jiffies == rdp->last_advance_all) |
1606 | return false; | 1605 | return false; |
1607 | rdtp->last_advance_all = jiffies; | 1606 | rdp->last_advance_all = jiffies; |
1608 | 1607 | ||
1609 | rdp = this_cpu_ptr(&rcu_data); | ||
1610 | rnp = rdp->mynode; | 1608 | rnp = rdp->mynode; |
1611 | 1609 | ||
1612 | /* | 1610 | /* |
@@ -1635,6 +1633,7 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void) | |||
1635 | */ | 1633 | */ |
1636 | int rcu_needs_cpu(u64 basemono, u64 *nextevt) | 1634 | int rcu_needs_cpu(u64 basemono, u64 *nextevt) |
1637 | { | 1635 | { |
1636 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | ||
1638 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); | 1637 | struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); |
1639 | unsigned long dj; | 1638 | unsigned long dj; |
1640 | 1639 | ||
@@ -1655,7 +1654,7 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt) | |||
1655 | invoke_rcu_core(); | 1654 | invoke_rcu_core(); |
1656 | return 1; | 1655 | return 1; |
1657 | } | 1656 | } |
1658 | rdtp->last_accelerate = jiffies; | 1657 | rdp->last_accelerate = jiffies; |
1659 | 1658 | ||
1660 | /* Request timer delay depending on laziness, and round. */ | 1659 | /* Request timer delay depending on laziness, and round. */ |
1661 | if (!rdtp->all_lazy) { | 1660 | if (!rdtp->all_lazy) { |
@@ -1718,9 +1717,9 @@ static void rcu_prepare_for_idle(void) | |||
1718 | * If we have not yet accelerated this jiffy, accelerate all | 1717 | * If we have not yet accelerated this jiffy, accelerate all |
1719 | * callbacks on this CPU. | 1718 | * callbacks on this CPU. |
1720 | */ | 1719 | */ |
1721 | if (rdtp->last_accelerate == jiffies) | 1720 | if (rdp->last_accelerate == jiffies) |
1722 | return; | 1721 | return; |
1723 | rdtp->last_accelerate = jiffies; | 1722 | rdp->last_accelerate = jiffies; |
1724 | if (rcu_segcblist_pend_cbs(&rdp->cblist)) { | 1723 | if (rcu_segcblist_pend_cbs(&rdp->cblist)) { |
1725 | rnp = rdp->mynode; | 1724 | rnp = rdp->mynode; |
1726 | raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ | 1725 | raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ |
@@ -1769,7 +1768,7 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu) | |||
1769 | unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap; | 1768 | unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap; |
1770 | 1769 | ||
1771 | sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c", | 1770 | sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c", |
1772 | rdtp->last_accelerate & 0xffff, jiffies & 0xffff, | 1771 | rdp->last_accelerate & 0xffff, jiffies & 0xffff, |
1773 | ulong2long(nlpd), | 1772 | ulong2long(nlpd), |
1774 | rdtp->all_lazy ? 'L' : '.', | 1773 | rdtp->all_lazy ? 'L' : '.', |
1775 | rdp->tick_nohz_enabled_snap ? '.' : 'D'); | 1774 | rdp->tick_nohz_enabled_snap ? '.' : 'D'); |