aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/rcu/tree.h4
-rw-r--r--kernel/rcu/tree_plugin.h17
2 files changed, 8 insertions, 13 deletions
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 5d447ceba769..69bd6bec05bb 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -49,10 +49,6 @@ struct rcu_dynticks {
49 /* # times non-lazy CBs posted to CPU. */ 49 /* # times non-lazy CBs posted to CPU. */
50 unsigned long nonlazy_posted_snap; 50 unsigned long nonlazy_posted_snap;
51 /* idle-period nonlazy_posted snapshot. */ 51 /* idle-period nonlazy_posted snapshot. */
52 unsigned long last_accelerate;
53 /* Last jiffy CBs were accelerated. */
54 unsigned long last_advance_all;
55 /* Last jiffy CBs were all advanced. */
56#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ 52#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
57}; 53};
58 54
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 6511032371c1..45708164ddf9 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1597,16 +1597,14 @@ module_param(rcu_idle_lazy_gp_delay, int, 0644);
1597static bool __maybe_unused rcu_try_advance_all_cbs(void) 1597static bool __maybe_unused rcu_try_advance_all_cbs(void)
1598{ 1598{
1599 bool cbs_ready = false; 1599 bool cbs_ready = false;
1600 struct rcu_data *rdp; 1600 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1601 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1602 struct rcu_node *rnp; 1601 struct rcu_node *rnp;
1603 1602
1604 /* Exit early if we advanced recently. */ 1603 /* Exit early if we advanced recently. */
1605 if (jiffies == rdtp->last_advance_all) 1604 if (jiffies == rdp->last_advance_all)
1606 return false; 1605 return false;
1607 rdtp->last_advance_all = jiffies; 1606 rdp->last_advance_all = jiffies;
1608 1607
1609 rdp = this_cpu_ptr(&rcu_data);
1610 rnp = rdp->mynode; 1608 rnp = rdp->mynode;
1611 1609
1612 /* 1610 /*
@@ -1635,6 +1633,7 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
1635 */ 1633 */
1636int rcu_needs_cpu(u64 basemono, u64 *nextevt) 1634int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1637{ 1635{
1636 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1638 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); 1637 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1639 unsigned long dj; 1638 unsigned long dj;
1640 1639
@@ -1655,7 +1654,7 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1655 invoke_rcu_core(); 1654 invoke_rcu_core();
1656 return 1; 1655 return 1;
1657 } 1656 }
1658 rdtp->last_accelerate = jiffies; 1657 rdp->last_accelerate = jiffies;
1659 1658
1660 /* Request timer delay depending on laziness, and round. */ 1659 /* Request timer delay depending on laziness, and round. */
1661 if (!rdtp->all_lazy) { 1660 if (!rdtp->all_lazy) {
@@ -1718,9 +1717,9 @@ static void rcu_prepare_for_idle(void)
1718 * If we have not yet accelerated this jiffy, accelerate all 1717 * If we have not yet accelerated this jiffy, accelerate all
1719 * callbacks on this CPU. 1718 * callbacks on this CPU.
1720 */ 1719 */
1721 if (rdtp->last_accelerate == jiffies) 1720 if (rdp->last_accelerate == jiffies)
1722 return; 1721 return;
1723 rdtp->last_accelerate = jiffies; 1722 rdp->last_accelerate = jiffies;
1724 if (rcu_segcblist_pend_cbs(&rdp->cblist)) { 1723 if (rcu_segcblist_pend_cbs(&rdp->cblist)) {
1725 rnp = rdp->mynode; 1724 rnp = rdp->mynode;
1726 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ 1725 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
@@ -1769,7 +1768,7 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1769 unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap; 1768 unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1770 1769
1771 sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c", 1770 sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1772 rdtp->last_accelerate & 0xffff, jiffies & 0xffff, 1771 rdp->last_accelerate & 0xffff, jiffies & 0xffff,
1773 ulong2long(nlpd), 1772 ulong2long(nlpd),
1774 rdtp->all_lazy ? 'L' : '.', 1773 rdtp->all_lazy ? 'L' : '.',
1775 rdp->tick_nohz_enabled_snap ? '.' : 'D'); 1774 rdp->tick_nohz_enabled_snap ? '.' : 'D');