rcu: Conditionally compile RCU's eqs warnings

This commit applies some warning-omission micro-optimizations to RCU's
various extended-quiescent-state functions, which are on the kernel/user
hotpath for CONFIG_NO_HZ_FULL=y.

Reported-by: Rik van Riel <riel@redhat.com>
Reported by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

2 files changed, 26 insertions, 8 deletions

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 0e9ce1272971..991fa5c5dc5e 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -620,7 +620,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 
         trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
-        if (!user && !is_idle_task(current)) {
+        if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+            !user && !is_idle_task(current)) {
                 struct task_struct *idle __maybe_unused =
                         idle_task(smp_processor_id());
 
@@ -639,7 +640,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
         smp_mb__before_atomic();  /* See above. */
         atomic_inc(&rdtp->dynticks);
         smp_mb__after_atomic();  /* Force ordering with next sojourn. */
-        WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                     atomic_read(&rdtp->dynticks) & 0x1);
         rcu_dynticks_task_enter();
 
         /*
@@ -665,7 +667,8 @@ static void rcu_eqs_enter(bool user)
 
         rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
-        WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
+        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                     (oldval & DYNTICK_TASK_NEST_MASK) == 0);
         if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {
                 rdtp->dynticks_nesting = 0;
                 rcu_eqs_enter_common(oldval, user);
@@ -738,7 +741,8 @@ void rcu_irq_exit(void)
         rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
         rdtp->dynticks_nesting--;
-        WARN_ON_ONCE(rdtp->dynticks_nesting < 0);
+        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                     rdtp->dynticks_nesting < 0);
         if (rdtp->dynticks_nesting)
                 trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
         else
@@ -763,10 +767,12 @@ static void rcu_eqs_exit_common(long long oldval, int user)
         atomic_inc(&rdtp->dynticks);
         /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
         smp_mb__after_atomic();  /* See above. */
-        WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                     !(atomic_read(&rdtp->dynticks) & 0x1));
         rcu_cleanup_after_idle();
         trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
-        if (!user && !is_idle_task(current)) {
+        if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+            !user && !is_idle_task(current)) {
                 struct task_struct *idle __maybe_unused =
                         idle_task(smp_processor_id());
 
@@ -790,7 +796,7 @@ static void rcu_eqs_exit(bool user)
 
         rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
-        WARN_ON_ONCE(oldval < 0);
+        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
         if (oldval & DYNTICK_TASK_NEST_MASK) {
                 rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
         } else {
@@ -863,7 +869,8 @@ void rcu_irq_enter(void)
         rdtp = this_cpu_ptr(&rcu_dynticks);
         oldval = rdtp->dynticks_nesting;
         rdtp->dynticks_nesting++;
-        WARN_ON_ONCE(rdtp->dynticks_nesting == 0);
+        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+                     rdtp->dynticks_nesting == 0);
         if (oldval)
                 trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
         else
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index c4e1cf04cf57..b908048f8d6a 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1373,6 +1373,17 @@ config RCU_TRACE
           Say Y here if you want to enable RCU tracing
           Say N if you are unsure.
 
+config RCU_EQS_DEBUG
+        bool "Use this when adding any sort of NO_HZ support to your arch"
+        depends on DEBUG_KERNEL
+        help
+          This option provides consistency checks in RCU's handling of
+          NO_HZ.  These checks have proven quite helpful in detecting
+          bugs in arch-specific NO_HZ code.
+
+          Say N here if you need ultimate kernel/user switch latencies
+          Say Y if you are unsure
+
 endmenu # "RCU Debugging"
 
 config DEBUG_BLOCK_EXT_DEVT