rcu: Remove local_irq_disable() in rcu_preempt_note_context_switch()

The rcu_preempt_note_context_switch() function is on a scheduling fast
path, so it would be good to avoid disabling irqs.  The reason that irqs
are disabled is to synchronize process-level and irq-handler access to
the task_struct ->rcu_read_unlock_special bitmask.  This commit therefore
makes ->rcu_read_unlock_special instead be a union of bools with a short
allowing single-access checks in RCU's __rcu_read_unlock().  This results
in the process-level and irq-handler accesses being simple loads and
stores, so that irqs need no longer be disabled.  This commit therefore
removes the irq disabling from rcu_preempt_note_context_switch().

Reported-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

1 files changed, 15 insertions, 17 deletions

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index e466b40052a7..0981c0cd70fe 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -155,9 +155,8 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed);
  * not in a quiescent state.  There might be any number of tasks blocked
  * while in an RCU read-side critical section.
  *
- * Unlike the other rcu_*_qs() functions, callers to this function
- * must disable irqs in order to protect the assignment to
- * ->rcu_read_unlock_special.
+ * As with the other rcu_*_qs() functions, callers to this function
+ * must disable preemption.
  */
 static void rcu_preempt_qs(int cpu)
 {
@@ -166,7 +165,7 @@ static void rcu_preempt_qs(int cpu)
         if (rdp->passed_quiesce == 0)
                 trace_rcu_grace_period(TPS("rcu_preempt"), rdp->gpnum, TPS("cpuqs"));
         rdp->passed_quiesce = 1;
-        current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
+        current->rcu_read_unlock_special.b.need_qs = false;
 }
 
 /*
@@ -190,14 +189,14 @@ static void rcu_preempt_note_context_switch(int cpu)
         struct rcu_node *rnp;
 
         if (t->rcu_read_lock_nesting > 0 &&
-            (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
+            !t->rcu_read_unlock_special.b.blocked) {
 
                 /* Possibly blocking in an RCU read-side critical section. */
                 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
                 rnp = rdp->mynode;
                 raw_spin_lock_irqsave(&rnp->lock, flags);
                 smp_mb__after_unlock_lock();
-                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
+                t->rcu_read_unlock_special.b.blocked = true;
                 t->rcu_blocked_node = rnp;
 
                 /*
@@ -239,7 +238,7 @@ static void rcu_preempt_note_context_switch(int cpu)
                                        : rnp->gpnum + 1);
                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
         } else if (t->rcu_read_lock_nesting < 0 &&
-                   t->rcu_read_unlock_special) {
+                   t->rcu_read_unlock_special.s) {
 
                 /*
                  * Complete exit from RCU read-side critical section on
@@ -257,9 +256,7 @@ static void rcu_preempt_note_context_switch(int cpu)
          * grace period, then the fact that the task has been enqueued
          * means that we continue to block the current grace period.
          */
-        local_irq_save(flags);
         rcu_preempt_qs(cpu);
-        local_irq_restore(flags);
 }
 
 /*
@@ -340,7 +337,7 @@ void rcu_read_unlock_special(struct task_struct *t)
         bool drop_boost_mutex = false;
 #endif /* #ifdef CONFIG_RCU_BOOST */
         struct rcu_node *rnp;
-        int special;
+        union rcu_special special;
 
         /* NMI handlers cannot block and cannot safely manipulate state. */
         if (in_nmi())
@@ -350,12 +347,13 @@ void rcu_read_unlock_special(struct task_struct *t)
 
         /*
          * If RCU core is waiting for this CPU to exit critical section,
-         * let it know that we have done so.
+         * let it know that we have done so.  Because irqs are disabled,
+         * t->rcu_read_unlock_special cannot change.
          */
         special = t->rcu_read_unlock_special;
-        if (special & RCU_READ_UNLOCK_NEED_QS) {
+        if (special.b.need_qs) {
                 rcu_preempt_qs(smp_processor_id());
-                if (!t->rcu_read_unlock_special) {
+                if (!t->rcu_read_unlock_special.s) {
                         local_irq_restore(flags);
                         return;
                 }
@@ -368,8 +366,8 @@ void rcu_read_unlock_special(struct task_struct *t)
         }
 
         /* Clean up if blocked during RCU read-side critical section. */
-        if (special & RCU_READ_UNLOCK_BLOCKED) {
-                t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
+        if (special.b.blocked) {
+                t->rcu_read_unlock_special.b.blocked = false;
 
                 /*
                  * Remove this task from the list it blocked on.  The
@@ -658,7 +656,7 @@ static void rcu_preempt_check_callbacks(int cpu)
         }
         if (t->rcu_read_lock_nesting > 0 &&
             per_cpu(rcu_preempt_data, cpu).qs_pending)
-                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
+                t->rcu_read_unlock_special.b.need_qs = true;
 }
 
 #ifdef CONFIG_RCU_BOOST
@@ -941,7 +939,7 @@ void exit_rcu(void)
                 return;
         t->rcu_read_lock_nesting = 1;
         barrier();
-        t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED;
+        t->rcu_read_unlock_special.b.blocked = true;
         __rcu_read_unlock();
 }