rcu: Simplify priority boosting by putting rt_mutex in rcu_node

RCU priority boosting currently checks for boosting via a pointer in
task_struct.  However, this is not needed: As Oleg noted, if the
rt_mutex is placed in the rcu_node instead of on the booster's stack,
the boostee can simply check it see if it owns the lock.  This commit
makes this change, shrinking task_struct by one pointer and the kernel
by thirteen lines.

Suggested-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

2 files changed, 14 insertions, 14 deletions

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 3eeb919e26a2..60fb0eaa2d16 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -177,6 +177,9 @@ struct rcu_node {
                                 /*  to carry out the boosting is fully */
                                 /*  released with no future boostee accesses */
                                 /*  before that rt_mutex is re-initialized. */
+        struct rt_mutex boost_mtx;
+                                /* Used only for the priority-boosting */
+                                /*  side effect, not as a lock. */
         unsigned long boost_time;
                                 /* When to start boosting (jiffies). */
         struct task_struct *boost_kthread_task;
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 9c811879d31e..719587af7b10 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -33,6 +33,7 @@
 #define RCU_KTHREAD_PRIO 1
 
 #ifdef CONFIG_RCU_BOOST
+#include "../locking/rtmutex_common.h"
 #define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO
 #else
 #define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
@@ -336,7 +337,7 @@ void rcu_read_unlock_special(struct task_struct *t)
         unsigned long flags;
         struct list_head *np;
 #ifdef CONFIG_RCU_BOOST
-        struct rt_mutex *rbmp = NULL;
+        bool drop_boost_mutex = false;
 #endif /* #ifdef CONFIG_RCU_BOOST */
         struct rcu_node *rnp;
         int special;
@@ -398,11 +399,8 @@ void rcu_read_unlock_special(struct task_struct *t)
 #ifdef CONFIG_RCU_BOOST
                 if (&t->rcu_node_entry == rnp->boost_tasks)
                         rnp->boost_tasks = np;
-                /* Snapshot/clear ->rcu_boost_mutex with rcu_node lock held. */
-                if (t->rcu_boost_mutex) {
-                        rbmp = t->rcu_boost_mutex;
-                        t->rcu_boost_mutex = NULL;
-                }
+                /* Snapshot ->boost_mtx ownership with rcu_node lock held. */
+                drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
 #endif /* #ifdef CONFIG_RCU_BOOST */
 
                 /*
@@ -427,8 +425,8 @@ void rcu_read_unlock_special(struct task_struct *t)
 
 #ifdef CONFIG_RCU_BOOST
                 /* Unboost if we were boosted. */
-                if (rbmp) {
-                        rt_mutex_unlock(rbmp);
+                if (drop_boost_mutex) {
+                        rt_mutex_unlock(&rnp->boost_mtx);
                         complete(&rnp->boost_completion);
                 }
 #endif /* #ifdef CONFIG_RCU_BOOST */
@@ -1151,7 +1149,6 @@ static void rcu_wake_cond(struct task_struct *t, int status)
 static int rcu_boost(struct rcu_node *rnp)
 {
         unsigned long flags;
-        struct rt_mutex mtx;
         struct task_struct *t;
         struct list_head *tb;
 
@@ -1202,14 +1199,14 @@ static int rcu_boost(struct rcu_node *rnp)
          * section.
          */
         t = container_of(tb, struct task_struct, rcu_node_entry);
-        rt_mutex_init_proxy_locked(&mtx, t);
-        t->rcu_boost_mutex = &mtx;
+        rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
         init_completion(&rnp->boost_completion);
         raw_spin_unlock_irqrestore(&rnp->lock, flags);
-        rt_mutex_lock(&mtx);  /* Side effect: boosts task t's priority. */
-        rt_mutex_unlock(&mtx);  /* Keep lockdep happy. */
+        /* Lock only for side effect: boosts task t's priority. */
+        rt_mutex_lock(&rnp->boost_mtx);
+        rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */
 
-        /* Wait until boostee is done accessing mtx before reinitializing. */
+        /* Wait for boostee to be done w/boost_mtx before reinitializing. */
         wait_for_completion(&rnp->boost_completion);
 
         return ACCESS_ONCE(rnp->exp_tasks) != NULL ||