1 files changed, 95 insertions, 108 deletions

diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 3cb8e362e883..f259c676195f 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -100,23 +100,28 @@ struct rcu_preempt_ctrlblk {
         u8 completed;           /* Last grace period completed. */
                                 /*  If all three are equal, RCU is idle. */
 #ifdef CONFIG_RCU_BOOST
-        s8 boosted_this_gp;     /* Has boosting already happened? */
         unsigned long boost_time; /* When to start boosting (jiffies) */
 #endif /* #ifdef CONFIG_RCU_BOOST */
 #ifdef CONFIG_RCU_TRACE
         unsigned long n_grace_periods;
 #ifdef CONFIG_RCU_BOOST
         unsigned long n_tasks_boosted;
+                                /* Total number of tasks boosted. */
         unsigned long n_exp_boosts;
+                                /* Number of tasks boosted for expedited GP. */
         unsigned long n_normal_boosts;
-        unsigned long n_normal_balk_blkd_tasks;
-        unsigned long n_normal_balk_gp_tasks;
-        unsigned long n_normal_balk_boost_tasks;
-        unsigned long n_normal_balk_boosted;
-        unsigned long n_normal_balk_notyet;
-        unsigned long n_normal_balk_nos;
-        unsigned long n_exp_balk_blkd_tasks;
-        unsigned long n_exp_balk_nos;
+                                /* Number of tasks boosted for normal GP. */
+        unsigned long n_balk_blkd_tasks;
+                                /* Refused to boost: no blocked tasks. */
+        unsigned long n_balk_exp_gp_tasks;
+                                /* Refused to boost: nothing blocking GP. */
+        unsigned long n_balk_boost_tasks;
+                                /* Refused to boost: already boosting. */
+        unsigned long n_balk_notyet;
+                                /* Refused to boost: not yet time. */
+        unsigned long n_balk_nos;
+                                /* Refused to boost: not sure why, though. */
+                                /*  This can happen due to race conditions. */
 #endif /* #ifdef CONFIG_RCU_BOOST */
 #endif /* #ifdef CONFIG_RCU_TRACE */
 };
@@ -201,7 +206,6 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t)
 
 #ifdef CONFIG_RCU_BOOST
 static void rcu_initiate_boost_trace(void);
-static void rcu_initiate_exp_boost_trace(void);
 #endif /* #ifdef CONFIG_RCU_BOOST */
 
 /*
@@ -219,41 +223,21 @@ static void show_tiny_preempt_stats(struct seq_file *m)
                    "N."[!rcu_preempt_ctrlblk.gp_tasks],
                    "E."[!rcu_preempt_ctrlblk.exp_tasks]);
 #ifdef CONFIG_RCU_BOOST
-        seq_printf(m, "             ttb=%c btg=",
-                   "B."[!rcu_preempt_ctrlblk.boost_tasks]);
-        switch (rcu_preempt_ctrlblk.boosted_this_gp) {
-        case -1:
-                seq_puts(m, "exp");
-                break;
-        case 0:
-                seq_puts(m, "no");
-                break;
-        case 1:
-                seq_puts(m, "begun");
-                break;
-        case 2:
-                seq_puts(m, "done");
-                break;
-        default:
-                seq_printf(m, "?%d?", rcu_preempt_ctrlblk.boosted_this_gp);
-        }
-        seq_printf(m, " ntb=%lu neb=%lu nnb=%lu j=%04x bt=%04x\n",
+        seq_printf(m, "%sttb=%c ntb=%lu neb=%lu nnb=%lu j=%04x bt=%04x\n",
+                   "             ",
+                   "B."[!rcu_preempt_ctrlblk.boost_tasks],
                    rcu_preempt_ctrlblk.n_tasks_boosted,
                    rcu_preempt_ctrlblk.n_exp_boosts,
                    rcu_preempt_ctrlblk.n_normal_boosts,
                    (int)(jiffies & 0xffff),
                    (int)(rcu_preempt_ctrlblk.boost_time & 0xffff));
-        seq_printf(m, "             %s: nt=%lu gt=%lu bt=%lu b=%lu ny=%lu nos=%lu\n",
-                   "normal balk",
-                   rcu_preempt_ctrlblk.n_normal_balk_blkd_tasks,
-                   rcu_preempt_ctrlblk.n_normal_balk_gp_tasks,
-                   rcu_preempt_ctrlblk.n_normal_balk_boost_tasks,
-                   rcu_preempt_ctrlblk.n_normal_balk_boosted,
-                   rcu_preempt_ctrlblk.n_normal_balk_notyet,
-                   rcu_preempt_ctrlblk.n_normal_balk_nos);
-        seq_printf(m, "             exp balk: bt=%lu nos=%lu\n",
-                   rcu_preempt_ctrlblk.n_exp_balk_blkd_tasks,
-                   rcu_preempt_ctrlblk.n_exp_balk_nos);
+        seq_printf(m, "%s: nt=%lu egt=%lu bt=%lu ny=%lu nos=%lu\n",
+                   "             balk",
+                   rcu_preempt_ctrlblk.n_balk_blkd_tasks,
+                   rcu_preempt_ctrlblk.n_balk_exp_gp_tasks,
+                   rcu_preempt_ctrlblk.n_balk_boost_tasks,
+                   rcu_preempt_ctrlblk.n_balk_notyet,
+                   rcu_preempt_ctrlblk.n_balk_nos);
 #endif /* #ifdef CONFIG_RCU_BOOST */
 }
 
@@ -271,25 +255,59 @@ static int rcu_boost(void)
 {
         unsigned long flags;
         struct rt_mutex mtx;
-        struct list_head *np;
         struct task_struct *t;
+        struct list_head *tb;
 
-        if (rcu_preempt_ctrlblk.boost_tasks == NULL)
+        if (rcu_preempt_ctrlblk.boost_tasks == NULL &&
+            rcu_preempt_ctrlblk.exp_tasks == NULL)
                 return 0;  /* Nothing to boost. */
+
         raw_local_irq_save(flags);
-        rcu_preempt_ctrlblk.boosted_this_gp++;
-        t = container_of(rcu_preempt_ctrlblk.boost_tasks, struct task_struct,
-                         rcu_node_entry);
-        np = rcu_next_node_entry(t);
+
+        /*
+         * Recheck with irqs disabled: all tasks in need of boosting
+         * might exit their RCU read-side critical sections on their own
+         * if we are preempted just before disabling irqs.
+         */
+        if (rcu_preempt_ctrlblk.boost_tasks == NULL &&
+            rcu_preempt_ctrlblk.exp_tasks == NULL) {
+                raw_local_irq_restore(flags);
+                return 0;
+        }
+
+        /*
+         * Preferentially boost tasks blocking expedited grace periods.
+         * This cannot starve the normal grace periods because a second
+         * expedited grace period must boost all blocked tasks, including
+         * those blocking the pre-existing normal grace period.
+         */
+        if (rcu_preempt_ctrlblk.exp_tasks != NULL) {
+                tb = rcu_preempt_ctrlblk.exp_tasks;
+                RCU_TRACE(rcu_preempt_ctrlblk.n_exp_boosts++);
+        } else {
+                tb = rcu_preempt_ctrlblk.boost_tasks;
+                RCU_TRACE(rcu_preempt_ctrlblk.n_normal_boosts++);
+        }
+        RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++);
+
+        /*
+         * We boost task t by manufacturing an rt_mutex that appears to
+         * be held by task t.  We leave a pointer to that rt_mutex where
+         * task t can find it, and task t will release the mutex when it
+         * exits its outermost RCU read-side critical section.  Then
+         * simply acquiring this artificial rt_mutex will boost task
+         * t's priority.  (Thanks to tglx for suggesting this approach!)
+         */
+        t = container_of(tb, struct task_struct, rcu_node_entry);
         rt_mutex_init_proxy_locked(&mtx, t);
         t->rcu_boost_mutex = &mtx;
         t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
         raw_local_irq_restore(flags);
         rt_mutex_lock(&mtx);
-        RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++);
-        rcu_preempt_ctrlblk.boosted_this_gp++;
-        rt_mutex_unlock(&mtx);
-        return rcu_preempt_ctrlblk.boost_tasks != NULL;
+        rt_mutex_unlock(&mtx);  /* Keep lockdep happy. */
+
+        return rcu_preempt_ctrlblk.boost_tasks != NULL ||
+               rcu_preempt_ctrlblk.exp_tasks != NULL;
 }
 
 /*
@@ -304,42 +322,25 @@ static int rcu_boost(void)
  */
 static int rcu_initiate_boost(void)
 {
-        if (!rcu_preempt_blocked_readers_cgp()) {
-                RCU_TRACE(rcu_preempt_ctrlblk.n_normal_balk_blkd_tasks++);
+        if (!rcu_preempt_blocked_readers_cgp() &&
+            rcu_preempt_ctrlblk.exp_tasks == NULL) {
+                RCU_TRACE(rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++);
                 return 0;