rcu: Move quiescent-state forcing into kthread

As the first step towards allowing quiescent-state forcing to be
preemptible, this commit moves RCU quiescent-state forcing into the
same kthread that is now used to initialize and clean up after grace
periods.  This is yet another step towards keeping scheduling
latency down to a dull roar.

Updated to change from raw_spin_lock_irqsave() to raw_spin_lock_irq()
and to remove the now-unused rcu_state structure fields as suggested by
Peter Zijlstra.

Reported-by: Mike Galbraith <mgalbraith@suse.de>
Reported-by: Dimitri Sivanich <sivanich@sgi.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

1 files changed, 4 insertions, 9 deletions

diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 5d92b80a0a28..2d04106d1533 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -378,13 +378,6 @@ struct rcu_state {
 
         u8      fqs_state ____cacheline_internodealigned_in_smp;
                                                 /* Force QS state. */
-        u8      fqs_active;                     /* force_quiescent_state() */
-                                                /*  is running. */
-        u8      fqs_need_gp;                    /* A CPU was prevented from */
-                                                /*  starting a new grace */
-                                                /*  period because */
-                                                /*  force_quiescent_state() */
-                                                /*  was running. */
         u8      boost;                          /* Subject to priority boost. */
         unsigned long gpnum;                    /* Current gp number. */
         unsigned long completed;                /* # of last completed gp. */
@@ -413,8 +406,6 @@ struct rcu_state {
         struct completion barrier_completion;   /* Wake at barrier end. */
         unsigned long n_barrier_done;           /* ++ at start and end of */
                                                 /*  _rcu_barrier(). */
-        raw_spinlock_t fqslock;                 /* Only one task forcing */
-                                                /*  quiescent states. */
         unsigned long jiffies_force_qs;         /* Time at which to invoke */
                                                 /*  force_quiescent_state(). */
         unsigned long n_force_qs;               /* Number of calls to */
@@ -433,6 +424,10 @@ struct rcu_state {
         struct list_head flavors;               /* List of RCU flavors. */
 };
 
+/* Values for rcu_state structure's gp_flags field. */
+#define RCU_GP_FLAG_INIT 0x1    /* Need grace-period initialization. */
+#define RCU_GP_FLAG_FQS  0x2    /* Need grace-period quiescent-state forcing. */
+
 extern struct list_head rcu_struct_flavors;
 #define for_each_rcu_flavor(rsp) \
         list_for_each_entry((rsp), &rcu_struct_flavors, flavors)