rcu: Create rcu_sync infrastructure

The rcu_sync infrastructure can be thought of as infrastructure to be
used to implement reader-writer primitives having extremely lightweight
readers during times when there are no writers.  The first use is in
the percpu_rwsem used by the VFS subsystem.

This infrastructure is functionally equivalent to

        struct rcu_sync_struct {
                atomic_t counter;
        };

	/* Check possibility of fast-path read-side operations. */
        static inline bool rcu_sync_is_idle(struct rcu_sync_struct *rss)
        {
                return atomic_read(&rss->counter) == 0;
        }

	/* Tell readers to use slowpaths. */
        static inline void rcu_sync_enter(struct rcu_sync_struct *rss)
        {
                atomic_inc(&rss->counter);
                synchronize_sched();
        }

	/* Allow readers to once again use fastpaths. */
        static inline void rcu_sync_exit(struct rcu_sync_struct *rss)
        {
                synchronize_sched();
                atomic_dec(&rss->counter);
        }

The main difference is that it records the state and only calls
synchronize_sched() if required.  At least some of the calls to
synchronize_sched() will be optimized away when rcu_sync_enter() and
rcu_sync_exit() are invoked repeatedly in quick succession.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

3 files changed, 270 insertions, 1 deletions

diff --git a/include/linux/rcu_sync.h b/include/linux/rcu_sync.h
new file mode 100644
index 000000000000..cb044df2e21c
--- /dev/null
+++ b/include/linux/rcu_sync.h
@@ -0,0 +1,94 @@
+/*
+ * RCU-based infrastructure for lightweight reader-writer locking
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * Copyright (c) 2015, Red Hat, Inc.
+ *
+ * Author: Oleg Nesterov <oleg@redhat.com>
+ */
+
+#ifndef _LINUX_RCU_SYNC_H_
+#define _LINUX_RCU_SYNC_H_
+
+#include <linux/wait.h>
+#include <linux/rcupdate.h>
+
+/* Structure to mediate between updaters and fastpath-using readers.  */
+struct rcu_sync {
+        int                     gp_state;
+        int                     gp_count;
+        wait_queue_head_t       gp_wait;
+
+        int                     cb_state;
+        struct rcu_head         cb_head;
+
+        void (*sync)(void);
+        void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+};
+
+#define ___RCU_SYNC_INIT(name)                                          \
+        .gp_state = 0,                                                  \
+        .gp_count = 0,                                                  \
+        .gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),         \
+        .cb_state = 0
+
+#define __RCU_SCHED_SYNC_INIT(name) {                                   \
+        ___RCU_SYNC_INIT(name),                                         \
+        .sync = synchronize_sched,                                      \
+        .call = call_rcu_sched,                                         \
+}
+
+#define __RCU_BH_SYNC_INIT(name) {                                      \
+        ___RCU_SYNC_INIT(name),                                         \
+        .sync = synchronize_rcu_bh,                                     \
+        .call = call_rcu_bh,                                            \
+}
+
+#define __RCU_SYNC_INIT(name) {                                         \
+        ___RCU_SYNC_INIT(name),                                         \
+        .sync = synchronize_rcu,                                        \
+        .call = call_rcu,                                               \
+}
+
+#define DEFINE_RCU_SCHED_SYNC(name)                                     \
+        struct rcu_sync name = __RCU_SCHED_SYNC_INIT(name)
+
+#define DEFINE_RCU_BH_SYNC(name)                                        \
+        struct rcu_sync name = __RCU_BH_SYNC_INIT(name)
+
+#define DEFINE_RCU_SYNC(name)                                           \
+        struct rcu_sync name = __RCU_SYNC_INIT(name)
+
+/**
+ * rcu_sync_is_idle() - Are readers permitted to use their fastpaths?
+ * @rsp: Pointer to rcu_sync structure to use for synchronization
+ *
+ * Returns true if readers are permitted to use their fastpaths.
+ * Must be invoked within an RCU read-side critical section whose
+ * flavor matches that of the rcu_sync struture.
+ */
+static inline bool rcu_sync_is_idle(struct rcu_sync *rsp)
+{
+        return !rsp->gp_state; /* GP_IDLE */
+}
+
+enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
+
+extern void rcu_sync_init(struct rcu_sync *, enum rcu_sync_type);
+extern void rcu_sync_enter(struct rcu_sync *);
+extern void rcu_sync_exit(struct rcu_sync *);
+
+#endif /* _LINUX_RCU_SYNC_H_ */
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
index 50a808424b06..61a16569ffbf 100644
--- a/kernel/rcu/Makefile
+++ b/kernel/rcu/Makefile
@@ -1,4 +1,4 @@
-obj-y += update.o
+obj-y += update.o sync.o
 obj-$(CONFIG_SRCU) += srcu.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
 obj-$(CONFIG_TREE_RCU) += tree.o
diff --git a/kernel/rcu/sync.c b/kernel/rcu/sync.c
new file mode 100644
index 000000000000..0a11df43be23
--- /dev/null
+++ b/kernel/rcu/sync.c
@@ -0,0 +1,175 @@
+/*
+ * RCU-based infrastructure for lightweight reader-writer locking
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * Copyright (c) 2015, Red Hat, Inc.
+ *
+ * Author: Oleg Nesterov <oleg@redhat.com>
+ */
+
+#include <linux/rcu_sync.h>
+#include <linux/sched.h>
+
+enum { GP_IDLE = 0, GP_PENDING, GP_PASSED };
+enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
+
+#define rss_lock        gp_wait.lock
+
+/**
+ * rcu_sync_init() - Initialize an rcu_sync structure
+ * @rsp: Pointer to rcu_sync structure to be initialized
+ * @type: Flavor of RCU with which to synchronize rcu_sync structure
+ */
+void rcu_sync_init(struct rcu_sync *rsp, enum rcu_sync_type type)
+{
+        memset(rsp, 0, sizeof(*rsp));
+        init_waitqueue_head(&rsp->gp_wait);
+
+        switch (type) {
+        case RCU_SYNC:
+                rsp->sync = synchronize_rcu;
+                rsp->call = call_rcu;
+                break;
+
+        case RCU_SCHED_SYNC:
+                rsp->sync = synchronize_sched;
+                rsp->call = call_rcu_sched;
+                break;
+
+        case RCU_BH_SYNC:
+                rsp->sync = synchronize_rcu_bh;
+                rsp->call = call_rcu_bh;
+                break;
+        }
+}
+
+/**
+ * rcu_sync_enter() - Force readers onto slowpath
+ * @rsp: Pointer to rcu_sync structure to use for synchronization
+ *
+ * This function is used by updaters who need readers to make use of
+ * a slowpath during the update.  After this function returns, all
+ * subsequent calls to rcu_sync_is_idle() will return false, which
+ * tells readers to stay off their fastpaths.  A later call to
+ * rcu_sync_exit() re-enables reader slowpaths.
+ *
+ * When called in isolation, rcu_sync_enter() must wait for a grace
+ * period, however, closely spaced calls to rcu_sync_enter() can
+ * optimize away the grace-period wait via a state machine implemented
+ * by rcu_sync_enter(), rcu_sync_exit(), and rcu_sync_func().
+ */
+void rcu_sync_enter(struct rcu_sync *rsp)
+{
+        bool need_wait, need_sync;
+
+        spin_lock_irq(&rsp->rss_lock);
+        need_wait = rsp->gp_count++;
+        need_sync = rsp->gp_state == GP_IDLE;
+        if (need_sync)
+                rsp->gp_state = GP_PENDING;
+        spin_unlock_irq(&rsp->rss_lock);
+
+        BUG_ON(need_wait && need_sync);
+
+        if (need_sync) {
+                rsp->sync();
+                rsp->gp_state = GP_PASSED;
+                wake_up_all(&rsp->gp_wait);
+        } else if (need_wait) {
+                wait_event(rsp->gp_wait, rsp->gp_state == GP_PASSED);
+        } else {
+                /*
+                 * Possible when there's a pending CB from a rcu_sync_exit().
+                 * Nobody has yet been allowed the 'fast' path and thus we can
+                 * avoid doing any sync(). The callback will get 'dropped'.
+                 */
+                BUG_ON(rsp->gp_state != GP_PASSED);
+        }
+}
+
+/**
+ * rcu_sync_func() - Callback function managing reader access to fastpath
+ * @rsp: Pointer to rcu_sync structure to use for synchronization
+ *
+ * This function is passed to one of the call_rcu() functions by
+ * rcu_sync_exit(), so that it is invoked after a grace period following the
+ * that invocation of rcu_sync_exit().  It takes action based on events that
+ * have taken place in the meantime, so that closely spaced rcu_sync_enter()
+ * and rcu_sync_exit() pairs need not wait for a grace period.
+ *
+ * If another rcu_sync_enter() is invoked before the grace period
+ * ended, reset state to allow the next rcu_sync_exit() to let the
+ * readers back onto their fastpaths (after a grace period).  If both
+ * another rcu_sync_enter() and its matching rcu_sync_exit() are invoked
+ * before the grace period ended, re-invoke call_rcu() on behalf of that
+ * rcu_sync_exit().  Otherwise, set all state back to idle so that readers
+ * can again use their fastpaths.
+ */
+static void rcu_sync_func(struct rcu_head *rcu)
+{
+        struct rcu_sync *rsp = container_of(rcu, struct rcu_sync, cb_head);
+        unsigned long flags;
+
+        BUG_ON(rsp->gp_state != GP_PASSED);
+        BUG_ON(rsp->cb_state == CB_IDLE);
+
+        spin_lock_irqsave(&rsp->rss_lock, flags);
+        if (rsp->gp_count) {
+                /*
+                 * A new rcu_sync_begin() has happened; drop the callback.
+                 */
+                rsp->cb_state = CB_IDLE;
+        } else if (rsp->cb_state == CB_REPLAY) {
+                /*
+                 * A new rcu_sync_exit() has happened; requeue the callback
+                 * to catch a later GP.
+                 */
+                rsp->cb_state = CB_PENDING;
+                rsp->call(&rsp->cb_head, rcu_sync_func);
+        } else {
+                /*
+                 * We're at least a GP after rcu_sync_exit(); eveybody will now
+                 * have observed the write side critical section. Let 'em rip!.
+                 */
+                rsp->cb_state = CB_IDLE;
+                rsp->gp_state = GP_IDLE;
+        }
+        spin_unlock_irqrestore(&rsp->rss_lock, flags);
+}
+
+/**
+ * rcu_sync_exit() - Allow readers back onto fast patch after grace period
+ * @rsp: Pointer to rcu_sync structure to use for synchronization
+ *
+ * This function is used by updaters who have completed, and can therefore
+ * now allow readers to make use of their fastpaths after a grace period
+ * has elapsed.  After this grace period has completed, all subsequent
+ * calls to rcu_sync_is_idle() will return true, which tells readers that
+ * they can once again use their fastpaths.
+ */
+void rcu_sync_exit(struct rcu_sync *rsp)
+{
+        spin_lock_irq(&rsp->rss_lock);
+        if (!--rsp->gp_count) {
+                if (rsp->cb_state == CB_IDLE) {
+                        rsp->cb_state = CB_PENDING;
+                        rsp->call(&rsp->cb_head, rcu_sync_func);
+                } else if (rsp->cb_state == CB_PENDING) {
+                        rsp->cb_state = CB_REPLAY;
+                }
+        }
+        spin_unlock_irq(&rsp->rss_lock);
+}