3 files changed, 166 insertions, 7 deletions

diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index 7ff386a6ae87..10a43ed0527e 100644
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -12,9 +12,10 @@
 #include <linux/errno.h>
 #include <linux/mutex.h>
 #include <linux/rwsem.h>
+#include <linux/srcu.h>
 
 /*
- * Notifier chains are of three types:
+ * Notifier chains are of four types:
  *
  *      Atomic notifier chains: Chain callbacks run in interrupt/atomic
  *              context. Callouts are not allowed to block.
@@ -23,13 +24,27 @@
  *      Raw notifier chains: There are no restrictions on callbacks,
  *              registration, or unregistration.  All locking and protection
  *              must be provided by the caller.
+ *      SRCU notifier chains: A variant of blocking notifier chains, with
+ *              the same restrictions.
  *
  * atomic_notifier_chain_register() may be called from an atomic context,
- * but blocking_notifier_chain_register() must be called from a process
- * context.  Ditto for the corresponding _unregister() routines.
+ * but blocking_notifier_chain_register() and srcu_notifier_chain_register()
+ * must be called from a process context.  Ditto for the corresponding
+ * _unregister() routines.
  *
- * atomic_notifier_chain_unregister() and blocking_notifier_chain_unregister()
- * _must not_ be called from within the call chain.
+ * atomic_notifier_chain_unregister(), blocking_notifier_chain_unregister(),
+ * and srcu_notifier_chain_unregister() _must not_ be called from within
+ * the call chain.
+ *
+ * SRCU notifier chains are an alternative form of blocking notifier chains.
+ * They use SRCU (Sleepable Read-Copy Update) instead of rw-semaphores for
+ * protection of the chain links.  This means there is _very_ low overhead
+ * in srcu_notifier_call_chain(): no cache bounces and no memory barriers.
+ * As compensation, srcu_notifier_chain_unregister() is rather expensive.
+ * SRCU notifier chains should be used when the chain will be called very
+ * often but notifier_blocks will seldom be removed.  Also, SRCU notifier
+ * chains are slightly more difficult to use because they require special
+ * runtime initialization.
  */
 
 struct notifier_block {
@@ -52,6 +67,12 @@ struct raw_notifier_head {
         struct notifier_block *head;
 };
 
+struct srcu_notifier_head {
+        struct mutex mutex;
+        struct srcu_struct srcu;
+        struct notifier_block *head;
+};
+
 #define ATOMIC_INIT_NOTIFIER_HEAD(name) do {    \
                 spin_lock_init(&(name)->lock);  \
                 (name)->head = NULL;            \
@@ -64,6 +85,11 @@ struct raw_notifier_head {
                 (name)->head = NULL;            \
         } while (0)
 
+/* srcu_notifier_heads must be initialized and cleaned up dynamically */
+extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
+#define srcu_cleanup_notifier_head(name)        \
+                cleanup_srcu_struct(&(name)->srcu);
+
 #define ATOMIC_NOTIFIER_INIT(name) {                            \
                 .lock = __SPIN_LOCK_UNLOCKED(name.lock),        \
                 .head = NULL }
@@ -72,6 +98,7 @@ struct raw_notifier_head {
                 .head = NULL }
 #define RAW_NOTIFIER_INIT(name) {                               \
                 .head = NULL }
+/* srcu_notifier_heads cannot be initialized statically */
 
 #define ATOMIC_NOTIFIER_HEAD(name)                              \
         struct atomic_notifier_head name =                      \
@@ -91,6 +118,8 @@ extern int blocking_notifier_chain_register(struct blocking_notifier_head *,
                 struct notifier_block *);
 extern int raw_notifier_chain_register(struct raw_notifier_head *,
                 struct notifier_block *);
+extern int srcu_notifier_chain_register(struct srcu_notifier_head *,
+                struct notifier_block *);
 
 extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *,
                 struct notifier_block *);
@@ -98,6 +127,8 @@ extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *,
                 struct notifier_block *);
 extern int raw_notifier_chain_unregister(struct raw_notifier_head *,
                 struct notifier_block *);
+extern int srcu_notifier_chain_unregister(struct srcu_notifier_head *,
+                struct notifier_block *);
 
 extern int atomic_notifier_call_chain(struct atomic_notifier_head *,
                 unsigned long val, void *v);
@@ -105,6 +136,8 @@ extern int blocking_notifier_call_chain(struct blocking_notifier_head *,
                 unsigned long val, void *v);
 extern int raw_notifier_call_chain(struct raw_notifier_head *,
                 unsigned long val, void *v);
+extern int srcu_notifier_call_chain(struct srcu_notifier_head *,
+                unsigned long val, void *v);
 
 #define NOTIFY_DONE             0x0000          /* Don't care */
 #define NOTIFY_OK               0x0001          /* Suits me */
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 947fdab2ddb0..8a45367b5f3a 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -24,6 +24,9 @@
  *
  */
 
+#ifndef _LINUX_SRCU_H
+#define _LINUX_SRCU_H
+
 struct srcu_struct_array {
         int c[2];
 };
@@ -46,4 +49,5 @@ int srcu_read_lock(struct srcu_struct *sp) __acquires(sp);
 void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);
 void synchronize_srcu(struct srcu_struct *sp);
 long srcu_batches_completed(struct srcu_struct *sp);
-void cleanup_srcu_struct(struct srcu_struct *sp);
+
+#endif
diff --git a/kernel/sys.c b/kernel/sys.c
index 2314867ae34f..fd5c71006775 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -153,7 +153,7 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl,
 
 /*
  *      Atomic notifier chain routines.  Registration and unregistration
- *      use a mutex, and call_chain is synchronized by RCU (no locks).
+ *      use a spinlock, and call_chain is synchronized by RCU (no locks).
  */
 
 /**
@@ -401,6 +401,128 @@ int raw_notifier_call_chain(struct raw_notifier_head *nh,
 
 EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
 
+/*
+ *      SRCU notifier chain routines.    Registration and unregistration
+ *      use a mutex, and call_chain is synchronized by SRCU (no locks).
+ */
+
+/**
+ *      srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
+ *      @nh: Pointer to head of the SRCU notifier chain
+ *      @n: New entry in notifier chain
+ *
+ *      Adds a notifier to an SRCU notifier chain.
+ *      Must be called in process context.
+ *
+ *      Currently always returns zero.
+ */
+
+int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
+                struct notifier_block *n)
+{
+        int ret;
+
+        /*
+         * This code gets used during boot-up, when task switching is
+         * not yet working and interrupts must remain disabled.  At
+         * such times we must not call mutex_lock().
+         */
+        if (unlikely(system_state == SYSTEM_BOOTING))
+                return notifier_chain_register(&nh->head, n);
+
+        mutex_lock(&nh->mutex);
+        ret = notifier_chain_register(&nh->head, n);
+        mutex_unlock(&nh->mutex);
+        return ret;
+}
+
+EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
+
+/**
+ *      srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
+ *      @nh: Pointer to head of the SRCU notifier chain
+ *      @n: Entry to remove from notifier chain
+ *
+ *      Removes a notifier from an SRCU notifier chain.
+ *      Must be called from process context.
+ *
+ *      Returns zero on success or %-ENOENT on failure.
+ */
+int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
+                struct notifier_block *n)
+{
+        int ret;
+
+        /*
+         * This code gets used during boot-up, when task switching is
+         * not yet working and interrupts must remain disabled.  At
+         * such times we must not call mutex_lock().
+         */
+        if (unlikely(system_state == SYSTEM_BOOTING))
+                return notifier_chain_unregister(&nh->head, n);
+
+        mutex_lock(&nh->mutex);
+        ret = notifier_chain_unregister(&nh->head, n);
+        mutex_unlock(&nh->mutex);
+        synchronize_srcu(&nh->srcu);
+        return ret;
+}
+
+EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
+
+/**
+ *      srcu_notifier_call_chain - Call functions in an SRCU notifier chain
+ *      @nh: Pointer to head of the SRCU notifier chain
+ *      @val: Value passed unmodified to notifier function
+ *      @v: Pointer passed unmodified to notifier function
+ *
+ *      Calls each function in a notifier chain in turn.  The functions
+ *      run in a process context, so they are allowed to block.
+ *
+ *      If the return value of the notifier can be and'ed
+ *      with %NOTIFY_STOP_MASK then srcu_notifier_call_chain
+ *      will return immediately, with the return value of
+ *      the notifier function which halted execution.
+ *      Otherwise the return value is the return value
+ *      of the last notifier function called.
+ */
+
+int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
+                unsigned long val, void *v)
+{
+        int ret;
+        int idx;
+
+        idx = srcu_read_lock(&nh->srcu);
+        ret = notifier_call_chain(&nh->head, val, v);
+        srcu_read_unlock(&nh->srcu, idx);
+        return ret;
+}
+
+EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
+
+/**
+ *      srcu_init_notifier_head - Initialize an SRCU notifier head
+ *      @nh: Pointer to head of the srcu notifier chain
+ *
+ *      Unlike other sorts of notifier heads, SRCU notifier heads require
+ *      dynamic initialization.  Be sure to call this routine before
+ *      calling any of the other SRCU notifier routines for this head.
+ *
+ *      If an SRCU notifier head is deallocated, it must first be cleaned
+ *      up by calling srcu_cleanup_notifier_head().  Otherwise the head's
+ *      per-cpu data (used by the SRCU mechanism) will leak.
+ */
+
+void srcu_init_notifier_head(struct srcu_notifier_head *nh)
+{
+        mutex_init(&nh->mutex);
+        init_srcu_struct(&nh->srcu);
+        nh->head = NULL;
+}
+
+EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
+
 /**
  *      register_reboot_notifier - Register function to be called at reboot time
  *      @nb: Info about notifier function to be called