6 files changed, 70 insertions, 31 deletions

diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index a1c81f80978e..49811cdddaa5 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -385,7 +385,7 @@ static __always_inline void __assign_once_size(volatile void *p, void *res, int
 
 /* Is this type a native word size -- useful for atomic operations */
 #ifndef __native_word
-# define __native_word(t) (sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
+# define __native_word(t) (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
 #endif
 
 /* Compile time object size, -1 for unknown */
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index 529bc946f450..a18b16f1dc0e 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -524,11 +524,11 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
  * @member:     the name of the hlist_node within the struct.
  */
 #define hlist_for_each_entry_continue_rcu(pos, member)                  \
-        for (pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\
-                        typeof(*(pos)), member);                        \
+        for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
+                        &(pos)->member)), typeof(*(pos)), member);      \
              pos;                                                       \
-             pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\
-                        typeof(*(pos)), member))
+             pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
+                        &(pos)->member)), typeof(*(pos)), member))
 
 /**
  * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
@@ -536,11 +536,11 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
  * @member:     the name of the hlist_node within the struct.
  */
 #define hlist_for_each_entry_continue_rcu_bh(pos, member)               \
-        for (pos = hlist_entry_safe(rcu_dereference_bh((pos)->member.next),\
-                        typeof(*(pos)), member);                        \
+        for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(  \
+                        &(pos)->member)), typeof(*(pos)), member);      \
              pos;                                                       \
-             pos = hlist_entry_safe(rcu_dereference_bh((pos)->member.next),\
-                        typeof(*(pos)), member))
+             pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(  \
+                        &(pos)->member)), typeof(*(pos)), member))
 
 /**
  * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index ed4f5939a452..78097491cd99 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -331,12 +331,13 @@ static inline void rcu_init_nohz(void)
 extern struct srcu_struct tasks_rcu_exit_srcu;
 #define rcu_note_voluntary_context_switch(t) \
         do { \
+                rcu_all_qs(); \
                 if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \
                         ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \
         } while (0)
 #else /* #ifdef CONFIG_TASKS_RCU */
 #define TASKS_RCU(x) do { } while (0)
-#define rcu_note_voluntary_context_switch(t)    do { } while (0)
+#define rcu_note_voluntary_context_switch(t)    rcu_all_qs()
 #endif /* #else #ifdef CONFIG_TASKS_RCU */
 
 /**
@@ -582,11 +583,11 @@ static inline void rcu_preempt_sleep_check(void)
 })
 #define __rcu_dereference_check(p, c, space) \
 ({ \
-        typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \
+        /* Dependency order vs. p above. */ \
+        typeof(*p) *________p1 = (typeof(*p) *__force)lockless_dereference(p); \
         rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \
         rcu_dereference_sparse(p, space); \
-        smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
-        ((typeof(*p) __force __kernel *)(_________p1)); \
+        ((typeof(*p) __force __kernel *)(________p1)); \
 })
 #define __rcu_dereference_protected(p, c, space) \
 ({ \
@@ -603,10 +604,10 @@ static inline void rcu_preempt_sleep_check(void)
 })
 #define __rcu_dereference_index_check(p, c) \
 ({ \
-        typeof(p) _________p1 = ACCESS_ONCE(p); \
+        /* Dependency order vs. p above. */ \
+        typeof(p) _________p1 = lockless_dereference(p); \
         rcu_lockdep_assert(c, \
                            "suspicious rcu_dereference_index_check() usage"); \
-        smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
         (_________p1); \
 })
 
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 0e5366200154..937edaeb150d 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -92,17 +92,49 @@ static inline void rcu_virt_note_context_switch(int cpu)
 }
 
 /*
- * Return the number of grace periods.
+ * Return the number of grace periods started.
  */
-static inline long rcu_batches_completed(void)
+static inline unsigned long rcu_batches_started(void)
 {
         return 0;
 }
 
 /*
- * Return the number of bottom-half grace periods.
+ * Return the number of bottom-half grace periods started.
  */
-static inline long rcu_batches_completed_bh(void)
+static inline unsigned long rcu_batches_started_bh(void)
+{
+        return 0;
+}
+
+/*
+ * Return the number of sched grace periods started.
+ */
+static inline unsigned long rcu_batches_started_sched(void)
+{
+        return 0;
+}
+
+/*
+ * Return the number of grace periods completed.
+ */
+static inline unsigned long rcu_batches_completed(void)
+{
+        return 0;
+}
+
+/*
+ * Return the number of bottom-half grace periods completed.
+ */
+static inline unsigned long rcu_batches_completed_bh(void)
+{
+        return 0;
+}
+
+/*
+ * Return the number of sched grace periods completed.
+ */
+static inline unsigned long rcu_batches_completed_sched(void)
 {
         return 0;
 }
@@ -154,7 +186,10 @@ static inline bool rcu_is_watching(void)
         return true;
 }
 
-
 #endif /* #else defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
 
+static inline void rcu_all_qs(void)
+{
+}
+
 #endif /* __LINUX_RCUTINY_H */
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 52953790dcca..d2e583a6aaca 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -81,9 +81,12 @@ void cond_synchronize_rcu(unsigned long oldstate);
 
 extern unsigned long rcutorture_testseq;
 extern unsigned long rcutorture_vernum;
-long rcu_batches_completed(void);
-long rcu_batches_completed_bh(void);
-long rcu_batches_completed_sched(void);
+unsigned long rcu_batches_started(void);
+unsigned long rcu_batches_started_bh(void);
+unsigned long rcu_batches_started_sched(void);
+unsigned long rcu_batches_completed(void);
+unsigned long rcu_batches_completed_bh(void);
+unsigned long rcu_batches_completed_sched(void);
 void show_rcu_gp_kthreads(void);
 
 void rcu_force_quiescent_state(void);
@@ -97,4 +100,6 @@ extern int rcu_scheduler_active __read_mostly;
 
 bool rcu_is_watching(void);
 
+void rcu_all_qs(void);
+
 #endif /* __LINUX_RCUTREE_H */
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index a2783cb5d275..9cfd9623fb03 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -45,7 +45,7 @@ struct rcu_batch {
 #define RCU_BATCH_INIT(name) { NULL, &(name.head) }
 
 struct srcu_struct {
-        unsigned completed;
+        unsigned long completed;
         struct srcu_struct_array __percpu *per_cpu_ref;
         spinlock_t queue_lock; /* protect ->batch_queue, ->running */
         bool running;
@@ -102,13 +102,11 @@ void process_srcu(struct work_struct *work);
  * define and init a srcu struct at build time.
  * dont't call init_srcu_struct() nor cleanup_srcu_struct() on it.
  */
-#define DEFINE_SRCU(name)                                               \
+#define __DEFINE_SRCU(name, is_static)                                  \
         static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\
-        struct srcu_struct name = __SRCU_STRUCT_INIT(name);
-
-#define DEFINE_STATIC_SRCU(name)                                        \
-        static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\
-        static struct srcu_struct name = __SRCU_STRUCT_INIT(name);
+        is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)
+#define DEFINE_SRCU(name)               __DEFINE_SRCU(name, /* not static */)
+#define DEFINE_STATIC_SRCU(name)        __DEFINE_SRCU(name, static)
 
 /**
  * call_srcu() - Queue a callback for invocation after an SRCU grace period
@@ -135,7 +133,7 @@ int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);
 void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);
 void synchronize_srcu(struct srcu_struct *sp);
 void synchronize_srcu_expedited(struct srcu_struct *sp);
-long srcu_batches_completed(struct srcu_struct *sp);
+unsigned long srcu_batches_completed(struct srcu_struct *sp);
 void srcu_barrier(struct srcu_struct *sp);
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC