1 files changed, 48 insertions, 2 deletions

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index d0ca524bf042..729a8706751d 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -51,6 +51,7 @@ module_param(counter_wrap_check, ulong, 0444);
 
 static void srcu_invoke_callbacks(struct work_struct *work);
 static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay);
+static void process_srcu(struct work_struct *work);
 
 /*
  * Initialize SRCU combining tree.  Note that statically allocated
@@ -896,6 +897,15 @@ static void __synchronize_srcu(struct srcu_struct *sp, bool do_norm)
         __call_srcu(sp, &rcu.head, wakeme_after_rcu, do_norm);
         wait_for_completion(&rcu.completion);
         destroy_rcu_head_on_stack(&rcu.head);
+
+        /*
+         * Make sure that later code is ordered after the SRCU grace
+         * period.  This pairs with the raw_spin_lock_irq_rcu_node()
+         * in srcu_invoke_callbacks().  Unlike Tree RCU, this is needed
+         * because the current CPU might have been totally uninvolved with
+         * (and thus unordered against) that grace period.
+         */
+        smp_mb();
 }
 
 /**
@@ -1194,7 +1204,7 @@ static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay)
 /*
  * This is the work-queue function that handles SRCU grace periods.
  */
-void process_srcu(struct work_struct *work)
+static void process_srcu(struct work_struct *work)
 {
         struct srcu_struct *sp;
 
@@ -1203,7 +1213,6 @@ void process_srcu(struct work_struct *work)
         srcu_advance_state(sp);
         srcu_reschedule(sp, srcu_get_delay(sp));
 }
-EXPORT_SYMBOL_GPL(process_srcu);
 
 void srcutorture_get_gp_data(enum rcutorture_type test_type,
                              struct srcu_struct *sp, int *flags,
@@ -1217,6 +1226,43 @@ void srcutorture_get_gp_data(enum rcutorture_type test_type,
 }
 EXPORT_SYMBOL_GPL(srcutorture_get_gp_data);
 
+void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf)
+{
+        int cpu;
+        int idx;
+        unsigned long s0 = 0, s1 = 0;
+
+        idx = sp->srcu_idx & 0x1;
+        pr_alert("%s%s Tree SRCU per-CPU(idx=%d):", tt, tf, idx);
+        for_each_possible_cpu(cpu) {
+                unsigned long l0, l1;
+                unsigned long u0, u1;
+                long c0, c1;
+                struct srcu_data *counts;
+
+                counts = per_cpu_ptr(sp->sda, cpu);
+                u0 = counts->srcu_unlock_count[!idx];
+                u1 = counts->srcu_unlock_count[idx];
+
+                /*
+                 * Make sure that a lock is always counted if the corresponding
+                 * unlock is counted.
+                 */
+                smp_rmb();
+
+                l0 = counts->srcu_lock_count[!idx];
+                l1 = counts->srcu_lock_count[idx];
+
+                c0 = l0 - u0;
+                c1 = l1 - u1;
+                pr_cont(" %d(%ld,%ld)", cpu, c0, c1);
+                s0 += c0;
+                s1 += c1;
+        }
+        pr_cont(" T(%ld,%ld)\n", s0, s1);
+}
+EXPORT_SYMBOL_GPL(srcu_torture_stats_print);
+
 static int __init srcu_bootup_announce(void)
 {
         pr_info("Hierarchical SRCU implementation.\n");