Merge with http://kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git

1 files changed, 54 insertions, 5 deletions

diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index c4d159a21e04..48d3bce465b8 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -116,6 +116,10 @@ void fastcall call_rcu(struct rcu_head *head,
         local_irq_restore(flags);
 }
 
+static atomic_t rcu_barrier_cpu_count;
+static struct semaphore rcu_barrier_sema;
+static struct completion rcu_barrier_completion;
+
 /**
  * call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
  * @head: structure to be used for queueing the RCU updates.
@@ -162,6 +166,42 @@ long rcu_batches_completed(void)
         return rcu_ctrlblk.completed;
 }
 
+static void rcu_barrier_callback(struct rcu_head *notused)
+{
+        if (atomic_dec_and_test(&rcu_barrier_cpu_count))
+                complete(&rcu_barrier_completion);
+}
+
+/*
+ * Called with preemption disabled, and from cross-cpu IRQ context.
+ */
+static void rcu_barrier_func(void *notused)
+{
+        int cpu = smp_processor_id();
+        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
+        struct rcu_head *head;
+
+        head = &rdp->barrier;
+        atomic_inc(&rcu_barrier_cpu_count);
+        call_rcu(head, rcu_barrier_callback);
+}
+
+/**
+ * rcu_barrier - Wait until all the in-flight RCUs are complete.
+ */
+void rcu_barrier(void)
+{
+        BUG_ON(in_interrupt());
+        /* Take cpucontrol semaphore to protect against CPU hotplug */
+        down(&rcu_barrier_sema);
+        init_completion(&rcu_barrier_completion);
+        atomic_set(&rcu_barrier_cpu_count, 0);
+        on_each_cpu(rcu_barrier_func, NULL, 0, 1);
+        wait_for_completion(&rcu_barrier_completion);
+        up(&rcu_barrier_sema);
+}
+EXPORT_SYMBOL_GPL(rcu_barrier);
+
 /*
  * Invoke the completed RCU callbacks. They are expected to be in
  * a per-cpu list.
@@ -217,15 +257,23 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp, struct rcu_state *rsp,
 
         if (rcp->next_pending &&
                         rcp->completed == rcp->cur) {
-                /* Can't change, since spin lock held. */
-                cpus_andnot(rsp->cpumask, cpu_online_map, nohz_cpu_mask);
-
                 rcp->next_pending = 0;
-                /* next_pending == 0 must be visible in __rcu_process_callbacks()
-                 * before it can see new value of cur.
+                /*
+                 * next_pending == 0 must be visible in
+                 * __rcu_process_callbacks() before it can see new value of cur.
                  */
                 smp_wmb();
                 rcp->cur++;
+
+                /*
+                 * Accessing nohz_cpu_mask before incrementing rcp->cur needs a
+                 * Barrier  Otherwise it can cause tickless idle CPUs to be
+                 * included in rsp->cpumask, which will extend graceperiods
+                 * unnecessarily.
+                 */
+                smp_mb();
+                cpus_andnot(rsp->cpumask, cpu_online_map, nohz_cpu_mask);
+
         }
 }
 
@@ -457,6 +505,7 @@ static struct notifier_block __devinitdata rcu_nb = {
  */
 void __init rcu_init(void)
 {
+        sema_init(&rcu_barrier_sema, 1);
         rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE,
                         (void *)(long)smp_processor_id());
         /* Register notifier for non-boot CPUs */