1 files changed, 97 insertions, 32 deletions

diff --git a/kernel/cpu.c b/kernel/cpu.c
index a343bde710b1..94bbe4695232 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -20,6 +20,7 @@
 #include <linux/gfp.h>
 #include <linux/suspend.h>
 #include <linux/lockdep.h>
+#include <linux/tick.h>
 #include <trace/events/power.h>
 
 #include "smpboot.h"
@@ -58,20 +59,23 @@ static int cpu_hotplug_disabled;
 
 static struct {
         struct task_struct *active_writer;
-        struct mutex lock; /* Synchronizes accesses to refcount, */
+        /* wait queue to wake up the active_writer */
+        wait_queue_head_t wq;
+        /* verifies that no writer will get active while readers are active */
+        struct mutex lock;
         /*
          * Also blocks the new readers during
          * an ongoing cpu hotplug operation.
          */
-        int refcount;
+        atomic_t refcount;
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
         struct lockdep_map dep_map;
 #endif
 } cpu_hotplug = {
         .active_writer = NULL,
+        .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
         .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
-        .refcount = 0,
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
         .dep_map = {.name = "cpu_hotplug.lock" },
 #endif
@@ -79,9 +83,12 @@ static struct {
 
 /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
 #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
+#define cpuhp_lock_acquire_tryread() \
+                                  lock_map_acquire_tryread(&cpu_hotplug.dep_map)
 #define cpuhp_lock_acquire()      lock_map_acquire(&cpu_hotplug.dep_map)
 #define cpuhp_lock_release()      lock_map_release(&cpu_hotplug.dep_map)
 
+
 void get_online_cpus(void)
 {
         might_sleep();
@@ -89,24 +96,38 @@ void get_online_cpus(void)
                 return;
         cpuhp_lock_acquire_read();
         mutex_lock(&cpu_hotplug.lock);
-        cpu_hotplug.refcount++;
+        atomic_inc(&cpu_hotplug.refcount);
         mutex_unlock(&cpu_hotplug.lock);
-
 }
 EXPORT_SYMBOL_GPL(get_online_cpus);
 
+bool try_get_online_cpus(void)
+{
+        if (cpu_hotplug.active_writer == current)
+                return true;
+        if (!mutex_trylock(&cpu_hotplug.lock))
+                return false;
+        cpuhp_lock_acquire_tryread();
+        atomic_inc(&cpu_hotplug.refcount);
+        mutex_unlock(&cpu_hotplug.lock);
+        return true;
+}
+EXPORT_SYMBOL_GPL(try_get_online_cpus);
+
 void put_online_cpus(void)
 {
+        int refcount;
+
         if (cpu_hotplug.active_writer == current)
                 return;
-        mutex_lock(&cpu_hotplug.lock);
 
-        if (WARN_ON(!cpu_hotplug.refcount))
-                cpu_hotplug.refcount++; /* try to fix things up */
+        refcount = atomic_dec_return(&cpu_hotplug.refcount);
+        if (WARN_ON(refcount < 0)) /* try to fix things up */
+                atomic_inc(&cpu_hotplug.refcount);
+
+        if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
+                wake_up(&cpu_hotplug.wq);
 
-        if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
-                wake_up_process(cpu_hotplug.active_writer);
-        mutex_unlock(&cpu_hotplug.lock);
         cpuhp_lock_release();
 
 }
@@ -136,17 +157,20 @@ EXPORT_SYMBOL_GPL(put_online_cpus);
  */
 void cpu_hotplug_begin(void)
 {
-        cpu_hotplug.active_writer = current;
+        DEFINE_WAIT(wait);
 
+        cpu_hotplug.active_writer = current;
         cpuhp_lock_acquire();
+
         for (;;) {
                 mutex_lock(&cpu_hotplug.lock);
-                if (likely(!cpu_hotplug.refcount))
-                        break;
-                __set_current_state(TASK_UNINTERRUPTIBLE);
+                prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
+                if (likely(!atomic_read(&cpu_hotplug.refcount)))
+                                break;
                 mutex_unlock(&cpu_hotplug.lock);
                 schedule();
         }
+        finish_wait(&cpu_hotplug.wq, &wait);
 }
 
 void cpu_hotplug_done(void)
@@ -274,21 +298,28 @@ void clear_tasks_mm_cpumask(int cpu)
         rcu_read_unlock();
 }
 
-static inline void check_for_tasks(int cpu)
+static inline void check_for_tasks(int dead_cpu)
 {
-        struct task_struct *p;
-        cputime_t utime, stime;
+        struct task_struct *g, *p;
 
-        write_lock_irq(&tasklist_lock);
-        for_each_process(p) {
-                task_cputime(p, &utime, &stime);
-                if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
-                    (utime || stime))
-                        pr_warn("Task %s (pid = %d) is on cpu %d (state = %ld, flags = %x)\n",
-                                p->comm, task_pid_nr(p), cpu,
-                                p->state, p->flags);
-        }
-        write_unlock_irq(&tasklist_lock);
+        read_lock_irq(&tasklist_lock);
+        do_each_thread(g, p) {
+                if (!p->on_rq)
+                        continue;
+                /*
+                 * We do the check with unlocked task_rq(p)->lock.
+                 * Order the reading to do not warn about a task,
+                 * which was running on this cpu in the past, and
+                 * it's just been woken on another cpu.
+                 */
+                rmb();
+                if (task_cpu(p) != dead_cpu)
+                        continue;
+
+                pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
+                        p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
+        } while_each_thread(g, p);
+        read_unlock_irq(&tasklist_lock);
 }
 
 struct take_cpu_down_param {
@@ -308,6 +339,8 @@ static int __ref take_cpu_down(void *_param)
                 return err;
 
         cpu_notify(CPU_DYING | param->mod, param->hcpu);
+        /* Give up timekeeping duties */
+        tick_handover_do_timer();
         /* Park the stopper thread */
         kthread_park(current);
         return 0;
@@ -378,13 +411,17 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
          *
          * Wait for the stop thread to go away.
          */
-        while (!idle_cpu(cpu))
+        while (!per_cpu(cpu_dead_idle, cpu))
                 cpu_relax();
+        smp_mb(); /* Read from cpu_dead_idle before __cpu_die(). */
+        per_cpu(cpu_dead_idle, cpu) = false;
 
+        hotplug_cpu__broadcast_tick_pull(cpu);
         /* This actually kills the CPU. */
         __cpu_die(cpu);
 
         /* CPU is completely dead: tell everyone.  Too late to complain. */
+        tick_cleanup_dead_cpu(cpu);
         cpu_notify_nofail(CPU_DEAD | mod, hcpu);
 
         check_for_tasks(cpu);
@@ -416,6 +453,37 @@ out:
 EXPORT_SYMBOL(cpu_down);
 #endif /*CONFIG_HOTPLUG_CPU*/
 
+/*
+ * Unpark per-CPU smpboot kthreads at CPU-online time.
+ */
+static int smpboot_thread_call(struct notifier_block *nfb,
+                               unsigned long action, void *hcpu)
+{
+        int cpu = (long)hcpu;
+
+        switch (action & ~CPU_TASKS_FROZEN) {
+
+        case CPU_ONLINE:
+                smpboot_unpark_threads(cpu);
+                break;
+
+        default:
+                break;
+        }
+
+        return NOTIFY_OK;
+}
+
+static struct notifier_block smpboot_thread_notifier = {
+        .notifier_call = smpboot_thread_call,
+        .priority = CPU_PRI_SMPBOOT,
+};
+
+void __cpuinit smpboot_thread_init(void)
+{
+        register_cpu_notifier(&smpboot_thread_notifier);
+}
+
 /* Requires cpu_add_remove_lock to be held */
 static int _cpu_up(unsigned int cpu, int tasks_frozen)
 {
@@ -455,9 +523,6 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen)
                 goto out_notify;
         BUG_ON(!cpu_online(cpu));
 
-        /* Wake the per cpu threads */
-        smpboot_unpark_threads(cpu);
-
         /* Now call notifier in preparation. */
         cpu_notify(CPU_ONLINE | mod, hcpu);