x86/x2apic: Convert to CPU hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
Cc: Len Brown <len.brown@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mathias Krause <minipli@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20160713153337.736898691@linutronix.de
Signed-off-by: Ingo Molnar <mingo@kernel.org>

2 files changed, 31 insertions, 50 deletions

diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index aca8b75c1552..b5da5a8e5e45 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -152,68 +152,48 @@ static void init_x2apic_ldr(void)
         }
 }
 
- /*
-  * At CPU state changes, update the x2apic cluster sibling info.
-  */
-static int
-update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu)
+/*
+ * At CPU state changes, update the x2apic cluster sibling info.
+ */
+int x2apic_prepare_cpu(unsigned int cpu)
 {
-        unsigned int this_cpu = (unsigned long)hcpu;
-        unsigned int cpu;
-        int err = 0;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-                if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, this_cpu),
-                                        GFP_KERNEL)) {
-                        err = -ENOMEM;
-                } else if (!zalloc_cpumask_var(&per_cpu(ipi_mask, this_cpu),
-                                               GFP_KERNEL)) {
-                        free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
-                        err = -ENOMEM;
-                }
-                break;
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
-        case CPU_DEAD:
-                for_each_online_cpu(cpu) {
-                        if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
-                                continue;
-                        cpumask_clear_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu));
-                        cpumask_clear_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu));
-                }
-                free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
-                free_cpumask_var(per_cpu(ipi_mask, this_cpu));
-                break;
+        if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL))
+                return -ENOMEM;
+
+        if (!zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL)) {
+                free_cpumask_var(per_cpu(cpus_in_cluster, cpu));
+                return -ENOMEM;
         }
 
-        return notifier_from_errno(err);
+        return 0;
 }
 
-static struct notifier_block x2apic_cpu_notifier = {
-        .notifier_call = update_clusterinfo,
-};
-
-static int x2apic_init_cpu_notifier(void)
+int x2apic_dead_cpu(unsigned int this_cpu)
 {
-        int cpu = smp_processor_id();
-
-        zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL);
-        zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL);
+        int cpu;
 
-        BUG_ON(!per_cpu(cpus_in_cluster, cpu) || !per_cpu(ipi_mask, cpu));
-
-        cpumask_set_cpu(cpu, per_cpu(cpus_in_cluster, cpu));
-        register_hotcpu_notifier(&x2apic_cpu_notifier);
-        return 1;
+        for_each_online_cpu(cpu) {
+                if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
+                        continue;
+                cpumask_clear_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu));
+                cpumask_clear_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu));
+        }
+        free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
+        free_cpumask_var(per_cpu(ipi_mask, this_cpu));
+        return 0;
 }
 
 static int x2apic_cluster_probe(void)
 {
-        if (x2apic_mode)
-                return x2apic_init_cpu_notifier();
-        else
+        int cpu = smp_processor_id();
+
+        if (!x2apic_mode)
                 return 0;
+
+        cpumask_set_cpu(cpu, per_cpu(cpus_in_cluster, cpu));
+        cpuhp_setup_state(CPUHP_X2APIC_PREPARE, "X2APIC_PREPARE",
+                          x2apic_prepare_cpu, x2apic_dead_cpu);
+        return 1;
 }
 
 static const struct cpumask *x2apic_cluster_target_cpus(void)
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index ace5ad0fc3ec..78170827a776 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -17,6 +17,7 @@ enum cpuhp_state {
         CPUHP_WORKQUEUE_PREP,
         CPUHP_HRTIMERS_PREPARE,
         CPUHP_PROFILE_PREPARE,
+        CPUHP_X2APIC_PREPARE,
         CPUHP_TIMERS_DEAD,
         CPUHP_NOTIFY_PREPARE,
         CPUHP_BRINGUP_CPU,