[PATCH] ppc32: support hotplug cpu on powermacs

This allows cpus to be off-lined on 32-bit SMP powermacs.  When a cpu
is off-lined, it is put into sleep mode with interrupts disabled.  It
can be on-lined again by asserting its soft-reset pin, which is
connected to a GPIO pin.

With this I can off-line the second cpu in my dual G4 powermac, which
means that I can then suspend the machine (the suspend/resume code
refuses to suspend if more than one cpu is online, and making it cope
with multiple cpus is surprisingly messy).

Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

7 files changed, 119 insertions, 61 deletions

diff --git a/arch/ppc/Kconfig b/arch/ppc/Kconfig
index e3f1ce33e64..347ea284140 100644
--- a/arch/ppc/Kconfig
+++ b/arch/ppc/Kconfig
@@ -265,6 +265,15 @@ config PPC601_SYNC_FIX
 
           If in doubt, say Y here.
 
+config HOTPLUG_CPU
+        bool "Support for enabling/disabling CPUs"
+        depends on SMP && HOTPLUG && EXPERIMENTAL && PPC_PMAC
+        ---help---
+          Say Y here to be able to disable and re-enable individual
+          CPUs at runtime on SMP machines.
+
+          Say N if you are unsure.
+
 source arch/ppc/platforms/4xx/Kconfig
 source arch/ppc/platforms/85xx/Kconfig
 
diff --git a/arch/ppc/kernel/head.S b/arch/ppc/kernel/head.S
index 55daf1210f3..1960fb8c259 100644
--- a/arch/ppc/kernel/head.S
+++ b/arch/ppc/kernel/head.S
@@ -1023,23 +1023,21 @@ __secondary_start_gemini:
         andc    r4,r4,r3
         mtspr   SPRN_HID0,r4
         sync
-        bl      gemini_prom_init
         b       __secondary_start
 #endif /* CONFIG_GEMINI */
-        .globl  __secondary_start_psurge
-__secondary_start_psurge:
-        li      r24,1                   /* cpu # */
-        b       __secondary_start_psurge99
-        .globl  __secondary_start_psurge2
-__secondary_start_psurge2:
-        li      r24,2                   /* cpu # */
-        b       __secondary_start_psurge99
-        .globl  __secondary_start_psurge3
-__secondary_start_psurge3:
-        li      r24,3                   /* cpu # */
-        b       __secondary_start_psurge99
-__secondary_start_psurge99:
-        /* we come in here with IR=0 and DR=1, and DBAT 0
+
+        .globl  __secondary_start_pmac_0
+__secondary_start_pmac_0:
+        /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
+        li      r24,0
+        b       1f
+        li      r24,1
+        b       1f
+        li      r24,2
+        b       1f
+        li      r24,3
+1:
+        /* on powersurge, we come in here with IR=0 and DR=1, and DBAT 0
            set to map the 0xf0000000 - 0xffffffff region */
         mfmsr   r0
         rlwinm  r0,r0,0,28,26           /* clear DR (0x10) */
diff --git a/arch/ppc/kernel/idle.c b/arch/ppc/kernel/idle.c
index 53547b6de45..fba29c876b6 100644
--- a/arch/ppc/kernel/idle.c
+++ b/arch/ppc/kernel/idle.c
@@ -22,6 +22,7 @@
 #include <linux/ptrace.h>
 #include <linux/slab.h>
 #include <linux/sysctl.h>
+#include <linux/cpu.h>
 
 #include <asm/pgtable.h>
 #include <asm/uaccess.h>
@@ -35,6 +36,7 @@
 void default_idle(void)
 {
         void (*powersave)(void);
+        int cpu = smp_processor_id();
 
         powersave = ppc_md.power_save;
 
@@ -44,7 +46,7 @@ void default_idle(void)
 #ifdef CONFIG_SMP
                 else {
                         set_thread_flag(TIF_POLLING_NRFLAG);
-                        while (!need_resched())
+                        while (!need_resched() && !cpu_is_offline(cpu))
                                 barrier();
                         clear_thread_flag(TIF_POLLING_NRFLAG);
                 }
@@ -52,6 +54,8 @@ void default_idle(void)
         }
         if (need_resched())
                 schedule();
+        if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
+                cpu_die();
 }
 
 /*
diff --git a/arch/ppc/kernel/smp.c b/arch/ppc/kernel/smp.c
index e70b587b9e5..726fe7ce174 100644
--- a/arch/ppc/kernel/smp.c
+++ b/arch/ppc/kernel/smp.c
@@ -45,6 +45,7 @@ cpumask_t cpu_online_map;
 cpumask_t cpu_possible_map;
 int smp_hw_index[NR_CPUS];
 struct thread_info *secondary_ti;
+static struct task_struct *idle_tasks[NR_CPUS];
 
 EXPORT_SYMBOL(cpu_online_map);
 EXPORT_SYMBOL(cpu_possible_map);
@@ -286,7 +287,8 @@ static void __devinit smp_store_cpu_info(int id)
 
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
-        int num_cpus, i;
+        int num_cpus, i, cpu;
+        struct task_struct *p;
 
         /* Fixup boot cpu */
         smp_store_cpu_info(smp_processor_id());
@@ -308,6 +310,17 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 
         if (smp_ops->space_timers)
                 smp_ops->space_timers(num_cpus);
+
+        for_each_cpu(cpu) {
+                if (cpu == smp_processor_id())
+                        continue;
+                /* create a process for the processor */
+                p = fork_idle(cpu);
+                if (IS_ERR(p))
+                        panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
+                p->thread_info->cpu = cpu;
+                idle_tasks[cpu] = p;
+        }
 }
 
 void __devinit smp_prepare_boot_cpu(void)
@@ -334,12 +347,17 @@ int __devinit start_secondary(void *unused)
         set_dec(tb_ticks_per_jiffy);
         cpu_callin_map[cpu] = 1;
 
-        printk("CPU %i done callin...\n", cpu);
+        printk("CPU %d done callin...\n", cpu);
         smp_ops->setup_cpu(cpu);
-        printk("CPU %i done setup...\n", cpu);
-        local_irq_enable();
+        printk("CPU %d done setup...\n", cpu);
         smp_ops->take_timebase();
-        printk("CPU %i done timebase take...\n", cpu);
+        printk("CPU %d done timebase take...\n", cpu);
+
+        spin_lock(&call_lock);
+        cpu_set(cpu, cpu_online_map);
+        spin_unlock(&call_lock);
+
+        local_irq_enable();
 
         cpu_idle();
         return 0;
@@ -347,17 +365,11 @@ int __devinit start_secondary(void *unused)
 
 int __cpu_up(unsigned int cpu)
 {
-        struct task_struct *p;
         char buf[32];
         int c;
 
-        /* create a process for the processor */
-        /* only regs.msr is actually used, and 0 is OK for it */
-        p = fork_idle(cpu);
-        if (IS_ERR(p))
-                panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
-        secondary_ti = p->thread_info;
-        p->thread_info->cpu = cpu;
+        secondary_ti = idle_tasks[cpu]->thread_info;
+        mb();
 
         /*
          * There was a cache flush loop here to flush the cache
@@ -389,7 +401,11 @@ int __cpu_up(unsigned int cpu)
         printk("Processor %d found.\n", cpu);
 
         smp_ops->give_timebase();
-        cpu_set(cpu, cpu_online_map);
+
+        /* Wait until cpu puts itself in the online map */
+        while (!cpu_online(cpu))
+                cpu_relax();
+
         return 0;
 }
 
diff --git a/arch/ppc/platforms/pmac_sleep.S b/arch/ppc/platforms/pmac_sleep.S
index 8d67adc7692..88419c77ac4 100644
--- a/arch/ppc/platforms/pmac_sleep.S
+++ b/arch/ppc/platforms/pmac_sleep.S
@@ -161,6 +161,8 @@ _GLOBAL(low_sleep_handler)
         addi r3,r3,sleep_storage@l
         stw r5,0(r3)
 
+        .globl  low_cpu_die
+low_cpu_die:
         /* Flush & disable all caches */
         bl      flush_disable_caches
 
diff --git a/arch/ppc/platforms/pmac_smp.c b/arch/ppc/platforms/pmac_smp.c
index 8e049dab4e6..794a23994b8 100644
--- a/arch/ppc/platforms/pmac_smp.c
+++ b/arch/ppc/platforms/pmac_smp.c
@@ -33,6 +33,7 @@
 #include <linux/spinlock.h>
 #include <linux/errno.h>
 #include <linux/hardirq.h>
+#include <linux/cpu.h>
 
 #include <asm/ptrace.h>
 #include <asm/atomic.h>
@@ -55,9 +56,7 @@
  * Powersurge (old powermac SMP) support.
  */
 
-extern void __secondary_start_psurge(void);
-extern void __secondary_start_psurge2(void);    /* Temporary horrible hack */
-extern void __secondary_start_psurge3(void);    /* Temporary horrible hack */
+extern void __secondary_start_pmac_0(void);
 
 /* Addresses for powersurge registers */
 #define HAMMERHEAD_BASE         0xf8000000
@@ -119,7 +118,7 @@ static volatile int sec_tb_reset = 0;
 static unsigned int pri_tb_hi, pri_tb_lo;
 static unsigned int pri_tb_stamp;
 
-static void __init core99_init_caches(int cpu)
+static void __devinit core99_init_caches(int cpu)
 {
         if (!cpu_has_feature(CPU_FTR_L2CR))
                 return;
@@ -346,7 +345,7 @@ static int __init smp_psurge_probe(void)
 
 static void __init smp_psurge_kick_cpu(int nr)
 {
-        void (*start)(void) = __secondary_start_psurge;
+        unsigned long start = __pa(__secondary_start_pmac_0) + nr * 8;
         unsigned long a;
 
         /* may need to flush here if secondary bats aren't setup */
@@ -356,17 +355,7 @@ static void __init smp_psurge_kick_cpu(int nr)
 
         if (ppc_md.progress) ppc_md.progress("smp_psurge_kick_cpu", 0x353);
 
-        /* setup entry point of secondary processor */
-        switch (nr) {
-        case 2:
-                start = __secondary_start_psurge2;
-                break;
-        case 3:
-                start = __secondary_start_psurge3;
-                break;
-        }
-
-        out_be32(psurge_start, __pa(start));
+        out_be32(psurge_start, start);
         mb();
 
         psurge_set_ipi(nr);
@@ -500,14 +489,14 @@ static int __init smp_core99_probe(void)
         return ncpus;
 }
 
-static void __init smp_core99_kick_cpu(int nr)
+static void __devinit smp_core99_kick_cpu(int nr)
 {
         unsigned long save_vector, new_vector;
         unsigned long flags;
 
         volatile unsigned long *vector
                  = ((volatile unsigned long *)(KERNELBASE+0x100));
-        if (nr < 1 || nr > 3)
+        if (nr < 0 || nr > 3)
                 return;
         if (ppc_md.progress) ppc_md.progress("smp_core99_kick_cpu", 0x346);
 
@@ -518,19 +507,9 @@ static void __init smp_core99_kick_cpu(int nr)
         save_vector = *vector;
 
         /* Setup fake reset vector that does    
-         *   b __secondary_start_psurge - KERNELBASE
+         *   b __secondary_start_pmac_0 + nr*8 - KERNELBASE
          */
-        switch(nr) {
-                case 1:
-                        new_vector = (unsigned long)__secondary_start_psurge;
-                        break;
-                case 2:
-                        new_vector = (unsigned long)__secondary_start_psurge2;
-                        break;
-                case 3:
-                        new_vector = (unsigned long)__secondary_start_psurge3;
-                        break;
-        }
+        new_vector = (unsigned long) __secondary_start_pmac_0 + nr * 8;
         *vector = 0x48000002 + new_vector - KERNELBASE;
 
         /* flush data cache and inval instruction cache */
@@ -554,7 +533,7 @@ static void __init smp_core99_kick_cpu(int nr)
         if (ppc_md.progress) ppc_md.progress("smp_core99_kick_cpu done", 0x347);
 }
 
-static void __init smp_core99_setup_cpu(int cpu_nr)
+static void __devinit smp_core99_setup_cpu(int cpu_nr)
 {
         /* Setup L2/L3 */
         if (cpu_nr != 0)
@@ -668,3 +647,47 @@ struct smp_ops_t core99_smp_ops __pmacdata = {
         .give_timebase  = smp_core99_give_timebase,
         .take_timebase  = smp_core99_take_timebase,
 };
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+int __cpu_disable(void)
+{
+        cpu_clear(smp_processor_id(), cpu_online_map);
+
+        /* XXX reset cpu affinity here */
+        openpic_set_priority(0xf);
+        asm volatile("mtdec %0" : : "r" (0x7fffffff));
+        mb();
+        udelay(20);
+        asm volatile("mtdec %0" : : "r" (0x7fffffff));
+        return 0;
+}
+
+extern void low_cpu_die(void) __attribute__((noreturn)); /* in pmac_sleep.S */
+static int cpu_dead[NR_CPUS];
+
+void cpu_die(void)
+{
+        local_irq_disable();
+        cpu_dead[smp_processor_id()] = 1;
+        mb();
+        low_cpu_die();
+}
+
+void __cpu_die(unsigned int cpu)
+{
+        int timeout;
+
+        timeout = 1000;
+        while (!cpu_dead[cpu]) {
+                if (--timeout == 0) {
+                        printk("CPU %u refused to die!\n", cpu);
+                        break;
+                }
+                msleep(1);
+        }
+        cpu_callin_map[cpu] = 0;
+        cpu_dead[cpu] = 0;
+}
+
+#endif
diff --git a/include/asm-ppc/smp.h b/include/asm-ppc/smp.h
index 17530c232c7..829481c0a9d 100644
--- a/include/asm-ppc/smp.h
+++ b/include/asm-ppc/smp.h
@@ -41,6 +41,10 @@ extern void smp_send_xmon_break(int cpu);
 struct pt_regs;
 extern void smp_message_recv(int, struct pt_regs *);
 
+extern int __cpu_disable(void);
+extern void __cpu_die(unsigned int cpu);
+extern void cpu_die(void) __attribute__((noreturn));
+
 #define NO_PROC_ID              0xFF            /* No processor magic marker */
 #define PROC_CHANGE_PENALTY     20
 
@@ -64,6 +68,8 @@ extern struct klock_info_struct klock_info;
 
 #else /* !(CONFIG_SMP) */
 
+static inline void cpu_die(void) { }
+
 #endif /* !(CONFIG_SMP) */
 
 #endif /* !(_PPC_SMP_H) */