/* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. * GPL v2 and any later version. */ #include <linux/cpu.h> #include <linux/err.h> #include <linux/kthread.h> #include <linux/module.h> #include <linux/sched.h> #include <linux/stop_machine.h> #include <linux/syscalls.h> #include <linux/interrupt.h> #include <asm/atomic.h> #include <asm/uaccess.h> /* Since we effect priority and affinity (both of which are visible * to, and settable by outside processes) we do indirection via a * kthread. */ /* Thread to stop each CPU in user context. */ enum stopmachine_state { STOPMACHINE_WAIT, STOPMACHINE_PREPARE, STOPMACHINE_DISABLE_IRQ, STOPMACHINE_EXIT, }; static enum stopmachine_state stopmachine_state; static unsigned int stopmachine_num_threads; static atomic_t stopmachine_thread_ack; static int stopmachine(void *cpu) { int irqs_disabled = 0; int prepared = 0; set_cpus_allowed_ptr(current, &cpumask_of_cpu((int)(long)cpu)); /* Ack: we are alive */ smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ atomic_inc(&stopmachine_thread_ack); /* Simple state machine */ while (stopmachine_state != STOPMACHINE_EXIT) { if (stopmachine_state == STOPMACHINE_DISABLE_IRQ && !irqs_disabled) { local_irq_disable(); hard_irq_disable(); irqs_disabled = 1; /* Ack: irqs disabled. */ smp_mb(); /* Must read state first. */ atomic_inc(&stopmachine_thread_ack); } else if (stopmachine_state == STOPMACHINE_PREPARE && !prepared) { /* Everyone is in place, hold CPU. */ preempt_disable(); prepared = 1; smp_mb(); /* Must read state first. */ atomic_inc(&stopmachine_thread_ack); } /* Yield in first stage: migration threads need to * help our sisters onto their CPUs. */ if (!prepared && !irqs_disabled) yield(); cpu_relax(); } /* Ack: we are exiting. */ smp_mb(); /* Must read state first. */ atomic_inc(&stopmachine_thread_ack); if (irqs_disabled) local_irq_enable(); if (prepared) preempt_enable(); return 0; } /* Change the thread state */ static void stopmachine_set_state(enum stopmachine_state state) { atomic_set(&stopmachine_thread_ack, 0); smp_wmb(); stopmachine_state = state; while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) cpu_relax(); } static int stop_machine(void) { int i, ret = 0; atomic_set(&stopmachine_thread_ack, 0); stopmachine_num_threads = 0; stopmachine_state = STOPMACHINE_WAIT; for_each_online_cpu(i) { if (i == raw_smp_processor_id()) continue; ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL); if (ret < 0) break; stopmachine_num_threads++; } /* Wait for them all to come to life. */ while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) { yield(); cpu_relax(); } /* If some failed, kill them all. */ if (ret < 0) { stopmachine_set_state(STOPMACHINE_EXIT); return ret; } /* Now they are all started, make them hold the CPUs, ready. */ preempt_disable(); stopmachine_set_state(STOPMACHINE_PREPARE); /* Make them disable irqs. */ local_irq_disable(); hard_irq_disable(); stopmachine_set_state(STOPMACHINE_DISABLE_IRQ); return 0; } static void restart_machine(void) { stopmachine_set_state(STOPMACHINE_EXIT); local_irq_enable(); preempt_enable_no_resched(); } struct stop_machine_data { int (*fn)(void *); void *data; struct completion done; }; static int do_stop(void *_smdata) { struct stop_machine_data *smdata = _smdata; int ret; ret = stop_machine(); if (ret == 0) { ret = smdata->fn(smdata->data); restart_machine(); } /* We're done: you can kthread_stop us now */ complete(&smdata->done); /* Wait for kthread_stop */ set_current_state(TASK_INTERRUPTIBLE); while (!kthread_should_stop()) { schedule(); set_current_state(TASK_INTERRUPTIBLE); } __set_current_state(TASK_RUNNING); return ret; } struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) { static DEFINE_MUTEX(stopmachine_mutex); struct stop_machine_data smdata; struct task_struct *p; smdata.fn = fn; smdata.data = data; init_completion(&smdata.done); mutex_lock(&stopmachine_mutex); /* If they don't care which CPU fn runs on, bind to any online one. */ if (cpu == NR_CPUS) cpu = raw_smp_processor_id(); p = kthread_create(do_stop, &smdata, "kstopmachine"); if (!IS_ERR(p)) { struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; /* One high-prio thread per cpu. We'll do this one. */ sched_setscheduler_nocheck(p, SCHED_FIFO, ¶m); kthread_bind(p, cpu); wake_up_process(p); wait_for_completion(&smdata.done); } mutex_unlock(&stopmachine_mutex); return p; } int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) { struct task_struct *p; int ret; /* No CPUs can come up or down during this. */ get_online_cpus(); p = __stop_machine_run(fn, data, cpu); if (!IS_ERR(p)) ret = kthread_stop(p); else ret = PTR_ERR(p); put_online_cpus(); return ret; } EXPORT_SYMBOL_GPL(stop_machine_run);