aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/stop_machine.c
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2008-10-13 05:05:51 -0400
committerIngo Molnar <mingo@elte.hu>2008-10-13 05:05:51 -0400
commitaccba5f3965d6a9d1bf7c1e1a7995d17e9d521b6 (patch)
tree8fb40782e79472ed882ff2098d4dd295557278ee /kernel/stop_machine.c
parent6852fd9b86d05063c6ef49d2e12e061cc7f6a105 (diff)
parent4480f15b3306f43bbb0310d461142b4e897ca45b (diff)
Merge branch 'linus' into oprofile-v2
Conflicts: arch/x86/kernel/apic_32.c arch/x86/oprofile/nmi_int.c include/linux/pci_ids.h
Diffstat (limited to 'kernel/stop_machine.c')
-rw-r--r--kernel/stop_machine.c287
1 files changed, 130 insertions, 157 deletions
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 738b411ff2d3..af3c7cea258b 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -1,4 +1,4 @@
1/* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. 1/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2 * GPL v2 and any later version. 2 * GPL v2 and any later version.
3 */ 3 */
4#include <linux/cpu.h> 4#include <linux/cpu.h>
@@ -13,204 +13,177 @@
13#include <asm/atomic.h> 13#include <asm/atomic.h>
14#include <asm/uaccess.h> 14#include <asm/uaccess.h>
15 15
16/* Since we effect priority and affinity (both of which are visible 16/* This controls the threads on each CPU. */
17 * to, and settable by outside processes) we do indirection via a
18 * kthread. */
19
20/* Thread to stop each CPU in user context. */
21enum stopmachine_state { 17enum stopmachine_state {
22 STOPMACHINE_WAIT, 18 /* Dummy starting state for thread. */
19 STOPMACHINE_NONE,
20 /* Awaiting everyone to be scheduled. */
23 STOPMACHINE_PREPARE, 21 STOPMACHINE_PREPARE,
22 /* Disable interrupts. */
24 STOPMACHINE_DISABLE_IRQ, 23 STOPMACHINE_DISABLE_IRQ,
24 /* Run the function */
25 STOPMACHINE_RUN,
26 /* Exit */
25 STOPMACHINE_EXIT, 27 STOPMACHINE_EXIT,
26}; 28};
29static enum stopmachine_state state;
27 30
28static enum stopmachine_state stopmachine_state; 31struct stop_machine_data {
29static unsigned int stopmachine_num_threads; 32 int (*fn)(void *);
30static atomic_t stopmachine_thread_ack; 33 void *data;
31 34 int fnret;
32static int stopmachine(void *cpu) 35};
33{
34 int irqs_disabled = 0;
35 int prepared = 0;
36 cpumask_of_cpu_ptr(cpumask, (int)(long)cpu);
37
38 set_cpus_allowed_ptr(current, cpumask);
39
40 /* Ack: we are alive */
41 smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
42 atomic_inc(&stopmachine_thread_ack);
43
44 /* Simple state machine */
45 while (stopmachine_state != STOPMACHINE_EXIT) {
46 if (stopmachine_state == STOPMACHINE_DISABLE_IRQ
47 && !irqs_disabled) {
48 local_irq_disable();
49 hard_irq_disable();
50 irqs_disabled = 1;
51 /* Ack: irqs disabled. */
52 smp_mb(); /* Must read state first. */
53 atomic_inc(&stopmachine_thread_ack);
54 } else if (stopmachine_state == STOPMACHINE_PREPARE
55 && !prepared) {
56 /* Everyone is in place, hold CPU. */
57 preempt_disable();
58 prepared = 1;
59 smp_mb(); /* Must read state first. */
60 atomic_inc(&stopmachine_thread_ack);
61 }
62 /* Yield in first stage: migration threads need to
63 * help our sisters onto their CPUs. */
64 if (!prepared && !irqs_disabled)
65 yield();
66 cpu_relax();
67 }
68
69 /* Ack: we are exiting. */
70 smp_mb(); /* Must read state first. */
71 atomic_inc(&stopmachine_thread_ack);
72
73 if (irqs_disabled)
74 local_irq_enable();
75 if (prepared)
76 preempt_enable();
77 36
78 return 0; 37/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
79} 38static unsigned int num_threads;
39static atomic_t thread_ack;
40static struct completion finished;
41static DEFINE_MUTEX(lock);
80 42
81/* Change the thread state */ 43static void set_state(enum stopmachine_state newstate)
82static void stopmachine_set_state(enum stopmachine_state state)
83{ 44{
84 atomic_set(&stopmachine_thread_ack, 0); 45 /* Reset ack counter. */
46 atomic_set(&thread_ack, num_threads);
85 smp_wmb(); 47 smp_wmb();
86 stopmachine_state = state; 48 state = newstate;
87 while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads)
88 cpu_relax();
89} 49}
90 50
91static int stop_machine(void) 51/* Last one to ack a state moves to the next state. */
52static void ack_state(void)
92{ 53{
93 int i, ret = 0; 54 if (atomic_dec_and_test(&thread_ack)) {
94 55 /* If we're the last one to ack the EXIT, we're finished. */
95 atomic_set(&stopmachine_thread_ack, 0); 56 if (state == STOPMACHINE_EXIT)
96 stopmachine_num_threads = 0; 57 complete(&finished);
97 stopmachine_state = STOPMACHINE_WAIT; 58 else
98 59 set_state(state + 1);
99 for_each_online_cpu(i) {
100 if (i == raw_smp_processor_id())
101 continue;
102 ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL);
103 if (ret < 0)
104 break;
105 stopmachine_num_threads++;
106 }
107
108 /* Wait for them all to come to life. */
109 while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) {
110 yield();
111 cpu_relax();
112 } 60 }
61}
113 62
114 /* If some failed, kill them all. */ 63/* This is the actual thread which stops the CPU. It exits by itself rather
115 if (ret < 0) { 64 * than waiting for kthread_stop(), because it's easier for hotplug CPU. */
116 stopmachine_set_state(STOPMACHINE_EXIT); 65static int stop_cpu(struct stop_machine_data *smdata)
117 return ret; 66{
118 } 67 enum stopmachine_state curstate = STOPMACHINE_NONE;
119 68
120 /* Now they are all started, make them hold the CPUs, ready. */ 69 /* Simple state machine */
121 preempt_disable(); 70 do {
122 stopmachine_set_state(STOPMACHINE_PREPARE); 71 /* Chill out and ensure we re-read stopmachine_state. */
72 cpu_relax();
73 if (state != curstate) {
74 curstate = state;
75 switch (curstate) {
76 case STOPMACHINE_DISABLE_IRQ:
77 local_irq_disable();
78 hard_irq_disable();
79 break;
80 case STOPMACHINE_RUN:
81 /* |= allows error detection if functions on
82 * multiple CPUs. */
83 smdata->fnret |= smdata->fn(smdata->data);
84 break;
85 default:
86 break;
87 }
88 ack_state();
89 }
90 } while (curstate != STOPMACHINE_EXIT);
123 91
124 /* Make them disable irqs. */ 92 local_irq_enable();
125 local_irq_disable(); 93 do_exit(0);
126 hard_irq_disable(); 94}
127 stopmachine_set_state(STOPMACHINE_DISABLE_IRQ);
128 95
96/* Callback for CPUs which aren't supposed to do anything. */
97static int chill(void *unused)
98{
129 return 0; 99 return 0;
130} 100}
131 101
132static void restart_machine(void) 102int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
133{ 103{
134 stopmachine_set_state(STOPMACHINE_EXIT); 104 int i, err;
135 local_irq_enable(); 105 struct stop_machine_data active, idle;
136 preempt_enable_no_resched(); 106 struct task_struct **threads;
137} 107
108 active.fn = fn;
109 active.data = data;
110 active.fnret = 0;
111 idle.fn = chill;
112 idle.data = NULL;
113
114 /* This could be too big for stack on large machines. */
115 threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL);
116 if (!threads)
117 return -ENOMEM;
118
119 /* Set up initial state. */
120 mutex_lock(&lock);
121 init_completion(&finished);
122 num_threads = num_online_cpus();
123 set_state(STOPMACHINE_PREPARE);
138 124
139struct stop_machine_data { 125 for_each_online_cpu(i) {
140 int (*fn)(void *); 126 struct stop_machine_data *smdata = &idle;
141 void *data; 127 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
142 struct completion done;
143};
144 128
145static int do_stop(void *_smdata) 129 if (!cpus) {
146{ 130 if (i == first_cpu(cpu_online_map))
147 struct stop_machine_data *smdata = _smdata; 131 smdata = &active;
148 int ret; 132 } else {
133 if (cpu_isset(i, *cpus))
134 smdata = &active;
135 }
149 136
150 ret = stop_machine(); 137 threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u",
151 if (ret == 0) { 138 i);
152 ret = smdata->fn(smdata->data); 139 if (IS_ERR(threads[i])) {
153 restart_machine(); 140 err = PTR_ERR(threads[i]);
154 } 141 threads[i] = NULL;
142 goto kill_threads;
143 }
155 144
156 /* We're done: you can kthread_stop us now */ 145 /* Place it onto correct cpu. */
157 complete(&smdata->done); 146 kthread_bind(threads[i], i);
158 147
159 /* Wait for kthread_stop */ 148 /* Make it highest prio. */
160 set_current_state(TASK_INTERRUPTIBLE); 149 if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, &param))
161 while (!kthread_should_stop()) { 150 BUG();
162 schedule();
163 set_current_state(TASK_INTERRUPTIBLE);
164 } 151 }
165 __set_current_state(TASK_RUNNING);
166 return ret;
167}
168 152
169struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, 153 /* We've created all the threads. Wake them all: hold this CPU so one
170 unsigned int cpu) 154 * doesn't hit this CPU until we're ready. */
171{ 155 get_cpu();
172 static DEFINE_MUTEX(stopmachine_mutex); 156 for_each_online_cpu(i)
173 struct stop_machine_data smdata; 157 wake_up_process(threads[i]);
174 struct task_struct *p;
175 158
176 smdata.fn = fn; 159 /* This will release the thread on our CPU. */
177 smdata.data = data; 160 put_cpu();
178 init_completion(&smdata.done); 161 wait_for_completion(&finished);
162 mutex_unlock(&lock);
179 163
180 mutex_lock(&stopmachine_mutex); 164 kfree(threads);
181 165
182 /* If they don't care which CPU fn runs on, bind to any online one. */ 166 return active.fnret;
183 if (cpu == NR_CPUS)
184 cpu = raw_smp_processor_id();
185 167
186 p = kthread_create(do_stop, &smdata, "kstopmachine"); 168kill_threads:
187 if (!IS_ERR(p)) { 169 for_each_online_cpu(i)
188 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 170 if (threads[i])
171 kthread_stop(threads[i]);
172 mutex_unlock(&lock);
189 173
190 /* One high-prio thread per cpu. We'll do this one. */ 174 kfree(threads);
191 sched_setscheduler_nocheck(p, SCHED_FIFO, &param); 175 return err;
192 kthread_bind(p, cpu);
193 wake_up_process(p);
194 wait_for_completion(&smdata.done);
195 }
196 mutex_unlock(&stopmachine_mutex);
197 return p;
198} 176}
199 177
200int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) 178int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
201{ 179{
202 struct task_struct *p;
203 int ret; 180 int ret;
204 181
205 /* No CPUs can come up or down during this. */ 182 /* No CPUs can come up or down during this. */
206 get_online_cpus(); 183 get_online_cpus();
207 p = __stop_machine_run(fn, data, cpu); 184 ret = __stop_machine(fn, data, cpus);
208 if (!IS_ERR(p))
209 ret = kthread_stop(p);
210 else
211 ret = PTR_ERR(p);
212 put_online_cpus(); 185 put_online_cpus();
213 186
214 return ret; 187 return ret;
215} 188}
216EXPORT_SYMBOL_GPL(stop_machine_run); 189EXPORT_SYMBOL_GPL(stop_machine);
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-16 15:48:04 -0500