aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/stop_machine.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/stop_machine.c')
-rw-r--r--kernel/stop_machine.c288
1 files changed, 131 insertions, 157 deletions
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 738b411ff2d3..e446c7c7d6a9 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -1,4 +1,4 @@
1/* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. 1/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2 * GPL v2 and any later version. 2 * GPL v2 and any later version.
3 */ 3 */
4#include <linux/cpu.h> 4#include <linux/cpu.h>
@@ -13,204 +13,178 @@
13#include <asm/atomic.h> 13#include <asm/atomic.h>
14#include <asm/uaccess.h> 14#include <asm/uaccess.h>
15 15
16/* Since we effect priority and affinity (both of which are visible 16/* This controls the threads on each CPU. */
17 * to, and settable by outside processes) we do indirection via a
18 * kthread. */
19
20/* Thread to stop each CPU in user context. */
21enum stopmachine_state { 17enum stopmachine_state {
22 STOPMACHINE_WAIT, 18 /* Dummy starting state for thread. */
19 STOPMACHINE_NONE,
20 /* Awaiting everyone to be scheduled. */
23 STOPMACHINE_PREPARE, 21 STOPMACHINE_PREPARE,
22 /* Disable interrupts. */
24 STOPMACHINE_DISABLE_IRQ, 23 STOPMACHINE_DISABLE_IRQ,
24 /* Run the function */
25 STOPMACHINE_RUN,
26 /* Exit */
25 STOPMACHINE_EXIT, 27 STOPMACHINE_EXIT,
26}; 28};
29static enum stopmachine_state state;
27 30
28static enum stopmachine_state stopmachine_state; 31struct stop_machine_data {
29static unsigned int stopmachine_num_threads; 32 int (*fn)(void *);
30static atomic_t stopmachine_thread_ack; 33 void *data;
31 34 int fnret;
32static int stopmachine(void *cpu) 35};
33{
34 int irqs_disabled = 0;
35 int prepared = 0;
36 cpumask_of_cpu_ptr(cpumask, (int)(long)cpu);
37
38 set_cpus_allowed_ptr(current, cpumask);
39
40 /* Ack: we are alive */
41 smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
42 atomic_inc(&stopmachine_thread_ack);
43
44 /* Simple state machine */
45 while (stopmachine_state != STOPMACHINE_EXIT) {
46 if (stopmachine_state == STOPMACHINE_DISABLE_IRQ
47 && !irqs_disabled) {
48 local_irq_disable();
49 hard_irq_disable();
50 irqs_disabled = 1;
51 /* Ack: irqs disabled. */
52 smp_mb(); /* Must read state first. */
53 atomic_inc(&stopmachine_thread_ack);
54 } else if (stopmachine_state == STOPMACHINE_PREPARE
55 && !prepared) {
56 /* Everyone is in place, hold CPU. */
57 preempt_disable();
58 prepared = 1;
59 smp_mb(); /* Must read state first. */
60 atomic_inc(&stopmachine_thread_ack);
61 }
62 /* Yield in first stage: migration threads need to
63 * help our sisters onto their CPUs. */
64 if (!prepared && !irqs_disabled)
65 yield();
66 cpu_relax();
67 }
68
69 /* Ack: we are exiting. */
70 smp_mb(); /* Must read state first. */
71 atomic_inc(&stopmachine_thread_ack);
72
73 if (irqs_disabled)
74 local_irq_enable();
75 if (prepared)
76 preempt_enable();
77 36
78 return 0; 37/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
79} 38static unsigned int num_threads;
39static atomic_t thread_ack;
40static struct completion finished;
41static DEFINE_MUTEX(lock);
80 42
81/* Change the thread state */ 43static void set_state(enum stopmachine_state newstate)
82static void stopmachine_set_state(enum stopmachine_state state)
83{ 44{
84 atomic_set(&stopmachine_thread_ack, 0); 45 /* Reset ack counter. */
46 atomic_set(&thread_ack, num_threads);
85 smp_wmb(); 47 smp_wmb();
86 stopmachine_state = state; 48 state = newstate;
87 while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads)
88 cpu_relax();
89} 49}
90 50
91static int stop_machine(void) 51/* Last one to ack a state moves to the next state. */
52static void ack_state(void)
92{ 53{
93 int i, ret = 0; 54 if (atomic_dec_and_test(&thread_ack)) {
94 55 /* If we're the last one to ack the EXIT, we're finished. */
95 atomic_set(&stopmachine_thread_ack, 0); 56 if (state == STOPMACHINE_EXIT)
96 stopmachine_num_threads = 0; 57 complete(&finished);
97 stopmachine_state = STOPMACHINE_WAIT; 58 else
98 59 set_state(state + 1);
99 for_each_online_cpu(i) {
100 if (i == raw_smp_processor_id())
101 continue;
102 ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL);
103 if (ret < 0)
104 break;
105 stopmachine_num_threads++;
106 }
107
108 /* Wait for them all to come to life. */
109 while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) {
110 yield();
111 cpu_relax();
112 } 60 }
61}
113 62
114 /* If some failed, kill them all. */ 63/* This is the actual thread which stops the CPU. It exits by itself rather
115 if (ret < 0) { 64 * than waiting for kthread_stop(), because it's easier for hotplug CPU. */
116 stopmachine_set_state(STOPMACHINE_EXIT); 65static int stop_cpu(struct stop_machine_data *smdata)
117 return ret; 66{
118 } 67 enum stopmachine_state curstate = STOPMACHINE_NONE;
68 int uninitialized_var(ret);
119 69
120 /* Now they are all started, make them hold the CPUs, ready. */ 70 /* Simple state machine */
121 preempt_disable(); 71 do {
122 stopmachine_set_state(STOPMACHINE_PREPARE); 72 /* Chill out and ensure we re-read stopmachine_state. */
73 cpu_relax();
74 if (state != curstate) {
75 curstate = state;
76 switch (curstate) {
77 case STOPMACHINE_DISABLE_IRQ:
78 local_irq_disable();
79 hard_irq_disable();
80 break;
81 case STOPMACHINE_RUN:
82 /* |= allows error detection if functions on
83 * multiple CPUs. */
84 smdata->fnret |= smdata->fn(smdata->data);
85 break;
86 default:
87 break;
88 }
89 ack_state();
90 }
91 } while (curstate != STOPMACHINE_EXIT);
123 92
124 /* Make them disable irqs. */ 93 local_irq_enable();
125 local_irq_disable(); 94 do_exit(0);
126 hard_irq_disable(); 95}
127 stopmachine_set_state(STOPMACHINE_DISABLE_IRQ);
128 96
97/* Callback for CPUs which aren't supposed to do anything. */
98static int chill(void *unused)
99{
129 return 0; 100 return 0;
130} 101}
131 102
132static void restart_machine(void) 103int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
133{ 104{
134 stopmachine_set_state(STOPMACHINE_EXIT); 105 int i, err;
135 local_irq_enable(); 106 struct stop_machine_data active, idle;
136 preempt_enable_no_resched(); 107 struct task_struct **threads;
137} 108
109 active.fn = fn;
110 active.data = data;
111 active.fnret = 0;
112 idle.fn = chill;
113 idle.data = NULL;
114
115 /* This could be too big for stack on large machines. */
116 threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL);
117 if (!threads)
118 return -ENOMEM;
119
120 /* Set up initial state. */
121 mutex_lock(&lock);
122 init_completion(&finished);
123 num_threads = num_online_cpus();
124 set_state(STOPMACHINE_PREPARE);
138 125
139struct stop_machine_data { 126 for_each_online_cpu(i) {
140 int (*fn)(void *); 127 struct stop_machine_data *smdata = &idle;
141 void *data; 128 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
142 struct completion done;
143};
144 129
145static int do_stop(void *_smdata) 130 if (!cpus) {
146{ 131 if (i == first_cpu(cpu_online_map))
147 struct stop_machine_data *smdata = _smdata; 132 smdata = &active;
148 int ret; 133 } else {
134 if (cpu_isset(i, *cpus))
135 smdata = &active;
136 }
149 137
150 ret = stop_machine(); 138 threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u",
151 if (ret == 0) { 139 i);
152 ret = smdata->fn(smdata->data); 140 if (IS_ERR(threads[i])) {
153 restart_machine(); 141 err = PTR_ERR(threads[i]);
154 } 142 threads[i] = NULL;
143 goto kill_threads;
144 }
155 145
156 /* We're done: you can kthread_stop us now */ 146 /* Place it onto correct cpu. */
157 complete(&smdata->done); 147 kthread_bind(threads[i], i);
158 148
159 /* Wait for kthread_stop */ 149 /* Make it highest prio. */
160 set_current_state(TASK_INTERRUPTIBLE); 150 if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, &param))
161 while (!kthread_should_stop()) { 151 BUG();
162 schedule();
163 set_current_state(TASK_INTERRUPTIBLE);
164 } 152 }
165 __set_current_state(TASK_RUNNING);
166 return ret;
167}
168 153
169struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, 154 /* We've created all the threads. Wake them all: hold this CPU so one
170 unsigned int cpu) 155 * doesn't hit this CPU until we're ready. */
171{ 156 get_cpu();
172 static DEFINE_MUTEX(stopmachine_mutex); 157 for_each_online_cpu(i)
173 struct stop_machine_data smdata; 158 wake_up_process(threads[i]);
174 struct task_struct *p;
175 159
176 smdata.fn = fn; 160 /* This will release the thread on our CPU. */
177 smdata.data = data; 161 put_cpu();
178 init_completion(&smdata.done); 162 wait_for_completion(&finished);
163 mutex_unlock(&lock);
179 164
180 mutex_lock(&stopmachine_mutex); 165 kfree(threads);
181 166
182 /* If they don't care which CPU fn runs on, bind to any online one. */ 167 return active.fnret;
183 if (cpu == NR_CPUS)
184 cpu = raw_smp_processor_id();
185 168
186 p = kthread_create(do_stop, &smdata, "kstopmachine"); 169kill_threads:
187 if (!IS_ERR(p)) { 170 for_each_online_cpu(i)
188 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 171 if (threads[i])
172 kthread_stop(threads[i]);
173 mutex_unlock(&lock);
189 174
190 /* One high-prio thread per cpu. We'll do this one. */ 175 kfree(threads);
191 sched_setscheduler_nocheck(p, SCHED_FIFO, &param); 176 return err;
192 kthread_bind(p, cpu);
193 wake_up_process(p);
194 wait_for_completion(&smdata.done);
195 }
196 mutex_unlock(&stopmachine_mutex);
197 return p;
198} 177}
199 178
200int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) 179int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
201{ 180{
202 struct task_struct *p;
203 int ret; 181 int ret;
204 182
205 /* No CPUs can come up or down during this. */ 183 /* No CPUs can come up or down during this. */
206 get_online_cpus(); 184 get_online_cpus();
207 p = __stop_machine_run(fn, data, cpu); 185 ret = __stop_machine(fn, data, cpus);
208 if (!IS_ERR(p))
209 ret = kthread_stop(p);
210 else
211 ret = PTR_ERR(p);
212 put_online_cpus(); 186 put_online_cpus();
213 187
214 return ret; 188 return ret;
215} 189}
216EXPORT_SYMBOL_GPL(stop_machine_run); 190EXPORT_SYMBOL_GPL(stop_machine);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-17 01:09:44 -0500