diff options
Diffstat (limited to 'kernel/smp.c')
-rw-r--r-- | kernel/smp.c | 382 |
1 files changed, 382 insertions, 0 deletions
diff --git a/kernel/smp.c b/kernel/smp.c new file mode 100644 index 000000000000..ab10793b0707 --- /dev/null +++ b/kernel/smp.c | |||
@@ -0,0 +1,382 @@ | |||
1 | /* | ||
2 | * Generic helpers for smp ipi calls | ||
3 | * | ||
4 | * (C) Jens Axboe <jens.axboe@oracle.com> 2008 | ||
5 | * | ||
6 | */ | ||
7 | #include <linux/init.h> | ||
8 | #include <linux/module.h> | ||
9 | #include <linux/percpu.h> | ||
10 | #include <linux/rcupdate.h> | ||
11 | #include <linux/rculist.h> | ||
12 | #include <linux/smp.h> | ||
13 | |||
14 | static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); | ||
15 | static LIST_HEAD(call_function_queue); | ||
16 | __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock); | ||
17 | |||
18 | enum { | ||
19 | CSD_FLAG_WAIT = 0x01, | ||
20 | CSD_FLAG_ALLOC = 0x02, | ||
21 | }; | ||
22 | |||
23 | struct call_function_data { | ||
24 | struct call_single_data csd; | ||
25 | spinlock_t lock; | ||
26 | unsigned int refs; | ||
27 | cpumask_t cpumask; | ||
28 | struct rcu_head rcu_head; | ||
29 | }; | ||
30 | |||
31 | struct call_single_queue { | ||
32 | struct list_head list; | ||
33 | spinlock_t lock; | ||
34 | }; | ||
35 | |||
36 | void __cpuinit init_call_single_data(void) | ||
37 | { | ||
38 | int i; | ||
39 | |||
40 | for_each_possible_cpu(i) { | ||
41 | struct call_single_queue *q = &per_cpu(call_single_queue, i); | ||
42 | |||
43 | spin_lock_init(&q->lock); | ||
44 | INIT_LIST_HEAD(&q->list); | ||
45 | } | ||
46 | } | ||
47 | |||
48 | static void csd_flag_wait(struct call_single_data *data) | ||
49 | { | ||
50 | /* Wait for response */ | ||
51 | do { | ||
52 | /* | ||
53 | * We need to see the flags store in the IPI handler | ||
54 | */ | ||
55 | smp_mb(); | ||
56 | if (!(data->flags & CSD_FLAG_WAIT)) | ||
57 | break; | ||
58 | cpu_relax(); | ||
59 | } while (1); | ||
60 | } | ||
61 | |||
62 | /* | ||
63 | * Insert a previously allocated call_single_data element for execution | ||
64 | * on the given CPU. data must already have ->func, ->info, and ->flags set. | ||
65 | */ | ||
66 | static void generic_exec_single(int cpu, struct call_single_data *data) | ||
67 | { | ||
68 | struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); | ||
69 | int wait = data->flags & CSD_FLAG_WAIT, ipi; | ||
70 | unsigned long flags; | ||
71 | |||
72 | spin_lock_irqsave(&dst->lock, flags); | ||
73 | ipi = list_empty(&dst->list); | ||
74 | list_add_tail(&data->list, &dst->list); | ||
75 | spin_unlock_irqrestore(&dst->lock, flags); | ||
76 | |||
77 | if (ipi) | ||
78 | arch_send_call_function_single_ipi(cpu); | ||
79 | |||
80 | if (wait) | ||
81 | csd_flag_wait(data); | ||
82 | } | ||
83 | |||
84 | static void rcu_free_call_data(struct rcu_head *head) | ||
85 | { | ||
86 | struct call_function_data *data; | ||
87 | |||
88 | data = container_of(head, struct call_function_data, rcu_head); | ||
89 | |||
90 | kfree(data); | ||
91 | } | ||
92 | |||
93 | /* | ||
94 | * Invoked by arch to handle an IPI for call function. Must be called with | ||
95 | * interrupts disabled. | ||
96 | */ | ||
97 | void generic_smp_call_function_interrupt(void) | ||
98 | { | ||
99 | struct call_function_data *data; | ||
100 | int cpu = get_cpu(); | ||
101 | |||
102 | /* | ||
103 | * It's ok to use list_for_each_rcu() here even though we may delete | ||
104 | * 'pos', since list_del_rcu() doesn't clear ->next | ||
105 | */ | ||
106 | rcu_read_lock(); | ||
107 | list_for_each_entry_rcu(data, &call_function_queue, csd.list) { | ||
108 | int refs; | ||
109 | |||
110 | if (!cpu_isset(cpu, data->cpumask)) | ||
111 | continue; | ||
112 | |||
113 | data->csd.func(data->csd.info); | ||
114 | |||
115 | spin_lock(&data->lock); | ||
116 | cpu_clear(cpu, data->cpumask); | ||
117 | WARN_ON(data->refs == 0); | ||
118 | data->refs--; | ||
119 | refs = data->refs; | ||
120 | spin_unlock(&data->lock); | ||
121 | |||
122 | if (refs) | ||
123 | continue; | ||
124 | |||
125 | spin_lock(&call_function_lock); | ||
126 | list_del_rcu(&data->csd.list); | ||
127 | spin_unlock(&call_function_lock); | ||
128 | |||
129 | if (data->csd.flags & CSD_FLAG_WAIT) { | ||
130 | /* | ||
131 | * serialize stores to data with the flag clear | ||
132 | * and wakeup | ||
133 | */ | ||
134 | smp_wmb(); | ||
135 | data->csd.flags &= ~CSD_FLAG_WAIT; | ||
136 | } else | ||
137 | call_rcu(&data->rcu_head, rcu_free_call_data); | ||
138 | } | ||
139 | rcu_read_unlock(); | ||
140 | |||
141 | put_cpu(); | ||
142 | } | ||
143 | |||
144 | /* | ||
145 | * Invoked by arch to handle an IPI for call function single. Must be called | ||
146 | * from the arch with interrupts disabled. | ||
147 | */ | ||
148 | void generic_smp_call_function_single_interrupt(void) | ||
149 | { | ||
150 | struct call_single_queue *q = &__get_cpu_var(call_single_queue); | ||
151 | LIST_HEAD(list); | ||
152 | |||
153 | /* | ||
154 | * Need to see other stores to list head for checking whether | ||
155 | * list is empty without holding q->lock | ||
156 | */ | ||
157 | smp_mb(); | ||
158 | while (!list_empty(&q->list)) { | ||
159 | unsigned int data_flags; | ||
160 | |||
161 | spin_lock(&q->lock); | ||
162 | list_replace_init(&q->list, &list); | ||
163 | spin_unlock(&q->lock); | ||
164 | |||
165 | while (!list_empty(&list)) { | ||
166 | struct call_single_data *data; | ||
167 | |||
168 | data = list_entry(list.next, struct call_single_data, | ||
169 | list); | ||
170 | list_del(&data->list); | ||
171 | |||
172 | /* | ||
173 | * 'data' can be invalid after this call if | ||
174 | * flags == 0 (when called through | ||
175 | * generic_exec_single(), so save them away before | ||
176 | * making the call. | ||
177 | */ | ||
178 | data_flags = data->flags; | ||
179 | |||
180 | data->func(data->info); | ||
181 | |||
182 | if (data_flags & CSD_FLAG_WAIT) { | ||
183 | smp_wmb(); | ||
184 | data->flags &= ~CSD_FLAG_WAIT; | ||
185 | } else if (data_flags & CSD_FLAG_ALLOC) | ||
186 | kfree(data); | ||
187 | } | ||
188 | /* | ||
189 | * See comment on outer loop | ||
190 | */ | ||
191 | smp_mb(); | ||
192 | } | ||
193 | } | ||
194 | |||
195 | /* | ||
196 | * smp_call_function_single - Run a function on a specific CPU | ||
197 | * @func: The function to run. This must be fast and non-blocking. | ||
198 | * @info: An arbitrary pointer to pass to the function. | ||
199 | * @wait: If true, wait until function has completed on other CPUs. | ||
200 | * | ||
201 | * Returns 0 on success, else a negative status code. Note that @wait | ||
202 | * will be implicitly turned on in case of allocation failures, since | ||
203 | * we fall back to on-stack allocation. | ||
204 | */ | ||
205 | int smp_call_function_single(int cpu, void (*func) (void *info), void *info, | ||
206 | int wait) | ||
207 | { | ||
208 | struct call_single_data d; | ||
209 | unsigned long flags; | ||
210 | /* prevent preemption and reschedule on another processor */ | ||
211 | int me = get_cpu(); | ||
212 | |||
213 | /* Can deadlock when called with interrupts disabled */ | ||
214 | WARN_ON(irqs_disabled()); | ||
215 | |||
216 | if (cpu == me) { | ||
217 | local_irq_save(flags); | ||
218 | func(info); | ||
219 | local_irq_restore(flags); | ||
220 | } else { | ||
221 | struct call_single_data *data = NULL; | ||
222 | |||
223 | if (!wait) { | ||
224 | data = kmalloc(sizeof(*data), GFP_ATOMIC); | ||
225 | if (data) | ||
226 | data->flags = CSD_FLAG_ALLOC; | ||
227 | } | ||
228 | if (!data) { | ||
229 | data = &d; | ||
230 | data->flags = CSD_FLAG_WAIT; | ||
231 | } | ||
232 | |||
233 | data->func = func; | ||
234 | data->info = info; | ||
235 | generic_exec_single(cpu, data); | ||
236 | } | ||
237 | |||
238 | put_cpu(); | ||
239 | return 0; | ||
240 | } | ||
241 | EXPORT_SYMBOL(smp_call_function_single); | ||
242 | |||
243 | /** | ||
244 | * __smp_call_function_single(): Run a function on another CPU | ||
245 | * @cpu: The CPU to run on. | ||
246 | * @data: Pre-allocated and setup data structure | ||
247 | * | ||
248 | * Like smp_call_function_single(), but allow caller to pass in a pre-allocated | ||
249 | * data structure. Useful for embedding @data inside other structures, for | ||
250 | * instance. | ||
251 | * | ||
252 | */ | ||
253 | void __smp_call_function_single(int cpu, struct call_single_data *data) | ||
254 | { | ||
255 | /* Can deadlock when called with interrupts disabled */ | ||
256 | WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled()); | ||
257 | |||
258 | generic_exec_single(cpu, data); | ||
259 | } | ||
260 | |||
261 | /** | ||
262 | * smp_call_function_mask(): Run a function on a set of other CPUs. | ||
263 | * @mask: The set of cpus to run on. | ||
264 | * @func: The function to run. This must be fast and non-blocking. | ||
265 | * @info: An arbitrary pointer to pass to the function. | ||
266 | * @wait: If true, wait (atomically) until function has completed on other CPUs. | ||
267 | * | ||
268 | * Returns 0 on success, else a negative status code. | ||
269 | * | ||
270 | * If @wait is true, then returns once @func has returned. Note that @wait | ||
271 | * will be implicitly turned on in case of allocation failures, since | ||
272 | * we fall back to on-stack allocation. | ||
273 | * | ||
274 | * You must not call this function with disabled interrupts or from a | ||
275 | * hardware interrupt handler or from a bottom half handler. Preemption | ||
276 | * must be disabled when calling this function. | ||
277 | */ | ||
278 | int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, | ||
279 | int wait) | ||
280 | { | ||
281 | struct call_function_data d; | ||
282 | struct call_function_data *data = NULL; | ||
283 | cpumask_t allbutself; | ||
284 | unsigned long flags; | ||
285 | int cpu, num_cpus; | ||
286 | |||
287 | /* Can deadlock when called with interrupts disabled */ | ||
288 | WARN_ON(irqs_disabled()); | ||
289 | |||
290 | cpu = smp_processor_id(); | ||
291 | allbutself = cpu_online_map; | ||
292 | cpu_clear(cpu, allbutself); | ||
293 | cpus_and(mask, mask, allbutself); | ||
294 | num_cpus = cpus_weight(mask); | ||
295 | |||
296 | /* | ||
297 | * If zero CPUs, return. If just a single CPU, turn this request | ||
298 | * into a targetted single call instead since it's faster. | ||
299 | */ | ||
300 | if (!num_cpus) | ||
301 | return 0; | ||
302 | else if (num_cpus == 1) { | ||
303 | cpu = first_cpu(mask); | ||
304 | return smp_call_function_single(cpu, func, info, wait); | ||
305 | } | ||
306 | |||
307 | if (!wait) { | ||
308 | data = kmalloc(sizeof(*data), GFP_ATOMIC); | ||
309 | if (data) | ||
310 | data->csd.flags = CSD_FLAG_ALLOC; | ||
311 | } | ||
312 | if (!data) { | ||
313 | data = &d; | ||
314 | data->csd.flags = CSD_FLAG_WAIT; | ||
315 | } | ||
316 | |||
317 | spin_lock_init(&data->lock); | ||
318 | data->csd.func = func; | ||
319 | data->csd.info = info; | ||
320 | data->refs = num_cpus; | ||
321 | data->cpumask = mask; | ||
322 | |||
323 | spin_lock_irqsave(&call_function_lock, flags); | ||
324 | list_add_tail_rcu(&data->csd.list, &call_function_queue); | ||
325 | spin_unlock_irqrestore(&call_function_lock, flags); | ||
326 | |||
327 | /* Send a message to all CPUs in the map */ | ||
328 | arch_send_call_function_ipi(mask); | ||
329 | |||
330 | /* optionally wait for the CPUs to complete */ | ||
331 | if (wait) | ||
332 | csd_flag_wait(&data->csd); | ||
333 | |||
334 | return 0; | ||
335 | } | ||
336 | EXPORT_SYMBOL(smp_call_function_mask); | ||
337 | |||
338 | /** | ||
339 | * smp_call_function(): Run a function on all other CPUs. | ||
340 | * @func: The function to run. This must be fast and non-blocking. | ||
341 | * @info: An arbitrary pointer to pass to the function. | ||
342 | * @wait: If true, wait (atomically) until function has completed on other CPUs. | ||
343 | * | ||
344 | * Returns 0 on success, else a negative status code. | ||
345 | * | ||
346 | * If @wait is true, then returns once @func has returned; otherwise | ||
347 | * it returns just before the target cpu calls @func. In case of allocation | ||
348 | * failure, @wait will be implicitly turned on. | ||
349 | * | ||
350 | * You must not call this function with disabled interrupts or from a | ||
351 | * hardware interrupt handler or from a bottom half handler. | ||
352 | */ | ||
353 | int smp_call_function(void (*func)(void *), void *info, int wait) | ||
354 | { | ||
355 | int ret; | ||
356 | |||
357 | preempt_disable(); | ||
358 | ret = smp_call_function_mask(cpu_online_map, func, info, wait); | ||
359 | preempt_enable(); | ||
360 | return ret; | ||
361 | } | ||
362 | EXPORT_SYMBOL(smp_call_function); | ||
363 | |||
364 | void ipi_call_lock(void) | ||
365 | { | ||
366 | spin_lock(&call_function_lock); | ||
367 | } | ||
368 | |||
369 | void ipi_call_unlock(void) | ||
370 | { | ||
371 | spin_unlock(&call_function_lock); | ||
372 | } | ||
373 | |||
374 | void ipi_call_lock_irq(void) | ||
375 | { | ||
376 | spin_lock_irq(&call_function_lock); | ||
377 | } | ||
378 | |||
379 | void ipi_call_unlock_irq(void) | ||
380 | { | ||
381 | spin_unlock_irq(&call_function_lock); | ||
382 | } | ||