aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/Kconfig26
-rw-r--r--arch/arm/common/Makefile2
-rw-r--r--arch/arm/common/bL_switcher.c822
-rw-r--r--arch/arm/common/bL_switcher_dummy_if.c71
-rw-r--r--arch/arm/common/mcpm_entry.c12
-rw-r--r--arch/arm/common/mcpm_head.S16
-rw-r--r--arch/arm/include/asm/bL_switcher.h77
-rw-r--r--arch/arm/include/asm/hardirq.h2
-rw-r--r--arch/arm/include/asm/mcpm.h8
-rw-r--r--arch/arm/include/asm/smp.h2
-rw-r--r--arch/arm/kernel/sleep.S26
-rw-r--r--arch/arm/kernel/smp.c21
-rw-r--r--arch/arm/kernel/suspend.c8
-rw-r--r--drivers/irqchip/irq-gic.c151
-rw-r--r--include/linux/irqchip/arm-gic.h7
-rw-r--r--include/trace/events/power_cpu_migrate.h67
16 files changed, 1294 insertions, 24 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 1ad6fb6c094d..22efc5d9c952 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1549,6 +1549,32 @@ config MCPM
1549 for (multi-)cluster based systems, such as big.LITTLE based 1549 for (multi-)cluster based systems, such as big.LITTLE based
1550 systems. 1550 systems.
1551 1551
1552config BIG_LITTLE
1553 bool "big.LITTLE support (Experimental)"
1554 depends on CPU_V7 && SMP
1555 select MCPM
1556 help
1557 This option enables support selections for the big.LITTLE
1558 system architecture.
1559
1560config BL_SWITCHER
1561 bool "big.LITTLE switcher support"
1562 depends on BIG_LITTLE && MCPM && HOTPLUG_CPU
1563 select CPU_PM
1564 select ARM_CPU_SUSPEND
1565 help
1566 The big.LITTLE "switcher" provides the core functionality to
1567 transparently handle transition between a cluster of A15's
1568 and a cluster of A7's in a big.LITTLE system.
1569
1570config BL_SWITCHER_DUMMY_IF
1571 tristate "Simple big.LITTLE switcher user interface"
1572 depends on BL_SWITCHER && DEBUG_KERNEL
1573 help
1574 This is a simple and dummy char dev interface to control
1575 the big.LITTLE switcher core code. It is meant for
1576 debugging purposes only.
1577
1552choice 1578choice
1553 prompt "Memory split" 1579 prompt "Memory split"
1554 default VMSPLIT_3G 1580 default VMSPLIT_3G
diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile
index 8c60f473e976..5c8584c4944d 100644
--- a/arch/arm/common/Makefile
+++ b/arch/arm/common/Makefile
@@ -17,3 +17,5 @@ obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o
17AFLAGS_mcpm_head.o := -march=armv7-a 17AFLAGS_mcpm_head.o := -march=armv7-a
18AFLAGS_vlock.o := -march=armv7-a 18AFLAGS_vlock.o := -march=armv7-a
19obj-$(CONFIG_TI_PRIV_EDMA) += edma.o 19obj-$(CONFIG_TI_PRIV_EDMA) += edma.o
20obj-$(CONFIG_BL_SWITCHER) += bL_switcher.o
21obj-$(CONFIG_BL_SWITCHER_DUMMY_IF) += bL_switcher_dummy_if.o
diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c
new file mode 100644
index 000000000000..63bbc4f70564
--- /dev/null
+++ b/arch/arm/common/bL_switcher.c
@@ -0,0 +1,822 @@
1/*
2 * arch/arm/common/bL_switcher.c -- big.LITTLE cluster switcher core driver
3 *
4 * Created by: Nicolas Pitre, March 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/atomic.h>
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/sched.h>
17#include <linux/interrupt.h>
18#include <linux/cpu_pm.h>
19#include <linux/cpu.h>
20#include <linux/cpumask.h>
21#include <linux/kthread.h>
22#include <linux/wait.h>
23#include <linux/time.h>
24#include <linux/clockchips.h>
25#include <linux/hrtimer.h>
26#include <linux/tick.h>
27#include <linux/notifier.h>
28#include <linux/mm.h>
29#include <linux/mutex.h>
30#include <linux/smp.h>
31#include <linux/spinlock.h>
32#include <linux/string.h>
33#include <linux/sysfs.h>
34#include <linux/irqchip/arm-gic.h>
35#include <linux/moduleparam.h>
36
37#include <asm/smp_plat.h>
38#include <asm/cputype.h>
39#include <asm/suspend.h>
40#include <asm/mcpm.h>
41#include <asm/bL_switcher.h>
42
43#define CREATE_TRACE_POINTS
44#include <trace/events/power_cpu_migrate.h>
45
46
47/*
48 * Use our own MPIDR accessors as the generic ones in asm/cputype.h have
49 * __attribute_const__ and we don't want the compiler to assume any
50 * constness here as the value _does_ change along some code paths.
51 */
52
53static int read_mpidr(void)
54{
55 unsigned int id;
56 asm volatile ("mrc p15, 0, %0, c0, c0, 5" : "=r" (id));
57 return id & MPIDR_HWID_BITMASK;
58}
59
60/*
61 * Get a global nanosecond time stamp for tracing.
62 */
63static s64 get_ns(void)
64{
65 struct timespec ts;
66 getnstimeofday(&ts);
67 return timespec_to_ns(&ts);
68}
69
70/*
71 * bL switcher core code.
72 */
73
74static void bL_do_switch(void *_arg)
75{
76 unsigned ib_mpidr, ib_cpu, ib_cluster;
77 long volatile handshake, **handshake_ptr = _arg;
78
79 pr_debug("%s\n", __func__);
80
81 ib_mpidr = cpu_logical_map(smp_processor_id());
82 ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0);
83 ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1);
84
85 /* Advertise our handshake location */
86 if (handshake_ptr) {
87 handshake = 0;
88 *handshake_ptr = &handshake;
89 } else
90 handshake = -1;
91
92 /*
93 * Our state has been saved at this point. Let's release our
94 * inbound CPU.
95 */
96 mcpm_set_entry_vector(ib_cpu, ib_cluster, cpu_resume);
97 sev();
98
99 /*
100 * From this point, we must assume that our counterpart CPU might
101 * have taken over in its parallel world already, as if execution
102 * just returned from cpu_suspend(). It is therefore important to
103 * be very careful not to make any change the other guy is not
104 * expecting. This is why we need stack isolation.
105 *
106 * Fancy under cover tasks could be performed here. For now
107 * we have none.
108 */
109
110 /*
111 * Let's wait until our inbound is alive.
112 */
113 while (!handshake) {
114 wfe();
115 smp_mb();
116 }
117
118 /* Let's put ourself down. */
119 mcpm_cpu_power_down();
120
121 /* should never get here */
122 BUG();
123}
124
125/*
126 * Stack isolation. To ensure 'current' remains valid, we just use another
127 * piece of our thread's stack space which should be fairly lightly used.
128 * The selected area starts just above the thread_info structure located
129 * at the very bottom of the stack, aligned to a cache line, and indexed
130 * with the cluster number.
131 */
132#define STACK_SIZE 512
133extern void call_with_stack(void (*fn)(void *), void *arg, void *sp);
134static int bL_switchpoint(unsigned long _arg)
135{
136 unsigned int mpidr = read_mpidr();
137 unsigned int clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1);
138 void *stack = current_thread_info() + 1;
139 stack = PTR_ALIGN(stack, L1_CACHE_BYTES);
140 stack += clusterid * STACK_SIZE + STACK_SIZE;
141 call_with_stack(bL_do_switch, (void *)_arg, stack);
142 BUG();
143}
144
145/*
146 * Generic switcher interface
147 */
148
149static unsigned int bL_gic_id[MAX_CPUS_PER_CLUSTER][MAX_NR_CLUSTERS];
150static int bL_switcher_cpu_pairing[NR_CPUS];
151
152/*
153 * bL_switch_to - Switch to a specific cluster for the current CPU
154 * @new_cluster_id: the ID of the cluster to switch to.
155 *
156 * This function must be called on the CPU to be switched.
157 * Returns 0 on success, else a negative status code.
158 */
159static int bL_switch_to(unsigned int new_cluster_id)
160{
161 unsigned int mpidr, this_cpu, that_cpu;
162 unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster;
163 struct completion inbound_alive;
164 struct tick_device *tdev;
165 enum clock_event_mode tdev_mode;
166 long volatile *handshake_ptr;
167 int ipi_nr, ret;
168
169 this_cpu = smp_processor_id();
170 ob_mpidr = read_mpidr();
171 ob_cpu = MPIDR_AFFINITY_LEVEL(ob_mpidr, 0);
172 ob_cluster = MPIDR_AFFINITY_LEVEL(ob_mpidr, 1);
173 BUG_ON(cpu_logical_map(this_cpu) != ob_mpidr);
174
175 if (new_cluster_id == ob_cluster)
176 return 0;
177
178 that_cpu = bL_switcher_cpu_pairing[this_cpu];
179 ib_mpidr = cpu_logical_map(that_cpu);
180 ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0);
181 ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1);
182
183 pr_debug("before switch: CPU %d MPIDR %#x -> %#x\n",
184 this_cpu, ob_mpidr, ib_mpidr);
185
186 this_cpu = smp_processor_id();
187
188 /* Close the gate for our entry vectors */
189 mcpm_set_entry_vector(ob_cpu, ob_cluster, NULL);
190 mcpm_set_entry_vector(ib_cpu, ib_cluster, NULL);
191
192 /* Install our "inbound alive" notifier. */
193 init_completion(&inbound_alive);
194 ipi_nr = register_ipi_completion(&inbound_alive, this_cpu);
195 ipi_nr |= ((1 << 16) << bL_gic_id[ob_cpu][ob_cluster]);
196 mcpm_set_early_poke(ib_cpu, ib_cluster, gic_get_sgir_physaddr(), ipi_nr);
197
198 /*
199 * Let's wake up the inbound CPU now in case it requires some delay
200 * to come online, but leave it gated in our entry vector code.
201 */
202 ret = mcpm_cpu_power_up(ib_cpu, ib_cluster);
203 if (ret) {
204 pr_err("%s: mcpm_cpu_power_up() returned %d\n", __func__, ret);
205 return ret;
206 }
207
208 /*
209 * Raise a SGI on the inbound CPU to make sure it doesn't stall
210 * in a possible WFI, such as in bL_power_down().
211 */
212 gic_send_sgi(bL_gic_id[ib_cpu][ib_cluster], 0);
213
214 /*
215 * Wait for the inbound to come up. This allows for other
216 * tasks to be scheduled in the mean time.
217 */
218 wait_for_completion(&inbound_alive);
219 mcpm_set_early_poke(ib_cpu, ib_cluster, 0, 0);
220
221 /*
222 * From this point we are entering the switch critical zone
223 * and can't take any interrupts anymore.
224 */
225 local_irq_disable();
226 local_fiq_disable();
227 trace_cpu_migrate_begin(get_ns(), ob_mpidr);
228
229 /* redirect GIC's SGIs to our counterpart */
230 gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]);
231
232 tdev = tick_get_device(this_cpu);
233 if (tdev && !cpumask_equal(tdev->evtdev->cpumask, cpumask_of(this_cpu)))
234 tdev = NULL;
235 if (tdev) {
236 tdev_mode = tdev->evtdev->mode;
237 clockevents_set_mode(tdev->evtdev, CLOCK_EVT_MODE_SHUTDOWN);
238 }
239
240 ret = cpu_pm_enter();
241
242 /* we can not tolerate errors at this point */
243 if (ret)
244 panic("%s: cpu_pm_enter() returned %d\n", __func__, ret);
245
246 /* Swap the physical CPUs in the logical map for this logical CPU. */
247 cpu_logical_map(this_cpu) = ib_mpidr;
248 cpu_logical_map(that_cpu) = ob_mpidr;
249
250 /* Let's do the actual CPU switch. */
251 ret = cpu_suspend((unsigned long)&handshake_ptr, bL_switchpoint);
252 if (ret > 0)
253 panic("%s: cpu_suspend() returned %d\n", __func__, ret);
254
255 /* We are executing on the inbound CPU at this point */
256 mpidr = read_mpidr();
257 pr_debug("after switch: CPU %d MPIDR %#x\n", this_cpu, mpidr);
258 BUG_ON(mpidr != ib_mpidr);
259
260 mcpm_cpu_powered_up();
261
262 ret = cpu_pm_exit();
263
264 if (tdev) {
265 clockevents_set_mode(tdev->evtdev, tdev_mode);
266 clockevents_program_event(tdev->evtdev,
267 tdev->evtdev->next_event, 1);
268 }
269
270 trace_cpu_migrate_finish(get_ns(), ib_mpidr);
271 local_fiq_enable();
272 local_irq_enable();
273
274 *handshake_ptr = 1;
275 dsb_sev();
276
277 if (ret)
278 pr_err("%s exiting with error %d\n", __func__, ret);
279 return ret;
280}
281
282struct bL_thread {
283 spinlock_t lock;
284 struct task_struct *task;
285 wait_queue_head_t wq;
286 int wanted_cluster;
287 struct completion started;
288 bL_switch_completion_handler completer;
289 void *completer_cookie;
290};
291
292static struct bL_thread bL_threads[NR_CPUS];
293
294static int bL_switcher_thread(void *arg)
295{
296 struct bL_thread *t = arg;
297 struct sched_param param = { .sched_priority = 1 };
298 int cluster;
299 bL_switch_completion_handler completer;
300 void *completer_cookie;
301
302 sched_setscheduler_nocheck(current, SCHED_FIFO, &param);
303 complete(&t->started);
304
305 do {
306 if (signal_pending(current))
307 flush_signals(current);
308 wait_event_interruptible(t->wq,
309 t->wanted_cluster != -1 ||
310 kthread_should_stop());
311
312 spin_lock(&t->lock);
313 cluster = t->wanted_cluster;
314 completer = t->completer;
315 completer_cookie = t->completer_cookie;
316 t->wanted_cluster = -1;
317 t->completer = NULL;
318 spin_unlock(&t->lock);
319
320 if (cluster != -1) {
321 bL_switch_to(cluster);
322
323 if (completer)
324 completer(completer_cookie);
325 }
326 } while (!kthread_should_stop());
327
328 return 0;
329}
330
331static struct task_struct *bL_switcher_thread_create(int cpu, void *arg)
332{
333 struct task_struct *task;
334
335 task = kthread_create_on_node(bL_switcher_thread, arg,
336 cpu_to_node(cpu), "kswitcher_%d", cpu);
337 if (!IS_ERR(task)) {
338 kthread_bind(task, cpu);
339 wake_up_process(task);
340 } else
341 pr_err("%s failed for CPU %d\n", __func__, cpu);
342 return task;
343}
344
345/*
346 * bL_switch_request_cb - Switch to a specific cluster for the given CPU,
347 * with completion notification via a callback
348 *
349 * @cpu: the CPU to switch
350 * @new_cluster_id: the ID of the cluster to switch to.
351 * @completer: switch completion callback. if non-NULL,
352 * @completer(@completer_cookie) will be called on completion of
353 * the switch, in non-atomic context.
354 * @completer_cookie: opaque context argument for @completer.
355 *
356 * This function causes a cluster switch on the given CPU by waking up
357 * the appropriate switcher thread. This function may or may not return
358 * before the switch has occurred.
359 *
360 * If a @completer callback function is supplied, it will be called when
361 * the switch is complete. This can be used to determine asynchronously
362 * when the switch is complete, regardless of when bL_switch_request()
363 * returns. When @completer is supplied, no new switch request is permitted
364 * for the affected CPU until after the switch is complete, and @completer
365 * has returned.
366 */
367int bL_switch_request_cb(unsigned int cpu, unsigned int new_cluster_id,
368 bL_switch_completion_handler completer,
369 void *completer_cookie)
370{
371 struct bL_thread *t;
372
373 if (cpu >= ARRAY_SIZE(bL_threads)) {
374 pr_err("%s: cpu %d out of bounds\n", __func__, cpu);
375 return -EINVAL;
376 }
377
378 t = &bL_threads[cpu];
379
380 if (IS_ERR(t->task))
381 return PTR_ERR(t->task);
382 if (!t->task)
383 return -ESRCH;
384
385 spin_lock(&t->lock);
386 if (t->completer) {
387 spin_unlock(&t->lock);
388 return -EBUSY;
389 }
390 t->completer = completer;
391 t->completer_cookie = completer_cookie;
392 t->wanted_cluster = new_cluster_id;
393 spin_unlock(&t->lock);
394 wake_up(&t->wq);
395 return 0;
396}
397EXPORT_SYMBOL_GPL(bL_switch_request_cb);
398
399/*
400 * Activation and configuration code.
401 */
402
403static DEFINE_MUTEX(bL_switcher_activation_lock);
404static BLOCKING_NOTIFIER_HEAD(bL_activation_notifier);
405static unsigned int bL_switcher_active;
406static unsigned int bL_switcher_cpu_original_cluster[NR_CPUS];
407static cpumask_t bL_switcher_removed_logical_cpus;
408
409int bL_switcher_register_notifier(struct notifier_block *nb)
410{
411 return blocking_notifier_chain_register(&bL_activation_notifier, nb);
412}
413EXPORT_SYMBOL_GPL(bL_switcher_register_notifier);
414
415int bL_switcher_unregister_notifier(struct notifier_block *nb)
416{
417 return blocking_notifier_chain_unregister(&bL_activation_notifier, nb);
418}
419EXPORT_SYMBOL_GPL(bL_switcher_unregister_notifier);
420
421static int bL_activation_notify(unsigned long val)
422{
423 int ret;
424
425 ret = blocking_notifier_call_chain(&bL_activation_notifier, val, NULL);
426 if (ret & NOTIFY_STOP_MASK)
427 pr_err("%s: notifier chain failed with status 0x%x\n",
428 __func__, ret);
429 return notifier_to_errno(ret);
430}
431
432static void bL_switcher_restore_cpus(void)
433{
434 int i;
435
436 for_each_cpu(i, &bL_switcher_removed_logical_cpus)
437 cpu_up(i);
438}
439
440static int bL_switcher_halve_cpus(void)
441{
442 int i, j, cluster_0, gic_id, ret;
443 unsigned int cpu, cluster, mask;
444 cpumask_t available_cpus;
445
446 /* First pass to validate what we have */
447 mask = 0;
448 for_each_online_cpu(i) {
449 cpu = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0);
450 cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1);
451 if (cluster >= 2) {
452 pr_err("%s: only dual cluster systems are supported\n", __func__);
453 return -EINVAL;
454 }
455 if (WARN_ON(cpu >= MAX_CPUS_PER_CLUSTER))
456 return -EINVAL;
457 mask |= (1 << cluster);
458 }
459 if (mask != 3) {
460 pr_err("%s: no CPU pairing possible\n", __func__);
461 return -EINVAL;
462 }
463
464 /*
465 * Now let's do the pairing. We match each CPU with another CPU
466 * from a different cluster. To get a uniform scheduling behavior
467 * without fiddling with CPU topology and compute capacity data,
468 * we'll use logical CPUs initially belonging to the same cluster.
469 */
470 memset(bL_switcher_cpu_pairing, -1, sizeof(bL_switcher_cpu_pairing));
471 cpumask_copy(&available_cpus, cpu_online_mask);
472 cluster_0 = -1;
473 for_each_cpu(i, &available_cpus) {
474 int match = -1;
475 cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1);
476 if (cluster_0 == -1)
477 cluster_0 = cluster;
478 if (cluster != cluster_0)
479 continue;
480 cpumask_clear_cpu(i, &available_cpus);
481 for_each_cpu(j, &available_cpus) {
482 cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(j), 1);
483 /*
484 * Let's remember the last match to create "odd"
485 * pairings on purpose in order for other code not
486 * to assume any relation between physical and
487 * logical CPU numbers.
488 */
489 if (cluster != cluster_0)
490 match = j;
491 }
492 if (match != -1) {
493 bL_switcher_cpu_pairing[i] = match;
494 cpumask_clear_cpu(match, &available_cpus);
495 pr_info("CPU%d paired with CPU%d\n", i, match);
496 }
497 }
498
499 /*
500 * Now we disable the unwanted CPUs i.e. everything that has no
501 * pairing information (that includes the pairing counterparts).
502 */
503 cpumask_clear(&bL_switcher_removed_logical_cpus);
504 for_each_online_cpu(i) {
505 cpu = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0);
506 cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1);
507
508 /* Let's take note of the GIC ID for this CPU */
509 gic_id = gic_get_cpu_id(i);
510 if (gic_id < 0) {
511 pr_err("%s: bad GIC ID for CPU %d\n", __func__, i);
512 bL_switcher_restore_cpus();
513 return -EINVAL;
514 }
515 bL_gic_id[cpu][cluster] = gic_id;
516 pr_info("GIC ID for CPU %u cluster %u is %u\n",
517 cpu, cluster, gic_id);
518
519 if (bL_switcher_cpu_pairing[i] != -1) {
520 bL_switcher_cpu_original_cluster[i] = cluster;
521 continue;
522 }
523
524 ret = cpu_down(i);
525 if (ret) {
526 bL_switcher_restore_cpus();
527 return ret;
528 }
529 cpumask_set_cpu(i, &bL_switcher_removed_logical_cpus);
530 }
531
532 return 0;
533}
534
535/* Determine the logical CPU a given physical CPU is grouped on. */
536int bL_switcher_get_logical_index(u32 mpidr)
537{
538 int cpu;
539
540 if (!bL_switcher_active)
541 return -EUNATCH;
542
543 mpidr &= MPIDR_HWID_BITMASK;
544 for_each_online_cpu(cpu) {
545 int pairing = bL_switcher_cpu_pairing[cpu];
546 if (pairing == -1)
547 continue;
548 if ((mpidr == cpu_logical_map(cpu)) ||
549 (mpidr == cpu_logical_map(pairing)))
550 return cpu;
551 }
552 return -EINVAL;
553}
554
555static void bL_switcher_trace_trigger_cpu(void *__always_unused info)
556{
557 trace_cpu_migrate_current(get_ns(), read_mpidr());
558}
559
560int bL_switcher_trace_trigger(void)
561{
562 int ret;
563
564 preempt_disable();
565
566 bL_switcher_trace_trigger_cpu(NULL);
567 ret = smp_call_function(bL_switcher_trace_trigger_cpu, NULL, true);
568
569 preempt_enable();
570
571 return ret;
572}
573EXPORT_SYMBOL_GPL(bL_switcher_trace_trigger);
574
575static int bL_switcher_enable(void)
576{
577 int cpu, ret;
578
579 mutex_lock(&bL_switcher_activation_lock);
580 cpu_hotplug_driver_lock();
581 if (bL_switcher_active) {
582 cpu_hotplug_driver_unlock();
583 mutex_unlock(&bL_switcher_activation_lock);
584 return 0;
585 }
586
587 pr_info("big.LITTLE switcher initializing\n");
588
589 ret = bL_activation_notify(BL_NOTIFY_PRE_ENABLE);
590 if (ret)
591 goto error;
592
593 ret = bL_switcher_halve_cpus();
594 if (ret)
595 goto error;
596
597 bL_switcher_trace_trigger();
598
599 for_each_online_cpu(cpu) {
600 struct bL_thread *t = &bL_threads[cpu];
601 spin_lock_init(&t->lock);
602 init_waitqueue_head(&t->wq);
603 init_completion(&t->started);
604 t->wanted_cluster = -1;
605 t->task = bL_switcher_thread_create(cpu, t);
606 }
607
608 bL_switcher_active = 1;
609 bL_activation_notify(BL_NOTIFY_POST_ENABLE);
610 pr_info("big.LITTLE switcher initialized\n");
611 goto out;
612
613error:
614 pr_warn("big.LITTLE switcher initialization failed\n");
615 bL_activation_notify(BL_NOTIFY_POST_DISABLE);
616
617out:
618 cpu_hotplug_driver_unlock();
619 mutex_unlock(&bL_switcher_activation_lock);
620 return ret;
621}
622
623#ifdef CONFIG_SYSFS
624
625static void bL_switcher_disable(void)
626{
627 unsigned int cpu, cluster;
628 struct bL_thread *t;
629 struct task_struct *task;
630
631 mutex_lock(&bL_switcher_activation_lock);
632 cpu_hotplug_driver_lock();
633
634 if (!bL_switcher_active)
635 goto out;
636
637 if (bL_activation_notify(BL_NOTIFY_PRE_DISABLE) != 0) {
638 bL_activation_notify(BL_NOTIFY_POST_ENABLE);
639 goto out;
640 }
641
642 bL_switcher_active = 0;
643
644 /*
645 * To deactivate the switcher, we must shut down the switcher
646 * threads to prevent any other requests from being accepted.
647 * Then, if the final cluster for given logical CPU is not the
648 * same as the original one, we'll recreate a switcher thread
649 * just for the purpose of switching the CPU back without any
650 * possibility for interference from external requests.
651 */
652 for_each_online_cpu(cpu) {
653 t = &bL_threads[cpu];
654 task = t->task;
655 t->task = NULL;
656 if (!task || IS_ERR(task))
657 continue;
658 kthread_stop(task);
659 /* no more switch may happen on this CPU at this point */
660 cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 1);
661 if (cluster == bL_switcher_cpu_original_cluster[cpu])
662 continue;
663 init_completion(&t->started);
664 t->wanted_cluster = bL_switcher_cpu_original_cluster[cpu];
665 task = bL_switcher_thread_create(cpu, t);
666 if (!IS_ERR(task)) {
667 wait_for_completion(&t->started);
668 kthread_stop(task);
669 cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 1);
670 if (cluster == bL_switcher_cpu_original_cluster[cpu])
671 continue;
672 }
673 /* If execution gets here, we're in trouble. */
674 pr_crit("%s: unable to restore original cluster for CPU %d\n",
675 __func__, cpu);
676 pr_crit("%s: CPU %d can't be restored\n",
677 __func__, bL_switcher_cpu_pairing[cpu]);
678 cpumask_clear_cpu(bL_switcher_cpu_pairing[cpu],
679 &bL_switcher_removed_logical_cpus);
680 }
681
682 bL_switcher_restore_cpus();
683 bL_switcher_trace_trigger();
684
685 bL_activation_notify(BL_NOTIFY_POST_DISABLE);
686
687out:
688 cpu_hotplug_driver_unlock();
689 mutex_unlock(&bL_switcher_activation_lock);
690}
691
692static ssize_t bL_switcher_active_show(struct kobject *kobj,
693 struct kobj_attribute *attr, char *buf)
694{
695 return sprintf(buf, "%u\n", bL_switcher_active);
696}
697
698static ssize_t bL_switcher_active_store(struct kobject *kobj,
699 struct kobj_attribute *attr, const char *buf, size_t count)
700{
701 int ret;
702
703 switch (buf[0]) {
704 case '0':
705 bL_switcher_disable();
706 ret = 0;
707 break;
708 case '1':
709 ret = bL_switcher_enable();
710 break;
711 default:
712 ret = -EINVAL;
713 }
714
715 return (ret >= 0) ? count : ret;
716}
717
718static ssize_t bL_switcher_trace_trigger_store(struct kobject *kobj,
719 struct kobj_attribute *attr, const char *buf, size_t count)
720{
721 int ret = bL_switcher_trace_trigger();
722
723 return ret ? ret : count;
724}
725
726static struct kobj_attribute bL_switcher_active_attr =
727 __ATTR(active, 0644, bL_switcher_active_show, bL_switcher_active_store);
728
729static struct kobj_attribute bL_switcher_trace_trigger_attr =
730 __ATTR(trace_trigger, 0200, NULL, bL_switcher_trace_trigger_store);
731
732static struct attribute *bL_switcher_attrs[] = {
733 &bL_switcher_active_attr.attr,
734 &bL_switcher_trace_trigger_attr.attr,
735 NULL,
736};
737
738static struct attribute_group bL_switcher_attr_group = {
739 .attrs = bL_switcher_attrs,
740};
741
742static struct kobject *bL_switcher_kobj;
743
744static int __init bL_switcher_sysfs_init(void)
745{
746 int ret;
747
748 bL_switcher_kobj = kobject_create_and_add("bL_switcher", kernel_kobj);
749 if (!bL_switcher_kobj)
750 return -ENOMEM;
751 ret = sysfs_create_group(bL_switcher_kobj, &bL_switcher_attr_group);
752 if (ret)
753 kobject_put(bL_switcher_kobj);
754 return ret;
755}
756
757#endif /* CONFIG_SYSFS */
758
759bool bL_switcher_get_enabled(void)
760{
761 mutex_lock(&bL_switcher_activation_lock);
762
763 return bL_switcher_active;
764}
765EXPORT_SYMBOL_GPL(bL_switcher_get_enabled);
766
767void bL_switcher_put_enabled(void)
768{
769 mutex_unlock(&bL_switcher_activation_lock);
770}
771EXPORT_SYMBOL_GPL(bL_switcher_put_enabled);
772
773/*
774 * Veto any CPU hotplug operation on those CPUs we've removed
775 * while the switcher is active.
776 * We're just not ready to deal with that given the trickery involved.
777 */
778static int bL_switcher_hotplug_callback(struct notifier_block *nfb,
779 unsigned long action, void *hcpu)
780{
781 if (bL_switcher_active) {
782 int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu];
783 switch (action & 0xf) {
784 case CPU_UP_PREPARE:
785 case CPU_DOWN_PREPARE:
786 if (pairing == -1)
787 return NOTIFY_BAD;
788 }
789 }
790 return NOTIFY_DONE;
791}
792
793static bool no_bL_switcher;
794core_param(no_bL_switcher, no_bL_switcher, bool, 0644);
795
796static int __init bL_switcher_init(void)
797{
798 int ret;
799
800 if (MAX_NR_CLUSTERS != 2) {
801 pr_err("%s: only dual cluster systems are supported\n", __func__);
802 return -EINVAL;
803 }
804
805 cpu_notifier(bL_switcher_hotplug_callback, 0);
806
807 if (!no_bL_switcher) {
808 ret = bL_switcher_enable();
809 if (ret)
810 return ret;
811 }
812
813#ifdef CONFIG_SYSFS
814 ret = bL_switcher_sysfs_init();
815 if (ret)
816 pr_err("%s: unable to create sysfs entry\n", __func__);
817#endif
818
819 return 0;
820}
821
822late_initcall(bL_switcher_init);
diff --git a/arch/arm/common/bL_switcher_dummy_if.c b/arch/arm/common/bL_switcher_dummy_if.c
new file mode 100644
index 000000000000..3f47f1203c6b
--- /dev/null
+++ b/arch/arm/common/bL_switcher_dummy_if.c
@@ -0,0 +1,71 @@
1/*
2 * arch/arm/common/bL_switcher_dummy_if.c -- b.L switcher dummy interface
3 *
4 * Created by: Nicolas Pitre, November 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
6 *
7 * Dummy interface to user space for debugging purpose only.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/init.h>
15#include <linux/module.h>
16#include <linux/fs.h>
17#include <linux/miscdevice.h>
18#include <asm/uaccess.h>
19#include <asm/bL_switcher.h>
20
21static ssize_t bL_switcher_write(struct file *file, const char __user *buf,
22 size_t len, loff_t *pos)
23{
24 unsigned char val[3];
25 unsigned int cpu, cluster;
26 int ret;
27
28 pr_debug("%s\n", __func__);
29
30 if (len < 3)
31 return -EINVAL;
32
33 if (copy_from_user(val, buf, 3))
34 return -EFAULT;
35
36 /* format: <cpu#>,<cluster#> */
37 if (val[0] < '0' || val[0] > '9' ||
38 val[1] != ',' ||
39 val[2] < '0' || val[2] > '1')
40 return -EINVAL;
41
42 cpu = val[0] - '0';
43 cluster = val[2] - '0';
44 ret = bL_switch_request(cpu, cluster);
45
46 return ret ? : len;
47}
48
49static const struct file_operations bL_switcher_fops = {
50 .write = bL_switcher_write,
51 .owner = THIS_MODULE,
52};
53
54static struct miscdevice bL_switcher_device = {
55 MISC_DYNAMIC_MINOR,
56 "b.L_switcher",
57 &bL_switcher_fops
58};
59
60static int __init bL_switcher_dummy_if_init(void)
61{
62 return misc_register(&bL_switcher_device);
63}
64
65static void __exit bL_switcher_dummy_if_exit(void)
66{
67 misc_deregister(&bL_switcher_device);
68}
69
70module_init(bL_switcher_dummy_if_init);
71module_exit(bL_switcher_dummy_if_exit);
diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c
index 370236dd1a03..4a2b32fd53a1 100644
--- a/arch/arm/common/mcpm_entry.c
+++ b/arch/arm/common/mcpm_entry.c
@@ -27,6 +27,18 @@ void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
27 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]); 27 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
28} 28}
29 29
30extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
31
32void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
33 unsigned long poke_phys_addr, unsigned long poke_val)
34{
35 unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
36 poke[0] = poke_phys_addr;
37 poke[1] = poke_val;
38 __cpuc_flush_dcache_area((void *)poke, 8);
39 outer_clean_range(__pa(poke), __pa(poke + 2));
40}
41
30static const struct mcpm_platform_ops *platform_ops; 42static const struct mcpm_platform_ops *platform_ops;
31 43
32int __init mcpm_platform_register(const struct mcpm_platform_ops *ops) 44int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
diff --git a/arch/arm/common/mcpm_head.S b/arch/arm/common/mcpm_head.S
index 39c96df3477a..49dd5352fe70 100644
--- a/arch/arm/common/mcpm_head.S
+++ b/arch/arm/common/mcpm_head.S
@@ -71,12 +71,19 @@ ENTRY(mcpm_entry_point)
71 * position independent way. 71 * position independent way.
72 */ 72 */
73 adr r5, 3f 73 adr r5, 3f
74 ldmia r5, {r6, r7, r8, r11} 74 ldmia r5, {r0, r6, r7, r8, r11}
75 add r0, r5, r0 @ r0 = mcpm_entry_early_pokes
75 add r6, r5, r6 @ r6 = mcpm_entry_vectors 76 add r6, r5, r6 @ r6 = mcpm_entry_vectors
76 ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys 77 ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys
77 add r8, r5, r8 @ r8 = mcpm_sync 78 add r8, r5, r8 @ r8 = mcpm_sync
78 add r11, r5, r11 @ r11 = first_man_locks 79 add r11, r5, r11 @ r11 = first_man_locks
79 80
81 @ Perform an early poke, if any
82 add r0, r0, r4, lsl #3
83 ldmia r0, {r0, r1}
84 teq r0, #0
85 strne r1, [r0]
86
80 mov r0, #MCPM_SYNC_CLUSTER_SIZE 87 mov r0, #MCPM_SYNC_CLUSTER_SIZE
81 mla r8, r0, r10, r8 @ r8 = sync cluster base 88 mla r8, r0, r10, r8 @ r8 = sync cluster base
82 89
@@ -195,7 +202,8 @@ mcpm_entry_gated:
195 202
196 .align 2 203 .align 2
197 204
1983: .word mcpm_entry_vectors - . 2053: .word mcpm_entry_early_pokes - .
206 .word mcpm_entry_vectors - 3b
199 .word mcpm_power_up_setup_phys - 3b 207 .word mcpm_power_up_setup_phys - 3b
200 .word mcpm_sync - 3b 208 .word mcpm_sync - 3b
201 .word first_man_locks - 3b 209 .word first_man_locks - 3b
@@ -214,6 +222,10 @@ first_man_locks:
214ENTRY(mcpm_entry_vectors) 222ENTRY(mcpm_entry_vectors)
215 .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER 223 .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
216 224
225 .type mcpm_entry_early_pokes, #object
226ENTRY(mcpm_entry_early_pokes)
227 .space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
228
217 .type mcpm_power_up_setup_phys, #object 229 .type mcpm_power_up_setup_phys, #object
218ENTRY(mcpm_power_up_setup_phys) 230ENTRY(mcpm_power_up_setup_phys)
219 .space 4 @ set by mcpm_sync_init() 231 .space 4 @ set by mcpm_sync_init()
diff --git a/arch/arm/include/asm/bL_switcher.h b/arch/arm/include/asm/bL_switcher.h
new file mode 100644
index 000000000000..1714800fa113
--- /dev/null
+++ b/arch/arm/include/asm/bL_switcher.h
@@ -0,0 +1,77 @@
1/*
2 * arch/arm/include/asm/bL_switcher.h
3 *
4 * Created by: Nicolas Pitre, April 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#ifndef ASM_BL_SWITCHER_H
13#define ASM_BL_SWITCHER_H
14
15#include <linux/compiler.h>
16#include <linux/types.h>
17
18typedef void (*bL_switch_completion_handler)(void *cookie);
19
20int bL_switch_request_cb(unsigned int cpu, unsigned int new_cluster_id,
21 bL_switch_completion_handler completer,
22 void *completer_cookie);
23static inline int bL_switch_request(unsigned int cpu, unsigned int new_cluster_id)
24{
25 return bL_switch_request_cb(cpu, new_cluster_id, NULL, NULL);
26}
27
28/*
29 * Register here to be notified about runtime enabling/disabling of
30 * the switcher.
31 *
32 * The notifier chain is called with the switcher activation lock held:
33 * the switcher will not be enabled or disabled during callbacks.
34 * Callbacks must not call bL_switcher_{get,put}_enabled().
35 */
36#define BL_NOTIFY_PRE_ENABLE 0
37#define BL_NOTIFY_POST_ENABLE 1
38#define BL_NOTIFY_PRE_DISABLE 2
39#define BL_NOTIFY_POST_DISABLE 3
40
41#ifdef CONFIG_BL_SWITCHER
42
43int bL_switcher_register_notifier(struct notifier_block *nb);
44int bL_switcher_unregister_notifier(struct notifier_block *nb);
45
46/*
47 * Use these functions to temporarily prevent enabling/disabling of
48 * the switcher.
49 * bL_switcher_get_enabled() returns true if the switcher is currently
50 * enabled. Each call to bL_switcher_get_enabled() must be followed
51 * by a call to bL_switcher_put_enabled(). These functions are not
52 * recursive.
53 */
54bool bL_switcher_get_enabled(void);
55void bL_switcher_put_enabled(void);
56
57int bL_switcher_trace_trigger(void);
58int bL_switcher_get_logical_index(u32 mpidr);
59
60#else
61static inline int bL_switcher_register_notifier(struct notifier_block *nb)
62{
63 return 0;
64}
65
66static inline int bL_switcher_unregister_notifier(struct notifier_block *nb)
67{
68 return 0;
69}
70
71static inline bool bL_switcher_get_enabled(void) { return false; }
72static inline void bL_switcher_put_enabled(void) { }
73static inline int bL_switcher_trace_trigger(void) { return 0; }
74static inline int bL_switcher_get_logical_index(u32 mpidr) { return -EUNATCH; }
75#endif /* CONFIG_BL_SWITCHER */
76
77#endif
diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h
index 2740c2a2df63..3d7351c844aa 100644
--- a/arch/arm/include/asm/hardirq.h
+++ b/arch/arm/include/asm/hardirq.h
@@ -5,7 +5,7 @@
5#include <linux/threads.h> 5#include <linux/threads.h>
6#include <asm/irq.h> 6#include <asm/irq.h>
7 7
8#define NR_IPI 6 8#define NR_IPI 7
9 9
10typedef struct { 10typedef struct {
11 unsigned int __softirq_pending; 11 unsigned int __softirq_pending;
diff --git a/arch/arm/include/asm/mcpm.h b/arch/arm/include/asm/mcpm.h
index 0f7b7620e9a5..7626a7fd4938 100644
--- a/arch/arm/include/asm/mcpm.h
+++ b/arch/arm/include/asm/mcpm.h
@@ -42,6 +42,14 @@ extern void mcpm_entry_point(void);
42void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr); 42void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr);
43 43
44/* 44/*
45 * This sets an early poke i.e a value to be poked into some address
46 * from very early assembly code before the CPU is ungated. The
47 * address must be physical, and if 0 then nothing will happen.
48 */
49void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
50 unsigned long poke_phys_addr, unsigned long poke_val);
51
52/*
45 * CPU/cluster power operations API for higher subsystems to use. 53 * CPU/cluster power operations API for higher subsystems to use.
46 */ 54 */
47 55
diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h
index a8cae71caceb..22a3b9b5d4a1 100644
--- a/arch/arm/include/asm/smp.h
+++ b/arch/arm/include/asm/smp.h
@@ -84,6 +84,8 @@ extern void arch_send_call_function_single_ipi(int cpu);
84extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); 84extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
85extern void arch_send_wakeup_ipi_mask(const struct cpumask *mask); 85extern void arch_send_wakeup_ipi_mask(const struct cpumask *mask);
86 86
87extern int register_ipi_completion(struct completion *completion, int cpu);
88
87struct smp_operations { 89struct smp_operations {
88#ifdef CONFIG_SMP 90#ifdef CONFIG_SMP
89 /* 91 /*
diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S
index db1536b8b30b..622460201911 100644
--- a/arch/arm/kernel/sleep.S
+++ b/arch/arm/kernel/sleep.S
@@ -55,6 +55,7 @@
55 * specific registers and some other data for resume. 55 * specific registers and some other data for resume.
56 * r0 = suspend function arg0 56 * r0 = suspend function arg0
57 * r1 = suspend function 57 * r1 = suspend function
58 * r2 = MPIDR value the resuming CPU will use
58 */ 59 */
59ENTRY(__cpu_suspend) 60ENTRY(__cpu_suspend)
60 stmfd sp!, {r4 - r11, lr} 61 stmfd sp!, {r4 - r11, lr}
@@ -67,23 +68,18 @@ ENTRY(__cpu_suspend)
67 mov r5, sp @ current virtual SP 68 mov r5, sp @ current virtual SP
68 add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn 69 add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn
69 sub sp, sp, r4 @ allocate CPU state on stack 70 sub sp, sp, r4 @ allocate CPU state on stack
70 stmfd sp!, {r0, r1} @ save suspend func arg and pointer
71 add r0, sp, #8 @ save pointer to save block
72 mov r1, r4 @ size of save block
73 mov r2, r5 @ virtual SP
74 ldr r3, =sleep_save_sp 71 ldr r3, =sleep_save_sp
72 stmfd sp!, {r0, r1} @ save suspend func arg and pointer
75 ldr r3, [r3, #SLEEP_SAVE_SP_VIRT] 73 ldr r3, [r3, #SLEEP_SAVE_SP_VIRT]
76 ALT_SMP(mrc p15, 0, r9, c0, c0, 5) 74 ALT_SMP(ldr r0, =mpidr_hash)
77 ALT_UP_B(1f) 75 ALT_UP_B(1f)
78 ldr r8, =mpidr_hash 76 /* This ldmia relies on the memory layout of the mpidr_hash struct */
79 /* 77 ldmia r0, {r1, r6-r8} @ r1 = mpidr mask (r6,r7,r8) = l[0,1,2] shifts
80 * This ldmia relies on the memory layout of the mpidr_hash 78 compute_mpidr_hash r0, r6, r7, r8, r2, r1
81 * struct mpidr_hash. 79 add r3, r3, r0, lsl #2
82 */ 801: mov r2, r5 @ virtual SP
83 ldmia r8, {r4-r7} @ r4 = mpidr mask (r5,r6,r7) = l[0,1,2] shifts 81 mov r1, r4 @ size of save block
84 compute_mpidr_hash lr, r5, r6, r7, r9, r4 82 add r0, sp, #8 @ pointer to save block
85 add r3, r3, lr, lsl #2
861:
87 bl __cpu_suspend_save 83 bl __cpu_suspend_save
88 adr lr, BSYM(cpu_suspend_abort) 84 adr lr, BSYM(cpu_suspend_abort)
89 ldmfd sp!, {r0, pc} @ call suspend fn 85 ldmfd sp!, {r0, pc} @ call suspend fn
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 72024ea8a3a6..7d80a549cae5 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -66,6 +66,7 @@ enum ipi_msg_type {
66 IPI_CALL_FUNC, 66 IPI_CALL_FUNC,
67 IPI_CALL_FUNC_SINGLE, 67 IPI_CALL_FUNC_SINGLE,
68 IPI_CPU_STOP, 68 IPI_CPU_STOP,
69 IPI_COMPLETION,
69}; 70};
70 71
71static DECLARE_COMPLETION(cpu_running); 72static DECLARE_COMPLETION(cpu_running);
@@ -456,6 +457,7 @@ static const char *ipi_types[NR_IPI] = {
456 S(IPI_CALL_FUNC, "Function call interrupts"), 457 S(IPI_CALL_FUNC, "Function call interrupts"),
457 S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), 458 S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
458 S(IPI_CPU_STOP, "CPU stop interrupts"), 459 S(IPI_CPU_STOP, "CPU stop interrupts"),
460 S(IPI_COMPLETION, "completion interrupts"),
459}; 461};
460 462
461void show_ipi_list(struct seq_file *p, int prec) 463void show_ipi_list(struct seq_file *p, int prec)
@@ -515,6 +517,19 @@ static void ipi_cpu_stop(unsigned int cpu)
515 cpu_relax(); 517 cpu_relax();
516} 518}
517 519
520static DEFINE_PER_CPU(struct completion *, cpu_completion);
521
522int register_ipi_completion(struct completion *completion, int cpu)
523{
524 per_cpu(cpu_completion, cpu) = completion;
525 return IPI_COMPLETION;
526}
527
528static void ipi_complete(unsigned int cpu)
529{
530 complete(per_cpu(cpu_completion, cpu));
531}
532
518/* 533/*
519 * Main handler for inter-processor interrupts 534 * Main handler for inter-processor interrupts
520 */ 535 */
@@ -565,6 +580,12 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
565 irq_exit(); 580 irq_exit();
566 break; 581 break;
567 582
583 case IPI_COMPLETION:
584 irq_enter();
585 ipi_complete(cpu);
586 irq_exit();
587 break;
588
568 default: 589 default:
569 printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n", 590 printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
570 cpu, ipinr); 591 cpu, ipinr);
diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c
index 41cf3cbf756d..2835d35234ca 100644
--- a/arch/arm/kernel/suspend.c
+++ b/arch/arm/kernel/suspend.c
@@ -10,7 +10,7 @@
10#include <asm/suspend.h> 10#include <asm/suspend.h>
11#include <asm/tlbflush.h> 11#include <asm/tlbflush.h>
12 12
13extern int __cpu_suspend(unsigned long, int (*)(unsigned long)); 13extern int __cpu_suspend(unsigned long, int (*)(unsigned long), u32 cpuid);
14extern void cpu_resume_mmu(void); 14extern void cpu_resume_mmu(void);
15 15
16#ifdef CONFIG_MMU 16#ifdef CONFIG_MMU
@@ -21,6 +21,7 @@ extern void cpu_resume_mmu(void);
21int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) 21int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
22{ 22{
23 struct mm_struct *mm = current->active_mm; 23 struct mm_struct *mm = current->active_mm;
24 u32 __mpidr = cpu_logical_map(smp_processor_id());
24 int ret; 25 int ret;
25 26
26 if (!idmap_pgd) 27 if (!idmap_pgd)
@@ -32,7 +33,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
32 * resume (indicated by a zero return code), we need to switch 33 * resume (indicated by a zero return code), we need to switch
33 * back to the correct page tables. 34 * back to the correct page tables.
34 */ 35 */
35 ret = __cpu_suspend(arg, fn); 36 ret = __cpu_suspend(arg, fn, __mpidr);
36 if (ret == 0) { 37 if (ret == 0) {
37 cpu_switch_mm(mm->pgd, mm); 38 cpu_switch_mm(mm->pgd, mm);
38 local_flush_bp_all(); 39 local_flush_bp_all();
@@ -44,7 +45,8 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
44#else 45#else
45int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) 46int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
46{ 47{
47 return __cpu_suspend(arg, fn); 48 u32 __mpidr = cpu_logical_map(smp_processor_id());
49 return __cpu_suspend(arg, fn, __mpidr);
48} 50}
49#define idmap_pgd NULL 51#define idmap_pgd NULL
50#endif 52#endif
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
index d0e948084eaf..9031171c141b 100644
--- a/drivers/irqchip/irq-gic.c
+++ b/drivers/irqchip/irq-gic.c
@@ -253,10 +253,9 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
253 if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids) 253 if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
254 return -EINVAL; 254 return -EINVAL;
255 255
256 raw_spin_lock(&irq_controller_lock);
256 mask = 0xff << shift; 257 mask = 0xff << shift;
257 bit = gic_cpu_map[cpu] << shift; 258 bit = gic_cpu_map[cpu] << shift;
258
259 raw_spin_lock(&irq_controller_lock);
260 val = readl_relaxed(reg) & ~mask; 259 val = readl_relaxed(reg) & ~mask;
261 writel_relaxed(val | bit, reg); 260 writel_relaxed(val | bit, reg);
262 raw_spin_unlock(&irq_controller_lock); 261 raw_spin_unlock(&irq_controller_lock);
@@ -652,7 +651,9 @@ static void __init gic_pm_init(struct gic_chip_data *gic)
652void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) 651void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
653{ 652{
654 int cpu; 653 int cpu;
655 unsigned long map = 0; 654 unsigned long flags, map = 0;
655
656 raw_spin_lock_irqsave(&irq_controller_lock, flags);
656 657
657 /* Convert our logical CPU mask into a physical one. */ 658 /* Convert our logical CPU mask into a physical one. */
658 for_each_cpu(cpu, mask) 659 for_each_cpu(cpu, mask)
@@ -666,7 +667,149 @@ void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
666 667
667 /* this always happens on GIC0 */ 668 /* this always happens on GIC0 */
668 writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); 669 writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
670
671 raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
672}
673#endif
674
675#ifdef CONFIG_BL_SWITCHER
676/*
677 * gic_send_sgi - send a SGI directly to given CPU interface number
678 *
679 * cpu_id: the ID for the destination CPU interface
680 * irq: the IPI number to send a SGI for
681 */
682void gic_send_sgi(unsigned int cpu_id, unsigned int irq)
683{
684 BUG_ON(cpu_id >= NR_GIC_CPU_IF);
685 cpu_id = 1 << cpu_id;
686 /* this always happens on GIC0 */
687 writel_relaxed((cpu_id << 16) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
688}
689
690/*
691 * gic_get_cpu_id - get the CPU interface ID for the specified CPU
692 *
693 * @cpu: the logical CPU number to get the GIC ID for.
694 *
695 * Return the CPU interface ID for the given logical CPU number,
696 * or -1 if the CPU number is too large or the interface ID is
697 * unknown (more than one bit set).
698 */
699int gic_get_cpu_id(unsigned int cpu)
700{
701 unsigned int cpu_bit;
702
703 if (cpu >= NR_GIC_CPU_IF)
704 return -1;
705 cpu_bit = gic_cpu_map[cpu];
706 if (cpu_bit & (cpu_bit - 1))
707 return -1;
708 return __ffs(cpu_bit);
669} 709}
710
711/*
712 * gic_migrate_target - migrate IRQs to another CPU interface
713 *
714 * @new_cpu_id: the CPU target ID to migrate IRQs to
715 *
716 * Migrate all peripheral interrupts with a target matching the current CPU
717 * to the interface corresponding to @new_cpu_id. The CPU interface mapping
718 * is also updated. Targets to other CPU interfaces are unchanged.
719 * This must be called with IRQs locally disabled.
720 */
721void gic_migrate_target(unsigned int new_cpu_id)
722{
723 unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
724 void __iomem *dist_base;
725 int i, ror_val, cpu = smp_processor_id();
726 u32 val, cur_target_mask, active_mask;
727
728 if (gic_nr >= MAX_GIC_NR)
729 BUG();
730
731 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
732 if (!dist_base)
733 return;
734 gic_irqs = gic_data[gic_nr].gic_irqs;
735
736 cur_cpu_id = __ffs(gic_cpu_map[cpu]);
737 cur_target_mask = 0x01010101 << cur_cpu_id;
738 ror_val = (cur_cpu_id - new_cpu_id) & 31;
739
740 raw_spin_lock(&irq_controller_lock);
741
742 /* Update the target interface for this logical CPU */
743 gic_cpu_map[cpu] = 1 << new_cpu_id;
744
745 /*
746 * Find all the peripheral interrupts targetting the current
747 * CPU interface and migrate them to the new CPU interface.
748 * We skip DIST_TARGET 0 to 7 as they are read-only.
749 */
750 for (i = 8; i < DIV_ROUND_UP(gic_irqs, 4); i++) {
751 val = readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
752 active_mask = val & cur_target_mask;
753 if (active_mask) {
754 val &= ~active_mask;
755 val |= ror32(active_mask, ror_val);
756 writel_relaxed(val, dist_base + GIC_DIST_TARGET + i*4);
757 }
758 }
759
760 raw_spin_unlock(&irq_controller_lock);
761
762 /*
763 * Now let's migrate and clear any potential SGIs that might be
764 * pending for us (cur_cpu_id). Since GIC_DIST_SGI_PENDING_SET
765 * is a banked register, we can only forward the SGI using
766 * GIC_DIST_SOFTINT. The original SGI source is lost but Linux
767 * doesn't use that information anyway.
768 *
769 * For the same reason we do not adjust SGI source information
770 * for previously sent SGIs by us to other CPUs either.
771 */
772 for (i = 0; i < 16; i += 4) {
773 int j;
774 val = readl_relaxed(dist_base + GIC_DIST_SGI_PENDING_SET + i);
775 if (!val)
776 continue;
777 writel_relaxed(val, dist_base + GIC_DIST_SGI_PENDING_CLEAR + i);
778 for (j = i; j < i + 4; j++) {
779 if (val & 0xff)
780 writel_relaxed((1 << (new_cpu_id + 16)) | j,
781 dist_base + GIC_DIST_SOFTINT);
782 val >>= 8;
783 }
784 }
785}
786
787/*
788 * gic_get_sgir_physaddr - get the physical address for the SGI register
789 *
790 * REturn the physical address of the SGI register to be used
791 * by some early assembly code when the kernel is not yet available.
792 */
793static unsigned long gic_dist_physaddr;
794
795unsigned long gic_get_sgir_physaddr(void)
796{
797 if (!gic_dist_physaddr)
798 return 0;
799 return gic_dist_physaddr + GIC_DIST_SOFTINT;
800}
801
802void __init gic_init_physaddr(struct device_node *node)
803{
804 struct resource res;
805 if (of_address_to_resource(node, 0, &res) == 0) {
806 gic_dist_physaddr = res.start;
807 pr_info("GIC physical location is %#lx\n", gic_dist_physaddr);
808 }
809}
810
811#else
812#define gic_init_physaddr(node) do { } while (0)
670#endif 813#endif
671 814
672static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 815static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
@@ -850,6 +993,8 @@ int __init gic_of_init(struct device_node *node, struct device_node *parent)
850 percpu_offset = 0; 993 percpu_offset = 0;
851 994
852 gic_init_bases(gic_cnt, -1, dist_base, cpu_base, percpu_offset, node); 995 gic_init_bases(gic_cnt, -1, dist_base, cpu_base, percpu_offset, node);
996 if (!gic_cnt)
997 gic_init_physaddr(node);
853 998
854 if (parent) { 999 if (parent) {
855 irq = irq_of_parse_and_map(node, 0); 1000 irq = irq_of_parse_and_map(node, 0);
diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h
index 0e5d9ecdb2b6..cac496b1e279 100644
--- a/include/linux/irqchip/arm-gic.h
+++ b/include/linux/irqchip/arm-gic.h
@@ -31,6 +31,8 @@
31#define GIC_DIST_TARGET 0x800 31#define GIC_DIST_TARGET 0x800
32#define GIC_DIST_CONFIG 0xc00 32#define GIC_DIST_CONFIG 0xc00
33#define GIC_DIST_SOFTINT 0xf00 33#define GIC_DIST_SOFTINT 0xf00
34#define GIC_DIST_SGI_PENDING_CLEAR 0xf10
35#define GIC_DIST_SGI_PENDING_SET 0xf20
34 36
35#define GICH_HCR 0x0 37#define GICH_HCR 0x0
36#define GICH_VTR 0x4 38#define GICH_VTR 0x4
@@ -74,6 +76,11 @@ static inline void gic_init(unsigned int nr, int start,
74 gic_init_bases(nr, start, dist, cpu, 0, NULL); 76 gic_init_bases(nr, start, dist, cpu, 0, NULL);
75} 77}
76 78
79void gic_send_sgi(unsigned int cpu_id, unsigned int irq);
80int gic_get_cpu_id(unsigned int cpu);
81void gic_migrate_target(unsigned int new_cpu_id);
82unsigned long gic_get_sgir_physaddr(void);
83
77#endif /* __ASSEMBLY */ 84#endif /* __ASSEMBLY */
78 85
79#endif 86#endif
diff --git a/include/trace/events/power_cpu_migrate.h b/include/trace/events/power_cpu_migrate.h
new file mode 100644
index 000000000000..f76dd4de625e
--- /dev/null
+++ b/include/trace/events/power_cpu_migrate.h
@@ -0,0 +1,67 @@
1#undef TRACE_SYSTEM
2#define TRACE_SYSTEM power
3
4#if !defined(_TRACE_POWER_CPU_MIGRATE_H) || defined(TRACE_HEADER_MULTI_READ)
5#define _TRACE_POWER_CPU_MIGRATE_H
6
7#include <linux/tracepoint.h>
8
9#define __cpu_migrate_proto \
10 TP_PROTO(u64 timestamp, \
11 u32 cpu_hwid)
12#define __cpu_migrate_args \
13 TP_ARGS(timestamp, \
14 cpu_hwid)
15
16DECLARE_EVENT_CLASS(cpu_migrate,
17
18 __cpu_migrate_proto,
19 __cpu_migrate_args,
20
21 TP_STRUCT__entry(
22 __field(u64, timestamp )
23 __field(u32, cpu_hwid )
24 ),
25
26 TP_fast_assign(
27 __entry->timestamp = timestamp;
28 __entry->cpu_hwid = cpu_hwid;
29 ),
30
31 TP_printk("timestamp=%llu cpu_hwid=0x%08lX",
32 (unsigned long long)__entry->timestamp,
33 (unsigned long)__entry->cpu_hwid
34 )
35);
36
37#define __define_cpu_migrate_event(name) \
38 DEFINE_EVENT(cpu_migrate, cpu_migrate_##name, \
39 __cpu_migrate_proto, \
40 __cpu_migrate_args \
41 )
42
43__define_cpu_migrate_event(begin);
44__define_cpu_migrate_event(finish);
45__define_cpu_migrate_event(current);
46
47#undef __define_cpu_migrate
48#undef __cpu_migrate_proto
49#undef __cpu_migrate_args
50
51/* This file can get included multiple times, TRACE_HEADER_MULTI_READ at top */
52#ifndef _PWR_CPU_MIGRATE_EVENT_AVOID_DOUBLE_DEFINING
53#define _PWR_CPU_MIGRATE_EVENT_AVOID_DOUBLE_DEFINING
54
55/*
56 * Set from_phys_cpu and to_phys_cpu to CPU_MIGRATE_ALL_CPUS to indicate
57 * a whole-cluster migration:
58 */
59#define CPU_MIGRATE_ALL_CPUS 0x80000000U
60#endif
61
62#endif /* _TRACE_POWER_CPU_MIGRATE_H */
63
64/* This part must be outside protection */
65#undef TRACE_INCLUDE_FILE
66#define TRACE_INCLUDE_FILE power_cpu_migrate
67#include <trace/define_trace.h>
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-25 18:32:37 -0400